+
+## Enviroment setup
+
+The code can be run on multiple GPUs or TPUs with different distribution
+strategies. See the TensorFlow distributed training
+[guide](https://www.tensorflow.org/guide/distributed_training) for an overview
+of `tf.distribute`.
+
+The code is compatible with TensorFlow 2.4+. See requirements.txt for all
+prerequisites, and you can also install them using the following command. `pip
+install -r ./official/requirements.txt`
+
+## Pretraining
+To pretrain the model on Imagenet, try the following command:
+
+```
+python3 -m official.vision.beta.projects.simclr.train \
+ --mode=train_and_eval \
+ --experiment=simclr_pretraining \
+ --model_dir={MODEL_DIR} \
+ --config_file={CONFIG_FILE}
+```
+
+An example of the config file can be found [here](./configs/experiments/imagenet_simclr_pretrain_gpu.yaml)
+
+
+## Semi-supervised learning and fine-tuning the whole network
+
+You can access 1% and 10% ImageNet subsets used for semi-supervised learning via
+[tensorflow datasets](https://www.tensorflow.org/datasets/catalog/imagenet2012_subset).
+You can also find image IDs of these subsets in `imagenet_subsets/`.
+
+To fine-tune the whole network, refer to the following command:
+
+```
+python3 -m official.vision.beta.projects.simclr.train \
+ --mode=train_and_eval \
+ --experiment=simclr_finetuning \
+ --model_dir={MODEL_DIR} \
+ --config_file={CONFIG_FILE}
+```
+
+An example of the config file can be found [here](./configs/experiments/imagenet_simclr_finetune_gpu.yaml).
+
+## Cite
+
+[SimCLR paper](https://arxiv.org/abs/2002.05709):
+
+```
+@article{chen2020simple,
+ title={A Simple Framework for Contrastive Learning of Visual Representations},
+ author={Chen, Ting and Kornblith, Simon and Norouzi, Mohammad and Hinton, Geoffrey},
+ journal={arXiv preprint arXiv:2002.05709},
+ year={2020}
+}
+```
+
+[SimCLRv2 paper](https://arxiv.org/abs/2006.10029):
+
+```
+@article{chen2020big,
+ title={Big Self-Supervised Models are Strong Semi-Supervised Learners},
+ author={Chen, Ting and Kornblith, Simon and Swersky, Kevin and Norouzi, Mohammad and Hinton, Geoffrey},
+ journal={arXiv preprint arXiv:2006.10029},
+ year={2020}
+}
+```
diff --git a/official/vision/beta/projects/simclr/common/registry_imports.py b/official/vision/beta/projects/simclr/common/registry_imports.py
new file mode 100644
index 0000000000000000000000000000000000000000..11a3b290811b168f010d96852d0661c6a384b576
--- /dev/null
+++ b/official/vision/beta/projects/simclr/common/registry_imports.py
@@ -0,0 +1,36 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""All necessary imports for registration."""
+
+# pylint: disable=unused-import
+from official.common import registry_imports
+from official.vision.beta.projects.simclr.configs import simclr
+from official.vision.beta.projects.simclr.losses import contrastive_losses
+from official.vision.beta.projects.simclr.modeling import simclr_model
+from official.vision.beta.projects.simclr.tasks import simclr as simclr_task
diff --git a/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml b/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5d5bd642efa5c32930a3c413d927c733af9bbf3a
--- /dev/null
+++ b/official/vision/beta/projects/simclr/configs/experiments/cifar_simclr_pretrain.yaml
@@ -0,0 +1,79 @@
+# Cifar classification.
+runtime:
+ distribution_strategy: 'mirrored'
+ mixed_precision_dtype: 'float16'
+ loss_scale: 'dynamic'
+ num_gpus: 16
+task:
+ model:
+ mode: 'pretrain'
+ input_size: [32, 32, 3]
+ backbone:
+ type: 'resnet'
+ resnet:
+ model_id: 50
+ backbone_trainable: true
+ projection_head:
+ proj_output_dim: 64
+ num_proj_layers: 2
+ ft_proj_idx: 1
+ supervised_head:
+ num_classes: 10
+ norm_activation:
+ use_sync_bn: true
+ norm_momentum: 0.9
+ norm_epsilon: 0.00001
+ loss:
+ projection_norm: true
+ temperature: 0.2
+ evaluation:
+ top_k: 5
+ one_hot: true
+ train_data:
+ tfds_name: 'cifar10'
+ tfds_split: 'train'
+ input_path: ''
+ is_training: true
+ global_batch_size: 512
+ dtype: 'float16'
+ parser:
+ mode: 'pretrain'
+ aug_color_jitter_strength: 0.5
+ aug_rand_blur: false
+ decoder:
+ decode_label: true
+ validation_data:
+ tfds_name: 'cifar10'
+ tfds_split: 'test'
+ input_path: ''
+ is_training: false
+ global_batch_size: 512
+ dtype: 'float16'
+ drop_remainder: false
+ parser:
+ mode: 'pretrain'
+ decoder:
+ decode_label: true
+trainer:
+ train_steps: 48000 # 500 epochs
+ validation_steps: 18 # NUM_EXAMPLES (10000) // global_batch_size
+ validation_interval: 96
+ steps_per_loop: 96 # NUM_EXAMPLES (50000) // global_batch_size
+ summary_interval: 96
+ checkpoint_interval: 96
+ optimizer_config:
+ optimizer:
+ type: 'lars'
+ lars:
+ momentum: 0.9
+ weight_decay_rate: 0.000001
+ exclude_from_weight_decay: ['batch_normalization', 'bias']
+ learning_rate:
+ type: 'cosine'
+ cosine:
+ initial_learning_rate: 0.6 # 0.3 × BatchSize / 256
+ decay_steps: 43200 # train_steps - warmup_steps
+ warmup:
+ type: 'linear'
+ linear:
+ warmup_steps: 4800 # 10% of total epochs
diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c79c0af7f9027ecae8ea0a05fb31bd2465b70f77
--- /dev/null
+++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_finetune_gpu.yaml
@@ -0,0 +1,72 @@
+# ImageNet classification.
+runtime:
+ distribution_strategy: 'mirrored'
+ mixed_precision_dtype: 'float16'
+ loss_scale: 'dynamic'
+ num_gpus: 16
+task:
+ model:
+ mode: 'finetune'
+ input_size: [224, 224, 3]
+ backbone:
+ type: 'resnet'
+ resnet:
+ model_id: 50
+ backbone_trainable: true
+ projection_head:
+ proj_output_dim: 128
+ num_proj_layers: 3
+ ft_proj_idx: 1
+ supervised_head:
+ num_classes: 1001
+ zero_init: true
+ norm_activation:
+ use_sync_bn: false
+ norm_momentum: 0.9
+ norm_epsilon: 0.00001
+ loss:
+ label_smoothing: 0.0
+ one_hot: true
+ evaluation:
+ top_k: 5
+ one_hot: true
+ init_checkpoint: gs://tf_model_garden/official/simclr/r50_1x
+ init_checkpoint_modules: 'backbone_projection'
+ train_data:
+ tfds_name: 'imagenet2012_subset/10pct'
+ tfds_split: 'train'
+ input_path: ''
+ is_training: true
+ global_batch_size: 1024
+ dtype: 'float16'
+ parser:
+ mode: 'finetune'
+ validation_data:
+ tfds_name: 'imagenet2012_subset/10pct'
+ tfds_split: 'validation'
+ input_path: ''
+ is_training: false
+ global_batch_size: 1024
+ dtype: 'float16'
+ drop_remainder: false
+ parser:
+ mode: 'finetune'
+trainer:
+ train_steps: 12500 # 100 epochs
+ validation_steps: 49 # NUM_EXAMPLES (50000) // global_batch_size
+ validation_interval: 125
+ steps_per_loop: 125 # NUM_EXAMPLES (1281167) // global_batch_size
+ summary_interval: 125
+ checkpoint_interval: 125
+ optimizer_config:
+ optimizer:
+ type: 'lars'
+ lars:
+ momentum: 0.9
+ weight_decay_rate: 0.0
+ exclude_from_weight_decay: ['batch_normalization', 'bias']
+ learning_rate:
+ type: 'cosine'
+ cosine:
+ initial_learning_rate: 0.04 # 0.01 × BatchSize / 512
+ decay_steps: 12500 # train_steps
diff --git a/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9e7c326c3d68cfe89500c38d6d0e7a676145bca8
--- /dev/null
+++ b/official/vision/beta/projects/simclr/configs/experiments/imagenet_simclr_pretrain_gpu.yaml
@@ -0,0 +1,73 @@
+# ImageNet classification.
+runtime:
+ distribution_strategy: 'mirrored'
+ mixed_precision_dtype: 'float16'
+ loss_scale: 'dynamic'
+ num_gpus: 16
+task:
+ model:
+ mode: 'pretrain'
+ input_size: [224, 224, 3]
+ backbone:
+ type: 'resnet'
+ resnet:
+ model_id: 50
+ backbone_trainable: true
+ projection_head:
+ proj_output_dim: 128
+ num_proj_layers: 3
+ ft_proj_idx: 0
+ supervised_head:
+ num_classes: 1001
+ norm_activation:
+ use_sync_bn: true
+ norm_momentum: 0.9
+ norm_epsilon: 0.00001
+ loss:
+ projection_norm: true
+ temperature: 0.1
+ evaluation:
+ top_k: 5
+ one_hot: true
+ train_data:
+ input_path: '/readahead/200M/placer/prod/home/distbelief/imagenet-tensorflow/imagenet-2012-tfrecord/train*'
+ is_training: true
+ global_batch_size: 2048
+ dtype: 'float16'
+ parser:
+ mode: 'pretrain'
+ decoder:
+ decode_label: true
+ validation_data:
+ input_path: '/readahead/200M/placer/prod/home/distbelief/imagenet-tensorflow/imagenet-2012-tfrecord/valid*'
+ is_training: false
+ global_batch_size: 2048
+ dtype: 'float16'
+ drop_remainder: false
+ parser:
+ mode: 'pretrain'
+ decoder:
+ decode_label: true
+trainer:
+ train_steps: 187200 # 300 epochs
+ validation_steps: 24 # NUM_EXAMPLES (50000) // global_batch_size
+ validation_interval: 624
+ steps_per_loop: 624 # NUM_EXAMPLES (1281167) // global_batch_size
+ summary_interval: 624
+ checkpoint_interval: 624
+ optimizer_config:
+ optimizer:
+ type: 'lars'
+ lars:
+ momentum: 0.9
+ weight_decay_rate: 0.000001
+ exclude_from_weight_decay: ['batch_normalization', 'bias']
+ learning_rate:
+ type: 'cosine'
+ cosine:
+ initial_learning_rate: 1.6 # 0.2 * BatchSize / 256
+ decay_steps: 177840 # train_steps - warmup_steps
+ warmup:
+ type: 'linear'
+ linear:
+ warmup_steps: 9360 # 5% of total epochs
diff --git a/official/vision/beta/projects/simclr/configs/simclr.py b/official/vision/beta/projects/simclr/configs/simclr.py
new file mode 100644
index 0000000000000000000000000000000000000000..f3264b428238b5839441c2c0ea086b71d848d74e
--- /dev/null
+++ b/official/vision/beta/projects/simclr/configs/simclr.py
@@ -0,0 +1,332 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SimCLR configurations."""
+import os
+from typing import List, Optional
+
+import dataclasses
+
+from official.core import config_definitions as cfg
+from official.core import exp_factory
+from official.modeling import hyperparams
+from official.modeling import optimization
+from official.vision.beta.configs import backbones
+from official.vision.beta.configs import common
+from official.vision.beta.projects.simclr.modeling import simclr_model
+
+
+@dataclasses.dataclass
+class Decoder(hyperparams.Config):
+ decode_label: bool = True
+
+
+@dataclasses.dataclass
+class Parser(hyperparams.Config):
+ """Parser config."""
+ aug_rand_crop: bool = True
+ aug_rand_hflip: bool = True
+ aug_color_distort: bool = True
+ aug_color_jitter_strength: float = 1.0
+ aug_color_jitter_impl: str = 'simclrv2' # 'simclrv1' or 'simclrv2'
+ aug_rand_blur: bool = True
+ parse_label: bool = True
+ test_crop: bool = True
+ mode: str = simclr_model.PRETRAIN
+
+
+@dataclasses.dataclass
+class DataConfig(cfg.DataConfig):
+ """Training data config."""
+ input_path: str = ''
+ global_batch_size: int = 0
+ is_training: bool = True
+ dtype: str = 'float32'
+ shuffle_buffer_size: int = 10000
+ cycle_length: int = 10
+ # simclr specific configs
+ parser: Parser = Parser()
+ decoder: Decoder = Decoder()
+
+
+@dataclasses.dataclass
+class ProjectionHead(hyperparams.Config):
+ proj_output_dim: int = 128
+ num_proj_layers: int = 3
+ ft_proj_idx: int = 1 # layer of the projection head to use for fine-tuning.
+
+
+@dataclasses.dataclass
+class SupervisedHead(hyperparams.Config):
+ num_classes: int = 1001
+ zero_init: bool = False
+
+
+@dataclasses.dataclass
+class ContrastiveLoss(hyperparams.Config):
+ projection_norm: bool = True
+ temperature: float = 0.1
+ l2_weight_decay: float = 0.0
+
+
+@dataclasses.dataclass
+class ClassificationLosses(hyperparams.Config):
+ label_smoothing: float = 0.0
+ one_hot: bool = True
+ l2_weight_decay: float = 0.0
+
+
+@dataclasses.dataclass
+class Evaluation(hyperparams.Config):
+ top_k: int = 5
+ one_hot: bool = True
+
+
+@dataclasses.dataclass
+class SimCLRModel(hyperparams.Config):
+ """SimCLR model config."""
+ input_size: List[int] = dataclasses.field(default_factory=list)
+ backbone: backbones.Backbone = backbones.Backbone(
+ type='resnet', resnet=backbones.ResNet())
+ projection_head: ProjectionHead = ProjectionHead(
+ proj_output_dim=128,
+ num_proj_layers=3,
+ ft_proj_idx=1)
+ supervised_head: SupervisedHead = SupervisedHead(num_classes=1001)
+ norm_activation: common.NormActivation = common.NormActivation(
+ norm_momentum=0.9, norm_epsilon=1e-5, use_sync_bn=False)
+ mode: str = simclr_model.PRETRAIN
+ backbone_trainable: bool = True
+
+
+@dataclasses.dataclass
+class SimCLRPretrainTask(cfg.TaskConfig):
+ """SimCLR pretraining task config."""
+ model: SimCLRModel = SimCLRModel(mode=simclr_model.PRETRAIN)
+ train_data: DataConfig = DataConfig(
+ parser=Parser(mode=simclr_model.PRETRAIN), is_training=True)
+ validation_data: DataConfig = DataConfig(
+ parser=Parser(mode=simclr_model.PRETRAIN), is_training=False)
+ loss: ContrastiveLoss = ContrastiveLoss()
+ evaluation: Evaluation = Evaluation()
+ init_checkpoint: Optional[str] = None
+ # all or backbone
+ init_checkpoint_modules: str = 'all'
+
+
+@dataclasses.dataclass
+class SimCLRFinetuneTask(cfg.TaskConfig):
+ """SimCLR fine tune task config."""
+ model: SimCLRModel = SimCLRModel(
+ mode=simclr_model.FINETUNE,
+ supervised_head=SupervisedHead(num_classes=1001, zero_init=True))
+ train_data: DataConfig = DataConfig(
+ parser=Parser(mode=simclr_model.FINETUNE), is_training=True)
+ validation_data: DataConfig = DataConfig(
+ parser=Parser(mode=simclr_model.FINETUNE), is_training=False)
+ loss: ClassificationLosses = ClassificationLosses()
+ evaluation: Evaluation = Evaluation()
+ init_checkpoint: Optional[str] = None
+ # all, backbone_projection or backbone
+ init_checkpoint_modules: str = 'backbone_projection'
+
+
+@exp_factory.register_config_factory('simclr_pretraining')
+def simclr_pretraining() -> cfg.ExperimentConfig:
+ """Image classification general."""
+ return cfg.ExperimentConfig(
+ task=SimCLRPretrainTask(),
+ trainer=cfg.TrainerConfig(),
+ restrictions=[
+ 'task.train_data.is_training != None',
+ 'task.validation_data.is_training != None'
+ ])
+
+
+@exp_factory.register_config_factory('simclr_finetuning')
+def simclr_finetuning() -> cfg.ExperimentConfig:
+ """Image classification general."""
+ return cfg.ExperimentConfig(
+ task=SimCLRFinetuneTask(),
+ trainer=cfg.TrainerConfig(),
+ restrictions=[
+ 'task.train_data.is_training != None',
+ 'task.validation_data.is_training != None'
+ ])
+
+
+IMAGENET_TRAIN_EXAMPLES = 1281167
+IMAGENET_VAL_EXAMPLES = 50000
+IMAGENET_INPUT_PATH_BASE = 'imagenet-2012-tfrecord'
+
+
+@exp_factory.register_config_factory('simclr_pretraining_imagenet')
+def simclr_pretraining_imagenet() -> cfg.ExperimentConfig:
+ """Image classification general."""
+ train_batch_size = 4096
+ eval_batch_size = 4096
+ steps_per_epoch = IMAGENET_TRAIN_EXAMPLES // train_batch_size
+ return cfg.ExperimentConfig(
+ task=SimCLRPretrainTask(
+ model=SimCLRModel(
+ mode=simclr_model.PRETRAIN,
+ backbone_trainable=True,
+ input_size=[224, 224, 3],
+ backbone=backbones.Backbone(
+ type='resnet', resnet=backbones.ResNet(model_id=50)),
+ projection_head=ProjectionHead(
+ proj_output_dim=128,
+ num_proj_layers=3,
+ ft_proj_idx=1),
+ supervised_head=SupervisedHead(num_classes=1001),
+ norm_activation=common.NormActivation(
+ norm_momentum=0.9, norm_epsilon=1e-5, use_sync_bn=True)),
+ loss=ContrastiveLoss(),
+ evaluation=Evaluation(),
+ train_data=DataConfig(
+ parser=Parser(mode=simclr_model.PRETRAIN),
+ decoder=Decoder(decode_label=True),
+ input_path=os.path.join(IMAGENET_INPUT_PATH_BASE, 'train*'),
+ is_training=True,
+ global_batch_size=train_batch_size),
+ validation_data=DataConfig(
+ parser=Parser(mode=simclr_model.PRETRAIN),
+ decoder=Decoder(decode_label=True),
+ input_path=os.path.join(IMAGENET_INPUT_PATH_BASE, 'valid*'),
+ is_training=False,
+ global_batch_size=eval_batch_size),
+ ),
+ trainer=cfg.TrainerConfig(
+ steps_per_loop=steps_per_epoch,
+ summary_interval=steps_per_epoch,
+ checkpoint_interval=steps_per_epoch,
+ train_steps=500 * steps_per_epoch,
+ validation_steps=IMAGENET_VAL_EXAMPLES // eval_batch_size,
+ validation_interval=steps_per_epoch,
+ optimizer_config=optimization.OptimizationConfig({
+ 'optimizer': {
+ 'type': 'lars',
+ 'lars': {
+ 'momentum': 0.9,
+ 'weight_decay_rate': 0.000001,
+ 'exclude_from_weight_decay': [
+ 'batch_normalization', 'bias']
+ }
+ },
+ 'learning_rate': {
+ 'type': 'cosine',
+ 'cosine': {
+ # 0.2 * BatchSize / 256
+ 'initial_learning_rate': 0.2 * train_batch_size / 256,
+ # train_steps - warmup_steps
+ 'decay_steps': 475 * steps_per_epoch
+ }
+ },
+ 'warmup': {
+ 'type': 'linear',
+ 'linear': {
+ # 5% of total epochs
+ 'warmup_steps': 25 * steps_per_epoch
+ }
+ }
+ })),
+ restrictions=[
+ 'task.train_data.is_training != None',
+ 'task.validation_data.is_training != None'
+ ])
+
+
+@exp_factory.register_config_factory('simclr_finetuning_imagenet')
+def simclr_finetuning_imagenet() -> cfg.ExperimentConfig:
+ """Image classification general."""
+ train_batch_size = 1024
+ eval_batch_size = 1024
+ steps_per_epoch = IMAGENET_TRAIN_EXAMPLES // train_batch_size
+ pretrain_model_base = ''
+ return cfg.ExperimentConfig(
+ task=SimCLRFinetuneTask(
+ model=SimCLRModel(
+ mode=simclr_model.FINETUNE,
+ backbone_trainable=True,
+ input_size=[224, 224, 3],
+ backbone=backbones.Backbone(
+ type='resnet', resnet=backbones.ResNet(model_id=50)),
+ projection_head=ProjectionHead(
+ proj_output_dim=128,
+ num_proj_layers=3,
+ ft_proj_idx=1),
+ supervised_head=SupervisedHead(
+ num_classes=1001, zero_init=True),
+ norm_activation=common.NormActivation(
+ norm_momentum=0.9, norm_epsilon=1e-5, use_sync_bn=False)),
+ loss=ClassificationLosses(),
+ evaluation=Evaluation(),
+ train_data=DataConfig(
+ parser=Parser(mode=simclr_model.FINETUNE),
+ input_path=os.path.join(IMAGENET_INPUT_PATH_BASE, 'train*'),
+ is_training=True,
+ global_batch_size=train_batch_size),
+ validation_data=DataConfig(
+ parser=Parser(mode=simclr_model.FINETUNE),
+ input_path=os.path.join(IMAGENET_INPUT_PATH_BASE, 'valid*'),
+ is_training=False,
+ global_batch_size=eval_batch_size),
+ init_checkpoint=pretrain_model_base,
+ # all, backbone_projection or backbone
+ init_checkpoint_modules='backbone_projection'),
+ trainer=cfg.TrainerConfig(
+ steps_per_loop=steps_per_epoch,
+ summary_interval=steps_per_epoch,
+ checkpoint_interval=steps_per_epoch,
+ train_steps=60 * steps_per_epoch,
+ validation_steps=IMAGENET_VAL_EXAMPLES // eval_batch_size,
+ validation_interval=steps_per_epoch,
+ optimizer_config=optimization.OptimizationConfig({
+ 'optimizer': {
+ 'type': 'lars',
+ 'lars': {
+ 'momentum': 0.9,
+ 'weight_decay_rate': 0.0,
+ 'exclude_from_weight_decay': [
+ 'batch_normalization', 'bias']
+ }
+ },
+ 'learning_rate': {
+ 'type': 'cosine',
+ 'cosine': {
+ # 0.01 × BatchSize / 512
+ 'initial_learning_rate': 0.01 * train_batch_size / 512,
+ 'decay_steps': 60 * steps_per_epoch
+ }
+ }
+ })),
+ restrictions=[
+ 'task.train_data.is_training != None',
+ 'task.validation_data.is_training != None'
+ ])
diff --git a/official/vision/beta/projects/simclr/configs/simclr_test.py b/official/vision/beta/projects/simclr/configs/simclr_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..f232d7533f4effe5161611bc3e0862ab87c5f305
--- /dev/null
+++ b/official/vision/beta/projects/simclr/configs/simclr_test.py
@@ -0,0 +1,62 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for simclr."""
+# pylint: disable=unused-import
+from absl.testing import parameterized
+
+import tensorflow as tf
+
+from official.core import config_definitions as cfg
+from official.core import exp_factory
+from official.vision.beta.projects.simclr.common import registry_imports # pylint: disable=unused-import
+from official.vision.beta.projects.simclr.configs import simclr as exp_cfg
+
+
+class SimCLRConfigTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.parameters(
+ 'simclr_pretraining_imagenet', 'simclr_finetuning_imagenet')
+ def test_simclr_configs(self, config_name):
+ config = exp_factory.get_exp_config(config_name)
+ self.assertIsInstance(config, cfg.ExperimentConfig)
+ if config_name == 'simclr_pretrain_imagenet':
+ self.assertIsInstance(config.task, exp_cfg.SimCLRPretrainTask)
+ elif config_name == 'simclr_finetuning_imagenet':
+ self.assertIsInstance(config.task, exp_cfg.SimCLRFinetuneTask)
+ self.assertIsInstance(config.task.model,
+ exp_cfg.SimCLRModel)
+ self.assertIsInstance(config.task.train_data, exp_cfg.DataConfig)
+ config.task.train_data.is_training = None
+ with self.assertRaises(KeyError):
+ config.validate()
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/projects/simclr/dataloaders/preprocess_ops.py b/official/vision/beta/projects/simclr/dataloaders/preprocess_ops.py
new file mode 100644
index 0000000000000000000000000000000000000000..29a3a3eadd98557598305def18feaeeff720f31a
--- /dev/null
+++ b/official/vision/beta/projects/simclr/dataloaders/preprocess_ops.py
@@ -0,0 +1,363 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Preprocessing ops."""
+import functools
+import tensorflow as tf
+
+CROP_PROPORTION = 0.875 # Standard for ImageNet.
+
+
+def random_apply(func, p, x):
+ """Randomly apply function func to x with probability p."""
+ return tf.cond(
+ tf.less(
+ tf.random.uniform([], minval=0, maxval=1, dtype=tf.float32),
+ tf.cast(p, tf.float32)), lambda: func(x), lambda: x)
+
+
+def random_brightness(image, max_delta, impl='simclrv2'):
+ """A multiplicative vs additive change of brightness."""
+ if impl == 'simclrv2':
+ factor = tf.random.uniform([], tf.maximum(1.0 - max_delta, 0),
+ 1.0 + max_delta)
+ image = image * factor
+ elif impl == 'simclrv1':
+ image = tf.image.random_brightness(image, max_delta=max_delta)
+ else:
+ raise ValueError('Unknown impl {} for random brightness.'.format(impl))
+ return image
+
+
+def to_grayscale(image, keep_channels=True):
+ image = tf.image.rgb_to_grayscale(image)
+ if keep_channels:
+ image = tf.tile(image, [1, 1, 3])
+ return image
+
+
+def color_jitter_nonrand(image,
+ brightness=0,
+ contrast=0,
+ saturation=0,
+ hue=0,
+ impl='simclrv2'):
+ """Distorts the color of the image (jittering order is fixed).
+
+ Args:
+ image: The input image tensor.
+ brightness: A float, specifying the brightness for color jitter.
+ contrast: A float, specifying the contrast for color jitter.
+ saturation: A float, specifying the saturation for color jitter.
+ hue: A float, specifying the hue for color jitter.
+ impl: 'simclrv1' or 'simclrv2'. Whether to use simclrv1 or simclrv2's
+ version of random brightness.
+
+ Returns:
+ The distorted image tensor.
+ """
+ with tf.name_scope('distort_color'):
+ def apply_transform(i, x, brightness, contrast, saturation, hue):
+ """Apply the i-th transformation."""
+ if brightness != 0 and i == 0:
+ x = random_brightness(x, max_delta=brightness, impl=impl)
+ elif contrast != 0 and i == 1:
+ x = tf.image.random_contrast(
+ x, lower=1 - contrast, upper=1 + contrast)
+ elif saturation != 0 and i == 2:
+ x = tf.image.random_saturation(
+ x, lower=1 - saturation, upper=1 + saturation)
+ elif hue != 0:
+ x = tf.image.random_hue(x, max_delta=hue)
+ return x
+
+ for i in range(4):
+ image = apply_transform(i, image, brightness, contrast, saturation, hue)
+ image = tf.clip_by_value(image, 0., 1.)
+ return image
+
+
+def color_jitter_rand(image,
+ brightness=0,
+ contrast=0,
+ saturation=0,
+ hue=0,
+ impl='simclrv2'):
+ """Distorts the color of the image (jittering order is random).
+
+ Args:
+ image: The input image tensor.
+ brightness: A float, specifying the brightness for color jitter.
+ contrast: A float, specifying the contrast for color jitter.
+ saturation: A float, specifying the saturation for color jitter.
+ hue: A float, specifying the hue for color jitter.
+ impl: 'simclrv1' or 'simclrv2'. Whether to use simclrv1 or simclrv2's
+ version of random brightness.
+
+ Returns:
+ The distorted image tensor.
+ """
+ with tf.name_scope('distort_color'):
+ def apply_transform(i, x):
+ """Apply the i-th transformation."""
+
+ def brightness_foo():
+ if brightness == 0:
+ return x
+ else:
+ return random_brightness(x, max_delta=brightness, impl=impl)
+
+ def contrast_foo():
+ if contrast == 0:
+ return x
+ else:
+ return tf.image.random_contrast(x, lower=1 - contrast,
+ upper=1 + contrast)
+
+ def saturation_foo():
+ if saturation == 0:
+ return x
+ else:
+ return tf.image.random_saturation(
+ x, lower=1 - saturation, upper=1 + saturation)
+
+ def hue_foo():
+ if hue == 0:
+ return x
+ else:
+ return tf.image.random_hue(x, max_delta=hue)
+
+ x = tf.cond(tf.less(i, 2),
+ lambda: tf.cond(tf.less(i, 1), brightness_foo, contrast_foo),
+ lambda: tf.cond(tf.less(i, 3), saturation_foo, hue_foo))
+ return x
+
+ perm = tf.random.shuffle(tf.range(4))
+ for i in range(4):
+ image = apply_transform(perm[i], image)
+ image = tf.clip_by_value(image, 0., 1.)
+ return image
+
+
+def color_jitter(image, strength, random_order=True, impl='simclrv2'):
+ """Distorts the color of the image.
+
+ Args:
+ image: The input image tensor.
+ strength: the floating number for the strength of the color augmentation.
+ random_order: A bool, specifying whether to randomize the jittering order.
+ impl: 'simclrv1' or 'simclrv2'. Whether to use simclrv1 or simclrv2's
+ version of random brightness.
+
+ Returns:
+ The distorted image tensor.
+ """
+ brightness = 0.8 * strength
+ contrast = 0.8 * strength
+ saturation = 0.8 * strength
+ hue = 0.2 * strength
+ if random_order:
+ return color_jitter_rand(
+ image, brightness, contrast, saturation, hue, impl=impl)
+ else:
+ return color_jitter_nonrand(
+ image, brightness, contrast, saturation, hue, impl=impl)
+
+
+def random_color_jitter(image,
+ p=1.0,
+ color_jitter_strength=1.0,
+ impl='simclrv2'):
+ """Perform random color jitter."""
+ def _transform(image):
+ color_jitter_t = functools.partial(
+ color_jitter, strength=color_jitter_strength, impl=impl)
+ image = random_apply(color_jitter_t, p=0.8, x=image)
+ return random_apply(to_grayscale, p=0.2, x=image)
+
+ return random_apply(_transform, p=p, x=image)
+
+
+def gaussian_blur(image, kernel_size, sigma, padding='SAME'):
+ """Blurs the given image with separable convolution.
+
+
+ Args:
+ image: Tensor of shape [height, width, channels] and dtype float to blur.
+ kernel_size: Integer Tensor for the size of the blur kernel. This is should
+ be an odd number. If it is an even number, the actual kernel size will be
+ size + 1.
+ sigma: Sigma value for gaussian operator.
+ padding: Padding to use for the convolution. Typically 'SAME' or 'VALID'.
+
+ Returns:
+ A Tensor representing the blurred image.
+ """
+ radius = tf.cast(kernel_size / 2, dtype=tf.int32)
+ kernel_size = radius * 2 + 1
+ x = tf.cast(tf.range(-radius, radius + 1), dtype=tf.float32)
+ blur_filter = tf.exp(-tf.pow(x, 2.0) /
+ (2.0 * tf.pow(tf.cast(sigma, dtype=tf.float32), 2.0)))
+ blur_filter /= tf.reduce_sum(blur_filter)
+ # One vertical and one horizontal filter.
+ blur_v = tf.reshape(blur_filter, [kernel_size, 1, 1, 1])
+ blur_h = tf.reshape(blur_filter, [1, kernel_size, 1, 1])
+ num_channels = tf.shape(image)[-1]
+ blur_h = tf.tile(blur_h, [1, 1, num_channels, 1])
+ blur_v = tf.tile(blur_v, [1, 1, num_channels, 1])
+ expand_batch_dim = image.shape.ndims == 3
+ if expand_batch_dim:
+ # Tensorflow requires batched input to convolutions, which we can fake with
+ # an extra dimension.
+ image = tf.expand_dims(image, axis=0)
+ blurred = tf.nn.depthwise_conv2d(
+ image, blur_h, strides=[1, 1, 1, 1], padding=padding)
+ blurred = tf.nn.depthwise_conv2d(
+ blurred, blur_v, strides=[1, 1, 1, 1], padding=padding)
+ if expand_batch_dim:
+ blurred = tf.squeeze(blurred, axis=0)
+ return blurred
+
+
+def random_blur(image, height, width, p=0.5):
+ """Randomly blur an image.
+
+ Args:
+ image: `Tensor` representing an image of arbitrary size.
+ height: Height of output image.
+ width: Width of output image.
+ p: probability of applying this transformation.
+
+ Returns:
+ A preprocessed image `Tensor`.
+ """
+ del width
+
+ def _transform(image):
+ sigma = tf.random.uniform([], 0.1, 2.0, dtype=tf.float32)
+ return gaussian_blur(
+ image, kernel_size=height // 10, sigma=sigma, padding='SAME')
+
+ return random_apply(_transform, p=p, x=image)
+
+
+def distorted_bounding_box_crop(image,
+ bbox,
+ min_object_covered=0.1,
+ aspect_ratio_range=(0.75, 1.33),
+ area_range=(0.05, 1.0),
+ max_attempts=100,
+ scope=None):
+ """Generates cropped_image using one of the bboxes randomly distorted.
+
+ See `tf.image.sample_distorted_bounding_box` for more documentation.
+
+ Args:
+ image: `Tensor` of image data.
+ bbox: `Tensor` of bounding boxes arranged `[1, num_boxes, coords]`
+ where each coordinate is [0, 1) and the coordinates are arranged
+ as `[ymin, xmin, ymax, xmax]`. If num_boxes is 0 then use the whole
+ image.
+ min_object_covered: An optional `float`. Defaults to `0.1`. The cropped
+ area of the image must contain at least this fraction of any bounding
+ box supplied.
+ aspect_ratio_range: An optional list of `float`s. The cropped area of the
+ image must have an aspect ratio = width / height within this range.
+ area_range: An optional list of `float`s. The cropped area of the image
+ must contain a fraction of the supplied image within in this range.
+ max_attempts: An optional `int`. Number of attempts at generating a cropped
+ region of the image of the specified constraints. After `max_attempts`
+ failures, return the entire image.
+ scope: Optional `str` for name scope.
+ Returns:
+ (cropped image `Tensor`, distorted bbox `Tensor`).
+ """
+ with tf.name_scope(scope or 'distorted_bounding_box_crop'):
+ shape = tf.shape(image)
+ sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
+ shape,
+ bounding_boxes=bbox,
+ min_object_covered=min_object_covered,
+ aspect_ratio_range=aspect_ratio_range,
+ area_range=area_range,
+ max_attempts=max_attempts,
+ use_image_if_no_bounding_boxes=True)
+ bbox_begin, bbox_size, _ = sample_distorted_bounding_box
+
+ # Crop the image to the specified bounding box.
+ offset_y, offset_x, _ = tf.unstack(bbox_begin)
+ target_height, target_width, _ = tf.unstack(bbox_size)
+ image = tf.image.crop_to_bounding_box(
+ image, offset_y, offset_x, target_height, target_width)
+
+ return image
+
+
+def crop_and_resize(image, height, width):
+ """Make a random crop and resize it to height `height` and width `width`.
+
+ Args:
+ image: Tensor representing the image.
+ height: Desired image height.
+ width: Desired image width.
+
+ Returns:
+ A `height` x `width` x channels Tensor holding a random crop of `image`.
+ """
+ bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4])
+ aspect_ratio = width / height
+ image = distorted_bounding_box_crop(
+ image,
+ bbox,
+ min_object_covered=0.1,
+ aspect_ratio_range=(3. / 4 * aspect_ratio, 4. / 3. * aspect_ratio),
+ area_range=(0.08, 1.0),
+ max_attempts=100,
+ scope=None)
+ return tf.image.resize([image], [height, width],
+ method=tf.image.ResizeMethod.BICUBIC)[0]
+
+
+def random_crop_with_resize(image, height, width, p=1.0):
+ """Randomly crop and resize an image.
+
+ Args:
+ image: `Tensor` representing an image of arbitrary size.
+ height: Height of output image.
+ width: Width of output image.
+ p: Probability of applying this transformation.
+
+ Returns:
+ A preprocessed image `Tensor`.
+ """
+
+ def _transform(image): # pylint: disable=missing-docstring
+ image = crop_and_resize(image, height, width)
+ return image
+
+ return random_apply(_transform, p=p, x=image)
diff --git a/official/vision/beta/projects/simclr/dataloaders/simclr_input.py b/official/vision/beta/projects/simclr/dataloaders/simclr_input.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e7607af5170eacb489d294e4cb7297c746bec60
--- /dev/null
+++ b/official/vision/beta/projects/simclr/dataloaders/simclr_input.py
@@ -0,0 +1,242 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Data parser and processing for SimCLR.
+
+For pre-training:
+- Preprocessing:
+ -> random cropping
+ -> resize back to the original size
+ -> random color distortions
+ -> random Gaussian blur (sequential)
+- Each image need to be processed randomly twice
+
+```snippets
+ if train_mode == 'pretrain':
+ xs = []
+ for _ in range(2): # Two transformations
+ xs.append(preprocess_fn_pretrain(image))
+ image = tf.concat(xs, -1)
+ else:
+ image = preprocess_fn_finetune(image)
+```
+
+For fine-tuning:
+typical image classification input
+"""
+
+from typing import List
+
+import tensorflow as tf
+
+from official.vision.beta.dataloaders import decoder
+from official.vision.beta.dataloaders import parser
+from official.vision.beta.ops import preprocess_ops
+from official.vision.beta.projects.simclr.dataloaders import preprocess_ops as simclr_preprocess_ops
+from official.vision.beta.projects.simclr.modeling import simclr_model
+
+
+class Decoder(decoder.Decoder):
+ """A tf.Example decoder for classification task."""
+
+ def __init__(self, decode_label=True):
+ self._decode_label = decode_label
+
+ self._keys_to_features = {
+ 'image/encoded': tf.io.FixedLenFeature((), tf.string, default_value=''),
+ }
+ if self._decode_label:
+ self._keys_to_features.update({
+ 'image/class/label': (
+ tf.io.FixedLenFeature((), tf.int64, default_value=-1))
+ })
+
+ def decode(self, serialized_example):
+ return tf.io.parse_single_example(
+ serialized_example, self._keys_to_features)
+
+
+class TFDSDecoder(decoder.Decoder):
+ """A TFDS decoder for classification task."""
+
+ def __init__(self, decode_label=True):
+ self._decode_label = decode_label
+
+ def decode(self, serialized_example):
+ sample_dict = {
+ 'image/encoded': tf.io.encode_jpeg(
+ serialized_example['image'], quality=100),
+ }
+ if self._decode_label:
+ sample_dict.update({
+ 'image/class/label': serialized_example['label'],
+ })
+ return sample_dict
+
+
+class Parser(parser.Parser):
+ """Parser for SimCLR training."""
+
+ def __init__(self,
+ output_size: List[int],
+ aug_rand_crop: bool = True,
+ aug_rand_hflip: bool = True,
+ aug_color_distort: bool = True,
+ aug_color_jitter_strength: float = 1.0,
+ aug_color_jitter_impl: str = 'simclrv2',
+ aug_rand_blur: bool = True,
+ parse_label: bool = True,
+ test_crop: bool = True,
+ mode: str = simclr_model.PRETRAIN,
+ dtype: str = 'float32'):
+ """Initializes parameters for parsing annotations in the dataset.
+
+ Args:
+ output_size: `Tensor` or `list` for [height, width] of output image. The
+ output_size should be divided by the largest feature stride 2^max_level.
+ aug_rand_crop: `bool`, if Ture, augment training with random cropping.
+ aug_rand_hflip: `bool`, if True, augment training with random
+ horizontal flip.
+ aug_color_distort: `bool`, if True augment training with color distortion.
+ aug_color_jitter_strength: `float`, the floating number for the strength
+ of the color augmentation
+ aug_color_jitter_impl: `str`, 'simclrv1' or 'simclrv2'. Define whether
+ to use simclrv1 or simclrv2's version of random brightness.
+ aug_rand_blur: `bool`, if True, augment training with random blur.
+ parse_label: `bool`, if True, parse label together with image.
+ test_crop: `bool`, if True, augment eval with center cropping.
+ mode: `str`, 'pretain' or 'finetune'. Define training mode.
+ dtype: `str`, cast output image in dtype. It can be 'float32', 'float16',
+ or 'bfloat16'.
+ """
+ self._output_size = output_size
+ self._aug_rand_crop = aug_rand_crop
+ self._aug_rand_hflip = aug_rand_hflip
+ self._aug_color_distort = aug_color_distort
+ self._aug_color_jitter_strength = aug_color_jitter_strength
+ self._aug_color_jitter_impl = aug_color_jitter_impl
+ self._aug_rand_blur = aug_rand_blur
+ self._parse_label = parse_label
+ self._mode = mode
+ self._test_crop = test_crop
+ if max(self._output_size[0], self._output_size[1]) <= 32:
+ self._test_crop = False
+
+ if dtype == 'float32':
+ self._dtype = tf.float32
+ elif dtype == 'float16':
+ self._dtype = tf.float16
+ elif dtype == 'bfloat16':
+ self._dtype = tf.bfloat16
+ else:
+ raise ValueError('dtype {!r} is not supported!'.format(dtype))
+
+ def _parse_one_train_image(self, image_bytes):
+
+ image = tf.image.decode_jpeg(image_bytes, channels=3)
+ # This line convert the image to float 0.0 - 1.0
+ image = tf.image.convert_image_dtype(image, dtype=tf.float32)
+
+ if self._aug_rand_crop:
+ image = simclr_preprocess_ops.random_crop_with_resize(
+ image, self._output_size[0], self._output_size[1])
+
+ if self._aug_rand_hflip:
+ image = tf.image.random_flip_left_right(image)
+
+ if self._aug_color_distort and self._mode == simclr_model.PRETRAIN:
+ image = simclr_preprocess_ops.random_color_jitter(
+ image=image,
+ color_jitter_strength=self._aug_color_jitter_strength,
+ impl=self._aug_color_jitter_impl)
+
+ if self._aug_rand_blur and self._mode == simclr_model.PRETRAIN:
+ image = simclr_preprocess_ops.random_blur(
+ image, self._output_size[0], self._output_size[1])
+
+ image = tf.image.resize(
+ image, self._output_size, method=tf.image.ResizeMethod.BILINEAR)
+ image = tf.reshape(image, [self._output_size[0], self._output_size[1], 3])
+
+ image = tf.clip_by_value(image, 0., 1.)
+ # Convert image to self._dtype.
+ image = tf.image.convert_image_dtype(image, self._dtype)
+
+ return image
+
+ def _parse_train_data(self, decoded_tensors):
+ """Parses data for training."""
+ image_bytes = decoded_tensors['image/encoded']
+
+ if self._mode == simclr_model.FINETUNE:
+ image = self._parse_one_train_image(image_bytes)
+
+ elif self._mode == simclr_model.PRETRAIN:
+ # Transform each example twice using a combination of
+ # simple augmentations, resulting in 2N data points
+ xs = []
+ for _ in range(2):
+ xs.append(self._parse_one_train_image(image_bytes))
+ image = tf.concat(xs, -1)
+
+ else:
+ raise ValueError('The mode {} is not supported by the Parser.'
+ .format(self._mode))
+
+ if self._parse_label:
+ label = tf.cast(decoded_tensors['image/class/label'], dtype=tf.int32)
+ return image, label
+
+ return image
+
+ def _parse_eval_data(self, decoded_tensors):
+ """Parses data for evaluation."""
+ image_bytes = decoded_tensors['image/encoded']
+ image_shape = tf.image.extract_jpeg_shape(image_bytes)
+
+ if self._test_crop:
+ image = preprocess_ops.center_crop_image_v2(image_bytes, image_shape)
+ else:
+ image = tf.image.decode_jpeg(image_bytes, channels=3)
+ # This line convert the image to float 0.0 - 1.0
+ image = tf.image.convert_image_dtype(image, dtype=tf.float32)
+
+ image = tf.image.resize(
+ image, self._output_size, method=tf.image.ResizeMethod.BILINEAR)
+ image = tf.reshape(image, [self._output_size[0], self._output_size[1], 3])
+
+ image = tf.clip_by_value(image, 0., 1.)
+
+ # Convert image to self._dtype.
+ image = tf.image.convert_image_dtype(image, self._dtype)
+
+ if self._parse_label:
+ label = tf.cast(decoded_tensors['image/class/label'], dtype=tf.int32)
+ return image, label
+
+ return image
diff --git a/official/vision/beta/projects/simclr/heads/simclr_head.py b/official/vision/beta/projects/simclr/heads/simclr_head.py
new file mode 100644
index 0000000000000000000000000000000000000000..32032323a3aeab1a8a965357161c80daf2d25f0b
--- /dev/null
+++ b/official/vision/beta/projects/simclr/heads/simclr_head.py
@@ -0,0 +1,215 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Dense prediction heads."""
+
+from typing import Text, Optional
+
+import tensorflow as tf
+
+from official.vision.beta.projects.simclr.modeling.layers import nn_blocks
+
+regularizers = tf.keras.regularizers
+layers = tf.keras.layers
+
+
+@tf.keras.utils.register_keras_serializable(package='simclr')
+class ProjectionHead(tf.keras.layers.Layer):
+ """Projection head."""
+
+ def __init__(
+ self,
+ num_proj_layers: int = 3,
+ proj_output_dim: Optional[int] = None,
+ ft_proj_idx: int = 0,
+ kernel_initializer: Text = 'VarianceScaling',
+ kernel_regularizer: Optional[regularizers.Regularizer] = None,
+ bias_regularizer: Optional[regularizers.Regularizer] = None,
+ use_sync_bn: bool = False,
+ norm_momentum: float = 0.99,
+ norm_epsilon: float = 0.001,
+ **kwargs):
+ """The projection head used during pretraining of SimCLR.
+
+ Args:
+ num_proj_layers: `int` number of Dense layers used.
+ proj_output_dim: `int` output dimension of projection head, i.e., output
+ dimension of the final layer.
+ ft_proj_idx: `int` index of layer to use during fine-tuning. 0 means no
+ projection head during fine tuning, -1 means the final layer.
+ kernel_initializer: kernel_initializer for convolutional layers.
+ kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D.
+ Default to None.
+ bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d.
+ Default to None.
+ use_sync_bn: if True, use synchronized batch normalization.
+ norm_momentum: `float` normalization omentum for the moving average.
+ norm_epsilon: `float` small float added to variance to avoid dividing by
+ zero.
+ **kwargs: keyword arguments to be passed.
+ """
+ super(ProjectionHead, self).__init__(**kwargs)
+
+ assert proj_output_dim is not None or num_proj_layers == 0
+ assert ft_proj_idx <= num_proj_layers, (num_proj_layers, ft_proj_idx)
+
+ self._proj_output_dim = proj_output_dim
+ self._num_proj_layers = num_proj_layers
+ self._ft_proj_idx = ft_proj_idx
+ self._kernel_initializer = kernel_initializer
+ self._kernel_regularizer = kernel_regularizer
+ self._bias_regularizer = bias_regularizer
+ self._use_sync_bn = use_sync_bn
+ self._norm_momentum = norm_momentum
+ self._norm_epsilon = norm_epsilon
+ self._layers = []
+
+ def get_config(self):
+ config = {
+ 'proj_output_dim': self._proj_output_dim,
+ 'num_proj_layers': self._num_proj_layers,
+ 'ft_proj_idx': self._ft_proj_idx,
+ 'kernel_initializer': self._kernel_initializer,
+ 'kernel_regularizer': self._kernel_regularizer,
+ 'bias_regularizer': self._bias_regularizer,
+ 'use_normalization': self._use_normalization,
+ 'norm_momentum': self._norm_momentum,
+ 'norm_epsilon': self._norm_epsilon
+ }
+ base_config = super(ProjectionHead, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
+
+ def build(self, input_shape):
+ self._layers = []
+ if self._num_proj_layers > 0:
+ intermediate_dim = int(input_shape[-1])
+ for j in range(self._num_proj_layers):
+ if j != self._num_proj_layers - 1:
+ # for the middle layers, use bias and relu for the output.
+ layer = nn_blocks.DenseBN(
+ output_dim=intermediate_dim,
+ use_bias=True,
+ use_normalization=True,
+ activation='relu',
+ kernel_initializer=self._kernel_initializer,
+ kernel_regularizer=self._kernel_regularizer,
+ bias_regularizer=self._bias_regularizer,
+ use_sync_bn=self._use_sync_bn,
+ norm_momentum=self._norm_momentum,
+ norm_epsilon=self._norm_epsilon,
+ name='nl_%d' % j)
+ else:
+ # for the final layer, neither bias nor relu is used.
+ layer = nn_blocks.DenseBN(
+ output_dim=self._proj_output_dim,
+ use_bias=False,
+ use_normalization=True,
+ activation=None,
+ kernel_regularizer=self._kernel_regularizer,
+ kernel_initializer=self._kernel_initializer,
+ use_sync_bn=self._use_sync_bn,
+ norm_momentum=self._norm_momentum,
+ norm_epsilon=self._norm_epsilon,
+ name='nl_%d' % j)
+ self._layers.append(layer)
+ super(ProjectionHead, self).build(input_shape)
+
+ def call(self, inputs, training=None):
+ hiddens_list = [tf.identity(inputs, 'proj_head_input')]
+
+ if self._num_proj_layers == 0:
+ proj_head_output = inputs
+ proj_finetune_output = inputs
+ else:
+ for j in range(self._num_proj_layers):
+ hiddens = self._layers[j](hiddens_list[-1], training)
+ hiddens_list.append(hiddens)
+ proj_head_output = tf.identity(
+ hiddens_list[-1], 'proj_head_output')
+ proj_finetune_output = tf.identity(
+ hiddens_list[self._ft_proj_idx], 'proj_finetune_output')
+
+ # The first element is the output of the projection head.
+ # The second element is the input of the finetune head.
+ return proj_head_output, proj_finetune_output
+
+
+@tf.keras.utils.register_keras_serializable(package='simclr')
+class ClassificationHead(tf.keras.layers.Layer):
+ """Classification Head."""
+
+ def __init__(
+ self,
+ num_classes: int,
+ kernel_initializer: Text = 'random_uniform',
+ kernel_regularizer: Optional[regularizers.Regularizer] = None,
+ bias_regularizer: Optional[regularizers.Regularizer] = None,
+ name: Text = 'head_supervised',
+ **kwargs):
+ """The classification head used during pretraining or fine tuning.
+
+ Args:
+ num_classes: `int` size of the output dimension or number of classes
+ for classification task.
+ kernel_initializer: kernel_initializer for convolutional layers.
+ kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D.
+ Default to None.
+ bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d.
+ Default to None.
+ name: `str`, name of the layer.
+ **kwargs: keyword arguments to be passed.
+ """
+ super(ClassificationHead, self).__init__(name=name, **kwargs)
+ self._num_classes = num_classes
+ self._kernel_initializer = kernel_initializer
+ self._kernel_regularizer = kernel_regularizer
+ self._bias_regularizer = bias_regularizer
+ self._name = name
+
+ def get_config(self):
+ config = {
+ 'num_classes': self._num_classes,
+ 'kernel_initializer': self._kernel_initializer,
+ 'kernel_regularizer': self._kernel_regularizer,
+ 'bias_regularizer': self._bias_regularizer,
+ }
+ base_config = super(ClassificationHead, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
+
+ def build(self, input_shape):
+ self._dense0 = layers.Dense(
+ units=self._num_classes,
+ kernel_initializer=self._kernel_initializer,
+ kernel_regularizer=self._kernel_regularizer,
+ bias_regularizer=self._bias_regularizer,
+ activation=None)
+ super(ClassificationHead, self).build(input_shape)
+
+ def call(self, inputs, training=None):
+ inputs = self._dense0(inputs)
+ return inputs
diff --git a/official/vision/beta/projects/simclr/heads/simclr_head_test.py b/official/vision/beta/projects/simclr/heads/simclr_head_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce98eecaea3122c5517c0c7e3521916ed42d206e
--- /dev/null
+++ b/official/vision/beta/projects/simclr/heads/simclr_head_test.py
@@ -0,0 +1,117 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from absl.testing import parameterized
+
+import numpy as np
+import tensorflow as tf
+
+from official.vision.beta.projects.simclr.heads import simclr_head
+
+
+class ProjectionHeadTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.parameters(
+ (0, None),
+ (1, 128),
+ (2, 128),
+ )
+ def test_head_creation(self, num_proj_layers, proj_output_dim):
+ test_layer = simclr_head.ProjectionHead(
+ num_proj_layers=num_proj_layers,
+ proj_output_dim=proj_output_dim)
+
+ input_dim = 64
+ x = tf.keras.Input(shape=(input_dim,))
+ proj_head_output, proj_finetune_output = test_layer(x)
+
+ proj_head_output_dim = input_dim
+ if num_proj_layers > 0:
+ proj_head_output_dim = proj_output_dim
+ self.assertAllEqual(proj_head_output.shape.as_list(),
+ [None, proj_head_output_dim])
+
+ if num_proj_layers > 0:
+ proj_finetune_output_dim = input_dim
+ self.assertAllEqual(proj_finetune_output.shape.as_list(),
+ [None, proj_finetune_output_dim])
+
+ @parameterized.parameters(
+ (0, None, 0),
+ (1, 128, 0),
+ (2, 128, 1),
+ (2, 128, 2),
+ )
+ def test_outputs(self, num_proj_layers, proj_output_dim, ft_proj_idx):
+ test_layer = simclr_head.ProjectionHead(
+ num_proj_layers=num_proj_layers,
+ proj_output_dim=proj_output_dim,
+ ft_proj_idx=ft_proj_idx
+ )
+
+ input_dim = 64
+ batch_size = 2
+ inputs = np.random.rand(batch_size, input_dim)
+ proj_head_output, proj_finetune_output = test_layer(inputs)
+
+ if num_proj_layers == 0:
+ self.assertAllClose(inputs, proj_head_output)
+ self.assertAllClose(inputs, proj_finetune_output)
+ else:
+ self.assertAllEqual(proj_head_output.shape.as_list(),
+ [batch_size, proj_output_dim])
+ if ft_proj_idx == 0:
+ self.assertAllClose(inputs, proj_finetune_output)
+ elif ft_proj_idx < num_proj_layers:
+ self.assertAllEqual(proj_finetune_output.shape.as_list(),
+ [batch_size, input_dim])
+ else:
+ self.assertAllEqual(proj_finetune_output.shape.as_list(),
+ [batch_size, proj_output_dim])
+
+
+class ClassificationHeadTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.parameters(
+ 10, 20
+ )
+ def test_head_creation(self, num_classes):
+ test_layer = simclr_head.ClassificationHead(num_classes=num_classes)
+
+ input_dim = 64
+ x = tf.keras.Input(shape=(input_dim,))
+ out_x = test_layer(x)
+
+ self.assertAllEqual(out_x.shape.as_list(),
+ [None, num_classes])
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/projects/simclr/losses/contrastive_losses.py b/official/vision/beta/projects/simclr/losses/contrastive_losses.py
new file mode 100644
index 0000000000000000000000000000000000000000..c178f001601da6614e3303b7186bc2d8168c57a9
--- /dev/null
+++ b/official/vision/beta/projects/simclr/losses/contrastive_losses.py
@@ -0,0 +1,157 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Contrastive loss functions."""
+
+import functools
+
+import tensorflow as tf
+
+LARGE_NUM = 1e9
+
+
+def cross_replica_concat(tensor: tf.Tensor, num_replicas: int) -> tf.Tensor:
+ """Reduce a concatenation of the `tensor` across multiple replicas.
+
+ Args:
+ tensor: `tf.Tensor` to concatenate.
+ num_replicas: `int` number of replicas.
+
+ Returns:
+ Tensor of the same rank as `tensor` with first dimension `num_replicas`
+ times larger.
+ """
+ if num_replicas <= 1:
+ return tensor
+
+ replica_context = tf.distribute.get_replica_context()
+ with tf.name_scope('cross_replica_concat'):
+ # This creates a tensor that is like the input tensor but has an added
+ # replica dimension as the outermost dimension. On each replica it will
+ # contain the local values and zeros for all other values that need to be
+ # fetched from other replicas.
+ ext_tensor = tf.scatter_nd(
+ indices=[[replica_context.replica_id_in_sync_group]],
+ updates=[tensor],
+ shape=tf.concat([[num_replicas], tf.shape(tensor)], axis=0))
+
+ # As every value is only present on one replica and 0 in all others, adding
+ # them all together will result in the full tensor on all replicas.
+ ext_tensor = replica_context.all_reduce(tf.distribute.ReduceOp.SUM,
+ ext_tensor)
+
+ # Flatten the replica dimension.
+ # The first dimension size will be: tensor.shape[0] * num_replicas
+ # Using [-1] trick to support also scalar input.
+ return tf.reshape(ext_tensor, [-1] + ext_tensor.shape.as_list()[2:])
+
+
+@tf.keras.utils.register_keras_serializable(package='simclr')
+class ContrastiveLoss(object):
+ """Contrastive training loss function."""
+
+ def __init__(self, projection_norm: bool = True, temperature: float = 1.0):
+ """Initializes `ContrastiveLoss`.
+
+ Args:
+ projection_norm: whether or not to use normalization on the hidden vector.
+ temperature: a `floating` number for temperature scaling.
+ """
+ self._projection_norm = projection_norm
+ self._temperature = temperature
+
+ def __call__(self, projection1: tf.Tensor, projection2: tf.Tensor):
+ """Compute the contrastive loss for contrastive learning.
+
+ Note that projection2 is generated with the same batch (same order) of raw
+ images, but with different augmentation. More specifically:
+ image[i] -> random augmentation 1 -> projection -> projection1[i]
+ image[i] -> random augmentation 2 -> projection -> projection2[i]
+
+ Args:
+ projection1: projection vector of shape (bsz, dim).
+ projection2: projection vector of shape (bsz, dim).
+
+ Returns:
+ A loss scalar.
+ The logits for contrastive prediction task.
+ The labels for contrastive prediction task.
+ """
+ # Get (normalized) hidden1 and hidden2.
+ if self._projection_norm:
+ projection1 = tf.math.l2_normalize(projection1, -1)
+ projection2 = tf.math.l2_normalize(projection2, -1)
+ batch_size = tf.shape(projection1)[0]
+
+ p1_local, p2_local = projection1, projection2
+ # Gather projection1/projection2 across replicas and create local labels.
+ num_replicas_in_sync = tf.distribute.get_strategy().num_replicas_in_sync
+ if num_replicas_in_sync > 1:
+ p1_global = cross_replica_concat(p1_local, num_replicas_in_sync)
+ p2_global = cross_replica_concat(p2_local, num_replicas_in_sync)
+ global_batch_size = tf.shape(p1_global)[0]
+
+ replica_context = tf.distribute.get_replica_context()
+ replica_id = tf.cast(
+ tf.cast(replica_context.replica_id_in_sync_group, tf.uint32),
+ tf.int32)
+ labels_idx = tf.range(batch_size) + replica_id * batch_size
+ labels = tf.one_hot(labels_idx, global_batch_size * 2)
+ masks = tf.one_hot(labels_idx, global_batch_size)
+ else:
+ p1_global = p1_local
+ p2_global = p2_local
+ labels = tf.one_hot(tf.range(batch_size), batch_size * 2)
+ masks = tf.one_hot(tf.range(batch_size), batch_size)
+
+ tb_matmul = functools.partial(tf.matmul, transpose_b=True)
+
+ logits_aa = tb_matmul(p1_local, p1_global) / self._temperature
+ logits_aa = logits_aa - masks * LARGE_NUM
+
+ logits_bb = tb_matmul(p2_local, p2_global) / self._temperature
+ logits_bb = logits_bb - masks * LARGE_NUM
+
+ logits_ab = tb_matmul(p1_local, p2_global) / self._temperature
+ logits_ba = tb_matmul(p2_local, p1_global) / self._temperature
+
+ loss_a_local = tf.nn.softmax_cross_entropy_with_logits(
+ labels, tf.concat([logits_ab, logits_aa], 1))
+ loss_b_local = tf.nn.softmax_cross_entropy_with_logits(
+ labels, tf.concat([logits_ba, logits_bb], 1))
+ loss_local = tf.reduce_mean(loss_a_local + loss_b_local)
+
+ return loss_local, (logits_ab, labels)
+
+ def get_config(self):
+ config = {
+ 'projection_norm': self._projection_norm,
+ 'temperature': self._temperature,
+ }
+ return config
diff --git a/official/vision/beta/projects/simclr/losses/contrastive_losses_test.py b/official/vision/beta/projects/simclr/losses/contrastive_losses_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f7feddee5d1c240f7f3db830303a2718e27240d
--- /dev/null
+++ b/official/vision/beta/projects/simclr/losses/contrastive_losses_test.py
@@ -0,0 +1,93 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from absl.testing import parameterized
+
+import numpy as np
+import tensorflow as tf
+
+from official.vision.beta.projects.simclr.losses import contrastive_losses
+
+
+class ContrastiveLossesTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.parameters(1.0, 0.5)
+ def test_contrastive_loss_computation(self, temperature):
+ batch_size = 2
+ project_dim = 16
+ projection_norm = False
+
+ p_1_arr = np.random.rand(batch_size, project_dim)
+ p_1 = tf.constant(p_1_arr, dtype=tf.float32)
+ p_2_arr = np.random.rand(batch_size, project_dim)
+ p_2 = tf.constant(p_2_arr, dtype=tf.float32)
+
+ losses_obj = contrastive_losses.ContrastiveLoss(
+ projection_norm=projection_norm,
+ temperature=temperature)
+ comp_contrastive_loss = losses_obj(
+ projection1=p_1,
+ projection2=p_2)
+
+ def _exp_sim(p1, p2):
+ return np.exp(np.matmul(p1, p2) / temperature)
+
+ l11 = - np.log(
+ _exp_sim(p_1_arr[0], p_2_arr[0]) /
+ (_exp_sim(p_1_arr[0], p_1_arr[1])
+ + _exp_sim(p_1_arr[0], p_2_arr[1])
+ + _exp_sim(p_1_arr[0], p_2_arr[0]))
+ ) - np.log(
+ _exp_sim(p_1_arr[0], p_2_arr[0]) /
+ (_exp_sim(p_2_arr[0], p_2_arr[1])
+ + _exp_sim(p_2_arr[0], p_1_arr[1])
+ + _exp_sim(p_1_arr[0], p_2_arr[0]))
+ )
+
+ l22 = - np.log(
+ _exp_sim(p_1_arr[1], p_2_arr[1]) /
+ (_exp_sim(p_1_arr[1], p_1_arr[0])
+ + _exp_sim(p_1_arr[1], p_2_arr[0])
+ + _exp_sim(p_1_arr[1], p_2_arr[1]))
+ ) - np.log(
+ _exp_sim(p_1_arr[1], p_2_arr[1]) /
+ (_exp_sim(p_2_arr[1], p_2_arr[0])
+ + _exp_sim(p_2_arr[1], p_1_arr[0])
+ + _exp_sim(p_1_arr[1], p_2_arr[1]))
+ )
+
+ exp_contrastive_loss = (l11 + l22) / 2.0
+
+ self.assertAlmostEqual(comp_contrastive_loss[0].numpy(),
+ exp_contrastive_loss, places=5)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/projects/simclr/modeling/layers/nn_blocks.py b/official/vision/beta/projects/simclr/modeling/layers/nn_blocks.py
new file mode 100644
index 0000000000000000000000000000000000000000..65a9a7a2830b5bbaf41e73d173652dd75889a6dd
--- /dev/null
+++ b/official/vision/beta/projects/simclr/modeling/layers/nn_blocks.py
@@ -0,0 +1,150 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Contains common building blocks for simclr neural networks."""
+from typing import Text, Optional
+
+import tensorflow as tf
+
+from official.modeling import tf_utils
+
+regularizers = tf.keras.regularizers
+
+
+@tf.keras.utils.register_keras_serializable(package='simclr')
+class DenseBN(tf.keras.layers.Layer):
+ """Modified Dense layer to help build simclr system.
+
+ The layer is a standards combination of Dense, BatchNorm and Activation.
+ """
+
+ def __init__(
+ self,
+ output_dim: int,
+ use_bias: bool = True,
+ use_normalization: bool = False,
+ use_sync_bn: bool = False,
+ norm_momentum: float = 0.99,
+ norm_epsilon: float = 0.001,
+ activation: Optional[Text] = 'relu',
+ kernel_initializer: Text = 'VarianceScaling',
+ kernel_regularizer: Optional[regularizers.Regularizer] = None,
+ bias_regularizer: Optional[regularizers.Regularizer] = None,
+ name='linear_layer',
+ **kwargs):
+ """Customized Dense layer.
+
+ Args:
+ output_dim: `int` size of output dimension.
+ use_bias: if True, use biase in the dense layer.
+ use_normalization: if True, use batch normalization.
+ use_sync_bn: if True, use synchronized batch normalization.
+ norm_momentum: `float` normalization momentum for the moving average.
+ norm_epsilon: `float` small float added to variance to avoid dividing by
+ zero.
+ activation: `str` name of the activation function.
+ kernel_initializer: kernel_initializer for convolutional layers.
+ kernel_regularizer: tf.keras.regularizers.Regularizer object for Conv2D.
+ Default to None.
+ bias_regularizer: tf.keras.regularizers.Regularizer object for Conv2d.
+ Default to None.
+ name: `str`, name of the layer.
+ **kwargs: keyword arguments to be passed.
+ """
+ # Note: use_bias is ignored for the dense layer when use_bn=True.
+ # However, it is still used for batch norm.
+ super(DenseBN, self).__init__(**kwargs)
+ self._output_dim = output_dim
+ self._use_bias = use_bias
+ self._use_normalization = use_normalization
+ self._use_sync_bn = use_sync_bn
+ self._norm_momentum = norm_momentum
+ self._norm_epsilon = norm_epsilon
+ self._activation = activation
+ self._kernel_initializer = kernel_initializer
+ self._kernel_regularizer = kernel_regularizer
+ self._bias_regularizer = bias_regularizer
+ self._name = name
+
+ if use_sync_bn:
+ self._norm = tf.keras.layers.experimental.SyncBatchNormalization
+ else:
+ self._norm = tf.keras.layers.BatchNormalization
+ if tf.keras.backend.image_data_format() == 'channels_last':
+ self._bn_axis = -1
+ else:
+ self._bn_axis = 1
+ if activation:
+ self._activation_fn = tf_utils.get_activation(activation)
+ else:
+ self._activation_fn = None
+
+ def get_config(self):
+ config = {
+ 'output_dim': self._output_dim,
+ 'use_bias': self._use_bias,
+ 'activation': self._activation,
+ 'use_sync_bn': self._use_sync_bn,
+ 'use_normalization': self._use_normalization,
+ 'norm_momentum': self._norm_momentum,
+ 'norm_epsilon': self._norm_epsilon,
+ 'kernel_initializer': self._kernel_initializer,
+ 'kernel_regularizer': self._kernel_regularizer,
+ 'bias_regularizer': self._bias_regularizer,
+ }
+ base_config = super(DenseBN, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
+
+ def build(self, input_shape):
+ self._dense0 = tf.keras.layers.Dense(
+ self._output_dim,
+ kernel_initializer=self._kernel_initializer,
+ kernel_regularizer=self._kernel_regularizer,
+ bias_regularizer=self._bias_regularizer,
+ use_bias=self._use_bias and not self._use_normalization)
+
+ if self._use_normalization:
+ self._norm0 = self._norm(
+ axis=self._bn_axis,
+ momentum=self._norm_momentum,
+ epsilon=self._norm_epsilon,
+ center=self._use_bias,
+ scale=True)
+
+ super(DenseBN, self).build(input_shape)
+
+ def call(self, inputs, training=None):
+ assert inputs.shape.ndims == 2, inputs.shape
+ x = self._dense0(inputs)
+ if self._use_normalization:
+ x = self._norm0(x)
+ if self._activation:
+ x = self._activation_fn(x)
+ return x
diff --git a/official/vision/beta/projects/simclr/modeling/layers/nn_blocks_test.py b/official/vision/beta/projects/simclr/modeling/layers/nn_blocks_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..e3e0da20c41aff3105a24a2cc1ef158d27d1525c
--- /dev/null
+++ b/official/vision/beta/projects/simclr/modeling/layers/nn_blocks_test.py
@@ -0,0 +1,74 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from absl.testing import parameterized
+
+import tensorflow as tf
+
+from official.vision.beta.projects.simclr.modeling.layers import nn_blocks
+
+
+class DenseBNTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.parameters(
+ (64, True, True),
+ (64, True, False),
+ (64, False, True),
+ )
+ def test_pass_through(self, output_dim, use_bias, use_normalization):
+ test_layer = nn_blocks.DenseBN(
+ output_dim=output_dim,
+ use_bias=use_bias,
+ use_normalization=use_normalization
+ )
+
+ x = tf.keras.Input(shape=(64,))
+ out_x = test_layer(x)
+
+ self.assertAllEqual(out_x.shape.as_list(), [None, output_dim])
+
+ # kernel of the dense layer
+ train_var_len = 1
+ if use_normalization:
+ if use_bias:
+ # batch norm introduce two trainable variables
+ train_var_len += 2
+ else:
+ # center is set to False if not use bias
+ train_var_len += 1
+ else:
+ if use_bias:
+ # bias of dense layer
+ train_var_len += 1
+ self.assertLen(test_layer.trainable_variables, train_var_len)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/projects/simclr/modeling/simclr_model.py b/official/vision/beta/projects/simclr/modeling/simclr_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..06759f3b2e7b770af8189d92600b2600017e6779
--- /dev/null
+++ b/official/vision/beta/projects/simclr/modeling/simclr_model.py
@@ -0,0 +1,177 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Build simclr models."""
+
+from typing import Optional
+from absl import logging
+
+import tensorflow as tf
+
+layers = tf.keras.layers
+
+PRETRAIN = 'pretrain'
+FINETUNE = 'finetune'
+
+PROJECTION_OUTPUT_KEY = 'projection_outputs'
+SUPERVISED_OUTPUT_KEY = 'supervised_outputs'
+
+
+@tf.keras.utils.register_keras_serializable(package='simclr')
+class SimCLRModel(tf.keras.Model):
+ """A classification model based on SimCLR framework."""
+
+ def __init__(self,
+ backbone: tf.keras.models.Model,
+ projection_head: tf.keras.layers.Layer,
+ supervised_head: Optional[tf.keras.layers.Layer] = None,
+ input_specs=layers.InputSpec(shape=[None, None, None, 3]),
+ mode: str = PRETRAIN,
+ backbone_trainable: bool = True,
+ **kwargs):
+ """A classification model based on SimCLR framework.
+
+ Args:
+ backbone: a backbone network.
+ projection_head: a projection head network.
+ supervised_head: a head network for supervised learning, e.g.
+ classification head.
+ input_specs: `tf.keras.layers.InputSpec` specs of the input tensor.
+ mode: `str` indicates mode of training to be executed.
+ backbone_trainable: `bool` whether the backbone is trainable or not.
+ **kwargs: keyword arguments to be passed.
+ """
+ super(SimCLRModel, self).__init__(**kwargs)
+ self._config_dict = {
+ 'backbone': backbone,
+ 'projection_head': projection_head,
+ 'supervised_head': supervised_head,
+ 'input_specs': input_specs,
+ 'mode': mode,
+ 'backbone_trainable': backbone_trainable,
+ }
+ self._input_specs = input_specs
+ self._backbone = backbone
+ self._projection_head = projection_head
+ self._supervised_head = supervised_head
+ self._mode = mode
+ self._backbone_trainable = backbone_trainable
+
+ # Set whether the backbone is trainable
+ self._backbone.trainable = backbone_trainable
+
+ def call(self, inputs, training=None, **kwargs):
+ model_outputs = {}
+
+ if training and self._mode == PRETRAIN:
+ num_transforms = 2
+ else:
+ num_transforms = 1
+
+ # Split channels, and optionally apply extra batched augmentation.
+ # (bsz, h, w, c*num_transforms) -> [(bsz, h, w, c), ....]
+ features_list = tf.split(inputs, num_or_size_splits=num_transforms, axis=-1)
+ # (num_transforms * bsz, h, w, c)
+ features = tf.concat(features_list, 0)
+
+ # Base network forward pass.
+ endpoints = self._backbone(features, training=training)
+ features = endpoints[max(endpoints.keys())]
+ projection_inputs = layers.GlobalAveragePooling2D()(features)
+
+ # Add heads.
+ projection_outputs, supervised_inputs = self._projection_head(
+ projection_inputs, training)
+
+ if self._supervised_head is not None:
+ if self._mode == PRETRAIN:
+ logging.info('Ignoring gradient from supervised outputs !')
+ # When performing pretraining and supervised_head together, we do not
+ # want information from supervised evaluation flowing back into
+ # pretraining network. So we put a stop_gradient.
+ supervised_outputs = self._supervised_head(
+ tf.stop_gradient(supervised_inputs), training)
+ else:
+ supervised_outputs = self._supervised_head(supervised_inputs, training)
+ else:
+ supervised_outputs = None
+
+ model_outputs.update({
+ PROJECTION_OUTPUT_KEY: projection_outputs,
+ SUPERVISED_OUTPUT_KEY: supervised_outputs
+ })
+
+ return model_outputs
+
+ @property
+ def checkpoint_items(self):
+ """Returns a dictionary of items to be additionally checkpointed."""
+ if self._supervised_head is not None:
+ items = dict(backbone=self.backbone,
+ projection_head=self.projection_head,
+ supervised_head=self.supervised_head)
+ else:
+ items = dict(backbone=self.backbone,
+ projection_head=self.projection_head)
+ return items
+
+ @property
+ def backbone(self):
+ return self._backbone
+
+ @property
+ def projection_head(self):
+ return self._projection_head
+
+ @property
+ def supervised_head(self):
+ return self._supervised_head
+
+ @property
+ def mode(self):
+ return self._mode
+
+ @mode.setter
+ def mode(self, value):
+ self._mode = value
+
+ @property
+ def backbone_trainable(self):
+ return self._backbone_trainable
+
+ @backbone_trainable.setter
+ def backbone_trainable(self, value):
+ self._backbone_trainable = value
+ self._backbone.trainable = value
+
+ def get_config(self):
+ return self._config_dict
+
+ @classmethod
+ def from_config(cls, config, custom_objects=None):
+ return cls(**config)
diff --git a/official/vision/beta/projects/simclr/modeling/simclr_model_test.py b/official/vision/beta/projects/simclr/modeling/simclr_model_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a54e247252a0d6ac9e1f6a97f772651dbb74fdf
--- /dev/null
+++ b/official/vision/beta/projects/simclr/modeling/simclr_model_test.py
@@ -0,0 +1,87 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from absl.testing import parameterized
+
+import numpy as np
+import tensorflow as tf
+
+from official.vision.beta.modeling import backbones
+from official.vision.beta.projects.simclr.heads import simclr_head
+from official.vision.beta.projects.simclr.modeling import simclr_model
+
+
+class SimCLRModelTest(parameterized.TestCase, tf.test.TestCase):
+
+ @parameterized.parameters(
+ (128, 3, 0),
+ (128, 3, 1),
+ (128, 1, 0),
+ (128, 1, 1),
+ )
+ def test_model_creation(self, project_dim, num_proj_layers, ft_proj_idx):
+ input_size = 224
+ inputs = np.random.rand(2, input_size, input_size, 3)
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[None, input_size, input_size, 3])
+
+ tf.keras.backend.set_image_data_format('channels_last')
+
+ backbone = backbones.ResNet(model_id=50, activation='relu',
+ input_specs=input_specs)
+ projection_head = simclr_head.ProjectionHead(
+ proj_output_dim=project_dim,
+ num_proj_layers=num_proj_layers,
+ ft_proj_idx=ft_proj_idx
+ )
+ num_classes = 10
+ supervised_head = simclr_head.ClassificationHead(
+ num_classes=10
+ )
+
+ model = simclr_model.SimCLRModel(
+ input_specs=input_specs,
+ backbone=backbone,
+ projection_head=projection_head,
+ supervised_head=supervised_head,
+ mode=simclr_model.PRETRAIN
+ )
+ outputs = model(inputs)
+ projection_outputs = outputs[simclr_model.PROJECTION_OUTPUT_KEY]
+ supervised_outputs = outputs[simclr_model.SUPERVISED_OUTPUT_KEY]
+
+ self.assertAllEqual(projection_outputs.shape.as_list(),
+ [2, project_dim])
+ self.assertAllEqual([2, num_classes],
+ supervised_outputs.numpy().shape)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/projects/simclr/tasks/simclr.py b/official/vision/beta/projects/simclr/tasks/simclr.py
new file mode 100644
index 0000000000000000000000000000000000000000..b9501af0c1077c0af0c8ca80c12084e0a4f2bbc0
--- /dev/null
+++ b/official/vision/beta/projects/simclr/tasks/simclr.py
@@ -0,0 +1,640 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Image SimCLR task definition.
+
+SimCLR training two different modes:
+- pretrain
+- fine-tuning
+
+For the above two different modes, the following components are different in
+the task definition:
+- training data format
+- training loss
+- projection_head and/or supervised_head
+"""
+
+from typing import Dict, Optional
+
+from absl import logging
+import tensorflow as tf
+
+from official.core import base_task
+from official.core import config_definitions
+from official.core import input_reader
+from official.core import task_factory
+from official.modeling import optimization
+from official.modeling import performance
+from official.modeling import tf_utils
+from official.vision.beta.modeling import backbones
+from official.vision.beta.projects.simclr.configs import simclr as exp_cfg
+from official.vision.beta.projects.simclr.dataloaders import simclr_input
+from official.vision.beta.projects.simclr.heads import simclr_head
+from official.vision.beta.projects.simclr.losses import contrastive_losses
+from official.vision.beta.projects.simclr.modeling import simclr_model
+
+OptimizationConfig = optimization.OptimizationConfig
+RuntimeConfig = config_definitions.RuntimeConfig
+
+
+@task_factory.register_task_cls(exp_cfg.SimCLRPretrainTask)
+class SimCLRPretrainTask(base_task.Task):
+ """A task for image classification."""
+
+ def create_optimizer(self, optimizer_config: OptimizationConfig,
+ runtime_config: Optional[RuntimeConfig] = None):
+ """Creates an TF optimizer from configurations.
+
+ Args:
+ optimizer_config: the parameters of the Optimization settings.
+ runtime_config: the parameters of the runtime.
+
+ Returns:
+ A tf.optimizers.Optimizer object.
+ """
+ if (optimizer_config.optimizer.type == 'lars'
+ and self.task_config.loss.l2_weight_decay > 0.0):
+ raise ValueError('The l2_weight_decay cannot be used together with lars '
+ 'optimizer. Please set it to 0.')
+
+ opt_factory = optimization.OptimizerFactory(optimizer_config)
+ optimizer = opt_factory.build_optimizer(opt_factory.build_learning_rate())
+ # Configuring optimizer when loss_scale is set in runtime config. This helps
+ # avoiding overflow/underflow for float16 computations.
+ if runtime_config and runtime_config.loss_scale:
+ optimizer = performance.configure_optimizer(
+ optimizer,
+ use_float16=runtime_config.mixed_precision_dtype == 'float16',
+ loss_scale=runtime_config.loss_scale)
+
+ return optimizer
+
+ def build_model(self):
+ model_config = self.task_config.model
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[None] + model_config.input_size)
+
+ l2_weight_decay = self.task_config.loss.l2_weight_decay
+ # Divide weight decay by 2.0 to match the implementation of tf.nn.l2_loss.
+ # (https://www.tensorflow.org/api_docs/python/tf/keras/regularizers/l2)
+ # (https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss)
+ l2_regularizer = (tf.keras.regularizers.l2(
+ l2_weight_decay / 2.0) if l2_weight_decay else None)
+
+ # Build backbone
+ backbone = backbones.factory.build_backbone(
+ input_specs=input_specs,
+ model_config=model_config,
+ l2_regularizer=l2_regularizer)
+
+ # Build projection head
+ norm_activation_config = model_config.norm_activation
+ projection_head_config = model_config.projection_head
+ projection_head = simclr_head.ProjectionHead(
+ proj_output_dim=projection_head_config.proj_output_dim,
+ num_proj_layers=projection_head_config.num_proj_layers,
+ ft_proj_idx=projection_head_config.ft_proj_idx,
+ kernel_regularizer=l2_regularizer,
+ use_sync_bn=norm_activation_config.use_sync_bn,
+ norm_momentum=norm_activation_config.norm_momentum,
+ norm_epsilon=norm_activation_config.norm_epsilon)
+
+ # Build supervised head
+ supervised_head_config = model_config.supervised_head
+ if supervised_head_config:
+ if supervised_head_config.zero_init:
+ s_kernel_initializer = 'zeros'
+ else:
+ s_kernel_initializer = 'random_uniform'
+ supervised_head = simclr_head.ClassificationHead(
+ num_classes=supervised_head_config.num_classes,
+ kernel_initializer=s_kernel_initializer,
+ kernel_regularizer=l2_regularizer)
+ else:
+ supervised_head = None
+
+ model = simclr_model.SimCLRModel(
+ input_specs=input_specs,
+ backbone=backbone,
+ projection_head=projection_head,
+ supervised_head=supervised_head,
+ mode=model_config.mode,
+ backbone_trainable=model_config.backbone_trainable)
+
+ logging.info(model.get_config())
+
+ return model
+
+ def initialize(self, model: tf.keras.Model):
+ """Loading pretrained checkpoint."""
+ if not self.task_config.init_checkpoint:
+ return
+
+ ckpt_dir_or_file = self.task_config.init_checkpoint
+ if tf.io.gfile.isdir(ckpt_dir_or_file):
+ ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file)
+
+ # Restoring checkpoint.
+ if self.task_config.init_checkpoint_modules == 'all':
+ ckpt = tf.train.Checkpoint(**model.checkpoint_items)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.assert_consumed()
+ elif self.task_config.init_checkpoint_modules == 'backbone':
+ ckpt = tf.train.Checkpoint(backbone=model.backbone)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.expect_partial().assert_existing_objects_matched()
+ else:
+ assert "Only 'all' or 'backbone' can be used to initialize the model."
+
+ logging.info('Finished loading pretrained checkpoint from %s',
+ ckpt_dir_or_file)
+
+ def build_inputs(self, params, input_context=None):
+ input_size = self.task_config.model.input_size
+
+ if params.tfds_name:
+ decoder = simclr_input.TFDSDecoder(params.decoder.decode_label)
+ else:
+ decoder = simclr_input.Decoder(params.decoder.decode_label)
+
+ parser = simclr_input.Parser(
+ output_size=input_size[:2],
+ aug_rand_crop=params.parser.aug_rand_crop,
+ aug_rand_hflip=params.parser.aug_rand_hflip,
+ aug_color_distort=params.parser.aug_color_distort,
+ aug_color_jitter_strength=params.parser.aug_color_jitter_strength,
+ aug_color_jitter_impl=params.parser.aug_color_jitter_impl,
+ aug_rand_blur=params.parser.aug_rand_blur,
+ parse_label=params.parser.parse_label,
+ test_crop=params.parser.test_crop,
+ mode=params.parser.mode,
+ dtype=params.dtype)
+
+ reader = input_reader.InputReader(
+ params,
+ dataset_fn=tf.data.TFRecordDataset,
+ decoder_fn=decoder.decode,
+ parser_fn=parser.parse_fn(params.is_training))
+
+ dataset = reader.read(input_context=input_context)
+
+ return dataset
+
+ def build_losses(self,
+ labels,
+ model_outputs,
+ aux_losses=None) -> Dict[str, tf.Tensor]:
+ # Compute contrastive relative loss
+ con_losses_obj = contrastive_losses.ContrastiveLoss(
+ projection_norm=self.task_config.loss.projection_norm,
+ temperature=self.task_config.loss.temperature)
+ # The projection outputs from model has the size of
+ # (2 * bsz, project_dim)
+ projection_outputs = model_outputs[simclr_model.PROJECTION_OUTPUT_KEY]
+ projection1, projection2 = tf.split(projection_outputs, 2, 0)
+ contrast_loss, (contrast_logits, contrast_labels) = con_losses_obj(
+ projection1=projection1,
+ projection2=projection2)
+
+ contrast_accuracy = tf.equal(
+ tf.argmax(contrast_labels, axis=1), tf.argmax(contrast_logits, axis=1))
+ contrast_accuracy = tf.reduce_mean(tf.cast(contrast_accuracy, tf.float32))
+
+ contrast_prob = tf.nn.softmax(contrast_logits)
+ contrast_entropy = -tf.reduce_mean(
+ tf.reduce_sum(contrast_prob * tf.math.log(contrast_prob + 1e-8), -1))
+
+ model_loss = contrast_loss
+
+ losses = {
+ 'contrast_loss': contrast_loss,
+ 'contrast_accuracy': contrast_accuracy,
+ 'contrast_entropy': contrast_entropy
+ }
+
+ if self.task_config.model.supervised_head is not None:
+ outputs = model_outputs[simclr_model.SUPERVISED_OUTPUT_KEY]
+ labels = tf.concat([labels, labels], 0)
+
+ if self.task_config.evaluation.one_hot:
+ sup_loss = tf.keras.losses.CategoricalCrossentropy(
+ from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(labels,
+ outputs)
+ else:
+ sup_loss = tf.keras.losses.SparseCategoricalCrossentropy(
+ from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(labels,
+ outputs)
+ sup_loss = tf.reduce_mean(sup_loss)
+
+ label_acc = tf.equal(tf.argmax(labels, axis=1),
+ tf.argmax(outputs, axis=1))
+ label_acc = tf.reduce_mean(tf.cast(label_acc, tf.float32))
+
+ model_loss = contrast_loss + sup_loss
+
+ losses.update({
+ 'accuracy': label_acc,
+ 'supervised_loss': sup_loss,
+ })
+
+ total_loss = model_loss
+ if aux_losses:
+ reg_loss = tf.reduce_sum(aux_losses)
+ total_loss = model_loss + reg_loss
+
+ losses['total_loss'] = total_loss
+
+ return losses
+
+ def build_metrics(self, training=True):
+
+ if training:
+ metrics = []
+ metric_names = [
+ 'total_loss',
+ 'contrast_loss',
+ 'contrast_accuracy',
+ 'contrast_entropy'
+ ]
+ if self.task_config.model.supervised_head:
+ metric_names.extend(['supervised_loss', 'accuracy'])
+ for name in metric_names:
+ metrics.append(tf.keras.metrics.Mean(name, dtype=tf.float32))
+ else:
+ k = self.task_config.evaluation.top_k
+ if self.task_config.evaluation.one_hot:
+ metrics = [
+ tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
+ tf.keras.metrics.TopKCategoricalAccuracy(
+ k=k, name='top_{}_accuracy'.format(k))]
+ else:
+ metrics = [
+ tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
+ tf.keras.metrics.SparseTopKCategoricalAccuracy(
+ k=k, name='top_{}_accuracy'.format(k))]
+ return metrics
+
+ def train_step(self, inputs, model, optimizer, metrics=None):
+ features, labels = inputs
+ if (self.task_config.model.supervised_head is not None
+ and self.task_config.evaluation.one_hot):
+ num_classes = self.task_config.model.supervised_head.num_classes
+ labels = tf.one_hot(labels, num_classes)
+
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(features, training=True)
+ # Casting output layer as float32 is necessary when mixed_precision is
+ # mixed_float16 or mixed_bfloat16 to ensure output is casted as float32.
+ outputs = tf.nest.map_structure(
+ lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss.
+ losses = self.build_losses(
+ model_outputs=outputs, labels=labels, aux_losses=model.losses)
+
+ scaled_loss = losses['total_loss'] / num_replicas
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ logging.info('Trainable variables:')
+ for var in tvars:
+ logging.info(var.name)
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient when LossScaleOptimizer is used.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = {self.loss: losses['total_loss']}
+
+ for m in metrics:
+ m.update_state(losses[m.name])
+ logs.update({m.name: m.result()})
+
+ return logs
+
+ def validation_step(self, inputs, model, metrics=None):
+ if self.task_config.model.supervised_head is None:
+ assert 'Skipping eval during pretraining without supervised head.'
+
+ features, labels = inputs
+ if self.task_config.evaluation.one_hot:
+ num_classes = self.task_config.model.supervised_head.num_classes
+ labels = tf.one_hot(labels, num_classes)
+
+ outputs = model(
+ features, training=False)[simclr_model.SUPERVISED_OUTPUT_KEY]
+ outputs = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), outputs)
+
+ logs = {self.loss: 0}
+
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in metrics})
+ elif model.compiled_metrics:
+ self.process_compiled_metrics(model.compiled_metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in model.metrics})
+
+ return logs
+
+
+@task_factory.register_task_cls(exp_cfg.SimCLRFinetuneTask)
+class SimCLRFinetuneTask(base_task.Task):
+ """A task for image classification."""
+
+ def create_optimizer(self, optimizer_config: OptimizationConfig,
+ runtime_config: Optional[RuntimeConfig] = None):
+ """Creates an TF optimizer from configurations.
+
+ Args:
+ optimizer_config: the parameters of the Optimization settings.
+ runtime_config: the parameters of the runtime.
+
+ Returns:
+ A tf.optimizers.Optimizer object.
+ """
+ if (optimizer_config.optimizer.type == 'lars'
+ and self.task_config.loss.l2_weight_decay > 0.0):
+ raise ValueError('The l2_weight_decay cannot be used together with lars '
+ 'optimizer. Please set it to 0.')
+
+ opt_factory = optimization.OptimizerFactory(optimizer_config)
+ optimizer = opt_factory.build_optimizer(opt_factory.build_learning_rate())
+ # Configuring optimizer when loss_scale is set in runtime config. This helps
+ # avoiding overflow/underflow for float16 computations.
+ if runtime_config and runtime_config.loss_scale:
+ optimizer = performance.configure_optimizer(
+ optimizer,
+ use_float16=runtime_config.mixed_precision_dtype == 'float16',
+ loss_scale=runtime_config.loss_scale)
+
+ return optimizer
+
+ def build_model(self):
+ model_config = self.task_config.model
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[None] + model_config.input_size)
+
+ l2_weight_decay = self.task_config.loss.l2_weight_decay
+ # Divide weight decay by 2.0 to match the implementation of tf.nn.l2_loss.
+ # (https://www.tensorflow.org/api_docs/python/tf/keras/regularizers/l2)
+ # (https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss)
+ l2_regularizer = (tf.keras.regularizers.l2(
+ l2_weight_decay / 2.0) if l2_weight_decay else None)
+
+ backbone = backbones.factory.build_backbone(
+ input_specs=input_specs,
+ model_config=model_config,
+ l2_regularizer=l2_regularizer)
+
+ norm_activation_config = model_config.norm_activation
+ projection_head_config = model_config.projection_head
+ projection_head = simclr_head.ProjectionHead(
+ proj_output_dim=projection_head_config.proj_output_dim,
+ num_proj_layers=projection_head_config.num_proj_layers,
+ ft_proj_idx=projection_head_config.ft_proj_idx,
+ kernel_regularizer=l2_regularizer,
+ use_sync_bn=norm_activation_config.use_sync_bn,
+ norm_momentum=norm_activation_config.norm_momentum,
+ norm_epsilon=norm_activation_config.norm_epsilon)
+
+ supervised_head_config = model_config.supervised_head
+ if supervised_head_config.zero_init:
+ s_kernel_initializer = 'zeros'
+ else:
+ s_kernel_initializer = 'random_uniform'
+ supervised_head = simclr_head.ClassificationHead(
+ num_classes=supervised_head_config.num_classes,
+ kernel_initializer=s_kernel_initializer,
+ kernel_regularizer=l2_regularizer)
+
+ model = simclr_model.SimCLRModel(
+ input_specs=input_specs,
+ backbone=backbone,
+ projection_head=projection_head,
+ supervised_head=supervised_head,
+ mode=model_config.mode,
+ backbone_trainable=model_config.backbone_trainable)
+
+ logging.info(model.get_config())
+
+ return model
+
+ def initialize(self, model: tf.keras.Model):
+ """Loading pretrained checkpoint."""
+ if not self.task_config.init_checkpoint:
+ return
+
+ ckpt_dir_or_file = self.task_config.init_checkpoint
+ if tf.io.gfile.isdir(ckpt_dir_or_file):
+ ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file)
+
+ # Restoring checkpoint.
+ if self.task_config.init_checkpoint_modules == 'all':
+ ckpt = tf.train.Checkpoint(**model.checkpoint_items)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.assert_consumed()
+ elif self.task_config.init_checkpoint_modules == 'backbone_projection':
+ ckpt = tf.train.Checkpoint(backbone=model.backbone,
+ projection_head=model.projection_head)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.expect_partial().assert_existing_objects_matched()
+ elif self.task_config.init_checkpoint_modules == 'backbone':
+ ckpt = tf.train.Checkpoint(backbone=model.backbone)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.expect_partial().assert_existing_objects_matched()
+ else:
+ assert "Only 'all' or 'backbone' can be used to initialize the model."
+
+ # If the checkpoint is from pretraining, reset the following parameters
+ model.backbone_trainable = self.task_config.model.backbone_trainable
+ logging.info('Finished loading pretrained checkpoint from %s',
+ ckpt_dir_or_file)
+
+ def build_inputs(self, params, input_context=None):
+ input_size = self.task_config.model.input_size
+
+ if params.tfds_name:
+ decoder = simclr_input.TFDSDecoder(params.decoder.decode_label)
+ else:
+ decoder = simclr_input.Decoder(params.decoder.decode_label)
+ parser = simclr_input.Parser(
+ output_size=input_size[:2],
+ parse_label=params.parser.parse_label,
+ test_crop=params.parser.test_crop,
+ mode=params.parser.mode,
+ dtype=params.dtype)
+
+ reader = input_reader.InputReader(
+ params,
+ dataset_fn=tf.data.TFRecordDataset,
+ decoder_fn=decoder.decode,
+ parser_fn=parser.parse_fn(params.is_training))
+
+ dataset = reader.read(input_context=input_context)
+
+ return dataset
+
+ def build_losses(self, labels, model_outputs, aux_losses=None):
+ """Sparse categorical cross entropy loss.
+
+ Args:
+ labels: labels.
+ model_outputs: Output logits of the classifier.
+ aux_losses: auxiliarly loss tensors, i.e. `losses` in keras.Model.
+
+ Returns:
+ The total loss tensor.
+ """
+ losses_config = self.task_config.loss
+ if losses_config.one_hot:
+ total_loss = tf.keras.losses.categorical_crossentropy(
+ labels,
+ model_outputs,
+ from_logits=True,
+ label_smoothing=losses_config.label_smoothing)
+ else:
+ total_loss = tf.keras.losses.sparse_categorical_crossentropy(
+ labels, model_outputs, from_logits=True)
+
+ total_loss = tf_utils.safe_mean(total_loss)
+ if aux_losses:
+ total_loss += tf.add_n(aux_losses)
+
+ return total_loss
+
+ def build_metrics(self, training=True):
+ """Gets streaming metrics for training/validation."""
+ k = self.task_config.evaluation.top_k
+ if self.task_config.evaluation.one_hot:
+ metrics = [
+ tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
+ tf.keras.metrics.TopKCategoricalAccuracy(
+ k=k, name='top_{}_accuracy'.format(k))]
+ else:
+ metrics = [
+ tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
+ tf.keras.metrics.SparseTopKCategoricalAccuracy(
+ k=k, name='top_{}_accuracy'.format(k))]
+ return metrics
+
+ def train_step(self, inputs, model, optimizer, metrics=None):
+ """Does forward and backward.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the model, forward pass definition.
+ optimizer: the optimizer for this training step.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+ if self.task_config.loss.one_hot:
+ num_classes = self.task_config.model.supervised_head.num_classes
+ labels = tf.one_hot(labels, num_classes)
+
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(
+ features, training=True)[simclr_model.SUPERVISED_OUTPUT_KEY]
+ # Casting output layer as float32 is necessary when mixed_precision is
+ # mixed_float16 or mixed_bfloat16 to ensure output is casted as float32.
+ outputs = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss.
+ loss = self.build_losses(
+ model_outputs=outputs,
+ labels=labels, aux_losses=model.losses)
+ # Scales loss as the default gradients allreduce performs sum inside the
+ # optimizer.
+ scaled_loss = loss / num_replicas
+
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(
+ optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ logging.info('Trainable variables:')
+ for var in tvars:
+ logging.info(var.name)
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient before apply_gradients when LossScaleOptimizer is
+ # used.
+ if isinstance(
+ optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = {self.loss: loss}
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in metrics})
+ elif model.compiled_metrics:
+ self.process_compiled_metrics(model.compiled_metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in model.metrics})
+ return logs
+
+ def validation_step(self, inputs, model, metrics=None):
+ """Validatation step.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the keras.Model.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+ if self.task_config.loss.one_hot:
+ num_classes = self.task_config.model.supervised_head.num_classes
+ labels = tf.one_hot(labels, num_classes)
+
+ outputs = self.inference_step(
+ features, model)[simclr_model.SUPERVISED_OUTPUT_KEY]
+ outputs = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), outputs)
+ loss = self.build_losses(model_outputs=outputs,
+ labels=labels, aux_losses=model.losses)
+
+ logs = {self.loss: loss}
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in metrics})
+ elif model.compiled_metrics:
+ self.process_compiled_metrics(model.compiled_metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in model.metrics})
+ return logs
diff --git a/official/vision/beta/projects/simclr/train.py b/official/vision/beta/projects/simclr/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..ca7c92262db6f49365b15f3b985109487a7a8065
--- /dev/null
+++ b/official/vision/beta/projects/simclr/train.py
@@ -0,0 +1,81 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""TensorFlow Model Garden Vision SimCLR training driver."""
+from absl import app
+from absl import flags
+import gin
+
+from official.common import distribute_utils
+from official.common import flags as tfm_flags
+from official.core import task_factory
+from official.core import train_lib
+from official.core import train_utils
+from official.modeling import performance
+from official.vision.beta.projects.simclr.common import registry_imports # pylint: disable=unused-import
+
+FLAGS = flags.FLAGS
+
+
+def main(_):
+ gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params)
+ print(FLAGS.experiment)
+ params = train_utils.parse_configuration(FLAGS)
+
+ model_dir = FLAGS.model_dir
+ if 'train' in FLAGS.mode:
+ # Pure eval modes do not output yaml files. Otherwise continuous eval job
+ # may race against the train job for writing the same file.
+ train_utils.serialize_config(params, model_dir)
+
+ # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16'
+ # can have significant impact on model speeds by utilizing float16 in case of
+ # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when
+ # dtype is float16
+ if params.runtime.mixed_precision_dtype:
+ performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype)
+ distribution_strategy = distribute_utils.get_distribution_strategy(
+ distribution_strategy=params.runtime.distribution_strategy,
+ all_reduce_alg=params.runtime.all_reduce_alg,
+ num_gpus=params.runtime.num_gpus,
+ tpu_address=params.runtime.tpu)
+ with distribution_strategy.scope():
+ task = task_factory.get_task(params.task, logging_dir=model_dir)
+
+ train_lib.run_experiment(
+ distribution_strategy=distribution_strategy,
+ task=task,
+ mode=FLAGS.mode,
+ params=params,
+ model_dir=model_dir)
+
+
+if __name__ == '__main__':
+ tfm_flags.define_flags()
+ app.run(main)
diff --git a/official/vision/beta/projects/yolo/README.md b/official/vision/beta/projects/yolo/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..0a1e27fbe9010f08d1a94361792d84335a97e235
--- /dev/null
+++ b/official/vision/beta/projects/yolo/README.md
@@ -0,0 +1,76 @@
+# YOLO Object Detectors, You Only Look Once
+
+[](https://arxiv.org/abs/1804.02767)
+[](https://arxiv.org/abs/2004.10934)
+
+This repository is the unofficial implementation of the following papers.
+However, we spent painstaking hours ensuring that every aspect that we
+constructed was the exact same as the original paper and the original
+repository.
+
+* YOLOv3: An Incremental Improvement: [YOLOv3: An Incremental Improvement](https://arxiv.org/abs/1804.02767)
+
+* YOLOv4: Optimal Speed and Accuracy of Object Detection: [YOLOv4: Optimal Speed and Accuracy of Object Detection](https://arxiv.org/abs/2004.10934)
+
+## Description
+
+Yolo v1 the original implementation was released in 2015 providing a ground
+breaking algorithm that would quickly process images, and locate objects in a
+single pass through the detector. The original implementation based used a
+backbone derived from state of the art object classifier of the time, like
+[GoogLeNet](https://arxiv.org/abs/1409.4842) and
+[VGG](https://arxiv.org/abs/1409.1556). More attention was given to the novel
+Yolo Detection head that allowed for Object Detection with a single pass of an
+image. Though limited, the network could predict up to 90 bounding boxes per
+image, and was tested for about 80 classes per box. Also, the model could only
+make prediction at one scale. These attributes caused yolo v1 to be more
+limited, and less versatile, so as the year passed, the Developers continued to
+update and develop this model.
+
+Yolo v3 and v4 serve as the most up to date and capable versions of the Yolo
+network group. These model uses a custom backbone called Darknet53 that uses
+knowledge gained from the ResNet paper to improve its predictions. The new
+backbone also allows for objects to be detected at multiple scales. As for the
+new detection head, the model now predicts the bounding boxes using a set of
+anchor box priors (Anchor Boxes) as suggestions. The multiscale predictions in
+combination with the Anchor boxes allows for the network to make up to 1000
+object predictions on a single image. Finally, the new loss function forces the
+network to make better prediction by using Intersection Over Union (IOU) to
+inform the model's confidence rather than relying on the mean squared error for
+the entire output.
+
+## Authors
+
+* Vishnu Samardh Banna ([@GitHub vishnubanna](https://github.com/vishnubanna))
+* Anirudh Vegesana ([@GitHub anivegesana](https://github.com/anivegesana))
+* Akhil Chinnakotla ([@GitHub The-Indian-Chinna](https://github.com/The-Indian-Chinna))
+* Tristan Yan ([@GitHub Tyan3001](https://github.com/Tyan3001))
+* Naveen Vivek ([@GitHub naveen-vivek](https://github.com/naveen-vivek))
+
+## Table of Contents
+
+* [Our Goal](#our-goal)
+* [Models in the library](#models-in-the-library)
+* [References](#references)
+
+
+## Our Goal
+
+Our goal with this model conversion is to provide implementations of the
+Backbone and Yolo Head. We have built the model in such a way that the Yolo
+head could be connected to a new, more powerful backbone if a person chose to.
+
+## Models in the library
+
+| Object Detectors | Classifiers |
+| :--------------: | :--------------: |
+| Yolo-v3 | Darknet53 |
+| Yolo-v3 tiny | CSPDarknet53 |
+| Yolo-v3 spp |
+| Yolo-v4 |
+| Yolo-v4 tiny |
+
+## Requirements
+
+[](https://github.com/tensorflow/tensorflow/releases/tag/v2.2.0)
+[](https://www.python.org/downloads/release/python-380/)
diff --git a/official/vision/beta/projects/yolo/common/registry_imports.py b/official/vision/beta/projects/yolo/common/registry_imports.py
new file mode 100644
index 0000000000000000000000000000000000000000..4ee795b8cc964514d3a05e6f48ad3186b45f7b22
--- /dev/null
+++ b/official/vision/beta/projects/yolo/common/registry_imports.py
@@ -0,0 +1,21 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""All necessary imports for registration."""
+
+# pylint: disable=unused-import
+from official.common import registry_imports
+from official.vision.beta.projects.yolo.configs import darknet_classification
+from official.vision.beta.projects.yolo.modeling.backbones import darknet
+from official.vision.beta.projects.yolo.tasks import image_classification
diff --git a/official/vision/beta/projects/yolo/configs/backbones.py b/official/vision/beta/projects/yolo/configs/backbones.py
new file mode 100644
index 0000000000000000000000000000000000000000..a79cb09e17e6e7e633e652dd4965866b3d0e80b8
--- /dev/null
+++ b/official/vision/beta/projects/yolo/configs/backbones.py
@@ -0,0 +1,34 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+
+"""Backbones configurations."""
+
+import dataclasses
+
+from official.modeling import hyperparams
+
+from official.vision.beta.configs import backbones
+
+
+@dataclasses.dataclass
+class DarkNet(hyperparams.Config):
+ """DarkNet config."""
+ model_id: str = "darknet53"
+
+
+@dataclasses.dataclass
+class Backbone(backbones.Backbone):
+ darknet: DarkNet = DarkNet()
diff --git a/official/vision/beta/projects/yolo/configs/darknet_classification.py b/official/vision/beta/projects/yolo/configs/darknet_classification.py
new file mode 100644
index 0000000000000000000000000000000000000000..b33e149d484f850aa9fcada5f94612eca8bf754d
--- /dev/null
+++ b/official/vision/beta/projects/yolo/configs/darknet_classification.py
@@ -0,0 +1,70 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Image classification with darknet configs."""
+
+from typing import List, Optional
+
+import dataclasses
+
+from official.core import config_definitions as cfg
+from official.core import exp_factory
+from official.modeling import hyperparams
+from official.vision.beta.configs import common
+from official.vision.beta.configs import image_classification as imc
+from official.vision.beta.projects.yolo.configs import backbones
+
+
+@dataclasses.dataclass
+class ImageClassificationModel(hyperparams.Config):
+ num_classes: int = 0
+ input_size: List[int] = dataclasses.field(default_factory=list)
+ backbone: backbones.Backbone = backbones.Backbone(
+ type='darknet', resnet=backbones.DarkNet())
+ dropout_rate: float = 0.0
+ norm_activation: common.NormActivation = common.NormActivation()
+ # Adds a BatchNormalization layer pre-GlobalAveragePooling in classification
+ add_head_batch_norm: bool = False
+
+
+@dataclasses.dataclass
+class Losses(hyperparams.Config):
+ one_hot: bool = True
+ label_smoothing: float = 0.0
+ l2_weight_decay: float = 0.0
+
+
+@dataclasses.dataclass
+class ImageClassificationTask(cfg.TaskConfig):
+ """The model config."""
+ model: ImageClassificationModel = ImageClassificationModel()
+ train_data: imc.DataConfig = imc.DataConfig(is_training=True)
+ validation_data: imc.DataConfig = imc.DataConfig(is_training=False)
+ evaluation: imc.Evaluation = imc.Evaluation()
+ losses: Losses = Losses()
+ gradient_clip_norm: float = 0.0
+ logging_dir: Optional[str] = None
+
+
+@exp_factory.register_config_factory('darknet_classification')
+def image_classification() -> cfg.ExperimentConfig:
+ """Image classification general."""
+ return cfg.ExperimentConfig(
+ task=ImageClassificationTask(),
+ trainer=cfg.TrainerConfig(),
+ restrictions=[
+ 'task.train_data.is_training != None',
+ 'task.validation_data.is_training != None'
+ ])
diff --git a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..10dbdc5685583475106e4d9af37a4cabd59429b0
--- /dev/null
+++ b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53.yaml
@@ -0,0 +1,51 @@
+runtime:
+ distribution_strategy: 'mirrored'
+ mixed_precision_dtype: 'float32'
+task:
+ model:
+ num_classes: 1001
+ input_size: [256, 256, 3]
+ backbone:
+ type: 'darknet'
+ darknet:
+ model_id: 'cspdarknet53'
+ norm_activation:
+ activation: 'mish'
+ losses:
+ l2_weight_decay: 0.0005
+ one_hot: true
+ label_smoothing: 0.1
+ train_data:
+ input_path: 'imagenet-2012-tfrecord/train*'
+ is_training: true
+ global_batch_size: 128
+ dtype: 'float16'
+ validation_data:
+ input_path: 'imagenet-2012-tfrecord/valid*'
+ is_training: true
+ global_batch_size: 128
+ dtype: 'float16'
+ drop_remainder: false
+trainer:
+ train_steps: 1200000 # epochs: 120
+ validation_steps: 400 # size of validation data
+ validation_interval: 10000
+ steps_per_loop: 10000
+ summary_interval: 10000
+ checkpoint_interval: 10000
+ optimizer_config:
+ optimizer:
+ type: 'sgd'
+ sgd:
+ momentum: 0.9
+ learning_rate:
+ type: 'polynomial'
+ polynomial:
+ initial_learning_rate: 0.1
+ end_learning_rate: 0.0001
+ power: 4.0
+ decay_steps: 1200000
+ warmup:
+ type: 'linear'
+ linear:
+ warmup_steps: 1000 # learning rate rises from 0 to 0.1 over 1000 steps
diff --git a/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b27ff015708a5152e559775a485764cafe01c5cf
--- /dev/null
+++ b/official/vision/beta/projects/yolo/configs/experiments/csp_darknet53_tfds.yaml
@@ -0,0 +1,58 @@
+runtime:
+ distribution_strategy: 'mirrored'
+ mixed_precision_dtype: 'float16'
+ loss_scale: 'dynamic'
+ num_gpus: 2
+task:
+ model:
+ num_classes: 1001
+ input_size: [256, 256, 3]
+ backbone:
+ type: 'darknet'
+ darknet:
+ model_id: 'cspdarknet53'
+ norm_activation:
+ activation: 'mish'
+ losses:
+ l2_weight_decay: 0.0005
+ one_hot: true
+ train_data:
+ tfds_name: 'imagenet2012'
+ tfds_split: 'train'
+ tfds_data_dir: '~/tensorflow_datasets/'
+ is_training: true
+ global_batch_size: 16 # default = 128
+ dtype: 'float16'
+ shuffle_buffer_size: 100
+ validation_data:
+ tfds_name: 'imagenet2012'
+ tfds_split: 'validation'
+ tfds_data_dir: '~/tensorflow_datasets/'
+ is_training: true
+ global_batch_size: 16 # default = 128
+ dtype: 'float16'
+ drop_remainder: false
+ shuffle_buffer_size: 100
+trainer:
+ train_steps: 9600000 # epochs: 120, 1200000 * 128/batchsize
+ validation_steps: 3200 # size of validation data, 400 * 128/batchsize
+ validation_interval: 10000 # 10000
+ steps_per_loop: 10000
+ summary_interval: 10000
+ checkpoint_interval: 10000
+ optimizer_config:
+ optimizer:
+ type: 'sgd'
+ sgd:
+ momentum: 0.9
+ learning_rate:
+ type: 'polynomial'
+ polynomial:
+ initial_learning_rate: 0.0125 # 0.1 * batchsize/128, default = 0.1
+ end_learning_rate: 0.0000125 # 0.0001 * batchsize/128, default = 0.0001
+ power: 4.0
+ decay_steps: 9592000 # 790000 * 128/batchsize, default = 800000 - 1000 = 799000
+ warmup:
+ type: 'linear'
+ linear:
+ warmup_steps: 8000 # 0 to 0.1 over 1000 * 128/batchsize, default = 128
diff --git a/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml b/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a3333b599e305a5ba1c2001fa02db4a04ac66ef9
--- /dev/null
+++ b/official/vision/beta/projects/yolo/configs/experiments/darknet53.yaml
@@ -0,0 +1,50 @@
+runtime:
+ distribution_strategy: 'mirrored'
+ mixed_precision_dtype: 'float32'
+task:
+ model:
+ num_classes: 1001
+ input_size: [256, 256, 3]
+ backbone:
+ type: 'darknet'
+ darknet:
+ model_id: 'darknet53'
+ norm_activation:
+ activation: 'mish'
+ losses:
+ l2_weight_decay: 0.0005
+ one_hot: true
+ train_data:
+ input_path: 'imagenet-2012-tfrecord/train*'
+ is_training: true
+ global_batch_size: 128
+ dtype: 'float16'
+ validation_data:
+ input_path: 'imagenet-2012-tfrecord/valid*'
+ is_training: true
+ global_batch_size: 128
+ dtype: 'float16'
+ drop_remainder: false
+trainer:
+ train_steps: 800000 # epochs: 80
+ validation_steps: 400 # size of validation data
+ validation_interval: 10000
+ steps_per_loop: 10000
+ summary_interval: 10000
+ checkpoint_interval: 10000
+ optimizer_config:
+ optimizer:
+ type: 'sgd'
+ sgd:
+ momentum: 0.9
+ learning_rate:
+ type: 'polynomial'
+ polynomial:
+ initial_learning_rate: 0.1
+ end_learning_rate: 0.0001
+ power: 4.0
+ decay_steps: 800000
+ warmup:
+ type: 'linear'
+ linear:
+ warmup_steps: 1000 # learning rate rises from 0 to 0.1 over 1000 steps
diff --git a/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml b/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8f9fb2dfc6b32804f27560bb8cba68e325723435
--- /dev/null
+++ b/official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml
@@ -0,0 +1,58 @@
+runtime:
+ distribution_strategy: 'mirrored'
+ mixed_precision_dtype: 'float16'
+ loss_scale: 'dynamic'
+ num_gpus: 2
+task:
+ model:
+ num_classes: 1001
+ input_size: [256, 256, 3]
+ backbone:
+ type: 'darknet'
+ darknet:
+ model_id: 'darknet53'
+ norm_activation:
+ activation: 'mish'
+ losses:
+ l2_weight_decay: 0.0005
+ one_hot: true
+ train_data:
+ tfds_name: 'imagenet2012'
+ tfds_split: 'train'
+ tfds_data_dir: '~/tensorflow_datasets/'
+ is_training: true
+ global_batch_size: 16 # default = 128
+ dtype: 'float16'
+ shuffle_buffer_size: 100
+ validation_data:
+ tfds_name: 'imagenet2012'
+ tfds_split: 'validation'
+ tfds_data_dir: '~/tensorflow_datasets/'
+ is_training: true
+ global_batch_size: 16 # default = 128
+ dtype: 'float16'
+ drop_remainder: false
+ shuffle_buffer_size: 100
+trainer:
+ train_steps: 6400000 # epochs: 80, 800000 * 128/batchsize
+ validation_steps: 3200 # size of validation data, 400 * 128/batchsize
+ validation_interval: 10000 # 10000
+ steps_per_loop: 10000
+ summary_interval: 10000
+ checkpoint_interval: 10000
+ optimizer_config:
+ optimizer:
+ type: 'sgd'
+ sgd:
+ momentum: 0.9
+ learning_rate:
+ type: 'polynomial'
+ polynomial:
+ initial_learning_rate: 0.0125 # 0.1 * batchsize/128, default = 0.1
+ end_learning_rate: 0.0000125 # 0.0001 * batchsize/128, default = 0.0001
+ power: 4.0
+ decay_steps: 6392000 # 790000 * 128/batchsize, default = 800000 - 1000 = 799000
+ warmup:
+ type: 'linear'
+ linear:
+ warmup_steps: 8000 # 0 to 0.1 over 1000 * 128/batchsize, default = 128
diff --git a/official/vision/beta/projects/yolo/dataloaders/__init__.py b/official/vision/beta/projects/yolo/dataloaders/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a25710c222e3327cb20e000db5df5c5651c4a2cc
--- /dev/null
+++ b/official/vision/beta/projects/yolo/dataloaders/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
diff --git a/official/vision/beta/projects/yolo/dataloaders/classification_tfds_decoder.py b/official/vision/beta/projects/yolo/dataloaders/classification_tfds_decoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..0abe984fd6d92ecb8a117db4301128ebb7701ed7
--- /dev/null
+++ b/official/vision/beta/projects/yolo/dataloaders/classification_tfds_decoder.py
@@ -0,0 +1,34 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""TFDS Classification decoder."""
+
+import tensorflow as tf
+from official.vision.beta.dataloaders import decoder
+
+
+class Decoder(decoder.Decoder):
+ """A tf.Example decoder for classification task."""
+
+ def __init__(self):
+ return
+
+ def decode(self, serialized_example):
+ sample_dict = {
+ 'image/encoded':
+ tf.io.encode_jpeg(serialized_example['image'], quality=100),
+ 'image/class/label':
+ serialized_example['label'],
+ }
+ return sample_dict
diff --git a/official/vision/beta/projects/yolo/dataloaders/yolo_detection_input.py b/official/vision/beta/projects/yolo/dataloaders/yolo_detection_input.py
new file mode 100644
index 0000000000000000000000000000000000000000..c3bc171e725b5c38188a707badeb77ce8270d145
--- /dev/null
+++ b/official/vision/beta/projects/yolo/dataloaders/yolo_detection_input.py
@@ -0,0 +1,319 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Detection Data parser and processing for YOLO.
+
+Parse image and ground truths in a dataset to training targets and package them
+into (image, labels) tuple for RetinaNet.
+"""
+
+import tensorflow as tf
+
+from official.vision.beta.dataloaders import parser
+from official.vision.beta.ops import box_ops
+from official.vision.beta.ops import preprocess_ops
+from official.vision.beta.projects.yolo.ops import box_ops as yolo_box_ops
+from official.vision.beta.projects.yolo.ops import preprocess_ops as yolo_preprocess_ops
+
+
+class Parser(parser.Parser):
+ """Parser to parse an image and its annotations into a dictionary of tensors."""
+
+ def __init__(self,
+ output_size,
+ num_classes,
+ fixed_size=True,
+ jitter_im=0.1,
+ jitter_boxes=0.005,
+ use_tie_breaker=True,
+ min_level=3,
+ max_level=5,
+ masks=None,
+ max_process_size=608,
+ min_process_size=320,
+ max_num_instances=200,
+ random_flip=True,
+ aug_rand_saturation=True,
+ aug_rand_brightness=True,
+ aug_rand_zoom=True,
+ aug_rand_hue=True,
+ anchors=None,
+ seed=10,
+ dtype=tf.float32):
+ """Initializes parameters for parsing annotations in the dataset.
+
+ Args:
+ output_size: a `Tuple` for (width, height) of input image.
+ num_classes: a `Tensor` or `int` for the number of classes.
+ fixed_size: a `bool` if True all output images have the same size.
+ jitter_im: a `float` representing a pixel value that is the maximum jitter
+ applied to the image for data augmentation during training.
+ jitter_boxes: a `float` representing a pixel value that is the maximum
+ jitter applied to the bounding box for data augmentation during
+ training.
+ use_tie_breaker: boolean value for wether or not to use the tie_breaker.
+ min_level: `int` number of minimum level of the output feature pyramid.
+ max_level: `int` number of maximum level of the output feature pyramid.
+ masks: a `Tensor`, `List` or `numpy.ndarray` for anchor masks.
+ max_process_size: an `int` for maximum image width and height.
+ min_process_size: an `int` for minimum image width and height ,
+ max_num_instances: an `int` number of maximum number of instances in an
+ image.
+ random_flip: a `bool` if True, augment training with random horizontal
+ flip.
+ aug_rand_saturation: `bool`, if True, augment training with random
+ saturation.
+ aug_rand_brightness: `bool`, if True, augment training with random
+ brightness.
+ aug_rand_zoom: `bool`, if True, augment training with random zoom.
+ aug_rand_hue: `bool`, if True, augment training with random hue.
+ anchors: a `Tensor`, `List` or `numpy.ndarrray` for bounding box priors.
+ seed: an `int` for the seed used by tf.random
+ dtype: a `tf.dtypes.DType` object that represents the dtype the outputs
+ will be casted to. The available types are tf.float32, tf.float16, or
+ tf.bfloat16.
+ """
+ self._net_down_scale = 2**max_level
+
+ self._num_classes = num_classes
+ self._image_w = (output_size[0] //
+ self._net_down_scale) * self._net_down_scale
+ self._image_h = (output_size[1] //
+ self._net_down_scale) * self._net_down_scale
+
+ self._max_process_size = max_process_size
+ self._min_process_size = min_process_size
+ self._fixed_size = fixed_size
+
+ self._anchors = anchors
+ self._masks = {
+ key: tf.convert_to_tensor(value) for key, value in masks.items()
+ }
+ self._use_tie_breaker = use_tie_breaker
+
+ self._jitter_im = 0.0 if jitter_im is None else jitter_im
+ self._jitter_boxes = 0.0 if jitter_boxes is None else jitter_boxes
+ self._max_num_instances = max_num_instances
+ self._random_flip = random_flip
+
+ self._aug_rand_saturation = aug_rand_saturation
+ self._aug_rand_brightness = aug_rand_brightness
+ self._aug_rand_zoom = aug_rand_zoom
+ self._aug_rand_hue = aug_rand_hue
+
+ self._seed = seed
+ self._dtype = dtype
+
+ def _build_grid(self, raw_true, width, batch=False, use_tie_breaker=False):
+ mask = self._masks
+ for key in self._masks.keys():
+ if not batch:
+ mask[key] = yolo_preprocess_ops.build_grided_gt(
+ raw_true, self._masks[key], width // 2**int(key),
+ raw_true['bbox'].dtype, use_tie_breaker)
+ else:
+ mask[key] = yolo_preprocess_ops.build_batch_grided_gt(
+ raw_true, self._masks[key], width // 2**int(key),
+ raw_true['bbox'].dtype, use_tie_breaker)
+ return mask
+
+ def _parse_train_data(self, data):
+ """Generates images and labels that are usable for model training.
+
+ Args:
+ data: a dict of Tensors produced by the decoder.
+ Returns:
+ images: the image tensor.
+ labels: a dict of Tensors that contains labels.
+ """
+
+ shape = tf.shape(data['image'])
+ image = data['image'] / 255
+ boxes = data['groundtruth_boxes']
+ width = shape[0]
+ height = shape[1]
+
+ image, boxes = yolo_preprocess_ops.fit_preserve_aspect_ratio(
+ image,
+ boxes,
+ width=width,
+ height=height,
+ target_dim=self._max_process_size)
+
+ image_shape = tf.shape(image)[:2]
+
+ if self._random_flip:
+ image, boxes, _ = preprocess_ops.random_horizontal_flip(
+ image, boxes, seed=self._seed)
+
+ randscale = self._image_w // self._net_down_scale
+
+ if not self._fixed_size:
+ do_scale = tf.greater(
+ tf.random.uniform([], minval=0, maxval=1, seed=self._seed), 0.5)
+ if do_scale:
+ # This scales the image to a random multiple of net_down_scale
+ # between 320 to 608
+ randscale = tf.random.uniform(
+ [],
+ minval=self._min_process_size // self._net_down_scale,
+ maxval=self._max_process_size // self._net_down_scale,
+ seed=self._seed,
+ dtype=tf.int32) * self._net_down_scale
+
+ if self._jitter_boxes != 0.0:
+ boxes = box_ops.denormalize_boxes(boxes, image_shape)
+ boxes = box_ops.jitter_boxes(boxes, 0.025)
+ boxes = box_ops.normalize_boxes(boxes, image_shape)
+
+ # YOLO loss function uses x-center, y-center format
+ boxes = yolo_box_ops.yxyx_to_xcycwh(boxes)
+
+ if self._jitter_im != 0.0:
+ image, boxes = yolo_preprocess_ops.random_translate(
+ image, boxes, self._jitter_im, seed=self._seed)
+
+ if self._aug_rand_zoom:
+ image, boxes = yolo_preprocess_ops.resize_crop_filter(
+ image,
+ boxes,
+ default_width=self._image_w,
+ default_height=self._image_h,
+ target_width=randscale,
+ target_height=randscale)
+ image = tf.image.resize(image, (416, 416), preserve_aspect_ratio=False)
+
+ if self._aug_rand_brightness:
+ image = tf.image.random_brightness(
+ image=image, max_delta=.1) # Brightness
+ if self._aug_rand_saturation:
+ image = tf.image.random_saturation(
+ image=image, lower=0.75, upper=1.25) # Saturation
+ if self._aug_rand_hue:
+ image = tf.image.random_hue(image=image, max_delta=.3) # Hue
+ image = tf.clip_by_value(image, 0.0, 1.0)
+ # Find the best anchor for the ground truth labels to maximize the iou
+ best_anchors = yolo_preprocess_ops.get_best_anchor(
+ boxes, self._anchors, width=self._image_w, height=self._image_h)
+
+ # Padding
+ boxes = preprocess_ops.clip_or_pad_to_fixed_size(boxes,
+ self._max_num_instances, 0)
+ classes = preprocess_ops.clip_or_pad_to_fixed_size(
+ data['groundtruth_classes'], self._max_num_instances, -1)
+ best_anchors = preprocess_ops.clip_or_pad_to_fixed_size(
+ best_anchors, self._max_num_instances, 0)
+ area = preprocess_ops.clip_or_pad_to_fixed_size(data['groundtruth_area'],
+ self._max_num_instances, 0)
+ is_crowd = preprocess_ops.clip_or_pad_to_fixed_size(
+ tf.cast(data['groundtruth_is_crowd'], tf.int32),
+ self._max_num_instances, 0)
+
+ labels = {
+ 'source_id': data['source_id'],
+ 'bbox': tf.cast(boxes, self._dtype),
+ 'classes': tf.cast(classes, self._dtype),
+ 'area': tf.cast(area, self._dtype),
+ 'is_crowd': is_crowd,
+ 'best_anchors': tf.cast(best_anchors, self._dtype),
+ 'width': width,
+ 'height': height,
+ 'num_detections': tf.shape(data['groundtruth_classes'])[0],
+ }
+
+ if self._fixed_size:
+ grid = self._build_grid(
+ labels, self._image_w, use_tie_breaker=self._use_tie_breaker)
+ labels.update({'grid_form': grid})
+
+ return image, labels
+
+ def _parse_eval_data(self, data):
+ """Generates images and labels that are usable for model training.
+
+ Args:
+ data: a dict of Tensors produced by the decoder.
+ Returns:
+ images: the image tensor.
+ labels: a dict of Tensors that contains labels.
+ """
+
+ shape = tf.shape(data['image'])
+ image = data['image'] / 255
+ boxes = data['groundtruth_boxes']
+ width = shape[0]
+ height = shape[1]
+
+ image, boxes = yolo_preprocess_ops.fit_preserve_aspect_ratio(
+ image, boxes, width=width, height=height, target_dim=self._image_w)
+ boxes = yolo_box_ops.yxyx_to_xcycwh(boxes)
+
+ # Find the best anchor for the ground truth labels to maximize the iou
+ best_anchors = yolo_preprocess_ops.get_best_anchor(
+ boxes, self._anchors, width=self._image_w, height=self._image_h)
+ boxes = yolo_preprocess_ops.pad_max_instances(boxes,
+ self._max_num_instances, 0)
+ classes = yolo_preprocess_ops.pad_max_instances(data['groundtruth_classes'],
+ self._max_num_instances, 0)
+ best_anchors = yolo_preprocess_ops.pad_max_instances(
+ best_anchors, self._max_num_instances, 0)
+ area = yolo_preprocess_ops.pad_max_instances(data['groundtruth_area'],
+ self._max_num_instances, 0)
+ is_crowd = yolo_preprocess_ops.pad_max_instances(
+ tf.cast(data['groundtruth_is_crowd'], tf.int32),
+ self._max_num_instances, 0)
+
+ labels = {
+ 'source_id': data['source_id'],
+ 'bbox': tf.cast(boxes, self._dtype),
+ 'classes': tf.cast(classes, self._dtype),
+ 'area': tf.cast(area, self._dtype),
+ 'is_crowd': is_crowd,
+ 'best_anchors': tf.cast(best_anchors, self._dtype),
+ 'width': width,
+ 'height': height,
+ 'num_detections': tf.shape(data['groundtruth_classes'])[0],
+ }
+
+ grid = self._build_grid(
+ labels,
+ self._image_w,
+ batch=False,
+ use_tie_breaker=self._use_tie_breaker)
+ labels.update({'grid_form': grid})
+ return image, labels
+
+ def _postprocess_fn(self, image, label):
+ randscale = self._image_w // self._net_down_scale
+ if not self._fixed_size:
+ do_scale = tf.greater(
+ tf.random.uniform([], minval=0, maxval=1, seed=self._seed), 0.5)
+ if do_scale:
+ # This scales the image to a random multiple of net_down_scale
+ # between 320 to 608
+ randscale = tf.random.uniform(
+ [],
+ minval=self._min_process_size // self._net_down_scale,
+ maxval=self._max_process_size // self._net_down_scale,
+ seed=self._seed,
+ dtype=tf.int32) * self._net_down_scale
+ width = randscale
+ image = tf.image.resize(image, (width, width))
+ grid = self._build_grid(
+ label, width, batch=True, use_tie_breaker=self._use_tie_breaker)
+ label.update({'grid_form': grid})
+ return image, label
+
+ def postprocess_fn(self, is_training=True):
+ return self._postprocess_fn if not self._fixed_size and is_training else None
diff --git a/official/vision/beta/projects/yolo/dataloaders/yolo_detection_input_test.py b/official/vision/beta/projects/yolo/dataloaders/yolo_detection_input_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..53be13d6fae831b1b925e765de6a5943deac8995
--- /dev/null
+++ b/official/vision/beta/projects/yolo/dataloaders/yolo_detection_input_test.py
@@ -0,0 +1,103 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test case for YOLO detection dataloader configuration definition."""
+from absl.testing import parameterized
+import dataclasses
+import tensorflow as tf
+
+from official.core import config_definitions as cfg
+from official.core import input_reader
+from official.modeling import hyperparams
+from official.vision.beta.dataloaders import tfds_detection_decoders
+from official.vision.beta.projects.yolo.dataloaders import yolo_detection_input
+
+
+@dataclasses.dataclass
+class Parser(hyperparams.Config):
+ """Dummy configuration for parser."""
+ output_size: int = (416, 416)
+ num_classes: int = 80
+ fixed_size: bool = True
+ jitter_im: float = 0.1
+ jitter_boxes: float = 0.005
+ min_process_size: int = 320
+ max_process_size: int = 608
+ max_num_instances: int = 200
+ random_flip: bool = True
+ seed: int = 10
+ shuffle_buffer_size: int = 10000
+
+
+@dataclasses.dataclass
+class DataConfig(cfg.DataConfig):
+ """Input config for training."""
+ input_path: str = ''
+ tfds_name: str = 'coco/2017'
+ tfds_split: str = 'train'
+ global_batch_size: int = 10
+ is_training: bool = True
+ dtype: str = 'float16'
+ decoder = None
+ parser: Parser = Parser()
+ shuffle_buffer_size: int = 10
+
+
+class YoloDetectionInputTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.named_parameters(('training', True), ('testing', False))
+ def test_yolo_input(self, is_training):
+ params = DataConfig(is_training=is_training)
+
+ decoder = tfds_detection_decoders.MSCOCODecoder()
+ anchors = [[12.0, 19.0], [31.0, 46.0], [96.0, 54.0], [46.0, 114.0],
+ [133.0, 127.0], [79.0, 225.0], [301.0, 150.0], [172.0, 286.0],
+ [348.0, 340.0]]
+ masks = {'3': [0, 1, 2], '4': [3, 4, 5], '5': [6, 7, 8]}
+
+ parser = yolo_detection_input.Parser(
+ output_size=params.parser.output_size,
+ num_classes=params.parser.num_classes,
+ fixed_size=params.parser.fixed_size,
+ jitter_im=params.parser.jitter_im,
+ jitter_boxes=params.parser.jitter_boxes,
+ min_process_size=params.parser.min_process_size,
+ max_process_size=params.parser.max_process_size,
+ max_num_instances=params.parser.max_num_instances,
+ random_flip=params.parser.random_flip,
+ seed=params.parser.seed,
+ anchors=anchors,
+ masks=masks)
+ postprocess_fn = parser.postprocess_fn(is_training=is_training)
+
+ reader = input_reader.InputReader(params,
+ dataset_fn=tf.data.TFRecordDataset,
+ decoder_fn=decoder.decode,
+ parser_fn=parser.parse_fn(
+ params.is_training))
+ dataset = reader.read(input_context=None).batch(10).take(1)
+ if postprocess_fn:
+ image, _ = postprocess_fn(
+ *tf.data.experimental.get_single_element(dataset))
+ else:
+ image, _ = tf.data.experimental.get_single_element(dataset)
+ print(image.shape)
+ self.assertAllEqual(image.shape, (10, 10, 416, 416, 3))
+ self.assertTrue(
+ tf.reduce_all(tf.math.logical_and(image >= 0, image <= 1)))
+
+
+if __name__ == '__main__':
+ tf.test.main()
+
diff --git a/official/vision/beta/projects/yolo/modeling/backbones/darknet.py b/official/vision/beta/projects/yolo/modeling/backbones/darknet.py
new file mode 100644
index 0000000000000000000000000000000000000000..10a614d80d5d81c21a5245ec91a87d98853d2a31
--- /dev/null
+++ b/official/vision/beta/projects/yolo/modeling/backbones/darknet.py
@@ -0,0 +1,445 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+
+"""Contains definitions of Darknet Backbone Networks.
+
+ The models are inspired by ResNet, and CSPNet
+
+Residual networks (ResNets) were proposed in:
+[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
+ Deep Residual Learning for Image Recognition. arXiv:1512.03385
+
+Cross Stage Partial networks (CSPNets) were proposed in:
+[1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang Chen,
+ Jun-Wei Hsieh
+ CSPNet: A New Backbone that can Enhance Learning Capability of CNN.
+ arXiv:1911.11929
+
+
+DarkNets Are used mainly for Object detection in:
+[1] Joseph Redmon, Ali Farhadi
+ YOLOv3: An Incremental Improvement. arXiv:1804.02767
+
+[2] Alexey Bochkovskiy, Chien-Yao Wang, Hong-Yuan Mark Liao
+ YOLOv4: Optimal Speed and Accuracy of Object Detection. arXiv:2004.10934
+"""
+import collections
+
+import tensorflow as tf
+
+from official.vision.beta.modeling.backbones import factory
+from official.vision.beta.projects.yolo.modeling.layers import nn_blocks
+
+
+class BlockConfig(object):
+ """Get layer config to make code more readable.
+
+ Args:
+ layer: string layer name
+ stack: the type of layer ordering to use for this specific level
+ repetitions: integer for the number of times to repeat block
+ bottelneck: boolean for does this stack have a bottle neck layer
+ filters: integer for the output depth of the level
+ pool_size: integer the pool_size of max pool layers
+ kernel_size: optional integer, for convolution kernel size
+ strides: integer or tuple to indicate convolution strides
+ padding: the padding to apply to layers in this stack
+ activation: string for the activation to use for this stack
+ route: integer for what level to route from to get the next input
+ output_name: the name to use for this output
+ is_output: is this layer an output in the default model
+ """
+
+ def __init__(self, layer, stack, reps, bottleneck, filters, pool_size,
+ kernel_size, strides, padding, activation, route, output_name,
+ is_output):
+ self.layer = layer
+ self.stack = stack
+ self.repetitions = reps
+ self.bottleneck = bottleneck
+ self.filters = filters
+ self.kernel_size = kernel_size
+ self.pool_size = pool_size
+ self.strides = strides
+ self.padding = padding
+ self.activation = activation
+ self.route = route
+ self.output_name = output_name
+ self.is_output = is_output
+
+
+def build_block_specs(config):
+ specs = []
+ for layer in config:
+ specs.append(BlockConfig(*layer))
+ return specs
+
+
+class LayerFactory(object):
+ """Class for quick look up of default layers.
+
+ Used by darknet to connect, introduce or exit a level. Used in place of an if
+ condition or switch to make adding new layers easier and to reduce redundant
+ code.
+ """
+
+ def __init__(self):
+ self._layer_dict = {
+ "ConvBN": (nn_blocks.ConvBN, self.conv_bn_config_todict),
+ "MaxPool": (tf.keras.layers.MaxPool2D, self.maxpool_config_todict)
+ }
+
+ def conv_bn_config_todict(self, config, kwargs):
+ dictvals = {
+ "filters": config.filters,
+ "kernel_size": config.kernel_size,
+ "strides": config.strides,
+ "padding": config.padding
+ }
+ dictvals.update(kwargs)
+ return dictvals
+
+ def darktiny_config_todict(self, config, kwargs):
+ dictvals = {"filters": config.filters, "strides": config.strides}
+ dictvals.update(kwargs)
+ return dictvals
+
+ def maxpool_config_todict(self, config, kwargs):
+ return {
+ "pool_size": config.pool_size,
+ "strides": config.strides,
+ "padding": config.padding,
+ "name": kwargs["name"]
+ }
+
+ def __call__(self, config, kwargs):
+ layer, get_param_dict = self._layer_dict[config.layer]
+ param_dict = get_param_dict(config, kwargs)
+ return layer(**param_dict)
+
+
+# model configs
+LISTNAMES = [
+ "default_layer_name", "level_type", "number_of_layers_in_level",
+ "bottleneck", "filters", "kernal_size", "pool_size", "strides", "padding",
+ "default_activation", "route", "level/name", "is_output"
+]
+
+# pylint: disable=line-too-long
+CSPDARKNET53 = {
+ "list_names": LISTNAMES,
+ "splits": {"backbone_split": 106,
+ "neck_split": 138},
+ "backbone": [
+ ["ConvBN", None, 1, False, 32, None, 3, 1, "same", "mish", -1, 0, False],
+ ["DarkRes", "csp", 1, True, 64, None, None, None, None, "mish", -1, 1, False],
+ ["DarkRes", "csp", 2, False, 128, None, None, None, None, "mish", -1, 2, False],
+ ["DarkRes", "csp", 8, False, 256, None, None, None, None, "mish", -1, 3, True],
+ ["DarkRes", "csp", 8, False, 512, None, None, None, None, "mish", -1, 4, True],
+ ["DarkRes", "csp", 4, False, 1024, None, None, None, None, "mish", -1, 5, True],
+ ]
+}
+
+DARKNET53 = {
+ "list_names": LISTNAMES,
+ "splits": {"backbone_split": 76},
+ "backbone": [
+ ["ConvBN", None, 1, False, 32, None, 3, 1, "same", "leaky", -1, 0, False],
+ ["DarkRes", "residual", 1, True, 64, None, None, None, None, "leaky", -1, 1, False],
+ ["DarkRes", "residual", 2, False, 128, None, None, None, None, "leaky", -1, 2, False],
+ ["DarkRes", "residual", 8, False, 256, None, None, None, None, "leaky", -1, 3, True],
+ ["DarkRes", "residual", 8, False, 512, None, None, None, None, "leaky", -1, 4, True],
+ ["DarkRes", "residual", 4, False, 1024, None, None, None, None, "leaky", -1, 5, True],
+ ]
+}
+
+CSPDARKNETTINY = {
+ "list_names": LISTNAMES,
+ "splits": {"backbone_split": 28},
+ "backbone": [
+ ["ConvBN", None, 1, False, 32, None, 3, 2, "same", "leaky", -1, 0, False],
+ ["ConvBN", None, 1, False, 64, None, 3, 2, "same", "leaky", -1, 1, False],
+ ["CSPTiny", "csp_tiny", 1, False, 64, None, 3, 2, "same", "leaky", -1, 2, False],
+ ["CSPTiny", "csp_tiny", 1, False, 128, None, 3, 2, "same", "leaky", -1, 3, False],
+ ["CSPTiny", "csp_tiny", 1, False, 256, None, 3, 2, "same", "leaky", -1, 4, True],
+ ["ConvBN", None, 1, False, 512, None, 3, 1, "same", "leaky", -1, 5, True],
+ ]
+}
+
+DARKNETTINY = {
+ "list_names": LISTNAMES,
+ "splits": {"backbone_split": 14},
+ "backbone": [
+ ["ConvBN", None, 1, False, 16, None, 3, 1, "same", "leaky", -1, 0, False],
+ ["DarkTiny", "tiny", 1, True, 32, None, 3, 2, "same", "leaky", -1, 1, False],
+ ["DarkTiny", "tiny", 1, True, 64, None, 3, 2, "same", "leaky", -1, 2, False],
+ ["DarkTiny", "tiny", 1, False, 128, None, 3, 2, "same", "leaky", -1, 3, False],
+ ["DarkTiny", "tiny", 1, False, 256, None, 3, 2, "same", "leaky", -1, 4, True],
+ ["DarkTiny", "tiny", 1, False, 512, None, 3, 2, "same", "leaky", -1, 5, False],
+ ["DarkTiny", "tiny", 1, False, 1024, None, 3, 1, "same", "leaky", -1, 5, True],
+ ]
+}
+# pylint: enable=line-too-long
+
+BACKBONES = {
+ "darknettiny": DARKNETTINY,
+ "darknet53": DARKNET53,
+ "cspdarknet53": CSPDARKNET53,
+ "cspdarknettiny": CSPDARKNETTINY
+}
+
+
+@tf.keras.utils.register_keras_serializable(package="yolo")
+class Darknet(tf.keras.Model):
+ """Darknet backbone."""
+
+ def __init__(
+ self,
+ model_id="darknet53",
+ input_specs=tf.keras.layers.InputSpec(shape=[None, None, None, 3]),
+ min_level=None,
+ max_level=5,
+ activation=None,
+ use_sync_bn=False,
+ norm_momentum=0.99,
+ norm_epsilon=0.001,
+ kernel_initializer="glorot_uniform",
+ kernel_regularizer=None,
+ bias_regularizer=None,
+ **kwargs):
+
+ layer_specs, splits = Darknet.get_model_config(model_id)
+
+ self._model_name = model_id
+ self._splits = splits
+ self._input_shape = input_specs
+ self._registry = LayerFactory()
+
+ # default layer look up
+ self._min_size = min_level
+ self._max_size = max_level
+ self._output_specs = None
+
+ self._kernel_initializer = kernel_initializer
+ self._bias_regularizer = bias_regularizer
+ self._norm_momentum = norm_momentum
+ self._norm_epislon = norm_epsilon
+ self._use_sync_bn = use_sync_bn
+ self._activation = activation
+ self._kernel_regularizer = kernel_regularizer
+
+ self._default_dict = {
+ "kernel_initializer": self._kernel_initializer,
+ "kernel_regularizer": self._kernel_regularizer,
+ "bias_regularizer": self._bias_regularizer,
+ "norm_momentum": self._norm_momentum,
+ "norm_epsilon": self._norm_epislon,
+ "use_sync_bn": self._use_sync_bn,
+ "activation": self._activation,
+ "name": None
+ }
+
+ inputs = tf.keras.layers.Input(shape=self._input_shape.shape[1:])
+ output = self._build_struct(layer_specs, inputs)
+ super().__init__(inputs=inputs, outputs=output, name=self._model_name)
+
+ @property
+ def input_specs(self):
+ return self._input_shape
+
+ @property
+ def output_specs(self):
+ return self._output_specs
+
+ @property
+ def splits(self):
+ return self._splits
+
+ def _build_struct(self, net, inputs):
+ endpoints = collections.OrderedDict()
+ stack_outputs = [inputs]
+ for i, config in enumerate(net):
+ if config.stack is None:
+ x = self._build_block(stack_outputs[config.route],
+ config,
+ name=f"{config.layer}_{i}")
+ stack_outputs.append(x)
+ elif config.stack == "residual":
+ x = self._residual_stack(stack_outputs[config.route],
+ config,
+ name=f"{config.layer}_{i}")
+ stack_outputs.append(x)
+ elif config.stack == "csp":
+ x = self._csp_stack(stack_outputs[config.route],
+ config,
+ name=f"{config.layer}_{i}")
+ stack_outputs.append(x)
+ elif config.stack == "csp_tiny":
+ x_pass, x = self._csp_tiny_stack(stack_outputs[config.route],
+ config, name=f"{config.layer}_{i}")
+ stack_outputs.append(x_pass)
+ elif config.stack == "tiny":
+ x = self._tiny_stack(stack_outputs[config.route],
+ config,
+ name=f"{config.layer}_{i}")
+ stack_outputs.append(x)
+ if (config.is_output and self._min_size is None):
+ endpoints[str(config.output_name)] = x
+ elif self._min_size is not None and config.output_name >= self._min_size and config.output_name <= self._max_size:
+ endpoints[str(config.output_name)] = x
+
+ self._output_specs = {l: endpoints[l].get_shape() for l in endpoints.keys()}
+ return endpoints
+
+ def _get_activation(self, activation):
+ if self._activation is None:
+ return activation
+ else:
+ return self._activation
+
+ def _csp_stack(self, inputs, config, name):
+ if config.bottleneck:
+ csp_filter_scale = 1
+ residual_filter_scale = 2
+ scale_filters = 1
+ else:
+ csp_filter_scale = 2
+ residual_filter_scale = 1
+ scale_filters = 2
+ self._default_dict["activation"] = self._get_activation(config.activation)
+ self._default_dict["name"] = f"{name}_csp_down"
+ x, x_route = nn_blocks.CSPRoute(filters=config.filters,
+ filter_scale=csp_filter_scale,
+ downsample=True,
+ **self._default_dict)(inputs)
+ for i in range(config.repetitions):
+ self._default_dict["name"] = f"{name}_{i}"
+ x = nn_blocks.DarkResidual(filters=config.filters // scale_filters,
+ filter_scale=residual_filter_scale,
+ **self._default_dict)(x)
+
+ self._default_dict["name"] = f"{name}_csp_connect"
+ output = nn_blocks.CSPConnect(filters=config.filters,
+ filter_scale=csp_filter_scale,
+ **self._default_dict)([x, x_route])
+ self._default_dict["activation"] = self._activation
+ self._default_dict["name"] = None
+ return output
+
+ def _csp_tiny_stack(self, inputs, config, name):
+ self._default_dict["activation"] = self._get_activation(config.activation)
+ self._default_dict["name"] = f"{name}_csp_tiny"
+ x, x_route = nn_blocks.CSPTiny(filters=config.filters,
+ **self._default_dict)(inputs)
+ self._default_dict["activation"] = self._activation
+ self._default_dict["name"] = None
+ return x, x_route
+
+ def _tiny_stack(self, inputs, config, name):
+ x = tf.keras.layers.MaxPool2D(pool_size=2,
+ strides=config.strides,
+ padding="same",
+ data_format=None,
+ name=f"{name}_tiny/pool")(inputs)
+ self._default_dict["activation"] = self._get_activation(config.activation)
+ self._default_dict["name"] = f"{name}_tiny/conv"
+ x = nn_blocks.ConvBN(
+ filters=config.filters,
+ kernel_size=(3, 3),
+ strides=(1, 1),
+ padding="same",
+ **self._default_dict)(
+ x)
+ self._default_dict["activation"] = self._activation
+ self._default_dict["name"] = None
+ return x
+
+ def _residual_stack(self, inputs, config, name):
+ self._default_dict["activation"] = self._get_activation(config.activation)
+ self._default_dict["name"] = f"{name}_residual_down"
+ x = nn_blocks.DarkResidual(filters=config.filters,
+ downsample=True,
+ **self._default_dict)(inputs)
+ for i in range(config.repetitions - 1):
+ self._default_dict["name"] = f"{name}_{i}"
+ x = nn_blocks.DarkResidual(filters=config.filters,
+ **self._default_dict)(x)
+ self._default_dict["activation"] = self._activation
+ self._default_dict["name"] = None
+ return x
+
+ def _build_block(self, inputs, config, name):
+ x = inputs
+ i = 0
+ self._default_dict["activation"] = self._get_activation(config.activation)
+ while i < config.repetitions:
+ self._default_dict["name"] = f"{name}_{i}"
+ layer = self._registry(config, self._default_dict)
+ x = layer(x)
+ i += 1
+ self._default_dict["activation"] = self._activation
+ self._default_dict["name"] = None
+ return x
+
+ @staticmethod
+ def get_model_config(name):
+ name = name.lower()
+ backbone = BACKBONES[name]["backbone"]
+ splits = BACKBONES[name]["splits"]
+ return build_block_specs(backbone), splits
+
+ @property
+ def model_id(self):
+ return self._model_name
+
+ @classmethod
+ def from_config(cls, config, custom_objects=None):
+ return cls(**config)
+
+ def get_config(self):
+ layer_config = {
+ "model_id": self._model_name,
+ "min_level": self._min_size,
+ "max_level": self._max_size,
+ "kernel_initializer": self._kernel_initializer,
+ "kernel_regularizer": self._kernel_regularizer,
+ "bias_regularizer": self._bias_regularizer,
+ "norm_momentum": self._norm_momentum,
+ "norm_epsilon": self._norm_epislon,
+ "use_sync_bn": self._use_sync_bn,
+ "activation": self._activation
+ }
+ return layer_config
+
+
+@factory.register_backbone_builder("darknet")
+def build_darknet(
+ input_specs: tf.keras.layers.InputSpec,
+ model_config,
+ l2_regularizer: tf.keras.regularizers.Regularizer = None) -> tf.keras.Model:
+ """Builds darknet backbone."""
+
+ backbone_cfg = model_config.backbone.get()
+ norm_activation_config = model_config.norm_activation
+ model = Darknet(
+ model_id=backbone_cfg.model_id,
+ input_shape=input_specs,
+ activation=norm_activation_config.activation,
+ use_sync_bn=norm_activation_config.use_sync_bn,
+ norm_momentum=norm_activation_config.norm_momentum,
+ norm_epsilon=norm_activation_config.norm_epsilon,
+ kernel_regularizer=l2_regularizer)
+ return model
diff --git a/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py b/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..76c595f2dd7d05086e419ef2cc54a334b68de503
--- /dev/null
+++ b/official/vision/beta/projects/yolo/modeling/backbones/darknet_test.py
@@ -0,0 +1,117 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Tests for resnet."""
+
+from absl.testing import parameterized
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.python.distribute import combinations
+from tensorflow.python.distribute import strategy_combinations
+from official.vision.beta.projects.yolo.modeling.backbones import darknet
+
+
+class DarkNetTest(parameterized.TestCase, tf.test.TestCase):
+
+ @parameterized.parameters(
+ (224, "darknet53", 2, 1),
+ (224, "darknettiny", 1, 2),
+ (224, "cspdarknettiny", 1, 1),
+ (224, "cspdarknet53", 2, 1),
+ )
+ def test_network_creation(self, input_size, model_id,
+ endpoint_filter_scale, scale_final):
+ """Test creation of ResNet family models."""
+ tf.keras.backend.set_image_data_format("channels_last")
+
+ network = darknet.Darknet(model_id=model_id, min_level=3, max_level=5)
+ self.assertEqual(network.model_id, model_id)
+
+ inputs = tf.keras.Input(shape=(input_size, input_size, 3), batch_size=1)
+ endpoints = network(inputs)
+
+ self.assertAllEqual(
+ [1, input_size / 2**3, input_size / 2**3, 128 * endpoint_filter_scale],
+ endpoints["3"].shape.as_list())
+ self.assertAllEqual(
+ [1, input_size / 2**4, input_size / 2**4, 256 * endpoint_filter_scale],
+ endpoints["4"].shape.as_list())
+ self.assertAllEqual([
+ 1, input_size / 2**5, input_size / 2**5,
+ 512 * endpoint_filter_scale * scale_final
+ ], endpoints["5"].shape.as_list())
+
+ @combinations.generate(
+ combinations.combine(
+ strategy=[
+ strategy_combinations.cloud_tpu_strategy,
+ strategy_combinations.one_device_strategy_gpu,
+ ],
+ use_sync_bn=[False, True],
+ ))
+ def test_sync_bn_multiple_devices(self, strategy, use_sync_bn):
+ """Test for sync bn on TPU and GPU devices."""
+ inputs = np.random.rand(1, 224, 224, 3)
+
+ tf.keras.backend.set_image_data_format("channels_last")
+
+ with strategy.scope():
+ network = darknet.Darknet(model_id="darknet53", min_size=3, max_size=5)
+ _ = network(inputs)
+
+ @parameterized.parameters(1, 3, 4)
+ def test_input_specs(self, input_dim):
+ """Test different input feature dimensions."""
+ tf.keras.backend.set_image_data_format("channels_last")
+
+ input_specs = tf.keras.layers.InputSpec(shape=[None, None, None, input_dim])
+ network = darknet.Darknet(
+ model_id="darknet53", min_level=3, max_level=5, input_specs=input_specs)
+
+ inputs = tf.keras.Input(shape=(224, 224, input_dim), batch_size=1)
+ _ = network(inputs)
+
+ def test_serialize_deserialize(self):
+ # Create a network object that sets all of its config options.
+ kwargs = dict(
+ model_id="darknet53",
+ min_level=3,
+ max_level=5,
+ use_sync_bn=False,
+ activation="relu",
+ norm_momentum=0.99,
+ norm_epsilon=0.001,
+ kernel_initializer="VarianceScaling",
+ kernel_regularizer=None,
+ bias_regularizer=None,
+ )
+ network = darknet.Darknet(**kwargs)
+
+ expected_config = dict(kwargs)
+ self.assertEqual(network.get_config(), expected_config)
+
+ # Create another network object from the first object's config.
+ new_network = darknet.Darknet.from_config(network.get_config())
+
+ # Validate that the config can be forced to JSON.
+ _ = new_network.to_json()
+
+ # If the serialization was successful, the new config should match the old.
+ self.assertAllEqual(network.get_config(), new_network.get_config())
+
+
+if __name__ == "__main__":
+ tf.test.main()
diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py
new file mode 100644
index 0000000000000000000000000000000000000000..8bc6a78078a1fb794e28ccf389ca7849b6ce48e3
--- /dev/null
+++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks.py
@@ -0,0 +1,821 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+
+"""Contains common building blocks for yolo neural networks."""
+
+from typing import Callable, List
+import tensorflow as tf
+from official.modeling import tf_utils
+
+
+@tf.keras.utils.register_keras_serializable(package="yolo")
+class Identity(tf.keras.layers.Layer):
+
+ def call(self, inputs):
+ return inputs
+
+
+@tf.keras.utils.register_keras_serializable(package="yolo")
+class ConvBN(tf.keras.layers.Layer):
+ """Modified Convolution layer to match that of the DarkNet Library.
+
+ The Layer is a standards combination of Conv BatchNorm Activation,
+ however, the use of bias in the conv is determined by the use of batch norm.
+
+ Cross Stage Partial networks (CSPNets) were proposed in:
+ [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang
+ Chen, Jun-Wei Hsieh.
+ CSPNet: A New Backbone that can Enhance Learning Capability of CNN.
+ arXiv:1911.11929
+ """
+
+ def __init__(self,
+ filters=1,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ padding="same",
+ dilation_rate=(1, 1),
+ kernel_initializer="glorot_uniform",
+ bias_initializer="zeros",
+ kernel_regularizer=None,
+ bias_regularizer=None,
+ use_bn=True,
+ use_sync_bn=False,
+ norm_momentum=0.99,
+ norm_epsilon=0.001,
+ activation="leaky",
+ leaky_alpha=0.1,
+ **kwargs):
+ """Initializes ConvBN layer.
+
+ Args:
+ filters: integer for output depth, or the number of features to learn
+ kernel_size: integer or tuple for the shape of the weight matrix or kernel
+ to learn.
+ strides: integer of tuple how much to move the kernel after each kernel
+ use padding: string 'valid' or 'same', if same, then pad the image, else
+ do not.
+ padding: `str`, padding method for conv layers.
+ dilation_rate: tuple to indicate how much to modulate kernel weights and
+ how many pixels in a feature map to skip.
+ kernel_initializer: string to indicate which function to use to initialize
+ weights.
+ bias_initializer: string to indicate which function to use to initialize
+ bias.
+ kernel_regularizer: string to indicate which function to use to
+ regularizer weights.
+ bias_regularizer: string to indicate which function to use to regularizer
+ bias.
+ use_bn: boolean for whether to use batch normalization.
+ use_sync_bn: boolean for whether sync batch normalization.
+ norm_momentum: float for moment to use for batch normalization
+ norm_epsilon: float for batch normalization epsilon
+ activation: string or None for activation function to use in layer,
+ if None activation is replaced by linear.
+ leaky_alpha: float to use as alpha if activation function is leaky.
+ **kwargs: Keyword Arguments
+ """
+ # convolution params
+ self._filters = filters
+ self._kernel_size = kernel_size
+ self._strides = strides
+ self._padding = padding
+ self._dilation_rate = dilation_rate
+ self._kernel_initializer = kernel_initializer
+ self._bias_initializer = bias_initializer
+ self._kernel_regularizer = kernel_regularizer
+ self._bias_regularizer = bias_regularizer
+
+ # batch normalization params
+ self._use_bn = use_bn
+ self._use_sync_bn = use_sync_bn
+ self._norm_moment = norm_momentum
+ self._norm_epsilon = norm_epsilon
+
+ if tf.keras.backend.image_data_format() == "channels_last":
+ # format: (batch_size, height, width, channels)
+ self._bn_axis = -1
+ else:
+ # format: (batch_size, channels, width, height)
+ self._bn_axis = 1
+
+ # activation params
+ self._activation = activation
+ self._leaky_alpha = leaky_alpha
+
+ super(ConvBN, self).__init__(**kwargs)
+
+ def build(self, input_shape):
+ use_bias = not self._use_bn
+
+ self.conv = tf.keras.layers.Conv2D(
+ filters=self._filters,
+ kernel_size=self._kernel_size,
+ strides=self._strides,
+ padding=self._padding,
+ dilation_rate=self._dilation_rate,
+ use_bias=use_bias,
+ kernel_initializer=self._kernel_initializer,
+ bias_initializer=self._bias_initializer,
+ kernel_regularizer=self._kernel_regularizer,
+ bias_regularizer=self._bias_regularizer)
+
+ if self._use_bn:
+ if self._use_sync_bn:
+ self.bn = tf.keras.layers.experimental.SyncBatchNormalization(
+ momentum=self._norm_moment,
+ epsilon=self._norm_epsilon,
+ axis=self._bn_axis)
+ else:
+ self.bn = tf.keras.layers.BatchNormalization(
+ momentum=self._norm_moment,
+ epsilon=self._norm_epsilon,
+ axis=self._bn_axis)
+ else:
+ self.bn = Identity()
+
+ if self._activation == "leaky":
+ self._activation_fn = tf.keras.layers.LeakyReLU(alpha=self._leaky_alpha)
+ elif self._activation == "mish":
+ self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x))
+ else:
+ self._activation_fn = tf_utils.get_activation(self._activation)
+
+ def call(self, x):
+ x = self.conv(x)
+ x = self.bn(x)
+ x = self._activation_fn(x)
+ return x
+
+ def get_config(self):
+ # used to store/share parameters to reconstruct the model
+ layer_config = {
+ "filters": self._filters,
+ "kernel_size": self._kernel_size,
+ "strides": self._strides,
+ "padding": self._padding,
+ "dilation_rate": self._dilation_rate,
+ "kernel_initializer": self._kernel_initializer,
+ "bias_initializer": self._bias_initializer,
+ "bias_regularizer": self._bias_regularizer,
+ "kernel_regularizer": self._kernel_regularizer,
+ "use_bn": self._use_bn,
+ "use_sync_bn": self._use_sync_bn,
+ "norm_moment": self._norm_moment,
+ "norm_epsilon": self._norm_epsilon,
+ "activation": self._activation,
+ "leaky_alpha": self._leaky_alpha
+ }
+ layer_config.update(super(ConvBN, self).get_config())
+ return layer_config
+
+ def __repr__(self):
+ return repr(self.get_config())
+
+
+@tf.keras.utils.register_keras_serializable(package="yolo")
+class DarkResidual(tf.keras.layers.Layer):
+ """DarkNet block with Residual connection for Yolo v3 Backbone.
+ """
+
+ def __init__(self,
+ filters=1,
+ filter_scale=2,
+ kernel_initializer="glorot_uniform",
+ bias_initializer="zeros",
+ kernel_regularizer=None,
+ bias_regularizer=None,
+ use_bn=True,
+ use_sync_bn=False,
+ norm_momentum=0.99,
+ norm_epsilon=0.001,
+ activation="leaky",
+ leaky_alpha=0.1,
+ sc_activation="linear",
+ downsample=False,
+ **kwargs):
+ """Initializes DarkResidual.
+
+ Args:
+ filters: integer for output depth, or the number of features to learn.
+ filter_scale: `int`, scale factor for number of filters.
+ kernel_initializer: string to indicate which function to use to initialize
+ weights
+ bias_initializer: string to indicate which function to use to initialize
+ bias
+ kernel_regularizer: string to indicate which function to use to
+ regularizer weights
+ bias_regularizer: string to indicate which function to use to regularizer
+ bias
+ use_bn: boolean for whether to use batch normalization
+ use_sync_bn: boolean for whether sync batch normalization.
+ norm_momentum: float for moment to use for batch normalization
+ norm_epsilon: float for batch normalization epsilon
+ activation: string for activation function to use in conv layers.
+ leaky_alpha: float to use as alpha if activation function is leaky
+ sc_activation: string for activation function to use in layer
+ downsample: boolean for if image input is larger than layer output, set
+ downsample to True so the dimensions are forced to match
+ **kwargs: Keyword Arguments
+ """
+ # downsample
+ self._downsample = downsample
+
+ # ConvBN params
+ self._filters = filters
+ self._filter_scale = filter_scale
+ self._kernel_initializer = kernel_initializer
+ self._bias_initializer = bias_initializer
+ self._bias_regularizer = bias_regularizer
+ self._use_bn = use_bn
+ self._use_sync_bn = use_sync_bn
+ self._kernel_regularizer = kernel_regularizer
+
+ # normal params
+ self._norm_moment = norm_momentum
+ self._norm_epsilon = norm_epsilon
+
+ # activation params
+ self._conv_activation = activation
+ self._leaky_alpha = leaky_alpha
+ self._sc_activation = sc_activation
+
+ super().__init__(**kwargs)
+
+ def build(self, input_shape):
+ self._dark_conv_args = {
+ "kernel_initializer": self._kernel_initializer,
+ "bias_initializer": self._bias_initializer,
+ "bias_regularizer": self._bias_regularizer,
+ "use_bn": self._use_bn,
+ "use_sync_bn": self._use_sync_bn,
+ "norm_momentum": self._norm_moment,
+ "norm_epsilon": self._norm_epsilon,
+ "activation": self._conv_activation,
+ "kernel_regularizer": self._kernel_regularizer,
+ "leaky_alpha": self._leaky_alpha
+ }
+ if self._downsample:
+ self._dconv = ConvBN(
+ filters=self._filters,
+ kernel_size=(3, 3),
+ strides=(2, 2),
+ padding="same",
+ **self._dark_conv_args)
+ else:
+ self._dconv = Identity()
+
+ self._conv1 = ConvBN(
+ filters=self._filters // self._filter_scale,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ padding="same",
+ **self._dark_conv_args)
+
+ self._conv2 = ConvBN(
+ filters=self._filters,
+ kernel_size=(3, 3),
+ strides=(1, 1),
+ padding="same",
+ **self._dark_conv_args)
+
+ self._shortcut = tf.keras.layers.Add()
+ if self._sc_activation == "leaky":
+ self._activation_fn = tf.keras.layers.LeakyReLU(
+ alpha=self._leaky_alpha)
+ elif self._sc_activation == "mish":
+ self._activation_fn = lambda x: x * tf.math.tanh(tf.math.softplus(x))
+ else:
+ self._activation_fn = tf_utils.get_activation(self._sc_activation)
+ super().build(input_shape)
+
+ def call(self, inputs):
+ shortcut = self._dconv(inputs)
+ x = self._conv1(shortcut)
+ x = self._conv2(x)
+ x = self._shortcut([x, shortcut])
+ return self._activation_fn(x)
+
+ def get_config(self):
+ # used to store/share parameters to reconstruct the model
+ layer_config = {
+ "filters": self._filters,
+ "kernel_initializer": self._kernel_initializer,
+ "bias_initializer": self._bias_initializer,
+ "kernel_regularizer": self._kernel_regularizer,
+ "use_bn": self._use_bn,
+ "use_sync_bn": self._use_sync_bn,
+ "norm_moment": self._norm_moment,
+ "norm_epsilon": self._norm_epsilon,
+ "activation": self._conv_activation,
+ "leaky_alpha": self._leaky_alpha,
+ "sc_activation": self._sc_activation,
+ "downsample": self._downsample
+ }
+ layer_config.update(super().get_config())
+ return layer_config
+
+
+@tf.keras.utils.register_keras_serializable(package="yolo")
+class CSPTiny(tf.keras.layers.Layer):
+ """A Small size convolution block proposed in the CSPNet.
+
+ The layer uses shortcuts, routing(concatnation), and feature grouping
+ in order to improve gradient variablity and allow for high efficency, low
+ power residual learning for small networtf.keras.
+
+ Cross Stage Partial networks (CSPNets) were proposed in:
+ [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang
+ Chen, Jun-Wei Hsieh
+ CSPNet: A New Backbone that can Enhance Learning Capability of CNN.
+ arXiv:1911.11929
+ """
+
+ def __init__(self,
+ filters=1,
+ kernel_initializer="glorot_uniform",
+ bias_initializer="zeros",
+ kernel_regularizer=None,
+ bias_regularizer=None,
+ use_bn=True,
+ use_sync_bn=False,
+ group_id=1,
+ groups=2,
+ norm_momentum=0.99,
+ norm_epsilon=0.001,
+ activation="leaky",
+ downsample=True,
+ leaky_alpha=0.1,
+ **kwargs):
+ """Initializes CSPTiny.
+
+ Args:
+ filters: integer for output depth, or the number of features to learn
+ kernel_initializer: string to indicate which function to use to initialize
+ weights
+ bias_initializer: string to indicate which function to use to initialize
+ bias
+ kernel_regularizer: string to indicate which function to use to
+ regularizer weights
+ bias_regularizer: string to indicate which function to use to regularizer
+ bias
+ use_bn: boolean for whether to use batch normalization
+ use_sync_bn: boolean for whether sync batch normalization statistics of
+ all batch norm layers to the models global statistics (across all input
+ batches)
+ group_id: integer for which group of features to pass through the csp tiny
+ stack.
+ groups: integer for how many splits there should be in the convolution
+ feature stack output
+ norm_momentum: float for moment to use for batch normalization
+ norm_epsilon: float for batch normalization epsilon
+ activation: string or None for activation function to use in layer,
+ if None activation is replaced by linear
+ downsample: boolean for if image input is larger than layer output, set
+ downsample to True so the dimensions are forced to match
+ leaky_alpha: float to use as alpha if activation function is leaky
+ **kwargs: Keyword Arguments
+ """
+
+ # ConvBN params
+ self._filters = filters
+ self._kernel_initializer = kernel_initializer
+ self._bias_initializer = bias_initializer
+ self._bias_regularizer = bias_regularizer
+ self._use_bn = use_bn
+ self._use_sync_bn = use_sync_bn
+ self._kernel_regularizer = kernel_regularizer
+ self._groups = groups
+ self._group_id = group_id
+ self._downsample = downsample
+
+ # normal params
+ self._norm_moment = norm_momentum
+ self._norm_epsilon = norm_epsilon
+
+ # activation params
+ self._conv_activation = activation
+ self._leaky_alpha = leaky_alpha
+
+ super().__init__(**kwargs)
+
+ def build(self, input_shape):
+ self._dark_conv_args = {
+ "kernel_initializer": self._kernel_initializer,
+ "bias_initializer": self._bias_initializer,
+ "bias_regularizer": self._bias_regularizer,
+ "use_bn": self._use_bn,
+ "use_sync_bn": self._use_sync_bn,
+ "norm_momentum": self._norm_moment,
+ "norm_epsilon": self._norm_epsilon,
+ "activation": self._conv_activation,
+ "kernel_regularizer": self._kernel_regularizer,
+ "leaky_alpha": self._leaky_alpha
+ }
+ self._convlayer1 = ConvBN(
+ filters=self._filters,
+ kernel_size=(3, 3),
+ strides=(1, 1),
+ padding="same",
+ **self._dark_conv_args)
+
+ self._convlayer2 = ConvBN(
+ filters=self._filters // 2,
+ kernel_size=(3, 3),
+ strides=(1, 1),
+ padding="same",
+ kernel_initializer=self._kernel_initializer,
+ bias_initializer=self._bias_initializer,
+ bias_regularizer=self._bias_regularizer,
+ kernel_regularizer=self._kernel_regularizer,
+ use_bn=self._use_bn,
+ use_sync_bn=self._use_sync_bn,
+ norm_momentum=self._norm_moment,
+ norm_epsilon=self._norm_epsilon,
+ activation=self._conv_activation,
+ leaky_alpha=self._leaky_alpha)
+
+ self._convlayer3 = ConvBN(
+ filters=self._filters // 2,
+ kernel_size=(3, 3),
+ strides=(1, 1),
+ padding="same",
+ **self._dark_conv_args)
+
+ self._convlayer4 = ConvBN(
+ filters=self._filters,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ padding="same",
+ **self._dark_conv_args)
+
+ self._maxpool = tf.keras.layers.MaxPool2D(
+ pool_size=2, strides=2, padding="same", data_format=None)
+
+ super().build(input_shape)
+
+ def call(self, inputs):
+ x1 = self._convlayer1(inputs)
+ x1_group = tf.split(x1, self._groups, axis=-1)[self._group_id]
+ x2 = self._convlayer2(x1_group) # grouping
+ x3 = self._convlayer3(x2)
+ x4 = tf.concat([x3, x2], axis=-1) # csp partial using grouping
+ x5 = self._convlayer4(x4)
+ x = tf.concat([x1, x5], axis=-1) # csp connect
+ if self._downsample:
+ x = self._maxpool(x)
+ return x, x5
+
+ def get_config(self):
+ # used to store/share parameters to reconsturct the model
+ layer_config = {
+ "filters": self._filters,
+ "strides": self._strides,
+ "kernel_initializer": self._kernel_initializer,
+ "bias_initializer": self._bias_initializer,
+ "kernel_regularizer": self._kernel_regularizer,
+ "use_bn": self._use_bn,
+ "use_sync_bn": self._use_sync_bn,
+ "norm_moment": self._norm_moment,
+ "norm_epsilon": self._norm_epsilon,
+ "activation": self._conv_activation,
+ "leaky_alpha": self._leaky_alpha,
+ "sc_activation": self._sc_activation,
+ }
+ layer_config.update(super().get_config())
+ return layer_config
+
+
+@tf.keras.utils.register_keras_serializable(package="yolo")
+class CSPRoute(tf.keras.layers.Layer):
+ """Down sampling layer to take the place of down sampleing.
+
+ It is applied in Residual networks. This is the first of 2 layers needed to
+ convert any Residual Network model to a CSPNet. At the start of a new level
+ change, this CSPRoute layer creates a learned identity that will act as a
+ cross stage connection, that is used to inform the inputs to the next stage.
+ It is called cross stage partial because the number of filters required in
+ every intermitent Residual layer is reduced by half. The sister layer will
+ take the partial generated by this layer and concatnate it with the output of
+ the final residual layer in the stack to create a fully feature level output.
+ This concatnation merges the partial blocks of 2 levels as input to the next
+ allowing the gradients of each level to be more unique, and reducing the
+ number of parameters required by each level by 50% while keeping accuracy
+ consistent.
+
+ Cross Stage Partial networks (CSPNets) were proposed in:
+ [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang
+ Chen, Jun-Wei Hsieh.
+ CSPNet: A New Backbone that can Enhance Learning Capability of CNN.
+ arXiv:1911.11929
+ """
+
+ def __init__(self,
+ filters,
+ filter_scale=2,
+ activation="mish",
+ downsample=True,
+ kernel_initializer="glorot_uniform",
+ bias_initializer="zeros",
+ kernel_regularizer=None,
+ bias_regularizer=None,
+ use_bn=True,
+ use_sync_bn=False,
+ norm_momentum=0.99,
+ norm_epsilon=0.001,
+ **kwargs):
+ """Initializes CSPRoute.
+
+ Args:
+ filters: integer for output depth, or the number of features to learn
+ filter_scale: integer dicating (filters//2) or the number of filters in
+ the partial feature stack.
+ activation: string for activation function to use in layer
+ downsample: down_sample the input.
+ kernel_initializer: string to indicate which function to use to initialize
+ weights.
+ bias_initializer: string to indicate which function to use to initialize
+ bias.
+ kernel_regularizer: string to indicate which function to use to
+ regularizer weights.
+ bias_regularizer: string to indicate which function to use to regularizer
+ bias.
+ use_bn: boolean for whether to use batch normalization.
+ use_sync_bn: boolean for whether sync batch normalization.
+ norm_momentum: float for moment to use for batch normalization
+ norm_epsilon: float for batch normalization epsilon
+ **kwargs: Keyword Arguments
+ """
+
+ super().__init__(**kwargs)
+ # Layer params.
+ self._filters = filters
+ self._filter_scale = filter_scale
+ self._activation = activation
+
+ # Convoultion params.
+ self._kernel_initializer = kernel_initializer
+ self._bias_initializer = bias_initializer
+ self._kernel_regularizer = kernel_regularizer
+ self._bias_regularizer = bias_regularizer
+ self._use_bn = use_bn
+ self._use_sync_bn = use_sync_bn
+ self._norm_moment = norm_momentum
+ self._norm_epsilon = norm_epsilon
+ self._downsample = downsample
+
+ def build(self, input_shape):
+ self._dark_conv_args = {
+ "kernel_initializer": self._kernel_initializer,
+ "bias_initializer": self._bias_initializer,
+ "bias_regularizer": self._bias_regularizer,
+ "use_bn": self._use_bn,
+ "use_sync_bn": self._use_sync_bn,
+ "norm_momentum": self._norm_moment,
+ "norm_epsilon": self._norm_epsilon,
+ "activation": self._activation,
+ "kernel_regularizer": self._kernel_regularizer,
+ }
+ if self._downsample:
+ self._conv1 = ConvBN(filters=self._filters,
+ kernel_size=(3, 3),
+ strides=(2, 2),
+ **self._dark_conv_args)
+ else:
+ self._conv1 = ConvBN(filters=self._filters,
+ kernel_size=(3, 3),
+ strides=(1, 1),
+ **self._dark_conv_args)
+ self._conv2 = ConvBN(filters=self._filters // self._filter_scale,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ **self._dark_conv_args)
+
+ self._conv3 = ConvBN(filters=self._filters // self._filter_scale,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ **self._dark_conv_args)
+
+ def call(self, inputs):
+ x = self._conv1(inputs)
+ y = self._conv2(x)
+ x = self._conv3(x)
+ return (x, y)
+
+
+@tf.keras.utils.register_keras_serializable(package="yolo")
+class CSPConnect(tf.keras.layers.Layer):
+ """Sister Layer to the CSPRoute layer.
+
+ Merges the partial feature stacks generated by the CSPDownsampling layer,
+ and the finaly output of the residual stack. Suggested in the CSPNet paper.
+
+ Cross Stage Partial networks (CSPNets) were proposed in:
+ [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang
+ Chen, Jun-Wei Hsieh.
+ CSPNet: A New Backbone that can Enhance Learning Capability of CNN.
+ arXiv:1911.11929
+ """
+
+ def __init__(self,
+ filters,
+ filter_scale=2,
+ activation="mish",
+ kernel_initializer="glorot_uniform",
+ bias_initializer="zeros",
+ kernel_regularizer=None,
+ bias_regularizer=None,
+ use_bn=True,
+ use_sync_bn=False,
+ norm_momentum=0.99,
+ norm_epsilon=0.001,
+ **kwargs):
+ """Initializes CSPConnect.
+
+ Args:
+ filters: integer for output depth, or the number of features to learn.
+ filter_scale: integer dicating (filters//2) or the number of filters in
+ the partial feature stack.
+ activation: string for activation function to use in layer.
+ kernel_initializer: string to indicate which function to use to initialize
+ weights.
+ bias_initializer: string to indicate which function to use to initialize
+ bias.
+ kernel_regularizer: string to indicate which function to use to
+ regularizer weights.
+ bias_regularizer: string to indicate which function to use to regularizer
+ bias.
+ use_bn: boolean for whether to use batch normalization.
+ use_sync_bn: boolean for whether sync batch normalization.
+ norm_momentum: float for moment to use for batch normalization
+ norm_epsilon: float for batch normalization epsilon
+ **kwargs: Keyword Arguments
+ """
+ super().__init__(**kwargs)
+ # layer params.
+ self._filters = filters
+ self._filter_scale = filter_scale
+ self._activation = activation
+
+ # Convoultion params.
+ self._kernel_initializer = kernel_initializer
+ self._bias_initializer = bias_initializer
+ self._kernel_regularizer = kernel_regularizer
+ self._bias_regularizer = bias_regularizer
+ self._use_bn = use_bn
+ self._use_sync_bn = use_sync_bn
+ self._norm_moment = norm_momentum
+ self._norm_epsilon = norm_epsilon
+
+ def build(self, input_shape):
+ self._dark_conv_args = {
+ "kernel_initializer": self._kernel_initializer,
+ "bias_initializer": self._bias_initializer,
+ "bias_regularizer": self._bias_regularizer,
+ "use_bn": self._use_bn,
+ "use_sync_bn": self._use_sync_bn,
+ "norm_momentum": self._norm_moment,
+ "norm_epsilon": self._norm_epsilon,
+ "activation": self._activation,
+ "kernel_regularizer": self._kernel_regularizer,
+ }
+ self._conv1 = ConvBN(filters=self._filters // self._filter_scale,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ **self._dark_conv_args)
+ self._concat = tf.keras.layers.Concatenate(axis=-1)
+ self._conv2 = ConvBN(filters=self._filters,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ **self._dark_conv_args)
+
+ def call(self, inputs):
+ x_prev, x_csp = inputs
+ x = self._conv1(x_prev)
+ x = self._concat([x, x_csp])
+ x = self._conv2(x)
+ return x
+
+
+class CSPStack(tf.keras.layers.Layer):
+ """CSP full stack.
+
+ Combines the route and the connect in case you dont want to just quickly wrap
+ an existing callable or list of layers to make it a cross stage partial.
+ Added for ease of use. you should be able to wrap any layer stack with a CSP
+ independent of wether it belongs to the Darknet family. if filter_scale = 2,
+ then the blocks in the stack passed into the the CSP stack should also have
+ filters = filters/filter_scale.
+
+ Cross Stage Partial networks (CSPNets) were proposed in:
+ [1] Chien-Yao Wang, Hong-Yuan Mark Liao, I-Hau Yeh, Yueh-Hua Wu, Ping-Yang
+ Chen, Jun-Wei Hsieh
+ CSPNet: A New Backbone that can Enhance Learning Capability of CNN.
+ arXiv:1911.11929
+ """
+
+ def __init__(self,
+ filters,
+ model_to_wrap=None,
+ filter_scale=2,
+ activation="mish",
+ kernel_initializer="glorot_uniform",
+ bias_initializer="zeros",
+ kernel_regularizer=None,
+ bias_regularizer=None,
+ downsample=True,
+ use_bn=True,
+ use_sync_bn=False,
+ norm_momentum=0.99,
+ norm_epsilon=0.001,
+ **kwargs):
+ """Initializes CSPStack.
+
+ Args:
+ filters: integer for output depth, or the number of features to learn.
+ model_to_wrap: callable Model or a list of callable objects that will
+ process the output of CSPRoute, and be input into CSPConnect. List will
+ be called sequentially.
+ filter_scale: integer dicating (filters//2) or the number of filters in
+ the partial feature stack.
+ activation: string for activation function to use in layer.
+ kernel_initializer: string to indicate which function to use to initialize
+ weights.
+ bias_initializer: string to indicate which function to use to initialize
+ bias.
+ kernel_regularizer: string to indicate which function to use to
+ regularizer weights.
+ bias_regularizer: string to indicate which function to use to regularizer
+ bias.
+ downsample: down_sample the input.
+ use_bn: boolean for whether to use batch normalization
+ use_sync_bn: boolean for whether sync batch normalization.
+ norm_momentum: float for moment to use for batch normalization
+ norm_epsilon: float for batch normalization epsilon
+ **kwargs: Keyword Arguments
+ """
+ super().__init__(**kwargs)
+ # Layer params.
+ self._filters = filters
+ self._filter_scale = filter_scale
+ self._activation = activation
+ self._downsample = downsample
+
+ # Convoultion params.
+ self._kernel_initializer = kernel_initializer
+ self._bias_initializer = bias_initializer
+ self._kernel_regularizer = kernel_regularizer
+ self._bias_regularizer = bias_regularizer
+ self._use_bn = use_bn
+ self._use_sync_bn = use_sync_bn
+ self._norm_moment = norm_momentum
+ self._norm_epsilon = norm_epsilon
+
+ if model_to_wrap is not None:
+ if isinstance(model_to_wrap, Callable):
+ self._model_to_wrap = [model_to_wrap]
+ elif isinstance(model_to_wrap, List):
+ self._model_to_wrap = model_to_wrap
+ else:
+ raise ValueError("The input to the CSPStack must be a list of layers"
+ "that we can iterate through, or \n a callable")
+ else:
+ self._model_to_wrap = []
+
+ def build(self, input_shape):
+ self._dark_conv_args = {
+ "filters": self._filters,
+ "filter_scale": self._filter_scale,
+ "activation": self._activation,
+ "kernel_initializer": self._kernel_initializer,
+ "bias_initializer": self._bias_initializer,
+ "bias_regularizer": self._bias_regularizer,
+ "use_bn": self._use_bn,
+ "use_sync_bn": self._use_sync_bn,
+ "norm_momentum": self._norm_moment,
+ "norm_epsilon": self._norm_epsilon,
+ "kernel_regularizer": self._kernel_regularizer,
+ }
+ self._route = CSPRoute(downsample=self._downsample, **self._dark_conv_args)
+ self._connect = CSPConnect(**self._dark_conv_args)
+ return
+
+ def call(self, inputs):
+ x, x_route = self._route(inputs)
+ for layer in self._model_to_wrap:
+ x = layer(x)
+ x = self._connect([x, x_route])
+ return x
diff --git a/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..5df28a4f3fb1377b2a74c825e779d8b25b7d481b
--- /dev/null
+++ b/official/vision/beta/projects/yolo/modeling/layers/nn_blocks_test.py
@@ -0,0 +1,285 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+
+from absl.testing import parameterized
+import numpy as np
+import tensorflow as tf
+
+from official.vision.beta.projects.yolo.modeling.layers import nn_blocks
+
+
+class CSPConnectTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.named_parameters(("same", 224, 224, 64, 1),
+ ("downsample", 224, 224, 64, 2))
+ def test_pass_through(self, width, height, filters, mod):
+ x = tf.keras.Input(shape=(width, height, filters))
+ test_layer = nn_blocks.CSPRoute(filters=filters, filter_scale=mod)
+ test_layer2 = nn_blocks.CSPConnect(filters=filters, filter_scale=mod)
+ outx, px = test_layer(x)
+ outx = test_layer2([outx, px])
+ print(outx)
+ print(outx.shape.as_list())
+ self.assertAllEqual(
+ outx.shape.as_list(),
+ [None, np.ceil(width // 2),
+ np.ceil(height // 2), (filters)])
+
+ @parameterized.named_parameters(("same", 224, 224, 64, 1),
+ ("downsample", 224, 224, 128, 2))
+ def test_gradient_pass_though(self, filters, width, height, mod):
+ loss = tf.keras.losses.MeanSquaredError()
+ optimizer = tf.keras.optimizers.SGD()
+ test_layer = nn_blocks.CSPRoute(filters, filter_scale=mod)
+ path_layer = nn_blocks.CSPConnect(filters, filter_scale=mod)
+
+ init = tf.random_normal_initializer()
+ x = tf.Variable(
+ initial_value=init(shape=(1, width, height, filters), dtype=tf.float32))
+ y = tf.Variable(initial_value=init(shape=(1, int(np.ceil(width // 2)),
+ int(np.ceil(height // 2)),
+ filters),
+ dtype=tf.float32))
+
+ with tf.GradientTape() as tape:
+ x_hat, x_prev = test_layer(x)
+ x_hat = path_layer([x_hat, x_prev])
+ grad_loss = loss(x_hat, y)
+ grad = tape.gradient(grad_loss, test_layer.trainable_variables)
+ optimizer.apply_gradients(zip(grad, test_layer.trainable_variables))
+
+ self.assertNotIn(None, grad)
+
+
+class CSPRouteTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.named_parameters(("same", 224, 224, 64, 1),
+ ("downsample", 224, 224, 64, 2))
+ def test_pass_through(self, width, height, filters, mod):
+ x = tf.keras.Input(shape=(width, height, filters))
+ test_layer = nn_blocks.CSPRoute(filters=filters, filter_scale=mod)
+ outx, _ = test_layer(x)
+ print(outx)
+ print(outx.shape.as_list())
+ self.assertAllEqual(
+ outx.shape.as_list(),
+ [None, np.ceil(width // 2),
+ np.ceil(height // 2), (filters / mod)])
+
+ @parameterized.named_parameters(("same", 224, 224, 64, 1),
+ ("downsample", 224, 224, 128, 2))
+ def test_gradient_pass_though(self, filters, width, height, mod):
+ loss = tf.keras.losses.MeanSquaredError()
+ optimizer = tf.keras.optimizers.SGD()
+ test_layer = nn_blocks.CSPRoute(filters, filter_scale=mod)
+ path_layer = nn_blocks.CSPConnect(filters, filter_scale=mod)
+
+ init = tf.random_normal_initializer()
+ x = tf.Variable(
+ initial_value=init(shape=(1, width, height, filters), dtype=tf.float32))
+ y = tf.Variable(initial_value=init(shape=(1, int(np.ceil(width // 2)),
+ int(np.ceil(height // 2)),
+ filters),
+ dtype=tf.float32))
+
+ with tf.GradientTape() as tape:
+ x_hat, x_prev = test_layer(x)
+ x_hat = path_layer([x_hat, x_prev])
+ grad_loss = loss(x_hat, y)
+ grad = tape.gradient(grad_loss, test_layer.trainable_variables)
+ optimizer.apply_gradients(zip(grad, test_layer.trainable_variables))
+
+ self.assertNotIn(None, grad)
+
+
+class CSPStackTest(tf.test.TestCase, parameterized.TestCase):
+
+ def build_layer(
+ self, layer_type, filters, filter_scale, count, stack_type, downsample):
+ if stack_type is not None:
+ layers = []
+ if layer_type == "residual":
+ for _ in range(count):
+ layers.append(
+ nn_blocks.DarkResidual(
+ filters=filters // filter_scale, filter_scale=filter_scale))
+ else:
+ for _ in range(count):
+ layers.append(nn_blocks.ConvBN(filters=filters))
+
+ if stack_type == "model":
+ layers = tf.keras.Sequential(layers=layers)
+ else:
+ layers = None
+
+ stack = nn_blocks.CSPStack(
+ filters=filters,
+ filter_scale=filter_scale,
+ downsample=downsample,
+ model_to_wrap=layers)
+ return stack
+
+ @parameterized.named_parameters(
+ ("no_stack", 224, 224, 64, 2, "residual", None, 0, True),
+ ("residual_stack", 224, 224, 64, 2, "residual", "list", 2, True),
+ ("conv_stack", 224, 224, 64, 2, "conv", "list", 3, False),
+ ("callable_no_scale", 224, 224, 64, 1, "residual", "model", 5, False))
+ def test_pass_through(self, width, height, filters, mod, layer_type,
+ stack_type, count, downsample):
+ x = tf.keras.Input(shape=(width, height, filters))
+ test_layer = self.build_layer(layer_type, filters, mod, count, stack_type,
+ downsample)
+ outx = test_layer(x)
+ print(outx)
+ print(outx.shape.as_list())
+ if downsample:
+ self.assertAllEqual(outx.shape.as_list(),
+ [None, width // 2, height // 2, filters])
+ else:
+ self.assertAllEqual(outx.shape.as_list(), [None, width, height, filters])
+
+ @parameterized.named_parameters(
+ ("no_stack", 224, 224, 64, 2, "residual", None, 0, True),
+ ("residual_stack", 224, 224, 64, 2, "residual", "list", 2, True),
+ ("conv_stack", 224, 224, 64, 2, "conv", "list", 3, False),
+ ("callable_no_scale", 224, 224, 64, 1, "residual", "model", 5, False))
+ def test_gradient_pass_though(self, width, height, filters, mod, layer_type,
+ stack_type, count, downsample):
+ loss = tf.keras.losses.MeanSquaredError()
+ optimizer = tf.keras.optimizers.SGD()
+
+ init = tf.random_normal_initializer()
+ x = tf.Variable(
+ initial_value=init(shape=(1, width, height, filters), dtype=tf.float32))
+
+ if not downsample:
+ y = tf.Variable(
+ initial_value=init(
+ shape=(1, width, height, filters), dtype=tf.float32))
+ else:
+ y = tf.Variable(
+ initial_value=init(
+ shape=(1, width // 2, height // 2, filters), dtype=tf.float32))
+ test_layer = self.build_layer(layer_type, filters, mod, count, stack_type,
+ downsample)
+
+ with tf.GradientTape() as tape:
+ x_hat = test_layer(x)
+ grad_loss = loss(x_hat, y)
+ grad = tape.gradient(grad_loss, test_layer.trainable_variables)
+ optimizer.apply_gradients(zip(grad, test_layer.trainable_variables))
+
+ self.assertNotIn(None, grad)
+
+
+class ConvBNTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.named_parameters(
+ ("valid", (3, 3), "valid", (1, 1)), ("same", (3, 3), "same", (1, 1)),
+ ("downsample", (3, 3), "same", (2, 2)), ("test", (1, 1), "valid", (1, 1)))
+ def test_pass_through(self, kernel_size, padding, strides):
+ if padding == "same":
+ pad_const = 1
+ else:
+ pad_const = 0
+ x = tf.keras.Input(shape=(224, 224, 3))
+ test_layer = nn_blocks.ConvBN(
+ filters=64,
+ kernel_size=kernel_size,
+ padding=padding,
+ strides=strides,
+ trainable=False)
+ outx = test_layer(x)
+ print(outx.shape.as_list())
+ test = [
+ None,
+ int((224 - kernel_size[0] + (2 * pad_const)) / strides[0] + 1),
+ int((224 - kernel_size[1] + (2 * pad_const)) / strides[1] + 1), 64
+ ]
+ print(test)
+ self.assertAllEqual(outx.shape.as_list(), test)
+
+ @parameterized.named_parameters(("filters", 3))
+ def test_gradient_pass_though(self, filters):
+ loss = tf.keras.losses.MeanSquaredError()
+ optimizer = tf.keras.optimizers.SGD()
+ with tf.device("/CPU:0"):
+ test_layer = nn_blocks.ConvBN(filters, kernel_size=(3, 3), padding="same")
+
+ init = tf.random_normal_initializer()
+ x = tf.Variable(initial_value=init(shape=(1, 224, 224,
+ 3), dtype=tf.float32))
+ y = tf.Variable(
+ initial_value=init(shape=(1, 224, 224, filters), dtype=tf.float32))
+
+ with tf.GradientTape() as tape:
+ x_hat = test_layer(x)
+ grad_loss = loss(x_hat, y)
+ grad = tape.gradient(grad_loss, test_layer.trainable_variables)
+ optimizer.apply_gradients(zip(grad, test_layer.trainable_variables))
+ self.assertNotIn(None, grad)
+
+
+class DarkResidualTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.named_parameters(("same", 224, 224, 64, False),
+ ("downsample", 223, 223, 32, True),
+ ("oddball", 223, 223, 32, False))
+ def test_pass_through(self, width, height, filters, downsample):
+ mod = 1
+ if downsample:
+ mod = 2
+ x = tf.keras.Input(shape=(width, height, filters))
+ test_layer = nn_blocks.DarkResidual(filters=filters, downsample=downsample)
+ outx = test_layer(x)
+ print(outx)
+ print(outx.shape.as_list())
+ self.assertAllEqual(
+ outx.shape.as_list(),
+ [None, np.ceil(width / mod),
+ np.ceil(height / mod), filters])
+
+ @parameterized.named_parameters(("same", 64, 224, 224, False),
+ ("downsample", 32, 223, 223, True),
+ ("oddball", 32, 223, 223, False))
+ def test_gradient_pass_though(self, filters, width, height, downsample):
+ loss = tf.keras.losses.MeanSquaredError()
+ optimizer = tf.keras.optimizers.SGD()
+ test_layer = nn_blocks.DarkResidual(filters, downsample=downsample)
+
+ if downsample:
+ mod = 2
+ else:
+ mod = 1
+
+ init = tf.random_normal_initializer()
+ x = tf.Variable(
+ initial_value=init(shape=(1, width, height, filters), dtype=tf.float32))
+ y = tf.Variable(initial_value=init(shape=(1, int(np.ceil(width / mod)),
+ int(np.ceil(height / mod)),
+ filters),
+ dtype=tf.float32))
+
+ with tf.GradientTape() as tape:
+ x_hat = test_layer(x)
+ grad_loss = loss(x_hat, y)
+ grad = tape.gradient(grad_loss, test_layer.trainable_variables)
+ optimizer.apply_gradients(zip(grad, test_layer.trainable_variables))
+
+ self.assertNotIn(None, grad)
+
+if __name__ == "__main__":
+ tf.test.main()
diff --git a/official/vision/beta/projects/yolo/ops/__init__.py b/official/vision/beta/projects/yolo/ops/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a25710c222e3327cb20e000db5df5c5651c4a2cc
--- /dev/null
+++ b/official/vision/beta/projects/yolo/ops/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
diff --git a/official/vision/beta/projects/yolo/ops/box_ops.py b/official/vision/beta/projects/yolo/ops/box_ops.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d49177d656906f063f1f0963a1f485be689cdff
--- /dev/null
+++ b/official/vision/beta/projects/yolo/ops/box_ops.py
@@ -0,0 +1,297 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Bounding box utils."""
+
+import math
+
+import tensorflow as tf
+
+
+def yxyx_to_xcycwh(box: tf.Tensor):
+ """Converts boxes from ymin, xmin, ymax, xmax.
+
+ to x_center, y_center, width, height.
+
+ Args:
+ box: `Tensor` whose shape is [..., 4] and represents the coordinates
+ of boxes in ymin, xmin, ymax, xmax.
+
+ Returns:
+ `Tensor` whose shape is [..., 4] and contains the new format.
+
+ Raises:
+ ValueError: If the last dimension of box is not 4 or if box's dtype isn't
+ a floating point type.
+ """
+ with tf.name_scope('yxyx_to_xcycwh'):
+ ymin, xmin, ymax, xmax = tf.split(box, 4, axis=-1)
+ x_center = (xmax + xmin) / 2
+ y_center = (ymax + ymin) / 2
+ width = xmax - xmin
+ height = ymax - ymin
+ box = tf.concat([x_center, y_center, width, height], axis=-1)
+ return box
+
+
+def xcycwh_to_yxyx(box: tf.Tensor, split_min_max: bool = False):
+ """Converts boxes from x_center, y_center, width, height.
+
+ to ymin, xmin, ymax, xmax.
+
+ Args:
+ box: a `Tensor` whose shape is [..., 4] and represents the coordinates
+ of boxes in x_center, y_center, width, height.
+ split_min_max: bool, whether or not to split x, y min and max values.
+
+ Returns:
+ box: a `Tensor` whose shape is [..., 4] and contains the new format.
+
+ Raises:
+ ValueError: If the last dimension of box is not 4 or if box's dtype isn't
+ a floating point type.
+ """
+ with tf.name_scope('xcycwh_to_yxyx'):
+ xy, wh = tf.split(box, 2, axis=-1)
+ xy_min = xy - wh / 2
+ xy_max = xy + wh / 2
+ x_min, y_min = tf.split(xy_min, 2, axis=-1)
+ x_max, y_max = tf.split(xy_max, 2, axis=-1)
+ box = tf.concat([y_min, x_min, y_max, x_max], axis=-1)
+ if split_min_max:
+ box = tf.split(box, 2, axis=-1)
+ return box
+
+
+def xcycwh_to_xyxy(box: tf.Tensor, split_min_max: bool = False):
+ """Converts boxes from x_center, y_center, width, height to.
+
+ xmin, ymin, xmax, ymax.
+
+ Args:
+ box: box: a `Tensor` whose shape is [..., 4] and represents the
+ coordinates of boxes in x_center, y_center, width, height.
+ split_min_max: bool, whether or not to split x, y min and max values.
+
+ Returns:
+ box: a `Tensor` whose shape is [..., 4] and contains the new format.
+
+ Raises:
+ ValueError: If the last dimension of box is not 4 or if box's dtype isn't
+ a floating point type.
+ """
+ with tf.name_scope('xcycwh_to_yxyx'):
+ xy, wh = tf.split(box, 2, axis=-1)
+ xy_min = xy - wh / 2
+ xy_max = xy + wh / 2
+ box = (xy_min, xy_max)
+ if not split_min_max:
+ box = tf.concat(box, axis=-1)
+ return box
+
+
+def center_distance(center_1: tf.Tensor, center_2: tf.Tensor):
+ """Calculates the squared distance between two points.
+
+ This function is mathematically equivalent to the following code, but has
+ smaller rounding errors.
+
+ tf.norm(center_1 - center_2, axis=-1)**2
+
+ Args:
+ center_1: a `Tensor` whose shape is [..., 2] and represents a point.
+ center_2: a `Tensor` whose shape is [..., 2] and represents a point.
+
+ Returns:
+ dist: a `Tensor` whose shape is [...] and value represents the squared
+ distance between center_1 and center_2.
+
+ Raises:
+ ValueError: If the last dimension of either center_1 or center_2 is not 2.
+ """
+ with tf.name_scope('center_distance'):
+ dist = (center_1[..., 0] - center_2[..., 0])**2 + (center_1[..., 1] -
+ center_2[..., 1])**2
+ return dist
+
+
+def compute_iou(box1, box2, yxyx=False):
+ """Calculates the intersection of union between box1 and box2.
+
+ Args:
+ box1: a `Tensor` whose shape is [..., 4] and represents the coordinates of
+ boxes in x_center, y_center, width, height.
+ box2: a `Tensor` whose shape is [..., 4] and represents the coordinates of
+ boxes in x_center, y_center, width, height.
+ yxyx: `bool`, whether or not box1, and box2 are in yxyx format.
+
+ Returns:
+ iou: a `Tensor` whose shape is [...] and value represents the intersection
+ over union.
+
+ Raises:
+ ValueError: If the last dimension of either box1 or box2 is not 4.
+ """
+ # Get box corners
+ with tf.name_scope('iou'):
+ if not yxyx:
+ box1 = xcycwh_to_yxyx(box1)
+ box2 = xcycwh_to_yxyx(box2)
+
+ b1mi, b1ma = tf.split(box1, 2, axis=-1)
+ b2mi, b2ma = tf.split(box2, 2, axis=-1)
+ intersect_mins = tf.math.maximum(b1mi, b2mi)
+ intersect_maxes = tf.math.minimum(b1ma, b2ma)
+ intersect_wh = tf.math.maximum(intersect_maxes - intersect_mins,
+ tf.zeros_like(intersect_mins))
+ intersection = tf.reduce_prod(
+ intersect_wh, axis=-1) # intersect_wh[..., 0] * intersect_wh[..., 1]
+
+ box1_area = tf.math.abs(tf.reduce_prod(b1ma - b1mi, axis=-1))
+ box2_area = tf.math.abs(tf.reduce_prod(b2ma - b2mi, axis=-1))
+ union = box1_area + box2_area - intersection
+
+ iou = intersection / (union + 1e-7)
+ iou = tf.clip_by_value(iou, clip_value_min=0.0, clip_value_max=1.0)
+ return iou
+
+
+def compute_giou(box1, box2):
+ """Calculates the generalized intersection of union between box1 and box2.
+
+ Args:
+ box1: a `Tensor` whose shape is [..., 4] and represents the coordinates of
+ boxes in x_center, y_center, width, height.
+ box2: a `Tensor` whose shape is [..., 4] and represents the coordinates of
+ boxes in x_center, y_center, width, height.
+
+ Returns:
+ iou: a `Tensor` whose shape is [...] and value represents the generalized
+ intersection over union.
+
+ Raises:
+ ValueError: If the last dimension of either box1 or box2 is not 4.
+ """
+ with tf.name_scope('giou'):
+ # get box corners
+ box1 = xcycwh_to_yxyx(box1)
+ box2 = xcycwh_to_yxyx(box2)
+
+ # compute IOU
+ intersect_mins = tf.math.maximum(box1[..., 0:2], box2[..., 0:2])
+ intersect_maxes = tf.math.minimum(box1[..., 2:4], box2[..., 2:4])
+ intersect_wh = tf.math.maximum(intersect_maxes - intersect_mins,
+ tf.zeros_like(intersect_mins))
+ intersection = intersect_wh[..., 0] * intersect_wh[..., 1]
+
+ box1_area = tf.math.abs(
+ tf.reduce_prod(box1[..., 2:4] - box1[..., 0:2], axis=-1))
+ box2_area = tf.math.abs(
+ tf.reduce_prod(box2[..., 2:4] - box2[..., 0:2], axis=-1))
+ union = box1_area + box2_area - intersection
+
+ iou = tf.math.divide_no_nan(intersection, union)
+ iou = tf.clip_by_value(iou, clip_value_min=0.0, clip_value_max=1.0)
+
+ # find the smallest box to encompase both box1 and box2
+ c_mins = tf.math.minimum(box1[..., 0:2], box2[..., 0:2])
+ c_maxes = tf.math.maximum(box1[..., 2:4], box2[..., 2:4])
+ c = tf.math.abs(tf.reduce_prod(c_mins - c_maxes, axis=-1))
+
+ # compute giou
+ giou = iou - tf.math.divide_no_nan((c - union), c)
+ return iou, giou
+
+
+def compute_diou(box1, box2):
+ """Calculates the distance intersection of union between box1 and box2.
+
+ Args:
+ box1: a `Tensor` whose shape is [..., 4] and represents the coordinates of
+ boxes in x_center, y_center, width, height.
+ box2: a `Tensor` whose shape is [..., 4] and represents the coordinates of
+ boxes in x_center, y_center, width, height.
+
+ Returns:
+ iou: a `Tensor` whose shape is [...] and value represents the distance
+ intersection over union.
+
+ Raises:
+ ValueError: If the last dimension of either box1 or box2 is not 4.
+ """
+ with tf.name_scope('diou'):
+ # compute center distance
+ dist = center_distance(box1[..., 0:2], box2[..., 0:2])
+
+ # get box corners
+ box1 = xcycwh_to_yxyx(box1)
+ box2 = xcycwh_to_yxyx(box2)
+
+ # compute IOU
+ intersect_mins = tf.math.maximum(box1[..., 0:2], box2[..., 0:2])
+ intersect_maxes = tf.math.minimum(box1[..., 2:4], box2[..., 2:4])
+ intersect_wh = tf.math.maximum(intersect_maxes - intersect_mins,
+ tf.zeros_like(intersect_mins))
+ intersection = intersect_wh[..., 0] * intersect_wh[..., 1]
+
+ box1_area = tf.math.abs(
+ tf.reduce_prod(box1[..., 2:4] - box1[..., 0:2], axis=-1))
+ box2_area = tf.math.abs(
+ tf.reduce_prod(box2[..., 2:4] - box2[..., 0:2], axis=-1))
+ union = box1_area + box2_area - intersection
+
+ iou = tf.math.divide_no_nan(intersection, union)
+ iou = tf.clip_by_value(iou, clip_value_min=0.0, clip_value_max=1.0)
+
+ # compute max diagnal of the smallest enclosing box
+ c_mins = tf.math.minimum(box1[..., 0:2], box2[..., 0:2])
+ c_maxes = tf.math.maximum(box1[..., 2:4], box2[..., 2:4])
+
+ diag_dist = tf.reduce_sum((c_maxes - c_mins)**2, axis=-1)
+
+ regularization = tf.math.divide_no_nan(dist, diag_dist)
+ diou = iou + regularization
+ return iou, diou
+
+
+def compute_ciou(box1, box2):
+ """Calculates the complete intersection of union between box1 and box2.
+
+ Args:
+ box1: a `Tensor` whose shape is [..., 4] and represents the coordinates
+ of boxes in x_center, y_center, width, height.
+ box2: a `Tensor` whose shape is [..., 4] and represents the coordinates of
+ boxes in x_center, y_center, width, height.
+
+ Returns:
+ iou: a `Tensor` whose shape is [...] and value represents the complete
+ intersection over union.
+
+ Raises:
+ ValueError: If the last dimension of either box1 or box2 is not 4.
+ """
+ with tf.name_scope('ciou'):
+ # compute DIOU and IOU
+ iou, diou = compute_diou(box1, box2)
+
+ # computer aspect ratio consistency
+ arcterm = (
+ tf.math.atan(tf.math.divide_no_nan(box1[..., 2], box1[..., 3])) -
+ tf.math.atan(tf.math.divide_no_nan(box2[..., 2], box2[..., 3])))**2
+ v = 4 * arcterm / (math.pi)**2
+
+ # compute IOU regularization
+ a = tf.math.divide_no_nan(v, ((1 - iou) + v))
+ ciou = diou + v * a
+ return iou, ciou
diff --git a/official/vision/beta/projects/yolo/ops/box_ops_test.py b/official/vision/beta/projects/yolo/ops/box_ops_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..b10e53235b731bbe839551d473831d9f28fe94b0
--- /dev/null
+++ b/official/vision/beta/projects/yolo/ops/box_ops_test.py
@@ -0,0 +1,56 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from absl.testing import parameterized
+import numpy as np
+import tensorflow as tf
+
+from official.vision.beta.projects.yolo.ops import box_ops
+
+
+class InputUtilsTest(parameterized.TestCase, tf.test.TestCase):
+
+ @parameterized.parameters((1), (4))
+ def test_box_conversions(self, num_boxes):
+ boxes = tf.convert_to_tensor(np.random.rand(num_boxes, 4))
+ expected_shape = np.array([num_boxes, 4])
+ xywh_box = box_ops.yxyx_to_xcycwh(boxes)
+ yxyx_box = box_ops.xcycwh_to_yxyx(boxes)
+ xyxy_box = box_ops.xcycwh_to_xyxy(boxes)
+ self.assertAllEqual(tf.shape(xywh_box).numpy(), expected_shape)
+ self.assertAllEqual(tf.shape(yxyx_box).numpy(), expected_shape)
+ self.assertAllEqual(tf.shape(xyxy_box).numpy(), expected_shape)
+
+ @parameterized.parameters((1), (5), (7))
+ def test_ious(self, num_boxes):
+ boxes = tf.convert_to_tensor(np.random.rand(num_boxes, 4))
+ expected_shape = np.array([
+ num_boxes,
+ ])
+ expected_iou = np.ones([
+ num_boxes,
+ ])
+ iou = box_ops.compute_iou(boxes, boxes)
+ _, giou = box_ops.compute_giou(boxes, boxes)
+ _, ciou = box_ops.compute_ciou(boxes, boxes)
+ _, diou = box_ops.compute_diou(boxes, boxes)
+ self.assertAllEqual(tf.shape(iou).numpy(), expected_shape)
+ self.assertArrayNear(iou, expected_iou, 0.001)
+ self.assertArrayNear(giou, expected_iou, 0.001)
+ self.assertArrayNear(ciou, expected_iou, 0.001)
+ self.assertArrayNear(diou, expected_iou, 0.001)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/projects/yolo/ops/preprocess_ops.py b/official/vision/beta/projects/yolo/ops/preprocess_ops.py
new file mode 100644
index 0000000000000000000000000000000000000000..562b4fe0a90099f6a8fd240f92fb56b33ba8523f
--- /dev/null
+++ b/official/vision/beta/projects/yolo/ops/preprocess_ops.py
@@ -0,0 +1,524 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Yolo preprocess ops."""
+
+import tensorflow as tf
+import tensorflow_addons as tfa
+
+from official.vision.beta.projects.yolo.ops import box_ops
+
+
+def resize_crop_filter(image, boxes, default_width, default_height,
+ target_width, target_height):
+ """Apply zooming to the image and boxes.
+
+ Args:
+ image: a `Tensor` representing the image.
+ boxes: a `Tensor` represeting the boxes.
+ default_width: a `Tensor` representing the width of the image.
+ default_height: a `Tensor` representing the height of the image.
+ target_width: a `Tensor` representing the desired width of the image.
+ target_height: a `Tensor` representing the desired height of the image.
+ Returns:
+ images: a `Tensor` representing the augmented image.
+ boxes: a `Tensor` representing the augmented boxes.
+ """
+ with tf.name_scope('resize_crop_filter'):
+ image = tf.image.resize(image, (target_width, target_height))
+ image = tf.image.resize_with_crop_or_pad(image,
+ target_height=default_height,
+ target_width=default_width)
+
+ default_width = tf.cast(default_width, boxes.dtype)
+ default_height = tf.cast(default_height, boxes.dtype)
+ target_width = tf.cast(target_width, boxes.dtype)
+ target_height = tf.cast(target_height, boxes.dtype)
+
+ aspect_change_width = target_width / default_width
+ aspect_change_height = target_height / default_height
+
+ x, y, width, height = tf.split(boxes, 4, axis=-1)
+ x = (x - 0.5) * target_width / default_width + 0.5
+ y = (y - 0.5) * target_height / default_height + 0.5
+ width = width * aspect_change_width
+ height = height * aspect_change_height
+ boxes = tf.concat([x, y, width, height], axis=-1)
+ return image, boxes
+
+
+def random_translate(image, box, t, seed=None):
+ """Randomly translate the image and boxes.
+
+ Args:
+ image: a `Tensor` representing the image.
+ box: a `Tensor` represeting the boxes.
+ t: an `int` representing the translation factor
+ seed: an optional seed for tf.random operations
+ Returns:
+ image: a `Tensor` representing the augmented image.
+ box: a `Tensor` representing the augmented boxes.
+ """
+ t_x = tf.random.uniform(minval=-t,
+ maxval=t,
+ shape=(),
+ dtype=tf.float32,
+ seed=seed)
+ t_y = tf.random.uniform(minval=-t,
+ maxval=t,
+ shape=(),
+ dtype=tf.float32,
+ seed=seed)
+ box = translate_boxes(box, t_x, t_y)
+ image = translate_image(image, t_x, t_y)
+ return image, box
+
+
+def translate_boxes(box, translate_x, translate_y):
+ """Randomly translate the boxes.
+
+ Args:
+ box: a `Tensor` represeitng the boxes.
+ translate_x: a `Tensor` represting the translation on the x-axis.
+ translate_y: a `Tensor` represting the translation on the y-axis.
+ Returns:
+ box: a `Tensor` representing the augmented boxes.
+ """
+ with tf.name_scope('translate_boxs'):
+ x = box[..., 0] + translate_x
+ y = box[..., 1] + translate_y
+ box = tf.stack([x, y, box[..., 2], box[..., 3]], axis=-1)
+ box.set_shape([None, 4])
+ return box
+
+
+def translate_image(image, translate_x, translate_y):
+ """Randomly translate the image.
+
+ Args:
+ image: a `Tensor` representing the image.
+ translate_x: a `Tensor` represting the translation on the x-axis.
+ translate_y: a `Tensor` represting the translation on the y-axis.
+ Returns:
+ box: a `Tensor` representing the augmented boxes.
+ """
+ with tf.name_scope('translate_image'):
+ if (translate_x != 0 and translate_y != 0):
+ image_jitter = tf.convert_to_tensor([translate_x, translate_y])
+ image_jitter.set_shape([2])
+ image = tfa.image.translate(
+ image, image_jitter * tf.cast(tf.shape(image)[1], tf.float32))
+ return image
+
+
+def pad_max_instances(value, instances, pad_value=0, pad_axis=0):
+ """Pads tensors to max number of instances."""
+ shape = tf.shape(value)
+ dim1 = shape[pad_axis]
+ take = tf.math.reduce_min([instances, dim1])
+ value, _ = tf.split(value, [take, -1],
+ axis=pad_axis) # value[:instances, ...]
+ pad = tf.convert_to_tensor([tf.math.reduce_max([instances - dim1, 0])])
+ nshape = tf.concat([shape[:pad_axis], pad, shape[(pad_axis + 1):]], axis=0)
+ pad_tensor = tf.fill(nshape, tf.cast(pad_value, dtype=value.dtype))
+ value = tf.concat([value, pad_tensor], axis=pad_axis)
+ return value
+
+
+def fit_preserve_aspect_ratio(image,
+ boxes,
+ width=None,
+ height=None,
+ target_dim=None):
+ """Resizes the image while peserving the image aspect ratio.
+
+ Args:
+ image: a `Tensor` representing the image.
+ boxes: a `Tensor` representing the boxes.
+ width: int for the image width.
+ height: int for the image height.
+ target_dim: list or a Tensor of height and width.
+ Returns:
+ image: a `Tensor` representing the image.
+ box: a `Tensor` representing the boxes.
+ """
+ if width is None or height is None:
+ shape = tf.shape(image)
+ if tf.shape(shape)[0] == 4:
+ width = shape[1]
+ height = shape[2]
+ else:
+ width = shape[0]
+ height = shape[1]
+
+ clipper = tf.math.maximum(width, height)
+ if target_dim is None:
+ target_dim = clipper
+
+ pad_width = clipper - width
+ pad_height = clipper - height
+ image = tf.image.pad_to_bounding_box(image, pad_width // 2, pad_height // 2,
+ clipper, clipper)
+
+ boxes = box_ops.yxyx_to_xcycwh(boxes)
+ x, y, w, h = tf.split(boxes, 4, axis=-1)
+
+ y *= tf.cast(width / clipper, tf.float32)
+ x *= tf.cast(height / clipper, tf.float32)
+
+ y += tf.cast((pad_width / clipper) / 2, tf.float32)
+ x += tf.cast((pad_height / clipper) / 2, tf.float32)
+
+ h *= tf.cast(width / clipper, tf.float32)
+ w *= tf.cast(height / clipper, tf.float32)
+
+ boxes = tf.concat([x, y, w, h], axis=-1)
+
+ boxes = box_ops.xcycwh_to_yxyx(boxes)
+ image = tf.image.resize(image, (target_dim, target_dim))
+ return image, boxes
+
+
+def get_best_anchor(y_true, anchors, width=1, height=1):
+ """Gets the correct anchor that is assoiciated with each box using IOU.
+
+ Args:
+ y_true: tf.Tensor[] for the list of bounding boxes in the yolo format
+ anchors: list or tensor for the anchor boxes to be used in prediction
+ found via Kmeans
+ width: int for the image width
+ height: int for the image height
+
+ Returns:
+ tf.Tensor: y_true with the anchor associated with each ground truth
+ box known.
+ """
+ with tf.name_scope('get_anchor'):
+ width = tf.cast(width, dtype=tf.float32)
+ height = tf.cast(height, dtype=tf.float32)
+
+ # split the boxes into center and width height
+ anchor_xy = y_true[..., 0:2]
+
+ # scale thhe boxes
+ anchors = tf.convert_to_tensor(anchors, dtype=tf.float32)
+ anchors_x = anchors[..., 0] / width
+ anchors_y = anchors[..., 1] / height
+ anchors = tf.stack([anchors_x, anchors_y], axis=-1)
+ k = tf.shape(anchors)[0]
+
+ # build a matrix of anchor boxes of shape [num_anchors, num_boxes, 4]
+ anchors = tf.transpose(anchors, perm=[1, 0])
+ anchor_xy = tf.tile(tf.expand_dims(anchor_xy, axis=-1),
+ [1, 1, tf.shape(anchors)[-1]])
+ anchors = tf.tile(tf.expand_dims(anchors, axis=0),
+ [tf.shape(anchor_xy)[0], 1, 1])
+
+ # stack the xy so, each anchor is asscoaited once with each center from
+ # the ground truth input
+ anchors = tf.concat([anchor_xy, anchors], axis=1)
+ anchors = tf.transpose(anchors, perm=[2, 0, 1])
+
+ # copy the gt n times so that each anchor from above can be compared to
+ # input ground truth to shape: [num_anchors, num_boxes, 4]
+ truth_comp = tf.tile(tf.expand_dims(y_true[..., 0:4], axis=-1),
+ [1, 1, tf.shape(anchors)[0]])
+ truth_comp = tf.transpose(truth_comp, perm=[2, 0, 1])
+
+ # compute intersection over union of the boxes, and take the argmax of
+ # comuted iou for each box. thus each box is associated with the
+ # largest interection over union
+ iou_raw = box_ops.compute_iou(truth_comp, anchors)
+ values, indexes = tf.math.top_k(tf.transpose(iou_raw, perm=[1, 0]),
+ k=tf.cast(k, dtype=tf.int32),
+ sorted=True)
+ ind_mask = tf.cast(values > 0.213, dtype=indexes.dtype)
+
+ # pad the indexs such that all values less than the thresh are -1
+ # add one, multiply the mask to zeros all the bad locations
+ # subtract 1 makeing all the bad locations 0.
+ iou_index = tf.concat([
+ tf.keras.backend.expand_dims(indexes[..., 0], axis=-1),
+ ((indexes[..., 1:] + 1) * ind_mask[..., 1:]) - 1
+ ],
+ axis=-1)
+ iou_index = iou_index[..., :6]
+
+ return tf.cast(iou_index, dtype=tf.float32)
+
+
+def build_grided_gt(y_true, mask, size, dtype, use_tie_breaker):
+ """Converts ground truth for use in loss functions.
+
+ Args:
+ y_true: tf.Tensor[] ground truth
+ [box coords[0:4], classes_onehot[0:-1], best_fit_anchor_box]
+ mask: list of the anchor boxes choresponding to the output,
+ ex. [1, 2, 3] tells this layer to predict only the first 3
+ anchors in the total.
+ size: The dimensions of this output, for regular, it progresses
+ from 13, to 26, to 52.
+ dtype: The expected output dtype.
+ use_tie_breaker: boolean value for wether or not to use the tie_breaker.
+
+ Returns:
+ tf.Tensor[] of shape [size, size, #of_anchors, 4, 1, num_classes]
+ """
+ # unpack required components from the input ground truth
+ boxes = tf.cast(y_true['bbox'], dtype)
+ classes = tf.expand_dims(tf.cast(y_true['classes'], dtype=dtype), axis=-1)
+ anchors = tf.cast(y_true['best_anchors'], dtype)
+
+ # get the number of boxes in the ground truth boxs
+ num_boxes = tf.shape(boxes)[0]
+ # get the number of anchor boxes used for this anchor scale
+ len_masks = tf.shape(mask)[0]
+
+ # init a fixed memeory size grid for this prediction scale
+ # [size, size, # of anchors, 1 + 1 + number of anchors per scale]
+ full = tf.zeros([size, size, len_masks, 6], dtype=dtype)
+ # init a grid to use to track which locations have already
+ # been used before (for the tie breaker)
+ depth_track = tf.zeros((size, size, len_masks), dtype=tf.int32)
+
+ # rescale the x and y centers to the size of the grid [size, size]
+ x = tf.cast(boxes[..., 0] * tf.cast(size, dtype=dtype), dtype=tf.int32)
+ y = tf.cast(boxes[..., 1] * tf.cast(size, dtype=dtype), dtype=tf.int32)
+
+ # init all the tensorArrays to be used in storeing the index
+ # and the values to be used to update both depth_track and full
+ update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True)
+ update = tf.TensorArray(dtype, size=0, dynamic_size=True)
+
+ # init constants and match data types before entering loop
+ i = 0
+ anchor_id = 0
+ const = tf.cast(tf.convert_to_tensor([1.]), dtype=dtype)
+ mask = tf.cast(mask, dtype=dtype)
+ rand_update = 0.0
+
+ for box_id in range(num_boxes):
+ # If the width or height of the box is zero, skip it.
+ # After pre processing, if the box is not in the i image bounds anymore,
+ # skip it.
+ if tf.keras.backend.all(tf.math.equal(
+ boxes[box_id, 2:4], 0)) or tf.keras.backend.any(
+ tf.math.less(boxes[box_id, 0:2], 0.0)) or tf.keras.backend.any(
+ tf.math.greater_equal(boxes[box_id, 0:2], 1.0)):
+ continue
+ if use_tie_breaker:
+ for anchor_id in range(tf.shape(anchors)[-1]):
+ index = tf.math.equal(anchors[box_id, anchor_id], mask)
+ if tf.keras.backend.any(index):
+ # using the boolean index mask to determine exactly which
+ # anchor box was used
+ p = tf.cast(
+ tf.keras.backend.argmax(tf.cast(index, dtype=tf.int32)),
+ dtype=tf.int32)
+ # determine if the index was used or not
+ used = depth_track[y[box_id], x[box_id], p]
+ # defualt used upadte value
+ uid = 1
+
+ # if anchor_id is 0, this is the best matched anchor for this box
+ # with the highest IOU
+ if anchor_id == 0:
+ # write the box to the update list
+ # create random numbr to trigger a replacment if the cell
+ # is used already
+ if tf.math.equal(used, 1):
+ rand_update = tf.random.uniform([], maxval=1)
+ else:
+ rand_update = 1.0
+
+ if rand_update > 0.5:
+ # write the box to the update list
+ update_index = update_index.write(i, [y[box_id], x[box_id], p])
+ value = tf.concat([boxes[box_id], const, classes[box_id]],
+ axis=-1)
+ update = update.write(i, value)
+
+ # if used is 2, this cell is filled with a non-optimal box
+ # if used is 0, the cell in the ground truth is not yet consumed
+ # in either case you can replace that cell with a new box, as long
+ # as it is not consumed by an optimal box with anchor_id = 0
+ elif tf.math.equal(used, 2) or tf.math.equal(used, 0):
+ uid = 2
+ # write the box to the update list
+ update_index = update_index.write(i, [y[box_id], x[box_id], p])
+ value = tf.concat([boxes[box_id], const, classes[box_id]], axis=-1)
+ update = update.write(i, value)
+
+ depth_track = tf.tensor_scatter_nd_update(
+ depth_track, [(y[box_id], x[box_id], p)], [uid])
+ i += 1
+ else:
+ index = tf.math.equal(anchors[box_id, 0], mask)
+ # if any there is an index match
+ if tf.keras.backend.any(index):
+ # find the index
+ p = tf.cast(
+ tf.keras.backend.argmax(tf.cast(index, dtype=tf.int32)),
+ dtype=tf.int32)
+ # update the list of used boxes
+ update_index = update_index.write(i, [y[box_id], x[box_id], p])
+ value = tf.concat([boxes[box_id], const, classes[box_id]], axis=-1)
+ update = update.write(i, value)
+ i += 1
+
+ # if the size of the update list is not 0, do an update, other wise,
+ # no boxes and pass an empty grid
+ if tf.math.greater(update_index.size(), 0):
+ update_index = update_index.stack()
+ update = update.stack()
+ full = tf.tensor_scatter_nd_update(full, update_index, update)
+ return full
+
+
+def build_batch_grided_gt(y_true, mask, size, dtype, use_tie_breaker):
+ """Converts ground truth for use in loss functions.
+
+ Args:
+ y_true: tf.Tensor[] ground truth
+ [batch, box coords[0:4], classes_onehot[0:-1], best_fit_anchor_box]
+ mask: list of the anchor boxes choresponding to the output,
+ ex. [1, 2, 3] tells this layer to predict only the first 3 anchors
+ in the total.
+ size: the dimensions of this output, for regular, it progresses from
+ 13, to 26, to 52
+ dtype: expected output datatype
+ use_tie_breaker: boolean value for wether or not to use the tie
+ breaker
+
+ Returns:
+ tf.Tensor[] of shape [batch, size, size, #of_anchors, 4, 1, num_classes]
+ """
+ # unpack required components from the input ground truth
+ boxes = tf.cast(y_true['bbox'], dtype)
+ classes = tf.expand_dims(tf.cast(y_true['classes'], dtype=dtype), axis=-1)
+ anchors = tf.cast(y_true['best_anchors'], dtype)
+
+ # get the batch size
+ batches = tf.shape(boxes)[0]
+ # get the number of boxes in the ground truth boxs
+ num_boxes = tf.shape(boxes)[1]
+ # get the number of anchor boxes used for this anchor scale
+ len_masks = tf.shape(mask)[0]
+
+ # init a fixed memeory size grid for this prediction scale
+ # [batch, size, size, # of anchors, 1 + 1 + number of anchors per scale]
+ full = tf.zeros([batches, size, size, len_masks, 1 + 4 + 1], dtype=dtype)
+ # init a grid to use to track which locations have already
+ # been used before (for the tie breaker)
+ depth_track = tf.zeros((batches, size, size, len_masks), dtype=tf.int32)
+
+ # rescale the x and y centers to the size of the grid [size, size]
+ x = tf.cast(boxes[..., 0] * tf.cast(size, dtype=dtype), dtype=tf.int32)
+ y = tf.cast(boxes[..., 1] * tf.cast(size, dtype=dtype), dtype=tf.int32)
+
+ # init all the tensorArrays to be used in storeing the index and the values
+ # to be used to update both depth_track and full
+ update_index = tf.TensorArray(tf.int32, size=0, dynamic_size=True)
+ update = tf.TensorArray(dtype, size=0, dynamic_size=True)
+
+ # init constants and match data types before entering loop
+ i = 0
+ anchor_id = 0
+ const = tf.cast(tf.convert_to_tensor([1.]), dtype=dtype)
+ mask = tf.cast(mask, dtype=dtype)
+ rand_update = 0.0
+
+ for batch in range(batches):
+ for box_id in range(num_boxes):
+ # if the width or height of the box is zero, skip it
+ if tf.keras.backend.all(tf.math.equal(boxes[batch, box_id, 2:4], 0)):
+ continue
+ # after pre processing, if the box is not in the image bounds anymore
+ # skip the box
+ if tf.keras.backend.any(tf.math.less(
+ boxes[batch, box_id, 0:2], 0.0)) or tf.keras.backend.any(
+ tf.math.greater_equal(boxes[batch, box_id, 0:2], 1.0)):
+ continue
+ if use_tie_breaker:
+ for anchor_id in range(tf.shape(anchors)[-1]):
+ index = tf.math.equal(anchors[batch, box_id, anchor_id], mask)
+ if tf.keras.backend.any(index):
+ # using the boolean index mask to determine exactly which anchor
+ # box was used
+ p = tf.cast(tf.keras.backend.argmax(tf.cast(index, dtype=tf.int32)),
+ dtype=tf.int32)
+ # determine if the index was used or not
+ used = depth_track[batch, y[batch, box_id], x[batch, box_id], p]
+ # defualt used upadte value
+ uid = 1
+
+ # if anchor_id is 0, this is the best matched anchor for this box
+ # with the highest IOU
+ if anchor_id == 0:
+ # create random number to trigger a replacment if the cell
+ # is used already
+ if tf.math.equal(used, 1):
+ rand_update = tf.random.uniform([], maxval=1)
+ else:
+ rand_update = 1.0
+
+ if rand_update > 0.5:
+ # write the box to the update list
+ update_index = update_index.write(
+ i, [batch, y[batch, box_id], x[batch, box_id], p])
+ value = tf.concat(
+ [boxes[batch, box_id], const, classes[batch, box_id]],
+ axis=-1)
+ update = update.write(i, value)
+
+ # if used is 2, this cell is filled with a non-optimal box
+ # if used is 0, the cell in the ground truth is not yet consumed
+ # in either case you can replace that cell with a new box, as long
+ # as it is not consumed by an optimal box with anchor_id = 0
+ elif tf.math.equal(used, 2) or tf.math.equal(used, 0):
+ uid = 2
+ # write the box to the update list
+ update_index = update_index.write(
+ i, [batch, y[batch, box_id], x[batch, box_id], p])
+ value = ([boxes[batch, box_id], const, classes[batch, box_id]])
+ update = update.write(i, value)
+
+ # update the used index for where and how the box was placed
+ depth_track = tf.tensor_scatter_nd_update(
+ depth_track, [(batch, y[batch, box_id], x[batch, box_id], p)],
+ [uid])
+ i += 1
+ else:
+ index = tf.math.equal(anchors[batch, box_id, 0], mask)
+ if tf.keras.backend.any(index):
+ # if any there is an index match
+ p = tf.cast(
+ tf.keras.backend.argmax(tf.cast(index, dtype=tf.int32)),
+ dtype=tf.int32)
+ # write the box to the update list
+ update_index = update_index.write(
+ i, [batch, y[batch, box_id], x[batch, box_id], p])
+ value = tf.concat(
+ [boxes[batch, box_id], const, classes[batch, box_id]], axis=-1)
+ update = update.write(i, value)
+ i += 1
+
+ # if the size of the update list is not 0, do an update, other wise,
+ # no boxes and pass an empty grid
+ if tf.math.greater(update_index.size(), 0):
+ update_index = update_index.stack()
+ update = update.stack()
+ full = tf.tensor_scatter_nd_update(full, update_index, update)
+ return full
+
diff --git a/official/vision/beta/projects/yolo/ops/preprocess_ops_test.py b/official/vision/beta/projects/yolo/ops/preprocess_ops_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..11973d9787454985653b2cbfe04f022a0fb3eb61
--- /dev/null
+++ b/official/vision/beta/projects/yolo/ops/preprocess_ops_test.py
@@ -0,0 +1,67 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from absl.testing import parameterized
+import numpy as np
+import tensorflow as tf
+
+from official.vision.beta.projects.yolo.ops import preprocess_ops
+
+
+class PreprocessOpsTest(parameterized.TestCase, tf.test.TestCase):
+
+ @parameterized.parameters((416, 416, 5, 300, 300), (100, 200, 6, 50, 50))
+ def test_resize_crop_filter(self, default_width, default_height, num_boxes,
+ target_width, target_height):
+ image = tf.convert_to_tensor(
+ np.random.rand(default_width, default_height, 3))
+ boxes = tf.convert_to_tensor(np.random.rand(num_boxes, 4))
+ resized_image, resized_boxes = preprocess_ops.resize_crop_filter(
+ image, boxes, default_width, default_height, target_width,
+ target_height)
+ resized_image_shape = tf.shape(resized_image)
+ resized_boxes_shape = tf.shape(resized_boxes)
+ self.assertAllEqual([default_height, default_width, 3],
+ resized_image_shape.numpy())
+ self.assertAllEqual([num_boxes, 4], resized_boxes_shape.numpy())
+
+ @parameterized.parameters((7, 7., 5.), (25, 35., 45.))
+ def test_translate_boxes(self, num_boxes, translate_x, translate_y):
+ boxes = tf.convert_to_tensor(np.random.rand(num_boxes, 4))
+ translated_boxes = preprocess_ops.translate_boxes(
+ boxes, translate_x, translate_y)
+ translated_boxes_shape = tf.shape(translated_boxes)
+ self.assertAllEqual([num_boxes, 4], translated_boxes_shape.numpy())
+
+ @parameterized.parameters((100, 200, 75., 25.), (400, 600, 25., 75.))
+ def test_translate_image(self, image_height, image_width, translate_x,
+ translate_y):
+ image = tf.convert_to_tensor(np.random.rand(image_height, image_width, 4))
+ translated_image = preprocess_ops.translate_image(
+ image, translate_x, translate_y)
+ translated_image_shape = tf.shape(translated_image)
+ self.assertAllEqual([image_height, image_width, 4],
+ translated_image_shape.numpy())
+
+ @parameterized.parameters(([1, 2], 20, 0), ([13, 2, 4], 15, 0))
+ def test_pad_max_instances(self, input_shape, instances, pad_axis):
+ expected_output_shape = input_shape
+ expected_output_shape[pad_axis] = instances
+ output = preprocess_ops.pad_max_instances(
+ np.ones(input_shape), instances, pad_axis=pad_axis)
+ self.assertAllEqual(expected_output_shape, tf.shape(output).numpy())
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/projects/yolo/tasks/image_classification.py b/official/vision/beta/projects/yolo/tasks/image_classification.py
new file mode 100644
index 0000000000000000000000000000000000000000..2dd3ce3a71c0d10dd0ce7bec6e9b356116e75a25
--- /dev/null
+++ b/official/vision/beta/projects/yolo/tasks/image_classification.py
@@ -0,0 +1,114 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Image classification task definition."""
+import tensorflow as tf
+
+from official.core import input_reader
+from official.core import task_factory
+from official.vision.beta.dataloaders import classification_input
+from official.vision.beta.projects.yolo.configs import darknet_classification as exp_cfg
+from official.vision.beta.projects.yolo.dataloaders import classification_tfds_decoder as cli
+from official.vision.beta.tasks import image_classification
+
+
+@task_factory.register_task_cls(exp_cfg.ImageClassificationTask)
+class ImageClassificationTask(image_classification.ImageClassificationTask):
+ """A task for image classification."""
+
+ def build_inputs(self, params, input_context=None):
+ """Builds classification input."""
+
+ num_classes = self.task_config.model.num_classes
+ input_size = self.task_config.model.input_size
+
+ if params.tfds_name:
+ decoder = cli.Decoder()
+ else:
+ decoder = classification_input.Decoder()
+
+ parser = classification_input.Parser(
+ output_size=input_size[:2],
+ num_classes=num_classes,
+ dtype=params.dtype)
+
+ reader = input_reader.InputReader(
+ params,
+ dataset_fn=tf.data.TFRecordDataset,
+ decoder_fn=decoder.decode,
+ parser_fn=parser.parse_fn(params.is_training))
+
+ dataset = reader.read(input_context=input_context)
+ return dataset
+
+ def train_step(self, inputs, model, optimizer, metrics=None):
+ """Does forward and backward.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the model, forward pass definition.
+ optimizer: the optimizer for this training step.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+ if self.task_config.losses.one_hot:
+ labels = tf.one_hot(labels, self.task_config.model.num_classes)
+
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(features, training=True)
+ # Casting output layer as float32 is necessary when mixed_precision is
+ # mixed_float16 or mixed_bfloat16 to ensure output is casted as float32.
+ outputs = tf.nest.map_structure(
+ lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss.
+ loss = self.build_losses(
+ model_outputs=outputs, labels=labels, aux_losses=model.losses)
+ # Scales loss as the default gradients allreduce performs sum inside the
+ # optimizer.
+ scaled_loss = loss / num_replicas
+
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient before apply_gradients when LossScaleOptimizer is
+ # used.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+
+ # Apply gradient clipping.
+ if self.task_config.gradient_clip_norm > 0:
+ grads, _ = tf.clip_by_global_norm(
+ grads, self.task_config.gradient_clip_norm)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = {self.loss: loss}
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in metrics})
+ elif model.compiled_metrics:
+ self.process_compiled_metrics(model.compiled_metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in model.metrics})
+ return logs
+
+
diff --git a/official/vision/beta/projects/yolo/train.py b/official/vision/beta/projects/yolo/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..93c499013b77654130940164d1c8142408cc746f
--- /dev/null
+++ b/official/vision/beta/projects/yolo/train.py
@@ -0,0 +1,73 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""TensorFlow Model Garden Vision training driver."""
+
+from absl import app
+from absl import flags
+import gin
+
+from official.common import distribute_utils
+from official.common import flags as tfm_flags
+from official.core import task_factory
+from official.core import train_lib
+from official.core import train_utils
+from official.modeling import performance
+from official.vision.beta.projects.yolo.common import registry_imports # pylint: disable=unused-import
+
+FLAGS = flags.FLAGS
+
+'''
+python3 -m official.vision.beta.projects.yolo.train --mode=train_and_eval --experiment=darknet_classification --model_dir=training_dir --config_file=official/vision/beta/projects/yolo/configs/experiments/darknet53_tfds.yaml
+'''
+
+
+def main(_):
+ gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params)
+ print(FLAGS.experiment)
+ params = train_utils.parse_configuration(FLAGS)
+
+ model_dir = FLAGS.model_dir
+ if 'train' in FLAGS.mode:
+ # Pure eval modes do not output yaml files. Otherwise continuous eval job
+ # may race against the train job for writing the same file.
+ train_utils.serialize_config(params, model_dir)
+
+ # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16'
+ # can have significant impact on model speeds by utilizing float16 in case of
+ # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when
+ # dtype is float16
+ if params.runtime.mixed_precision_dtype:
+ performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype)
+ distribution_strategy = distribute_utils.get_distribution_strategy(
+ distribution_strategy=params.runtime.distribution_strategy,
+ all_reduce_alg=params.runtime.all_reduce_alg,
+ num_gpus=params.runtime.num_gpus,
+ tpu_address=params.runtime.tpu)
+ with distribution_strategy.scope():
+ task = task_factory.get_task(params.task, logging_dir=model_dir)
+
+ train_lib.run_experiment(
+ distribution_strategy=distribution_strategy,
+ task=task,
+ mode=FLAGS.mode,
+ params=params,
+ model_dir=model_dir)
+
+ train_utils.save_gin_config(FLAGS.mode, model_dir)
+
+if __name__ == '__main__':
+ tfm_flags.define_flags()
+ app.run(main)
diff --git a/official/vision/beta/projects/yt8m/README.md b/official/vision/beta/projects/yt8m/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..2f4cf2ab4b079c3473089c4457d71a3f7b1e2793
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/README.md
@@ -0,0 +1,154 @@
+# YouTube-8M Tensorflow Starter Code
+
+DISCLAIMER: This project is still under development.
+No support will be provided during the development phase.
+
+This repo contains starter code (written in TensorFlow 2.x) for training and
+evaluating machine learning models over the [YouTube-8M][1] dataset.
+This is the Tensorflow2 version of the original starter code:
+[YouTube-8M Tensorflow Starter Code][2]
+which was tested on Tensorflow 1.14. (The code gives an end-to-end
+working example for reading the dataset, training a TensorFlow model,
+and evaluating the performance of the model). Functionalities are maintained,
+while necessary migrations were done to accomodate running on tf2 environment.
+
+### Requirements
+
+The starter code requires Tensorflow. If you haven't installed it yet, follow
+the instructions on [tensorflow.org][3].
+This code has been tested with Tensorflow 2.4.0. Going forward,
+we will continue to target the latest released version of Tensorflow.
+
+Please verify that you have Python 3.6+ and Tensorflow 2.4.0 or higher
+installed by running the following commands:
+
+```sh
+python --version
+python -c 'import tensorflow as tf; print(tf.__version__)'
+```
+
+Refer to the [instructions here][4]
+for using the model in this repo. Make sure to add the models folder to your
+Python path.
+
+[1]: https://research.google.com/youtube8m/
+[2]: https://github.com/google/youtube-8m
+[3]: https://www.tensorflow.org/install/
+[4]:
+https://github.com/tensorflow/models/tree/master/official#running-the-models
+
+#### Using GPUs
+
+If your Tensorflow installation has GPU support
+(which should have been provided with `pip install tensorflow` for any version
+above Tensorflow 1.15), this code will make use of all of your compatible GPUs.
+You can verify your installation by running
+
+```
+tf.config.list_physical_devices('GPU')
+```
+
+This will print out something like the following for each of your compatible
+GPUs.
+
+```
+I tensorflow/core/common_runtime/gpu/gpu_device.cc:1720]
+Found device 0 with properties:
+pciBusID: 0000:00:04.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
+coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 15.90GiB
+deviceMemoryBandwidth: 681.88GiB/s
+...
+```
+
+### Train and inference
+Train video-level model on frame-level features and inference at segment-level.
+
+#### Train using the config file.
+Create a YAML or JSON file for specifying the parameters to be overridden.
+Working examples can be found in yt8m/experiments directory.
+```sh
+task:
+ model:
+ cluster_size: 2048
+ hidden_size: 2048
+ add_batch_norm: true
+ sample_random_frames: true
+ is_training: true
+ activation: "relu6"
+ pooling_method: "average"
+ yt8m_agg_classifier_model: "MoeModel"
+ train_data:
+ segment_labels: false
+ temporal_stride: 1
+ num_devices: 1
+ input_path: 'gs://youtube8m-ml/2/frame/train/train*.tfrecord'
+ num_examples: 3888919
+ ...
+```
+
+The code can be run in different modes: `train / train_and_eval / eval`.
+Run `train.py` and specify which mode you wish to execute.
+Training is done using frame-level features with video-level labels,
+while inference can be done at segment-level.
+Setting `segment_labels=True` in your configuration forces
+the segment level labels to be used in the evaluation/validation phrase.
+If set to `False`, video level labels are used for inference.
+
+The following commands will train a model on Google Cloud over frame-level
+features.
+
+```bash
+python3 train.py --mode='train' \
+ --experiment='yt8m_experiment' \
+ --model_dir=$MODEL_DIR \
+ --config_file=$CONFIG_FILE
+```
+
+In order to run evaluation after each training epoch,
+set the mode to `train_and_eval`.
+Paths to both train and validation dataset on Google Cloud are set as
+train: `input_path=gs://youtube8m-ml/2/frame/train/train*.tfrecord`
+validation:`input_path=gs://youtube8m-ml/3/frame/validate/validate*.tfrecord`
+as default.
+
+```bash
+python3 train.py --mode='train_and_eval' \
+ --experiment='yt8m_experiment' \
+ --model_dir=$MODEL_DIR \
+ --config_file=$CONFIG_FILE \
+```
+
+Running on evaluation mode loads saved checkpoint from specified path
+and runs inference task.
+```bash
+python3 train.py --mode='eval' \
+ --experiment='yt8m_experiment' \
+ --model_dir=$MODEL_DIR \
+ --config_file=$CONFIG_FILE
+```
+
+
+Once these job starts executing you will see outputs similar to the following:
+```
+train | step: 15190 | training until step 22785...
+train | step: 22785 | steps/sec: 0.4 | output:
+ {'learning_rate': 0.0049961056,
+ 'model_loss': 0.0012011167,
+ 'total_loss': 0.0013538885,
+ 'training_loss': 0.0013538885}
+
+```
+
+and the following for evaluation:
+
+```
+eval | step: 22785 | running 2172 steps of evaluation...
+eval | step: 22785 | eval time: 1663.4 | output:
+ {'avg_hit_at_one': 0.5572835238737471,
+ 'avg_perr': 0.557277077999072,
+ 'gap': 0.768825760186494,
+ 'map': 0.19354554465020685,
+ 'model_loss': 0.0005052475,
+ 'total_loss': 0.0006564412,
+ 'validation_loss': 0.0006564412}
+```
diff --git a/official/vision/beta/projects/yt8m/__init__.py b/official/vision/beta/projects/yt8m/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/official/vision/beta/projects/yt8m/configs/__init__.py b/official/vision/beta/projects/yt8m/configs/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d620c7f1faddc7b962861e4c77f16c3f915d332
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/configs/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Configs package definition."""
+
+from official.vision.beta.projects.yt8m.configs import yt8m
diff --git a/official/vision/beta/projects/yt8m/configs/yt8m.py b/official/vision/beta/projects/yt8m/configs/yt8m.py
new file mode 100644
index 0000000000000000000000000000000000000000..df1e588272cfddc442050da2c72fae0aecb46ce9
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/configs/yt8m.py
@@ -0,0 +1,157 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Video classification configuration definition."""
+from typing import Optional, Tuple
+from absl import flags
+import dataclasses
+
+from official.core import config_definitions as cfg
+from official.core import exp_factory
+from official.modeling import hyperparams
+from official.modeling import optimization
+
+FLAGS = flags.FLAGS
+
+YT8M_TRAIN_EXAMPLES = 3888919
+YT8M_VAL_EXAMPLES = 1112356
+# 2/frame -> frame level
+# 3/frame -> segment level
+YT8M_TRAIN_PATH = 'gs://youtube8m-ml/2/frame/train/train*.tfrecord'
+YT8M_VAL_PATH = 'gs://youtube8m-ml/3/frame/validate/validate*.tfrecord'
+
+
+@dataclasses.dataclass
+class DataConfig(cfg.DataConfig):
+ """The base configuration for building datasets."""
+ name: Optional[str] = 'yt8m'
+ split: Optional[str] = None
+ feature_sizes: Tuple[int, ...] = (1024, 128)
+ feature_names: Tuple[str, ...] = ('rgb', 'audio')
+ segment_size: int = 1
+ segment_labels: bool = False
+ temporal_stride: int = 1
+ max_frames: int = 300
+ num_frames: int = 300 # set smaller to allow random sample (Parser)
+ num_classes: int = 3862
+ num_devices: int = 1
+ input_path: str = ''
+ is_training: bool = True
+ random_seed: int = 123
+ num_examples: int = -1
+
+
+def yt8m(is_training):
+ """YT8M dataset configs."""
+ return DataConfig(
+ num_frames=30,
+ temporal_stride=1,
+ segment_labels=False,
+ segment_size=5,
+ is_training=is_training,
+ split='train' if is_training else 'valid',
+ num_examples=YT8M_TRAIN_EXAMPLES if is_training else YT8M_VAL_EXAMPLES,
+ input_path=YT8M_TRAIN_PATH if is_training else YT8M_VAL_PATH)
+
+
+@dataclasses.dataclass
+class YT8MModel(hyperparams.Config):
+ """The model config."""
+ cluster_size: int = 2048
+ hidden_size: int = 2048
+ add_batch_norm: bool = True
+ sample_random_frames: bool = True
+ is_training: bool = True
+ activation: str = 'relu6'
+ pooling_method: str = 'average'
+ yt8m_agg_classifier_model: str = 'MoeModel'
+
+
+@dataclasses.dataclass
+class Losses(hyperparams.Config):
+ name: str = 'binary_crossentropy'
+ from_logits: bool = False
+ label_smoothing: float = 0.0
+
+
+@dataclasses.dataclass
+class YT8MTask(cfg.TaskConfig):
+ """The task config."""
+ model: YT8MModel = YT8MModel()
+ train_data: DataConfig = yt8m(is_training=True)
+ validation_data: DataConfig = yt8m(is_training=False)
+ losses: Losses = Losses()
+ gradient_clip_norm: float = 1.0
+ num_readers: int = 8
+ top_k: int = 20
+ top_n: Optional[int] = None
+
+
+def add_trainer(
+ experiment: cfg.ExperimentConfig,
+ train_batch_size: int,
+ eval_batch_size: int,
+ learning_rate: float = 0.005,
+ train_epochs: int = 44,
+):
+ """Add and config a trainer to the experiment config."""
+ if YT8M_TRAIN_EXAMPLES <= 0:
+ raise ValueError('Wrong train dataset size {!r}'.format(
+ experiment.task.train_data))
+ if YT8M_VAL_EXAMPLES <= 0:
+ raise ValueError('Wrong validation dataset size {!r}'.format(
+ experiment.task.validation_data))
+ experiment.task.train_data.global_batch_size = train_batch_size
+ experiment.task.validation_data.global_batch_size = eval_batch_size
+ steps_per_epoch = YT8M_TRAIN_EXAMPLES // train_batch_size
+ experiment.trainer = cfg.TrainerConfig(
+ steps_per_loop=steps_per_epoch,
+ summary_interval=steps_per_epoch,
+ checkpoint_interval=steps_per_epoch,
+ train_steps=train_epochs * steps_per_epoch,
+ validation_steps=YT8M_VAL_EXAMPLES // eval_batch_size,
+ validation_interval=steps_per_epoch,
+ optimizer_config=optimization.OptimizationConfig({
+ 'optimizer': {
+ 'type': 'adam',
+ 'adam': {}
+ },
+ 'learning_rate': {
+ 'type': 'exponential',
+ 'exponential': {
+ 'initial_learning_rate': learning_rate,
+ 'decay_rate': 0.95,
+ 'decay_steps': 1500000,
+ }
+ },
+ }))
+ return experiment
+
+
+@exp_factory.register_config_factory('yt8m_experiment')
+def yt8m_experiment() -> cfg.ExperimentConfig:
+ """Video classification general."""
+ exp_config = cfg.ExperimentConfig(
+ runtime=cfg.RuntimeConfig(mixed_precision_dtype='bfloat16'),
+ task=YT8MTask(),
+ trainer=cfg.TrainerConfig(),
+ restrictions=[
+ 'task.train_data.is_training != None',
+ 'task.validation_data.is_training != None',
+ 'task.train_data.num_classes == task.validation_data.num_classes',
+ 'task.train_data.feature_sizes != None',
+ 'task.train_data.feature_names != None',
+ ])
+
+ return add_trainer(exp_config, train_batch_size=512, eval_batch_size=512)
diff --git a/official/vision/beta/projects/yt8m/dataloaders/utils.py b/official/vision/beta/projects/yt8m/dataloaders/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..eda1a4ab23ac9e8ccf27f03ebcc86ae6be7b34e6
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/dataloaders/utils.py
@@ -0,0 +1,215 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains a collection of util functions for training and evaluating."""
+
+from absl import logging
+import numpy
+import tensorflow as tf
+
+
+def Dequantize(feat_vector, max_quantized_value=2, min_quantized_value=-2):
+ """Dequantize the feature from the byte format to the float format.
+
+ Args:
+ feat_vector: the input 1-d vector.
+ max_quantized_value: the maximum of the quantized value.
+ min_quantized_value: the minimum of the quantized value.
+
+ Returns:
+ A float vector which has the same shape as feat_vector.
+ """
+ assert max_quantized_value > min_quantized_value
+ quantized_range = max_quantized_value - min_quantized_value
+ scalar = quantized_range / 255.0
+ bias = (quantized_range / 512.0) + min_quantized_value
+ return feat_vector * scalar + bias
+
+
+def MakeSummary(name, value):
+ """Creates a tf.Summary proto with the given name and value."""
+ summary = tf.Summary()
+ val = summary.value.add()
+ val.tag = str(name)
+ val.simple_value = float(value)
+ return summary
+
+
+def AddGlobalStepSummary(summary_writer,
+ global_step_val,
+ global_step_info_dict,
+ summary_scope="Eval"):
+ """Add the global_step summary to the Tensorboard.
+
+ Args:
+ summary_writer: Tensorflow summary_writer.
+ global_step_val: a int value of the global step.
+ global_step_info_dict: a dictionary of the evaluation metrics calculated for
+ a mini-batch.
+ summary_scope: Train or Eval.
+
+ Returns:
+ A string of this global_step summary
+ """
+ this_hit_at_one = global_step_info_dict["hit_at_one"]
+ this_perr = global_step_info_dict["perr"]
+ this_loss = global_step_info_dict["loss"]
+ examples_per_second = global_step_info_dict.get("examples_per_second", -1)
+
+ summary_writer.add_summary(
+ MakeSummary("GlobalStep/" + summary_scope + "_Hit@1", this_hit_at_one),
+ global_step_val)
+ summary_writer.add_summary(
+ MakeSummary("GlobalStep/" + summary_scope + "_Perr", this_perr),
+ global_step_val)
+ summary_writer.add_summary(
+ MakeSummary("GlobalStep/" + summary_scope + "_Loss", this_loss),
+ global_step_val)
+
+ if examples_per_second != -1:
+ summary_writer.add_summary(
+ MakeSummary("GlobalStep/" + summary_scope + "_Example_Second",
+ examples_per_second), global_step_val)
+
+ summary_writer.flush()
+ info = (
+ "global_step {0} | Batch Hit@1: {1:.3f} | Batch PERR: {2:.3f} | Batch "
+ "Loss: {3:.3f} | Examples_per_sec: {4:.3f}").format(
+ global_step_val, this_hit_at_one, this_perr, this_loss,
+ examples_per_second)
+ return info
+
+
+def AddEpochSummary(summary_writer,
+ global_step_val,
+ epoch_info_dict,
+ summary_scope="Eval"):
+ """Add the epoch summary to the Tensorboard.
+
+ Args:
+ summary_writer: Tensorflow summary_writer.
+ global_step_val: a int value of the global step.
+ epoch_info_dict: a dictionary of the evaluation metrics calculated for the
+ whole epoch.
+ summary_scope: Train or Eval.
+
+ Returns:
+ A string of this global_step summary
+ """
+ epoch_id = epoch_info_dict["epoch_id"]
+ avg_hit_at_one = epoch_info_dict["avg_hit_at_one"]
+ avg_perr = epoch_info_dict["avg_perr"]
+ avg_loss = epoch_info_dict["avg_loss"]
+ aps = epoch_info_dict["aps"]
+ gap = epoch_info_dict["gap"]
+ mean_ap = numpy.mean(aps)
+
+ summary_writer.add_summary(
+ MakeSummary("Epoch/" + summary_scope + "_Avg_Hit@1", avg_hit_at_one),
+ global_step_val)
+ summary_writer.add_summary(
+ MakeSummary("Epoch/" + summary_scope + "_Avg_Perr", avg_perr),
+ global_step_val)
+ summary_writer.add_summary(
+ MakeSummary("Epoch/" + summary_scope + "_Avg_Loss", avg_loss),
+ global_step_val)
+ summary_writer.add_summary(
+ MakeSummary("Epoch/" + summary_scope + "_MAP", mean_ap), global_step_val)
+ summary_writer.add_summary(
+ MakeSummary("Epoch/" + summary_scope + "_GAP", gap), global_step_val)
+ summary_writer.flush()
+
+ info = ("epoch/eval number {0} | Avg_Hit@1: {1:.3f} | Avg_PERR: {2:.3f} "
+ "| MAP: {3:.3f} | GAP: {4:.3f} | Avg_Loss: {5:3f} | num_classes: {6}"
+ ).format(epoch_id, avg_hit_at_one, avg_perr, mean_ap, gap, avg_loss,
+ len(aps))
+ return info
+
+
+def GetListOfFeatureNamesAndSizes(feature_names, feature_sizes):
+ """Extract the list of feature names and the dimensionality.
+
+ Args:
+ feature_names: string containing comma separated list of feature names
+ feature_sizes: string containing comma separated list of feature sizes
+
+ Returns:
+ List of the feature names and list of the dimensionality of each feature.
+ Elements in the first/second list are strings/integers.
+ """
+ list_of_feature_names = [
+ feature_names.strip() for feature_names in feature_names.split(",")
+ ]
+ list_of_feature_sizes = [
+ int(feature_sizes) for feature_sizes in feature_sizes.split(",")
+ ]
+ if len(list_of_feature_names) != len(list_of_feature_sizes):
+ logging.error(
+ "length of the feature names (=%r) != length of feature "
+ "sizes (=%r)", str(len(list_of_feature_names)),
+ str(len(list_of_feature_sizes)))
+
+ return list_of_feature_names, list_of_feature_sizes
+
+
+def ClipGradientNorms(gradients_to_variables, max_norm):
+ """Clips the gradients by the given value.
+
+ Args:
+ gradients_to_variables: A list of gradient to variable pairs (tuples).
+ max_norm: the maximum norm value.
+
+ Returns:
+ A list of clipped gradient to variable pairs.
+ """
+ clipped_grads_and_vars = []
+ for grad, var in gradients_to_variables:
+ if grad is not None:
+ if isinstance(grad, tf.IndexedSlices):
+ tmp = tf.clip_by_norm(grad.values, max_norm)
+ grad = tf.IndexedSlices(tmp, grad.indices, grad.dense_shape)
+ else:
+ grad = tf.clip_by_norm(grad, max_norm)
+ clipped_grads_and_vars.append((grad, var))
+ return clipped_grads_and_vars
+
+
+def CombineGradients(tower_grads):
+ """Calculate the combined gradient for each shared variable across all towers.
+
+ Note that this function provides a synchronization point across all towers.
+
+ Args:
+ tower_grads: List of lists of (gradient, variable) tuples. The outer list is
+ over individual gradients. The inner list is over the gradient calculation
+ for each tower.
+
+ Returns:
+ List of pairs of (gradient, variable) where the gradient has been summed
+ across all towers.
+ """
+ filtered_grads = [
+ [x for x in grad_list if x[0] is not None] for grad_list in tower_grads
+ ]
+ final_grads = []
+ for i in range(len(filtered_grads[0])):
+ grads = [filtered_grads[t][i] for t in range(len(filtered_grads))]
+ grad = tf.stack([x[0] for x in grads], 0)
+ grad = tf.reduce_sum(grad, 0)
+ final_grads.append((
+ grad,
+ filtered_grads[0][i][1],
+ ))
+
+ return final_grads
diff --git a/official/vision/beta/projects/yt8m/dataloaders/yt8m_input.py b/official/vision/beta/projects/yt8m/dataloaders/yt8m_input.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d59e73854a988ce85a4e9d75c8b488802867397
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/dataloaders/yt8m_input.py
@@ -0,0 +1,419 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""class YT8MFrameFeatureReader(BaseReader).
+
+ Reads TFRecords of SequenceExamples.
+
+ The TFRecords must contain SequenceExamples with the sparse in64 'labels'
+ context feature and a fixed length byte-quantized feature vector, obtained
+ from the features in 'feature_names'. The quantized features will be mapped
+ back into a range between min_quantized_value and max_quantized_value.
+ link for details: https://research.google.com/youtube8m/download.html
+"""
+
+from typing import Dict
+
+import tensorflow as tf
+from official.vision.beta.configs import video_classification as exp_cfg
+from official.vision.beta.dataloaders import decoder
+from official.vision.beta.dataloaders import parser
+from official.vision.beta.projects.yt8m.dataloaders import utils
+
+
+def resize_axis(tensor, axis, new_size, fill_value=0):
+ """Truncates or pads a tensor to new_size on on a given axis.
+
+ Truncate or extend tensor such that tensor.shape[axis] == new_size. If the
+ size increases, the padding will be performed at the end, using fill_value.
+
+ Args:
+ tensor: The tensor to be resized.
+ axis: An integer representing the dimension to be sliced.
+ new_size: An integer or 0d tensor representing the new value for
+ tensor.shape[axis].
+ fill_value: Value to use to fill any new entries in the tensor. Will be cast
+ to the type of tensor.
+
+ Returns:
+ The resized tensor.
+ """
+ tensor = tf.convert_to_tensor(tensor)
+ shape = tf.unstack(tf.shape(tensor))
+
+ pad_shape = shape[:]
+ pad_shape[axis] = tf.maximum(0, new_size - shape[axis])
+
+ shape[axis] = tf.minimum(shape[axis], new_size)
+ shape = tf.stack(shape)
+
+ resized = tf.concat([
+ tf.slice(tensor, tf.zeros_like(shape), shape),
+ tf.fill(tf.stack(pad_shape), tf.cast(fill_value, tensor.dtype))
+ ], axis)
+
+ # Update shape.
+ new_shape = tensor.shape.as_list() # A copy is being made.
+ new_shape[axis] = new_size
+ resized = tf.ensure_shape(resized, new_shape)
+ return resized
+
+
+def _process_segment_and_label(video_matrix, num_frames, contexts,
+ segment_labels, segment_size,
+ num_classes) -> Dict[str, tf.Tensor]:
+ """Processes a batched Tensor of frames.
+
+ The same parameters used in process should be used here.
+ Args:
+ video_matrix: different features concatenated into one matrix
+ num_frames: Number of frames per subclip.
+ contexts: context information extracted from decoder
+ segment_labels: if we read segment labels instead.
+ segment_size: the segment_size used for reading segments.
+ num_classes: a positive integer for the number of classes.
+
+ Returns:
+ output: dictionary containing batch information
+ """
+ # Partition frame-level feature matrix to segment-level feature matrix.
+ if segment_labels:
+ start_times = contexts["segment_start_times"].values
+ # Here we assume all the segments that started at the same start time has
+ # the same segment_size.
+ uniq_start_times, seg_idxs = tf.unique(start_times, out_idx=tf.dtypes.int64)
+ # Range gather matrix, e.g., [[0,1,2],[1,2,3]] for segment_size == 3.
+ range_mtx = tf.expand_dims(
+ uniq_start_times, axis=-1) + tf.expand_dims(
+ tf.range(0, segment_size, dtype=tf.int64), axis=0)
+ # Shape: [num_segment, segment_size, feature_dim].
+ batch_video_matrix = tf.gather_nd(video_matrix,
+ tf.expand_dims(range_mtx, axis=-1))
+ num_segment = tf.shape(batch_video_matrix)[0]
+ batch_video_ids = tf.reshape(
+ tf.tile([contexts["id"]], [num_segment]), (num_segment,))
+ batch_frames = tf.reshape(
+ tf.tile([segment_size], [num_segment]), (num_segment,))
+ batch_frames = tf.cast(tf.expand_dims(batch_frames, 1), tf.float32)
+
+ # For segment labels, all labels are not exhaustively rated. So we only
+ # evaluate the rated labels.
+
+ # Label indices for each segment, shape: [num_segment, 2].
+ label_indices = tf.stack([seg_idxs, contexts["segment_labels"].values],
+ axis=-1)
+ label_values = contexts["segment_scores"].values
+ sparse_labels = tf.sparse.SparseTensor(label_indices, label_values,
+ (num_segment, num_classes))
+ batch_labels = tf.sparse.to_dense(sparse_labels, validate_indices=False)
+
+ sparse_label_weights = tf.sparse.SparseTensor(
+ label_indices, tf.ones_like(label_values, dtype=tf.float32),
+ (num_segment, num_classes))
+ batch_label_weights = tf.sparse.to_dense(
+ sparse_label_weights, validate_indices=False)
+ # output_dict = utils.get_segments(batch_video_matrix, batch_frames, 5)
+ else:
+ # Process video-level labels.
+ label_indices = contexts["labels"].values
+ sparse_labels = tf.sparse.SparseTensor(
+ tf.expand_dims(label_indices, axis=-1),
+ tf.ones_like(contexts["labels"].values, dtype=tf.bool), (num_classes,))
+ labels = tf.sparse.to_dense(
+ sparse_labels, default_value=False, validate_indices=False)
+
+ # convert to batch format.
+ batch_video_ids = tf.expand_dims(contexts["id"], 0)
+ batch_video_matrix = tf.expand_dims(video_matrix, 0)
+ batch_labels = tf.expand_dims(labels, 0)
+ batch_frames = tf.expand_dims(num_frames, 0)
+ batch_label_weights = None
+
+ output_dict = {
+ "video_ids": batch_video_ids,
+ "video_matrix": batch_video_matrix,
+ "labels": batch_labels,
+ "num_frames": batch_frames,
+ }
+ if batch_label_weights is not None:
+ output_dict["label_weights"] = batch_label_weights
+
+ return output_dict
+
+
+def _get_video_matrix(features, feature_size, max_frames, max_quantized_value,
+ min_quantized_value):
+ """Decodes features from an input string and quantizes it.
+
+ Args:
+ features: raw feature values
+ feature_size: length of each frame feature vector
+ max_frames: number of frames (rows) in the output feature_matrix
+ max_quantized_value: the maximum of the quantized value.
+ min_quantized_value: the minimum of the quantized value.
+
+ Returns:
+ feature_matrix: matrix of all frame-features
+ num_frames: number of frames in the sequence
+ """
+ decoded_features = tf.reshape(
+ tf.cast(tf.io.decode_raw(features, tf.uint8), tf.float32),
+ [-1, feature_size])
+
+ num_frames = tf.math.minimum(tf.shape(decoded_features)[0], max_frames)
+ feature_matrix = utils.Dequantize(decoded_features, max_quantized_value,
+ min_quantized_value)
+ feature_matrix = resize_axis(feature_matrix, 0, max_frames)
+ return feature_matrix, num_frames
+
+
+def _concat_features(features, feature_names, feature_sizes, max_frames,
+ max_quantized_value, min_quantized_value):
+ """Loads (potentially) different types of features and concatenates them.
+
+ Args:
+ features: raw feature values
+ feature_names: list of feature names
+ feature_sizes: list of features sizes
+ max_frames: number of frames in the sequence
+ max_quantized_value: the maximum of the quantized value.
+ min_quantized_value: the minimum of the quantized value.
+
+ Returns:
+ video_matrix: different features concatenated into one matrix
+ num_frames: the number of frames in the video
+ """
+
+ num_features = len(feature_names)
+ assert num_features > 0, "No feature selected: feature_names is empty!"
+
+ assert len(feature_names) == len(feature_sizes), (
+ "length of feature_names (={}) != length of feature_sizes (={})".format(
+ len(feature_names), len(feature_sizes)))
+
+ num_frames = -1 # the number of frames in the video
+ feature_matrices = [None] * num_features # an array of different features
+ for feature_index in range(num_features):
+ feature_matrix, num_frames_in_this_feature = _get_video_matrix(
+ features[feature_names[feature_index]], feature_sizes[feature_index],
+ max_frames, max_quantized_value, min_quantized_value)
+ if num_frames == -1:
+ num_frames = num_frames_in_this_feature
+
+ feature_matrices[feature_index] = feature_matrix
+
+ # cap the number of frames at self.max_frames
+ num_frames = tf.minimum(num_frames, max_frames)
+
+ # concatenate different features
+ video_matrix = tf.concat(feature_matrices, 1)
+
+ return video_matrix, num_frames
+
+
+class Decoder(decoder.Decoder):
+ """A tf.Example decoder for classification task."""
+
+ def __init__(
+ self,
+ input_params: exp_cfg.DataConfig,
+ ):
+
+ self._segment_labels = input_params.segment_labels
+ self._feature_names = input_params.feature_names
+ self._context_features = {
+ "id": tf.io.FixedLenFeature([], tf.string),
+ }
+ if self._segment_labels:
+ self._context_features.update({
+ # There is no need to read end-time given we always assume the segment
+ # has the same size.
+ "segment_labels": tf.io.VarLenFeature(tf.int64),
+ "segment_start_times": tf.io.VarLenFeature(tf.int64),
+ "segment_scores": tf.io.VarLenFeature(tf.float32)
+ })
+ else:
+ self._context_features.update({"labels": tf.io.VarLenFeature(tf.int64)})
+
+ self._sequence_features = {
+ feature_name: tf.io.FixedLenSequenceFeature([], dtype=tf.string)
+ for feature_name in self._feature_names
+ }
+
+ def decode(self, serialized_example):
+ """Parses a single tf.Example into image and label tensors."""
+
+ contexts, features = tf.io.parse_single_sequence_example(
+ serialized_example,
+ context_features=self._context_features,
+ sequence_features=self._sequence_features)
+
+ return {"contexts": contexts, "features": features}
+
+
+class Parser(parser.Parser):
+ """Parses a video and label dataset.
+
+ takes the decoded raw tensors dict
+ and parse them into a dictionary of tensors
+ that can be consumed by the model.
+ It will be executed after decoder.
+ """
+
+ def __init__(
+ self,
+ input_params: exp_cfg.DataConfig,
+ max_quantized_value=2,
+ min_quantized_value=-2,
+ ):
+ self._num_classes = input_params.num_classes
+ self._segment_size = input_params.segment_size
+ self._segment_labels = input_params.segment_labels
+ self._feature_names = input_params.feature_names
+ self._feature_sizes = input_params.feature_sizes
+ self.stride = input_params.temporal_stride
+ self._max_frames = input_params.max_frames
+ self._num_frames = input_params.num_frames
+ self._seed = input_params.random_seed
+ self._max_quantized_value = max_quantized_value
+ self._min_quantized_value = min_quantized_value
+
+ def _parse_train_data(self, decoded_tensors):
+ """Parses data for training."""
+ # loads (potentially) different types of features and concatenates them
+ self.video_matrix, self.num_frames = _concat_features(
+ decoded_tensors["features"], self._feature_names, self._feature_sizes,
+ self._max_frames, self._max_quantized_value, self._min_quantized_value)
+ output_dict = _process_segment_and_label(self.video_matrix, self.num_frames,
+ decoded_tensors["contexts"],
+ self._segment_labels,
+ self._segment_size,
+ self._num_classes)
+ return output_dict
+
+ def _parse_eval_data(self, decoded_tensors):
+ """Parses data for evaluation."""
+ # loads (potentially) different types of features and concatenates them
+ self.video_matrix, self.num_frames = _concat_features(
+ decoded_tensors["features"], self._feature_names, self._feature_sizes,
+ self._max_frames, self._max_quantized_value, self._min_quantized_value)
+ output_dict = _process_segment_and_label(self.video_matrix, self.num_frames,
+ decoded_tensors["contexts"],
+ self._segment_labels,
+ self._segment_size,
+ self._num_classes)
+ return output_dict # batched
+
+ def parse_fn(self, is_training):
+ """Returns a parse fn that reads and parses raw tensors from the decoder.
+
+ Args:
+ is_training: a `bool` to indicate whether it is in training mode.
+
+ Returns:
+ parse: a `callable` that takes the serialized example and generate the
+ images, labels tuple where labels is a dict of Tensors that contains
+ labels.
+ """
+
+ def parse(decoded_tensors):
+ """Parses the serialized example data."""
+ if is_training:
+ return self._parse_train_data(decoded_tensors)
+ else:
+ return self._parse_eval_data(decoded_tensors)
+
+ return parse
+
+
+class PostBatchProcessor():
+ """Processes a video and label dataset which is batched."""
+
+ def __init__(self, input_params: exp_cfg.DataConfig):
+ self.segment_labels = input_params.segment_labels
+ self.num_classes = input_params.num_classes
+ self.segment_size = input_params.segment_size
+
+ def post_fn(self, batched_tensors):
+ """Processes batched Tensors."""
+ video_ids = batched_tensors["video_ids"]
+ video_matrix = batched_tensors["video_matrix"]
+ labels = batched_tensors["labels"]
+ num_frames = batched_tensors["num_frames"]
+ label_weights = None
+
+ if self.segment_labels:
+ # [batch x num_segment x segment_size x num_features]
+ # -> [batch * num_segment x segment_size x num_features]
+ video_ids = tf.reshape(video_ids, [-1])
+ video_matrix = tf.reshape(video_matrix, [-1, self.segment_size, 1152])
+ labels = tf.reshape(labels, [-1, self.num_classes])
+ num_frames = tf.reshape(num_frames, [-1, 1])
+
+ label_weights = tf.reshape(batched_tensors["label_weights"],
+ [-1, self.num_classes])
+
+ else:
+ video_matrix = tf.squeeze(video_matrix)
+ labels = tf.squeeze(labels)
+
+ batched_tensors = {
+ "video_ids": video_ids,
+ "video_matrix": video_matrix,
+ "labels": labels,
+ "num_frames": num_frames,
+ }
+
+ if label_weights is not None:
+ batched_tensors["label_weights"] = label_weights
+
+ return batched_tensors
+
+
+class TransformBatcher():
+ """Performs manual batching on input dataset."""
+
+ def __init__(self, input_params: exp_cfg.DataConfig):
+ self._segment_labels = input_params.segment_labels
+ self._global_batch_size = input_params.global_batch_size
+ self._is_training = input_params.is_training
+
+ def batch_fn(self, dataset, input_context):
+ """Add padding when segment_labels is true."""
+ per_replica_batch_size = input_context.get_per_replica_batch_size(
+ self._global_batch_size) if input_context else self._global_batch_size
+ if not self._segment_labels:
+ dataset = dataset.batch(per_replica_batch_size, drop_remainder=True)
+ else:
+ # add padding
+ pad_shapes = {
+ "video_ids": [None],
+ "video_matrix": [None, None, None],
+ "labels": [None, None],
+ "num_frames": [None, None],
+ "label_weights": [None, None]
+ }
+ pad_values = {
+ "video_ids": None,
+ "video_matrix": 0.0,
+ "labels": -1.0,
+ "num_frames": 0.0,
+ "label_weights": 0.0
+ }
+ dataset = dataset.padded_batch(
+ per_replica_batch_size,
+ padded_shapes=pad_shapes,
+ drop_remainder=True,
+ padding_values=pad_values)
+ return dataset
diff --git a/official/vision/beta/projects/yt8m/eval_utils/average_precision_calculator.py b/official/vision/beta/projects/yt8m/eval_utils/average_precision_calculator.py
new file mode 100644
index 0000000000000000000000000000000000000000..9bf1123793d5ee8f726687a9681936c7d49553bf
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/eval_utils/average_precision_calculator.py
@@ -0,0 +1,273 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Calculate or keep track of the interpolated average precision.
+
+It provides an interface for calculating interpolated average precision for an
+entire list or the top-n ranked items. For the definition of the
+(non-)interpolated average precision:
+http://trec.nist.gov/pubs/trec15/appendices/CE.MEASURES06.pdf
+
+Example usages:
+1) Use it as a static function call to directly calculate average precision for
+a short ranked list in the memory.
+
+```
+import random
+
+p = np.array([random.random() for _ in xrange(10)])
+a = np.array([random.choice([0, 1]) for _ in xrange(10)])
+
+ap = average_precision_calculator.AveragePrecisionCalculator.ap(p, a)
+```
+
+2) Use it as an object for long ranked list that cannot be stored in memory or
+the case where partial predictions can be observed at a time (Tensorflow
+predictions). In this case, we first call the function accumulate many times
+to process parts of the ranked list. After processing all the parts, we call
+peek_interpolated_ap_at_n.
+```
+p1 = np.array([random.random() for _ in xrange(5)])
+a1 = np.array([random.choice([0, 1]) for _ in xrange(5)])
+p2 = np.array([random.random() for _ in xrange(5)])
+a2 = np.array([random.choice([0, 1]) for _ in xrange(5)])
+
+# interpolated average precision at 10 using 1000 break points
+calculator = average_precision_calculator.AveragePrecisionCalculator(10)
+calculator.accumulate(p1, a1)
+calculator.accumulate(p2, a2)
+ap3 = calculator.peek_ap_at_n()
+```
+"""
+
+import heapq
+import numbers
+import random
+
+import numpy
+
+
+class AveragePrecisionCalculator(object):
+ """Calculate the average precision and average precision at n."""
+
+ def __init__(self, top_n=None):
+ """Construct an AveragePrecisionCalculator to calculate average precision.
+
+ This class is used to calculate the average precision for a single label.
+
+ Args:
+ top_n: A positive Integer specifying the average precision at n, or None
+ to use all provided data points.
+
+ Raises:
+ ValueError: An error occurred when the top_n is not a positive integer.
+ """
+ if not ((isinstance(top_n, int) and top_n >= 0) or top_n is None):
+ raise ValueError("top_n must be a positive integer or None.")
+
+ self._top_n = top_n # average precision at n
+ self._total_positives = 0 # total number of positives have seen
+ self._heap = [] # max heap of (prediction, actual)
+
+ @property
+ def heap_size(self):
+ """Gets the heap size maintained in the class."""
+ return len(self._heap)
+
+ @property
+ def num_accumulated_positives(self):
+ """Gets the number of positive samples that have been accumulated."""
+ return self._total_positives
+
+ def accumulate(self, predictions, actuals, num_positives=None):
+ """Accumulate the predictions and their ground truth labels.
+
+ After the function call, we may call peek_ap_at_n to actually calculate
+ the average precision.
+ Note predictions and actuals must have the same shape.
+
+ Args:
+ predictions: a list storing the prediction scores.
+ actuals: a list storing the ground truth labels. Any value larger than 0
+ will be treated as positives, otherwise as negatives. num_positives = If
+ the 'predictions' and 'actuals' inputs aren't complete, then it's
+ possible some true positives were missed in them. In that case, you can
+ provide 'num_positives' in order to accurately track recall.
+ num_positives: number of positive examples.
+
+ Raises:
+ ValueError: An error occurred when the format of the input is not the
+ numpy 1-D array or the shape of predictions and actuals does not match.
+ """
+ if len(predictions) != len(actuals):
+ raise ValueError("the shape of predictions and actuals does not match.")
+
+ if num_positives is not None:
+ if not isinstance(num_positives, numbers.Number) or num_positives < 0:
+ raise ValueError(
+ "'num_positives' was provided but it was a negative number.")
+
+ if num_positives is not None:
+ self._total_positives += num_positives
+ else:
+ self._total_positives += numpy.size(
+ numpy.where(numpy.array(actuals) > 1e-5))
+ topk = self._top_n
+ heap = self._heap
+
+ for i in range(numpy.size(predictions)):
+ if topk is None or len(heap) < topk:
+ heapq.heappush(heap, (predictions[i], actuals[i]))
+ else:
+ if predictions[i] > heap[0][0]: # heap[0] is the smallest
+ heapq.heappop(heap)
+ heapq.heappush(heap, (predictions[i], actuals[i]))
+
+ def clear(self):
+ """Clear the accumulated predictions."""
+ self._heap = []
+ self._total_positives = 0
+
+ def peek_ap_at_n(self):
+ """Peek the non-interpolated average precision at n.
+
+ Returns:
+ The non-interpolated average precision at n (default 0).
+ If n is larger than the length of the ranked list,
+ the average precision will be returned.
+ """
+ if self.heap_size <= 0:
+ return 0
+ predlists = numpy.array(list(zip(*self._heap)))
+
+ ap = self.ap_at_n(
+ predlists[0],
+ predlists[1],
+ n=self._top_n,
+ total_num_positives=self._total_positives)
+ return ap
+
+ @staticmethod
+ def ap(predictions, actuals):
+ """Calculate the non-interpolated average precision.
+
+ Args:
+ predictions: a numpy 1-D array storing the sparse prediction scores.
+ actuals: a numpy 1-D array storing the ground truth labels. Any value
+ larger than 0 will be treated as positives, otherwise as negatives.
+
+ Returns:
+ The non-interpolated average precision at n.
+ If n is larger than the length of the ranked list,
+ the average precision will be returned.
+
+ Raises:
+ ValueError: An error occurred when the format of the input is not the
+ numpy 1-D array or the shape of predictions and actuals does not match.
+ """
+ return AveragePrecisionCalculator.ap_at_n(predictions, actuals, n=None)
+
+ @staticmethod
+ def ap_at_n(predictions, actuals, n=20, total_num_positives=None):
+ """Calculate the non-interpolated average precision.
+
+ Args:
+ predictions: a numpy 1-D array storing the sparse prediction scores.
+ actuals: a numpy 1-D array storing the ground truth labels. Any value
+ larger than 0 will be treated as positives, otherwise as negatives.
+ n: the top n items to be considered in ap@n.
+ total_num_positives : (optionally) you can specify the number of total
+ positive in the list. If specified, it will be used in calculation.
+
+ Returns:
+ The non-interpolated average precision at n.
+ If n is larger than the length of the ranked list,
+ the average precision will be returned.
+
+ Raises:
+ ValueError: An error occurred when
+ 1) the format of the input is not the numpy 1-D array;
+ 2) the shape of predictions and actuals does not match;
+ 3) the input n is not a positive integer.
+ """
+ if len(predictions) != len(actuals):
+ raise ValueError("the shape of predictions and actuals does not match.")
+
+ if n is not None:
+ if not isinstance(n, int) or n <= 0:
+ raise ValueError("n must be 'None' or a positive integer."
+ " It was '%s'." % n)
+
+ ap = 0.0
+
+ predictions = numpy.array(predictions)
+ actuals = numpy.array(actuals)
+
+ # add a shuffler to avoid overestimating the ap
+ predictions, actuals = AveragePrecisionCalculator._shuffle(
+ predictions, actuals)
+ sortidx = sorted(
+ range(len(predictions)), key=lambda k: predictions[k], reverse=True)
+
+ if total_num_positives is None:
+ numpos = numpy.size(numpy.where(actuals > 0))
+ else:
+ numpos = total_num_positives
+
+ if numpos == 0:
+ return 0
+
+ if n is not None:
+ numpos = min(numpos, n)
+ delta_recall = 1.0 / numpos
+ poscount = 0.0
+
+ # calculate the ap
+ r = len(sortidx)
+ if n is not None:
+ r = min(r, n)
+ for i in range(r):
+ if actuals[sortidx[i]] > 0:
+ poscount += 1
+ ap += poscount / (i + 1) * delta_recall
+ return ap
+
+ @staticmethod
+ def _shuffle(predictions, actuals):
+ random.seed(0)
+ suffidx = random.sample(range(len(predictions)), len(predictions))
+ predictions = predictions[suffidx]
+ actuals = actuals[suffidx]
+ return predictions, actuals
+
+ @staticmethod
+ def _zero_one_normalize(predictions, epsilon=1e-7):
+ """Normalize the predictions to the range between 0.0 and 1.0.
+
+ For some predictions like SVM predictions, we need to normalize them before
+ calculate the interpolated average precision. The normalization will not
+ change the rank in the original list and thus won't change the average
+ precision.
+
+ Args:
+ predictions: a numpy 1-D array storing the sparse prediction scores.
+ epsilon: a small constant to avoid denominator being zero.
+
+ Returns:
+ The normalized prediction.
+ """
+ denominator = numpy.max(predictions) - numpy.min(predictions)
+ ret = (predictions - numpy.min(predictions)) / numpy.max(
+ denominator, epsilon)
+ return ret
diff --git a/official/vision/beta/projects/yt8m/eval_utils/eval_util.py b/official/vision/beta/projects/yt8m/eval_utils/eval_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..b42660121d88b67ea77ecbfa4bcd6e5de6a66402
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/eval_utils/eval_util.py
@@ -0,0 +1,271 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides functions to help with evaluating models."""
+import numpy as np
+import tensorflow as tf
+from official.vision.beta.projects.yt8m.eval_utils import \
+ average_precision_calculator as ap_calculator
+from official.vision.beta.projects.yt8m.eval_utils import \
+ mean_average_precision_calculator as map_calculator
+
+
+def flatten(l):
+ """Merges a list of lists into a single list."""
+ # pylint: disable=g-complex-comprehension
+ return [item for sublist in l for item in sublist]
+ # pylint: enable=g-complex-comprehension
+
+
+def calculate_hit_at_one(predictions, actuals):
+ """Performs a local (numpy) calculation of the hit at one.
+
+ Args:
+ predictions: Matrix containing the outputs of the model. Dimensions are
+ 'batch' x 'num_classes'.
+ actuals: Matrix containing the ground truth labels. Dimensions are 'batch' x
+ 'num_classes'.
+
+ Returns:
+ float: The average hit at one across the entire batch.
+ """
+ top_prediction = np.argmax(predictions, 1)
+ hits = actuals[np.arange(actuals.shape[0]), top_prediction]
+ return np.average(hits)
+
+
+def calculate_precision_at_equal_recall_rate(predictions, actuals):
+ """Performs a local (numpy) calculation of the PERR.
+
+ Args:
+ predictions: Matrix containing the outputs of the model. Dimensions are
+ 'batch' x 'num_classes'.
+ actuals: Matrix containing the ground truth labels. Dimensions are 'batch' x
+ 'num_classes'.
+
+ Returns:
+ float: The average precision at equal recall rate across the entire batch.
+ """
+ aggregated_precision = 0.0
+ num_videos = actuals.shape[0]
+ for row in np.arange(num_videos):
+ num_labels = int(np.sum(actuals[row]))
+ top_indices = np.argpartition(predictions[row], -num_labels)[-num_labels:]
+ item_precision = 0.0
+ for label_index in top_indices:
+ if predictions[row][label_index] > 0:
+ item_precision += actuals[row][label_index]
+ item_precision /= top_indices.size
+ aggregated_precision += item_precision
+ aggregated_precision /= num_videos
+ return aggregated_precision
+
+
+def calculate_gap(predictions, actuals, top_k=20):
+ """Performs a local (numpy) calculation of the global average precision.
+
+ Only the top_k predictions are taken for each of the videos.
+
+ Args:
+ predictions: Matrix containing the outputs of the model. Dimensions are
+ 'batch' x 'num_classes'.
+ actuals: Matrix containing the ground truth labels. Dimensions are 'batch' x
+ 'num_classes'.
+ top_k: How many predictions to use per video.
+
+ Returns:
+ float: The global average precision.
+ """
+ gap_calculator = ap_calculator.AveragePrecisionCalculator()
+ sparse_predictions, sparse_labels, num_positives = top_k_by_class(
+ predictions, actuals, top_k)
+ gap_calculator.accumulate(
+ flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
+ return gap_calculator.peek_ap_at_n()
+
+
+def top_k_by_class(predictions, labels, k=20):
+ """Extracts the top k predictions for each video, sorted by class.
+
+ Args:
+ predictions: A numpy matrix containing the outputs of the model. Dimensions
+ are 'batch' x 'num_classes'.
+ labels: A numpy matrix containing the ground truth labels.
+ Dimensions are 'batch' x 'num_classes'.
+ k: the top k non-zero entries to preserve in each prediction.
+
+ Returns:
+ A tuple (predictions,labels, true_positives). 'predictions' and 'labels'
+ are lists of lists of floats. 'true_positives' is a list of scalars. The
+ length of the lists are equal to the number of classes. The entries in the
+ predictions variable are probability predictions, and
+ the corresponding entries in the labels variable are the ground truth for
+ those predictions. The entries in 'true_positives' are the number of true
+ positives for each class in the ground truth.
+
+ Raises:
+ ValueError: An error occurred when the k is not a positive integer.
+ """
+ if k <= 0:
+ raise ValueError("k must be a positive integer.")
+ k = min(k, predictions.shape[1])
+ num_classes = predictions.shape[1]
+ prediction_triplets = []
+ for video_index in range(predictions.shape[0]):
+ prediction_triplets.extend(
+ top_k_triplets(predictions[video_index], labels[video_index], k))
+ out_predictions = [[] for _ in range(num_classes)]
+ out_labels = [[] for _ in range(num_classes)]
+ for triplet in prediction_triplets:
+ out_predictions[triplet[0]].append(triplet[1])
+ out_labels[triplet[0]].append(triplet[2])
+ out_true_positives = [np.sum(labels[:, i]) for i in range(num_classes)]
+
+ return out_predictions, out_labels, out_true_positives
+
+
+def top_k_triplets(predictions, labels, k=20):
+ """Get the top_k for a 1-d numpy array.
+
+ Args:
+ predictions: A numpy matrix containing the outputs of the model. Dimensions
+ are 'batch' x 'num_classes'.
+ labels: A numpy matrix containing the ground truth labels.
+ Dimensions are 'batch' x 'num_classes'.
+ k: The number top predictions to pick.
+ Returns:
+ a sparse list of tuples in (prediction, class) format.
+ """
+ m = len(predictions)
+ k = min(k, m)
+ indices = np.argpartition(predictions, -k)[-k:]
+ return [(index, predictions[index], labels[index]) for index in indices]
+
+
+class EvaluationMetrics(object):
+ """A class to store the evaluation metrics."""
+
+ def __init__(self, num_class, top_k, top_n):
+ """Construct an EvaluationMetrics object to store the evaluation metrics.
+
+ Args:
+ num_class: A positive integer specifying the number of classes.
+ top_k: A positive integer specifying how many predictions are considered
+ per video.
+ top_n: A positive Integer specifying the average precision at n, or None
+ to use all provided data points.
+
+ Raises:
+ ValueError: An error occurred when MeanAveragePrecisionCalculator cannot
+ not be constructed.
+ """
+ self.sum_hit_at_one = 0.0
+ self.sum_perr = 0.0
+ self.map_calculator = map_calculator.MeanAveragePrecisionCalculator(
+ num_class, top_n=top_n)
+ self.global_ap_calculator = ap_calculator.AveragePrecisionCalculator()
+ self.top_k = top_k
+ self.num_examples = 0
+ self.num_class = num_class
+
+ def accumulate(self, predictions, labels):
+ """Accumulate the metrics calculated locally for this mini-batch.
+
+ Args:
+ predictions: A numpy matrix containing the outputs of the model.
+ Dimensions are 'batch' x 'num_classes'.
+ labels: A numpy matrix containing the ground truth labels. Dimensions are
+ 'batch' x 'num_classes'.
+
+ Returns:
+ dictionary: A dictionary storing the metrics for the mini-batch.
+
+ Raises:
+ ValueError: An error occurred when the shape of predictions and actuals
+ does not match.
+ """
+ predictions, labels = self._convert_to_numpy(
+ predictions=predictions[0], groundtruths=labels[0])
+ batch_size = labels.shape[0]
+ mean_hit_at_one = calculate_hit_at_one(predictions, labels)
+ mean_perr = calculate_precision_at_equal_recall_rate(predictions, labels)
+
+ # Take the top 20 predictions.
+ sparse_predictions, sparse_labels, num_positives = top_k_by_class(
+ predictions, labels, self.top_k)
+ self.map_calculator.accumulate(sparse_predictions, sparse_labels,
+ num_positives)
+ self.global_ap_calculator.accumulate(
+ flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
+
+ self.num_examples += batch_size
+ self.sum_hit_at_one += mean_hit_at_one * batch_size
+ self.sum_perr += mean_perr * batch_size
+
+ return {"hit_at_one": mean_hit_at_one, "perr": mean_perr}
+
+ def get(self):
+ """Calculate the evaluation metrics for the whole epoch.
+
+ Raises:
+ ValueError: If no examples were accumulated.
+
+ Returns:
+ dictionary: a dictionary storing the evaluation metrics for the epoch. The
+ dictionary has the fields: avg_hit_at_one, avg_perr, and
+ aps (default nan).
+ """
+ if self.num_examples <= 0:
+ raise ValueError("total_sample must be positive.")
+ avg_hit_at_one = self.sum_hit_at_one / self.num_examples
+ avg_perr = self.sum_perr / self.num_examples
+
+ aps = self.map_calculator.peek_map_at_n()
+ mean_ap = sum(aps) / self.num_class
+ gap = self.global_ap_calculator.peek_ap_at_n()
+
+ epoch_info_dict = {
+ "avg_hit_at_one": avg_hit_at_one,
+ "avg_perr": avg_perr,
+ "map": mean_ap,
+ "gap": gap
+ }
+ return epoch_info_dict
+
+ def clear(self):
+ """Clear the evaluation metrics and reset the EvaluationMetrics object."""
+ self.sum_hit_at_one = 0.0
+ self.sum_perr = 0.0
+ self.map_calculator.clear()
+ self.global_ap_calculator.clear()
+ self.num_examples = 0
+
+ @property
+ def name(self):
+ return "avg_prec_metric"
+
+ def _convert_to_numpy(self, groundtruths, predictions):
+ """Converts tesnors to numpy arrays."""
+ if groundtruths is not None:
+ labels = tf.nest.map_structure(lambda x: x.numpy(), groundtruths)
+ else:
+ labels = groundtruths
+
+ if predictions is not None:
+ outputs = tf.nest.map_structure(lambda x: x.numpy(), predictions)
+ else:
+ outputs = predictions
+
+ labels = labels * 1
+ return outputs, labels
diff --git a/official/vision/beta/projects/yt8m/eval_utils/mean_average_precision_calculator.py b/official/vision/beta/projects/yt8m/eval_utils/mean_average_precision_calculator.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d6a5d6ceaa1f2bff921c7f4f8ce3f66ff991597
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/eval_utils/mean_average_precision_calculator.py
@@ -0,0 +1,115 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Calculate the mean average precision.
+
+It provides an interface for calculating mean average precision
+for an entire list or the top-n ranked items.
+
+Example usages:
+We first call the function accumulate many times to process parts of the ranked
+list. After processing all the parts, we call peek_map_at_n
+to calculate the mean average precision.
+
+```
+import random
+
+p = np.array([[random.random() for _ in xrange(50)] for _ in xrange(1000)])
+a = np.array([[random.choice([0, 1]) for _ in xrange(50)]
+ for _ in xrange(1000)])
+
+# mean average precision for 50 classes.
+calculator = mean_average_precision_calculator.MeanAveragePrecisionCalculator(
+ num_class=50)
+calculator.accumulate(p, a)
+aps = calculator.peek_map_at_n()
+```
+"""
+
+from official.vision.beta.projects.yt8m.eval_utils import \
+ average_precision_calculator
+
+
+class MeanAveragePrecisionCalculator(object):
+ """This class is to calculate mean average precision."""
+
+ def __init__(self, num_class, filter_empty_classes=True, top_n=None):
+ """Construct a calculator to calculate the (macro) average precision.
+
+ Args:
+ num_class: A positive Integer specifying the number of classes.
+ filter_empty_classes: whether to filter classes without any positives.
+ top_n: A positive Integer specifying the average precision at n, or None
+ to use all provided data points.
+
+ Raises:
+ ValueError: An error occurred when num_class is not a positive integer;
+ or the top_n_array is not a list of positive integers.
+ """
+ if not isinstance(num_class, int) or num_class <= 1:
+ raise ValueError("num_class must be a positive integer.")
+
+ self._ap_calculators = [] # member of AveragePrecisionCalculator
+ self._num_class = num_class # total number of classes
+ self._filter_empty_classes = filter_empty_classes
+ for _ in range(num_class):
+ self._ap_calculators.append(
+ average_precision_calculator.AveragePrecisionCalculator(top_n=top_n))
+
+ def accumulate(self, predictions, actuals, num_positives=None):
+ """Accumulate the predictions and their ground truth labels.
+
+ Args:
+ predictions: A list of lists storing the prediction scores. The outer
+ dimension corresponds to classes.
+ actuals: A list of lists storing the ground truth labels. The dimensions
+ should correspond to the predictions input. Any value larger than 0 will
+ be treated as positives, otherwise as negatives.
+ num_positives: If provided, it is a list of numbers representing the
+ number of true positives for each class. If not provided, the number of
+ true positives will be inferred from the 'actuals' array.
+
+ Raises:
+ ValueError: An error occurred when the shape of predictions and actuals
+ does not match.
+ """
+ if not num_positives:
+ num_positives = [None for i in range(self._num_class)]
+
+ calculators = self._ap_calculators
+ for i in range(self._num_class):
+ calculators[i].accumulate(predictions[i], actuals[i], num_positives[i])
+
+ def clear(self):
+ for calculator in self._ap_calculators:
+ calculator.clear()
+
+ def is_empty(self):
+ return ([calculator.heap_size for calculator in self._ap_calculators
+ ] == [0 for _ in range(self._num_class)])
+
+ def peek_map_at_n(self):
+ """Peek the non-interpolated mean average precision at n.
+
+ Returns:
+ An array of non-interpolated average precision at n (default 0) for each
+ class.
+ """
+ aps = []
+ for i in range(self._num_class):
+ if (not self._filter_empty_classes or
+ self._ap_calculators[i].num_accumulated_positives > 0):
+ ap = self._ap_calculators[i].peek_ap_at_n()
+ aps.append(ap)
+ return aps
diff --git a/official/vision/beta/projects/yt8m/experiments/yt8m.yaml b/official/vision/beta/projects/yt8m/experiments/yt8m.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c099f23f90b3c94b874e60713b63de6ebd1c1c3a
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/experiments/yt8m.yaml
@@ -0,0 +1,56 @@
+# yt8m config file
+task:
+ model:
+ cluster_size: 8192
+ hidden_size: 1024
+ add_batch_norm: true
+ sample_random_frames: true
+ is_training: true
+ activation: "sigmoid"
+ pooling_method: "max"
+ yt8m_agg_classifier_model: "MoeModel"
+ train_data:
+ name: 'yt8m'
+ split: 'train'
+ feature_sizes: !!python/tuple
+ - 1024
+ - 128
+ feature_names: !!python/tuple
+ - "rgb"
+ - "audio"
+ segment_size: 1
+ segment_labels: false
+ temporal_stride: 1
+ max_frames: 300
+ num_frames: 300
+ num_classes: 3862
+ num_devices: 1
+ input_path: 'gs://youtube8m-ml/2/frame/train/train*.tfrecord'
+ is_training: true
+ random_seed: 123
+ validation_data:
+ name: 'yt8m'
+ split: 'train'
+ feature_sizes: !!python/tuple
+ - 1024
+ - 128
+ feature_names: !!python/tuple
+ - "rgb"
+ - "audio"
+ segment_size: 1
+ segment_labels: true
+ temporal_stride: 1
+ max_frames: 300
+ num_frames: 300
+ num_classes: 3862
+ num_devices: 1
+ input_path: 'gs://youtube8m-ml/3/frame/validate/validate*.tfrecord'
+ is_training: false
+ random_seed: 123
+ losses:
+ name: 'binary_crossentropy'
+ from_logits: false
+ label_smoothing: 0.0
+ gradient_clip_norm: 1.0
+ num_readers: 8
+ top_k: 20
diff --git a/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml b/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9a7ef94cc73373db1c6015d4a0b665a16fba4d90
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/experiments/yt8m_test.yaml
@@ -0,0 +1,31 @@
+# yt8m test config file
+task:
+ model:
+ cluster_size: 2048
+ hidden_size: 2048
+ add_batch_norm: true
+ sample_random_frames: true
+ is_training: true
+ activation: "relu6"
+ pooling_method: "average"
+ yt8m_agg_classifier_model: "MoeModel"
+ train_data:
+ segment_labels: false
+ temporal_stride: 1
+ num_devices: 1
+ input_path: 'gs://youtube8m-ml/2/frame/train/train*.tfrecord'
+ num_examples: 8000
+ validation_data:
+ segment_size: 5
+ segment_labels: true
+ temporal_stride: 1
+ num_devices: 1
+ input_path: 'gs://youtube8m-ml/3/frame/validate/validate*.tfrecord'
+ num_examples: 2000
+ losses:
+ name: 'binary_crossentropy'
+ from_logits: false
+ label_smoothing: 0.0
+ gradient_clip_norm: 1.0
+ num_readers: 8
+ top_k: 20
diff --git a/official/vision/beta/projects/yt8m/modeling/__init__.py b/official/vision/beta/projects/yt8m/modeling/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/modeling/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/official/vision/beta/projects/yt8m/modeling/yt8m_agg_models.py b/official/vision/beta/projects/yt8m/modeling/yt8m_agg_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..faf75beb9f050acfcb6da8142ac12bcca988b13f
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/modeling/yt8m_agg_models.py
@@ -0,0 +1,101 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains model definitions."""
+
+import tensorflow as tf
+
+layers = tf.keras.layers
+regularizers = tf.keras.regularizers
+# The number of mixtures (excluding the dummy 'expert') used for MoeModel.
+moe_num_mixtures = 2
+
+
+class LogisticModel():
+ """Logistic model with L2 regularization."""
+
+ def create_model(self, model_input, vocab_size, l2_penalty=1e-8):
+ """Creates a logistic model.
+
+ Args:
+ model_input: 'batch' x 'num_features' matrix of input features.
+ vocab_size: The number of classes in the dataset.
+ l2_penalty: L2 weight regularization ratio.
+
+ Returns:
+ A dictionary with a tensor containing the probability predictions of the
+ model in the 'predictions' key. The dimensions of the tensor are
+ batch_size x num_classes.
+ """
+ output = layers.Dense(
+ vocab_size,
+ activation=tf.nn.sigmoid,
+ kernel_regularizer=regularizers.l2(l2_penalty))(
+ model_input)
+ return {"predictions": output}
+
+
+class MoeModel():
+ """A softmax over a mixture of logistic models (with L2 regularization)."""
+
+ def create_model(self,
+ model_input,
+ vocab_size,
+ num_mixtures=None,
+ l2_penalty=1e-8):
+ """Creates a Mixture of (Logistic) Experts model.
+
+ The model consists of a per-class softmax distribution over a
+ configurable number of logistic classifiers. One of the classifiers
+ in the mixture is not trained, and always predicts 0.
+ Args:
+ model_input: 'batch_size' x 'num_features' matrix of input features.
+ vocab_size: The number of classes in the dataset.
+ num_mixtures: The number of mixtures (excluding a dummy 'expert' that
+ always predicts the non-existence of an entity).
+ l2_penalty: How much to penalize the squared magnitudes of parameter
+ values.
+
+ Returns:
+ A dictionary with a tensor containing the probability predictions
+ of the model in the 'predictions' key. The dimensions of the tensor
+ are batch_size x num_classes.
+ """
+ num_mixtures = num_mixtures or moe_num_mixtures
+
+ gate_activations = layers.Dense(
+ vocab_size * (num_mixtures + 1),
+ activation=None,
+ bias_initializer=None,
+ kernel_regularizer=regularizers.l2(l2_penalty))(
+ model_input)
+ expert_activations = layers.Dense(
+ vocab_size * num_mixtures,
+ activation=None,
+ kernel_regularizer=regularizers.l2(l2_penalty))(
+ model_input)
+
+ gating_distribution = tf.nn.softmax(
+ tf.reshape(
+ gate_activations,
+ [-1, num_mixtures + 1])) # (Batch * #Labels) x (num_mixtures + 1)
+ expert_distribution = tf.nn.sigmoid(
+ tf.reshape(expert_activations,
+ [-1, num_mixtures])) # (Batch * #Labels) x num_mixtures
+
+ final_probabilities_by_class_and_batch = tf.reduce_sum(
+ gating_distribution[:, :num_mixtures] * expert_distribution, 1)
+ final_probabilities = tf.reshape(final_probabilities_by_class_and_batch,
+ [-1, vocab_size])
+ return {"predictions": final_probabilities}
diff --git a/official/vision/beta/projects/yt8m/modeling/yt8m_model.py b/official/vision/beta/projects/yt8m/modeling/yt8m_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..87eb4ddf0ddae5610abd53b56c8f3e7637d90d76
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/modeling/yt8m_model.py
@@ -0,0 +1,148 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""YT8M model definition."""
+
+import tensorflow as tf
+from official.modeling import tf_utils
+from official.vision.beta.projects.yt8m.configs import yt8m as yt8m_cfg
+from official.vision.beta.projects.yt8m.modeling import yt8m_agg_models
+from official.vision.beta.projects.yt8m.modeling import yt8m_model_utils as utils
+
+layers = tf.keras.layers
+
+
+class YT8MModel(tf.keras.Model):
+ """A YT8M model class builder."""
+
+ def __init__(self,
+ input_params: yt8m_cfg.YT8MModel,
+ num_frames=30,
+ num_classes=3862,
+ input_specs=layers.InputSpec(shape=[None, None, 1152]),
+ **kwargs):
+ """YT8M initialization function.
+
+ Args:
+ input_params: model configuration parameters
+ num_frames: `int` number of frames in a single input.
+ num_classes: `int` number of classes in dataset.
+ input_specs: `tf.keras.layers.InputSpec` specs of the input tensor.
+ [batch_size x num_frames x num_features]
+ **kwargs: keyword arguments to be passed.
+ """
+
+ self._self_setattr_tracking = False
+ self._config_dict = {
+ "input_specs": input_specs,
+ "num_classes": num_classes,
+ "num_frames": num_frames,
+ "input_params": input_params
+ }
+ self._num_classes = num_classes
+ self._input_specs = input_specs
+ self._act_fn = tf_utils.get_activation(input_params.activation)
+ self._is_training = input_params.is_training
+
+ # [batch_size x num_frames x num_features]
+ feature_size = input_specs.shape[-1]
+ # shape 'excluding' batch_size
+ model_input = tf.keras.Input(shape=self._input_specs.shape[1:])
+ reshaped_input = tf.reshape(model_input, [-1, feature_size])
+ tf.summary.histogram("input_hist", model_input)
+
+ # configure model
+ if input_params.add_batch_norm:
+ reshaped_input = layers.BatchNormalization(
+ name="input_bn", scale=True, center=True,
+ trainable=self._is_training)(
+ reshaped_input)
+
+ # activation = reshaped input * cluster weights
+ activation = layers.Dense(
+ input_params.cluster_size,
+ kernel_initializer=tf.random_normal_initializer(
+ stddev=1 / tf.sqrt(tf.cast(feature_size, tf.float32))))(
+ reshaped_input)
+
+ if input_params.add_batch_norm:
+ activation = layers.BatchNormalization(
+ name="cluster_bn",
+ scale=True,
+ center=True,
+ trainable=self._is_training)(
+ activation)
+
+ else:
+ cluster_biases = tf.Variable(
+ tf.random_normal_initializer(stddev=1 / tf.math.sqrt(feature_size))(
+ shape=[input_params.cluster_size]),
+ name="cluster_biases")
+ tf.summary.histogram("cluster_biases", cluster_biases)
+ activation += cluster_biases
+
+ activation = self._act_fn(activation)
+ tf.summary.histogram("cluster_output", activation)
+
+ activation = tf.reshape(activation,
+ [-1, num_frames, input_params.cluster_size])
+ activation = utils.FramePooling(activation, input_params.pooling_method)
+
+ # activation = activation * hidden1_weights
+ activation = layers.Dense(
+ input_params.hidden_size,
+ kernel_initializer=tf.random_normal_initializer(
+ stddev=1 /
+ tf.sqrt(tf.cast(input_params.cluster_size, tf.float32))))(
+ activation)
+
+ if input_params.add_batch_norm:
+ activation = layers.BatchNormalization(
+ name="hidden1_bn",
+ scale=True,
+ center=True,
+ trainable=self._is_training)(
+ activation)
+
+ else:
+ hidden1_biases = tf.Variable(
+ tf.random_normal_initializer(stddev=0.01)(
+ shape=[input_params.hidden_size]),
+ name="hidden1_biases")
+
+ tf.summary.histogram("hidden1_biases", hidden1_biases)
+ activation += hidden1_biases
+
+ activation = self._act_fn(activation)
+ tf.summary.histogram("hidden1_output", activation)
+
+ aggregated_model = getattr(yt8m_agg_models,
+ input_params.yt8m_agg_classifier_model)
+ output = aggregated_model().create_model(
+ model_input=activation, vocab_size=self._num_classes)
+
+ super().__init__(
+ inputs=model_input, outputs=output.get("predictions"), **kwargs)
+
+ @property
+ def checkpoint_items(self):
+ """Returns a dictionary of items to be additionally checkpointed."""
+ return dict()
+
+ def get_config(self):
+ return self._config_dict
+
+ @classmethod
+ def from_config(cls, config):
+ return cls(**config)
diff --git a/official/vision/beta/projects/yt8m/modeling/yt8m_model_test.py b/official/vision/beta/projects/yt8m/modeling/yt8m_model_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7bd9d9974f70300c36066e1db39a4568da9d207
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/modeling/yt8m_model_test.py
@@ -0,0 +1,63 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for yt8m network."""
+
+from absl.testing import parameterized
+import numpy as np
+import tensorflow as tf
+
+from official.vision.beta.projects.yt8m.configs import yt8m as yt8m_cfg
+from official.vision.beta.projects.yt8m.modeling import yt8m_model
+
+
+class YT8MNetworkTest(parameterized.TestCase, tf.test.TestCase):
+ """Class for testing yt8m network."""
+
+ # test_yt8m_network_creation arbitrary params
+ @parameterized.parameters((32, 1152)) # 1152 = 1024 + 128
+ def test_yt8m_network_creation(self, num_frames, feature_dims):
+ """Test for creation of a YT8M Model.
+
+ Args:
+ num_frames: number of frames.
+ feature_dims: indicates total dimension size of the features.
+ """
+ input_specs = tf.keras.layers.InputSpec(shape=[num_frames, feature_dims])
+
+ num_classes = 3862
+ model = yt8m_model.YT8MModel(
+ input_params=yt8m_cfg.YT8MTask.model,
+ num_frames=num_frames,
+ num_classes=num_classes,
+ input_specs=input_specs)
+
+ # batch = 2 -> arbitrary value for test
+ inputs = np.random.rand(2 * num_frames, feature_dims)
+ logits = model(inputs)
+ self.assertAllEqual([2, num_classes], logits.numpy().shape)
+
+ def test_serialize_deserialize(self):
+ model = yt8m_model.YT8MModel(input_params=yt8m_cfg.YT8MTask.model)
+
+ config = model.get_config()
+ new_model = yt8m_model.YT8MModel.from_config(config)
+
+ # If the serialization was successful,
+ # the new config should match the old.
+ self.assertAllEqual(model.get_config(), new_model.get_config())
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/projects/yt8m/modeling/yt8m_model_utils.py b/official/vision/beta/projects/yt8m/modeling/yt8m_model_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..cebebb449d4979250383013b20734cffa3215901
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/modeling/yt8m_model_utils.py
@@ -0,0 +1,95 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains a collection of util functions for model construction."""
+import tensorflow as tf
+
+
+def SampleRandomSequence(model_input, num_frames, num_samples):
+ """Samples a random sequence of frames of size num_samples.
+
+ Args:
+ model_input: tensor of shape [batch_size x max_frames x feature_size]
+ num_frames: tensor of shape [batch_size x 1]
+ num_samples: a scalar indicating the number of samples
+
+ Returns:
+ reshaped model_input in [batch_size x 'num_samples' x feature_size]
+ """
+
+ batch_size = tf.shape(model_input)[0]
+ frame_index_offset = tf.tile(
+ tf.expand_dims(tf.range(num_samples), 0), [batch_size, 1])
+ max_start_frame_index = tf.maximum(num_frames - num_samples, 0)
+ start_frame_index = tf.cast(
+ tf.multiply(
+ tf.random_uniform([batch_size, 1]),
+ tf.cast(max_start_frame_index + 1, tf.float32)), tf.int32)
+ frame_index = tf.minimum(start_frame_index + frame_index_offset,
+ tf.cast(num_frames - 1, tf.int32))
+ batch_index = tf.tile(
+ tf.expand_dims(tf.range(batch_size), 1), [1, num_samples])
+ index = tf.stack([batch_index, frame_index], 2)
+ return tf.gather_nd(model_input, index)
+
+
+def SampleRandomFrames(model_input, num_frames, num_samples):
+ """Samples a random set of frames of size num_samples.
+
+ Args:
+ model_input: tensor of shape [batch_size x max_frames x feature_size]
+ num_frames: tensor of shape [batch_size x 1]
+ num_samples (int): a scalar indicating the number of samples
+
+ Returns:
+ reshaped model_input in [batch_size x 'num_samples' x feature_size]
+ """
+ batch_size = tf.shape(model_input)[0]
+ frame_index = tf.cast(
+ tf.multiply(
+ tf.random.uniform([batch_size, num_samples]),
+ tf.tile(tf.cast(num_frames, tf.float32), [1, num_samples])), tf.int32)
+ batch_index = tf.tile(
+ tf.expand_dims(tf.range(batch_size), 1), [1, num_samples])
+ index = tf.stack([batch_index, frame_index], 2)
+ return tf.gather_nd(model_input, index)
+
+
+def FramePooling(frames, method):
+ """Pools over the frames of a video.
+
+ Args:
+ frames: tensor of shape [batch_size, num_frames, feature_size].
+ method: string indicating pooling method, one of: "average", "max",
+ "attention", or "none".
+
+ Returns:
+ tensor of shape [batch_size, feature_size] for average, max, or
+ attention pooling, and shape [batch_size*num_frames, feature_size]
+ for none pooling.
+ Raises:
+ ValueError: if method is other than "average", "max", "attention", or
+ "none".
+ """
+ if method == "average":
+ reduced = tf.reduce_mean(frames, 1)
+ elif method == "max":
+ reduced = tf.reduce_max(frames, 1)
+ elif method == "none":
+ feature_size = frames.shape_as_list()[2]
+ reduced = tf.reshape(frames, [-1, feature_size])
+ else:
+ raise ValueError("Unrecognized pooling method: %s" % method)
+
+ return reduced
diff --git a/official/vision/beta/projects/yt8m/tasks/__init__.py b/official/vision/beta/projects/yt8m/tasks/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..226844ec2ff4b9581458306aea4c574f1b2f8230
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/tasks/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tasks package definition."""
+from official.vision.beta.projects.yt8m.tasks import yt8m_task
diff --git a/official/vision/beta/projects/yt8m/tasks/yt8m_task.py b/official/vision/beta/projects/yt8m/tasks/yt8m_task.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab79c732613535357f207cda5583f4090f0d0d50
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/tasks/yt8m_task.py
@@ -0,0 +1,282 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Video classification task definition."""
+from absl import logging
+import tensorflow as tf
+
+from official.core import base_task
+from official.core import input_reader
+from official.core import task_factory
+from official.modeling import tf_utils
+from official.vision.beta.projects.yt8m.configs import yt8m as yt8m_cfg
+from official.vision.beta.projects.yt8m.dataloaders import yt8m_input
+from official.vision.beta.projects.yt8m.eval_utils import eval_util
+from official.vision.beta.projects.yt8m.modeling import yt8m_model_utils as utils
+from official.vision.beta.projects.yt8m.modeling.yt8m_model import YT8MModel
+
+
+@task_factory.register_task_cls(yt8m_cfg.YT8MTask)
+class YT8MTask(base_task.Task):
+ """A task for video classification."""
+
+ def build_model(self):
+ """Builds model for YT8M Task."""
+ train_cfg = self.task_config.train_data
+ common_input_shape = [None, sum(train_cfg.feature_sizes)]
+
+ # [batch_size x num_frames x num_features]
+ input_specs = tf.keras.layers.InputSpec(shape=[None] + common_input_shape)
+ logging.info('Build model input %r', common_input_shape)
+
+ # Model configuration.
+ model_config = self.task_config.model
+ model = YT8MModel(
+ input_params=model_config,
+ input_specs=input_specs,
+ num_frames=train_cfg.num_frames,
+ num_classes=train_cfg.num_classes)
+ return model
+
+ def build_inputs(self, params: yt8m_cfg.DataConfig, input_context=None):
+ """Builds input.
+
+ Args:
+ params: configuration for input data
+ input_context: indicates information about the compute replicas and input
+ pipelines
+
+ Returns:
+ dataset: dataset fetched from reader
+ """
+
+ decoder = yt8m_input.Decoder(input_params=params)
+ decoder_fn = decoder.decode
+ parser = yt8m_input.Parser(input_params=params)
+ parser_fn = parser.parse_fn(params.is_training)
+ postprocess = yt8m_input.PostBatchProcessor(input_params=params)
+ postprocess_fn = postprocess.post_fn
+ transform_batch = yt8m_input.TransformBatcher(input_params=params)
+ batch_fn = transform_batch.batch_fn
+
+ reader = input_reader.InputReader(
+ params,
+ dataset_fn=tf.data.TFRecordDataset,
+ decoder_fn=decoder_fn,
+ parser_fn=parser_fn,
+ postprocess_fn=postprocess_fn,
+ transform_and_batch_fn=batch_fn)
+
+ dataset = reader.read(input_context=input_context)
+
+ return dataset
+
+ def build_losses(self, labels, model_outputs, aux_losses=None):
+ """Sigmoid Cross Entropy.
+
+ Args:
+ labels: tensor containing truth labels.
+ model_outputs: output logits of the classifier.
+ aux_losses: tensor containing auxiliarly loss tensors, i.e. `losses` in
+ keras.Model.
+
+ Returns:
+ Tensors: The total loss, model loss tensors.
+ """
+ losses_config = self.task_config.losses
+ model_loss = tf.keras.losses.binary_crossentropy(
+ labels,
+ model_outputs,
+ from_logits=losses_config.from_logits,
+ label_smoothing=losses_config.label_smoothing)
+
+ model_loss = tf_utils.safe_mean(model_loss)
+ total_loss = model_loss
+ if aux_losses:
+ total_loss += tf.add_n(aux_losses)
+
+ return total_loss, model_loss
+
+ def build_metrics(self, training=True):
+ """Gets streaming metrics for training/validation.
+
+ metric: mAP/gAP
+ top_k: A positive integer specifying how many predictions are considered
+ per video.
+ top_n: A positive Integer specifying the average precision at n, or None
+ to use all provided data points.
+ Args:
+ training: bool value, true for training mode, false for eval/validation.
+
+ Returns:
+ list of strings that indicate metrics to be used
+ """
+ metrics = []
+ metric_names = ['total_loss', 'model_loss']
+ for name in metric_names:
+ metrics.append(tf.keras.metrics.Mean(name, dtype=tf.float32))
+
+ if not training: # Cannot run in train step.
+ num_classes = self.task_config.validation_data.num_classes
+ top_k = self.task_config.top_k
+ top_n = self.task_config.top_n
+ self.avg_prec_metric = eval_util.EvaluationMetrics(
+ num_classes, top_k=top_k, top_n=top_n)
+
+ return metrics
+
+ def train_step(self, inputs, model, optimizer, metrics=None):
+ """Does forward and backward.
+
+ Args:
+ inputs: a dictionary of input tensors. output_dict = {
+ "video_ids": batch_video_ids,
+ "video_matrix": batch_video_matrix,
+ "labels": batch_labels,
+ "num_frames": batch_frames, }
+ model: the model, forward pass definition.
+ optimizer: the optimizer for this training step.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ a dictionary of logs.
+ """
+ features, labels = inputs['video_matrix'], inputs['labels']
+ num_frames = inputs['num_frames']
+
+ # Normalize input features.
+ feature_dim = len(features.shape) - 1
+ features = tf.nn.l2_normalize(features, feature_dim)
+
+ # sample random frames / random sequence
+ num_frames = tf.cast(num_frames, tf.float32)
+ sample_frames = self.task_config.train_data.num_frames
+ if self.task_config.model.sample_random_frames:
+ features = utils.SampleRandomFrames(features, num_frames, sample_frames)
+ else:
+ features = utils.SampleRandomSequence(features, num_frames, sample_frames)
+
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(features, training=True)
+ # Casting output layer as float32 is necessary when mixed_precision is
+ # mixed_float16 or mixed_bfloat16 to ensure output is casted as float32.
+ outputs = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss
+ loss, model_loss = self.build_losses(
+ model_outputs=outputs, labels=labels, aux_losses=model.losses)
+ # Scales loss as the default gradients allreduce performs sum inside the
+ # optimizer.
+ scaled_loss = loss / num_replicas
+
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(optimizer,
+ tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient before apply_gradients when LossScaleOptimizer is
+ # used.
+ if isinstance(optimizer,
+ tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+
+ # Apply gradient clipping.
+ if self.task_config.gradient_clip_norm > 0:
+ grads, _ = tf.clip_by_global_norm(grads,
+ self.task_config.gradient_clip_norm)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = {self.loss: loss}
+
+ all_losses = {'total_loss': loss, 'model_loss': model_loss}
+
+ if metrics:
+ for m in metrics:
+ m.update_state(all_losses[m.name])
+ logs.update({m.name: m.result()})
+
+ return logs
+
+ def validation_step(self, inputs, model, metrics=None):
+ """Validatation step.
+
+ Args:
+ inputs: a dictionary of input tensors. output_dict = {
+ "video_ids": batch_video_ids,
+ "video_matrix": batch_video_matrix,
+ "labels": batch_labels,
+ "num_frames": batch_frames, }
+ model: the model, forward definition
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ a dictionary of logs.
+ """
+ features, labels = inputs['video_matrix'], inputs['labels']
+ num_frames = inputs['num_frames']
+
+ # Normalize input features.
+ feature_dim = len(features.shape) - 1
+ features = tf.nn.l2_normalize(features, feature_dim)
+
+ # sample random frames (None, 5, 1152) -> (None, 30, 1152)
+ sample_frames = self.task_config.validation_data.num_frames
+ if self.task_config.model.sample_random_frames:
+ features = utils.SampleRandomFrames(features, num_frames, sample_frames)
+ else:
+ features = utils.SampleRandomSequence(features, num_frames, sample_frames)
+
+ outputs = self.inference_step(features, model)
+ outputs = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), outputs)
+ if self.task_config.validation_data.segment_labels:
+ # workaround to ignore the unrated labels.
+ outputs *= inputs['label_weights']
+ # remove padding
+ outputs = outputs[~tf.reduce_all(labels == -1, axis=1)]
+ labels = labels[~tf.reduce_all(labels == -1, axis=1)]
+ loss, model_loss = self.build_losses(
+ model_outputs=outputs, labels=labels, aux_losses=model.losses)
+
+ logs = {self.loss: loss}
+
+ all_losses = {'total_loss': loss, 'model_loss': model_loss}
+
+ logs.update({self.avg_prec_metric.name: (labels, outputs)})
+
+ if metrics:
+ for m in metrics:
+ m.update_state(all_losses[m.name])
+ logs.update({m.name: m.result()})
+ return logs
+
+ def inference_step(self, inputs, model):
+ """Performs the forward step."""
+ return model(inputs, training=False)
+
+ def aggregate_logs(self, state=None, step_logs=None):
+ if state is None:
+ state = self.avg_prec_metric
+ self.avg_prec_metric.accumulate(
+ labels=step_logs[self.avg_prec_metric.name][0],
+ predictions=step_logs[self.avg_prec_metric.name][1])
+ return state
+
+ def reduce_aggregated_logs(self, aggregated_logs):
+ avg_prec_metrics = self.avg_prec_metric.get()
+ self.avg_prec_metric.clear()
+ return avg_prec_metrics
diff --git a/official/vision/beta/projects/yt8m/train.py b/official/vision/beta/projects/yt8m/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..097ef1d19e3507c06c7b43497e7a47269e599a7b
--- /dev/null
+++ b/official/vision/beta/projects/yt8m/train.py
@@ -0,0 +1,68 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""YT8M model training driver."""
+
+from absl import app
+from absl import flags
+import gin
+
+from official.common import distribute_utils
+from official.common import flags as tfm_flags
+from official.core import task_factory
+from official.core import train_lib
+from official.core import train_utils
+from official.modeling import performance
+# pylint: disable=unused-import
+from official.vision.beta.projects.yt8m.configs import yt8m
+from official.vision.beta.projects.yt8m.tasks import yt8m_task
+# pylint: enable=unused-import
+
+FLAGS = flags.FLAGS
+
+
+def main(_):
+ gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params)
+ params = train_utils.parse_configuration(FLAGS)
+ model_dir = FLAGS.model_dir
+ if 'train' in FLAGS.mode:
+ # Pure eval modes do not output yaml files. Otherwise continuous eval job
+ # may race against the train job for writing the same file.
+ train_utils.serialize_config(params, model_dir)
+
+ # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16'
+ # can have significant impact on model speeds by utilizing float16 in case of
+ # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when
+ # dtype is float16
+ if params.runtime.mixed_precision_dtype:
+ performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype)
+ distribution_strategy = distribute_utils.get_distribution_strategy(
+ distribution_strategy=params.runtime.distribution_strategy,
+ all_reduce_alg=params.runtime.all_reduce_alg,
+ num_gpus=params.runtime.num_gpus,
+ tpu_address=params.runtime.tpu)
+ with distribution_strategy.scope():
+ task = task_factory.get_task(params.task, logging_dir=model_dir)
+
+ train_lib.run_experiment(
+ distribution_strategy=distribution_strategy,
+ task=task,
+ mode=FLAGS.mode,
+ params=params,
+ model_dir=model_dir)
+
+
+if __name__ == '__main__':
+ tfm_flags.define_flags()
+ app.run(main)
diff --git a/official/vision/beta/serving/__init__.py b/official/vision/beta/serving/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422
--- /dev/null
+++ b/official/vision/beta/serving/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/official/vision/beta/serving/detection.py b/official/vision/beta/serving/detection.py
new file mode 100644
index 0000000000000000000000000000000000000000..7061048e4b6fb66f58532488a370c5e1a9cecf86
--- /dev/null
+++ b/official/vision/beta/serving/detection.py
@@ -0,0 +1,151 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Detection input and model functions for serving/inference."""
+
+import tensorflow as tf
+
+from official.vision.beta import configs
+from official.vision.beta.modeling import factory
+from official.vision.beta.ops import anchor
+from official.vision.beta.ops import preprocess_ops
+from official.vision.beta.serving import export_base
+
+
+MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255)
+STDDEV_RGB = (0.229 * 255, 0.224 * 255, 0.225 * 255)
+
+
+class DetectionModule(export_base.ExportModule):
+ """Detection Module."""
+
+ def _build_model(self):
+
+ if self._batch_size is None:
+ ValueError("batch_size can't be None for detection models")
+ if not self.params.task.model.detection_generator.use_batched_nms:
+ ValueError('Only batched_nms is supported.')
+ input_specs = tf.keras.layers.InputSpec(shape=[self._batch_size] +
+ self._input_image_size + [3])
+
+ if isinstance(self.params.task.model, configs.maskrcnn.MaskRCNN):
+ model = factory.build_maskrcnn(
+ input_specs=input_specs, model_config=self.params.task.model)
+ elif isinstance(self.params.task.model, configs.retinanet.RetinaNet):
+ model = factory.build_retinanet(
+ input_specs=input_specs, model_config=self.params.task.model)
+ else:
+ raise ValueError('Detection module not implemented for {} model.'.format(
+ type(self.params.task.model)))
+
+ return model
+
+ def _build_inputs(self, image):
+ """Builds detection model inputs for serving."""
+ model_params = self.params.task.model
+ # Normalizes image with mean and std pixel values.
+ image = preprocess_ops.normalize_image(image,
+ offset=MEAN_RGB,
+ scale=STDDEV_RGB)
+
+ image, image_info = preprocess_ops.resize_and_crop_image(
+ image,
+ self._input_image_size,
+ padded_size=preprocess_ops.compute_padded_size(
+ self._input_image_size, 2**model_params.max_level),
+ aug_scale_min=1.0,
+ aug_scale_max=1.0)
+
+ input_anchor = anchor.build_anchor_generator(
+ min_level=model_params.min_level,
+ max_level=model_params.max_level,
+ num_scales=model_params.anchor.num_scales,
+ aspect_ratios=model_params.anchor.aspect_ratios,
+ anchor_size=model_params.anchor.anchor_size)
+ anchor_boxes = input_anchor(image_size=(self._input_image_size[0],
+ self._input_image_size[1]))
+
+ return image, anchor_boxes, image_info
+
+ def serve(self, images: tf.Tensor):
+ """Cast image to float and run inference.
+
+ Args:
+ images: uint8 Tensor of shape [batch_size, None, None, 3]
+ Returns:
+ Tensor holding detection output logits.
+ """
+ model_params = self.params.task.model
+ with tf.device('cpu:0'):
+ images = tf.cast(images, dtype=tf.float32)
+
+ # Tensor Specs for map_fn outputs (images, anchor_boxes, and image_info).
+ images_spec = tf.TensorSpec(shape=self._input_image_size + [3],
+ dtype=tf.float32)
+
+ num_anchors = model_params.anchor.num_scales * len(
+ model_params.anchor.aspect_ratios) * 4
+ anchor_shapes = []
+ for level in range(model_params.min_level, model_params.max_level + 1):
+ anchor_level_spec = tf.TensorSpec(
+ shape=[
+ self._input_image_size[0] // 2**level,
+ self._input_image_size[1] // 2**level, num_anchors
+ ],
+ dtype=tf.float32)
+ anchor_shapes.append((str(level), anchor_level_spec))
+
+ image_info_spec = tf.TensorSpec(shape=[4, 2], dtype=tf.float32)
+
+ images, anchor_boxes, image_info = tf.nest.map_structure(
+ tf.identity,
+ tf.map_fn(
+ self._build_inputs,
+ elems=images,
+ fn_output_signature=(images_spec, dict(anchor_shapes),
+ image_info_spec),
+ parallel_iterations=32))
+
+ input_image_shape = image_info[:, 1, :]
+
+ # To overcome keras.Model extra limitation to save a model with layers that
+ # have multiple inputs, we use `model.call` here to trigger the forward
+ # path. Note that, this disables some keras magics happens in `__call__`.
+ detections = self.model.call(
+ images=images,
+ image_shape=input_image_shape,
+ anchor_boxes=anchor_boxes,
+ training=False)
+
+ if self.params.task.model.detection_generator.apply_nms:
+ final_outputs = {
+ 'detection_boxes': detections['detection_boxes'],
+ 'detection_scores': detections['detection_scores'],
+ 'detection_classes': detections['detection_classes'],
+ 'num_detections': detections['num_detections']
+ }
+ else:
+ final_outputs = {
+ 'decoded_boxes': detections['decoded_boxes'],
+ 'decoded_box_scores': detections['decoded_box_scores'],
+ 'cls_outputs': detections['cls_outputs'],
+ 'box_outputs': detections['box_outputs']
+ }
+
+ if 'detection_masks' in detections.keys():
+ final_outputs['detection_masks'] = detections['detection_masks']
+
+ final_outputs.update({'image_info': image_info})
+ return final_outputs
diff --git a/official/vision/beta/serving/detection_test.py b/official/vision/beta/serving/detection_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..26ec504cfa705108d4872180a288511625f2481a
--- /dev/null
+++ b/official/vision/beta/serving/detection_test.py
@@ -0,0 +1,123 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Test for image detection export lib."""
+
+import io
+import os
+
+from absl.testing import parameterized
+import numpy as np
+from PIL import Image
+import tensorflow as tf
+
+from official.common import registry_imports # pylint: disable=unused-import
+from official.core import exp_factory
+from official.vision.beta.serving import detection
+
+
+class DetectionExportTest(tf.test.TestCase, parameterized.TestCase):
+
+ def _get_detection_module(self, experiment_name):
+ params = exp_factory.get_exp_config(experiment_name)
+ params.task.model.backbone.resnet.model_id = 18
+ params.task.model.detection_generator.use_batched_nms = True
+ detection_module = detection.DetectionModule(
+ params, batch_size=1, input_image_size=[640, 640])
+ return detection_module
+
+ def _export_from_module(self, module, input_type, save_directory):
+ signatures = module.get_inference_signatures(
+ {input_type: 'serving_default'})
+ tf.saved_model.save(module, save_directory, signatures=signatures)
+
+ def _get_dummy_input(self, input_type, batch_size, image_size):
+ """Get dummy input for the given input type."""
+ h, w = image_size
+
+ if input_type == 'image_tensor':
+ return tf.zeros((batch_size, h, w, 3), dtype=np.uint8)
+ elif input_type == 'image_bytes':
+ image = Image.fromarray(np.zeros((h, w, 3), dtype=np.uint8))
+ byte_io = io.BytesIO()
+ image.save(byte_io, 'PNG')
+ return [byte_io.getvalue() for b in range(batch_size)]
+ elif input_type == 'tf_example':
+ image_tensor = tf.zeros((h, w, 3), dtype=tf.uint8)
+ encoded_jpeg = tf.image.encode_jpeg(tf.constant(image_tensor)).numpy()
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ tf.train.Feature(
+ bytes_list=tf.train.BytesList(value=[encoded_jpeg])),
+ })).SerializeToString()
+ return [example for b in range(batch_size)]
+
+ @parameterized.parameters(
+ ('image_tensor', 'fasterrcnn_resnetfpn_coco', [384, 384]),
+ ('image_bytes', 'fasterrcnn_resnetfpn_coco', [640, 640]),
+ ('tf_example', 'fasterrcnn_resnetfpn_coco', [640, 640]),
+ ('image_tensor', 'maskrcnn_resnetfpn_coco', [640, 640]),
+ ('image_bytes', 'maskrcnn_resnetfpn_coco', [640, 384]),
+ ('tf_example', 'maskrcnn_resnetfpn_coco', [640, 640]),
+ ('image_tensor', 'retinanet_resnetfpn_coco', [640, 640]),
+ ('image_bytes', 'retinanet_resnetfpn_coco', [640, 640]),
+ ('tf_example', 'retinanet_resnetfpn_coco', [384, 640]),
+ ('image_tensor', 'retinanet_resnetfpn_coco', [384, 384]),
+ ('image_bytes', 'retinanet_spinenet_coco', [640, 640]),
+ ('tf_example', 'retinanet_spinenet_coco', [640, 384]),
+ )
+ def test_export(self, input_type, experiment_name, image_size):
+ tmp_dir = self.get_temp_dir()
+ module = self._get_detection_module(experiment_name)
+
+ self._export_from_module(module, input_type, tmp_dir)
+
+ self.assertTrue(os.path.exists(os.path.join(tmp_dir, 'saved_model.pb')))
+ self.assertTrue(
+ os.path.exists(os.path.join(tmp_dir, 'variables', 'variables.index')))
+ self.assertTrue(
+ os.path.exists(
+ os.path.join(tmp_dir, 'variables',
+ 'variables.data-00000-of-00001')))
+
+ imported = tf.saved_model.load(tmp_dir)
+ detection_fn = imported.signatures['serving_default']
+
+ images = self._get_dummy_input(
+ input_type, batch_size=1, image_size=image_size)
+
+ processed_images, anchor_boxes, image_info = module._build_inputs(
+ tf.zeros((224, 224, 3), dtype=tf.uint8))
+ image_shape = image_info[1, :]
+ image_shape = tf.expand_dims(image_shape, 0)
+ processed_images = tf.expand_dims(processed_images, 0)
+ for l, l_boxes in anchor_boxes.items():
+ anchor_boxes[l] = tf.expand_dims(l_boxes, 0)
+
+ expected_outputs = module.model(
+ images=processed_images,
+ image_shape=image_shape,
+ anchor_boxes=anchor_boxes,
+ training=False)
+ outputs = detection_fn(tf.constant(images))
+
+ self.assertAllClose(outputs['num_detections'].numpy(),
+ expected_outputs['num_detections'].numpy())
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/serving/export_base.py b/official/vision/beta/serving/export_base.py
new file mode 100644
index 0000000000000000000000000000000000000000..08472b55670359e65bf37c581a12675022172f2f
--- /dev/null
+++ b/official/vision/beta/serving/export_base.py
@@ -0,0 +1,179 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Base class for model export."""
+
+import abc
+from typing import Dict, List, Mapping, Optional, Text
+
+import tensorflow as tf
+
+from official.core import export_base
+from official.modeling.hyperparams import config_definitions as cfg
+
+
+class ExportModule(export_base.ExportModule, metaclass=abc.ABCMeta):
+ """Base Export Module."""
+
+ def __init__(self,
+ params: cfg.ExperimentConfig,
+ *,
+ batch_size: int,
+ input_image_size: List[int],
+ num_channels: int = 3,
+ model: Optional[tf.keras.Model] = None):
+ """Initializes a module for export.
+
+ Args:
+ params: Experiment params.
+ batch_size: The batch size of the model input. Can be `int` or None.
+ input_image_size: List or Tuple of size of the input image. For 2D image,
+ it is [height, width].
+ num_channels: The number of the image channels.
+ model: A tf.keras.Model instance to be exported.
+ """
+ self.params = params
+ self._batch_size = batch_size
+ self._input_image_size = input_image_size
+ self._num_channels = num_channels
+ if model is None:
+ model = self._build_model() # pylint: disable=assignment-from-none
+ super().__init__(params=params, model=model)
+
+ def _decode_image(self, encoded_image_bytes: str) -> tf.Tensor:
+ """Decodes an image bytes to an image tensor.
+
+ Use `tf.image.decode_image` to decode an image if input is expected to be 2D
+ image; otherwise use `tf.io.decode_raw` to convert the raw bytes to tensor
+ and reshape it to desire shape.
+
+ Args:
+ encoded_image_bytes: An encoded image string to be decoded.
+
+ Returns:
+ A decoded image tensor.
+ """
+ if len(self._input_image_size) == 2:
+ # Decode an image if 2D input is expected.
+ image_tensor = tf.image.decode_image(
+ encoded_image_bytes, channels=self._num_channels)
+ image_tensor.set_shape((None, None, self._num_channels))
+ else:
+ # Convert raw bytes into a tensor and reshape it, if not 2D input.
+ image_tensor = tf.io.decode_raw(encoded_image_bytes, out_type=tf.uint8)
+ image_tensor = tf.reshape(image_tensor,
+ self._input_image_size + [self._num_channels])
+ return image_tensor
+
+ def _decode_tf_example(
+ self, tf_example_string_tensor: tf.train.Example) -> tf.Tensor:
+ """Decodes a TF Example to an image tensor.
+
+ Args:
+ tf_example_string_tensor: A tf.train.Example of encoded image and other
+ information.
+
+ Returns:
+ A decoded image tensor.
+ """
+ keys_to_features = {'image/encoded': tf.io.FixedLenFeature((), tf.string)}
+ parsed_tensors = tf.io.parse_single_example(
+ serialized=tf_example_string_tensor, features=keys_to_features)
+ image_tensor = self._decode_image(parsed_tensors['image/encoded'])
+ return image_tensor
+
+ def _build_model(self, **kwargs):
+ """Returns a model built from the params."""
+ return None
+
+ @tf.function
+ def inference_from_image_tensors(
+ self, inputs: tf.Tensor) -> Mapping[str, tf.Tensor]:
+ return self.serve(inputs)
+
+ @tf.function
+ def inference_from_image_bytes(self, inputs: tf.Tensor):
+ with tf.device('cpu:0'):
+ images = tf.nest.map_structure(
+ tf.identity,
+ tf.map_fn(
+ self._decode_image,
+ elems=inputs,
+ fn_output_signature=tf.TensorSpec(
+ shape=[None] * len(self._input_image_size) +
+ [self._num_channels],
+ dtype=tf.uint8),
+ parallel_iterations=32))
+ images = tf.stack(images)
+ return self.serve(images)
+
+ @tf.function
+ def inference_from_tf_example(self,
+ inputs: tf.Tensor) -> Mapping[str, tf.Tensor]:
+ with tf.device('cpu:0'):
+ images = tf.nest.map_structure(
+ tf.identity,
+ tf.map_fn(
+ self._decode_tf_example,
+ elems=inputs,
+ # Height/width of the shape of input images is unspecified (None)
+ # at the time of decoding the example, but the shape will
+ # be adjusted to conform to the input layer of the model,
+ # by _run_inference_on_image_tensors() below.
+ fn_output_signature=tf.TensorSpec(
+ shape=[None] * len(self._input_image_size) +
+ [self._num_channels],
+ dtype=tf.uint8),
+ dtype=tf.uint8,
+ parallel_iterations=32))
+ images = tf.stack(images)
+ return self.serve(images)
+
+ def get_inference_signatures(self, function_keys: Dict[Text, Text]):
+ """Gets defined function signatures.
+
+ Args:
+ function_keys: A dictionary with keys as the function to create signature
+ for and values as the signature keys when returns.
+
+ Returns:
+ A dictionary with key as signature key and value as concrete functions
+ that can be used for tf.saved_model.save.
+ """
+ signatures = {}
+ for key, def_name in function_keys.items():
+ if key == 'image_tensor':
+ input_signature = tf.TensorSpec(
+ shape=[self._batch_size] + [None] * len(self._input_image_size) +
+ [self._num_channels],
+ dtype=tf.uint8)
+ signatures[
+ def_name] = self.inference_from_image_tensors.get_concrete_function(
+ input_signature)
+ elif key == 'image_bytes':
+ input_signature = tf.TensorSpec(
+ shape=[self._batch_size], dtype=tf.string)
+ signatures[
+ def_name] = self.inference_from_image_bytes.get_concrete_function(
+ input_signature)
+ elif key == 'serve_examples' or key == 'tf_example':
+ input_signature = tf.TensorSpec(
+ shape=[self._batch_size], dtype=tf.string)
+ signatures[
+ def_name] = self.inference_from_tf_example.get_concrete_function(
+ input_signature)
+ else:
+ raise ValueError('Unrecognized `input_type`')
+ return signatures
diff --git a/official/vision/beta/serving/export_saved_model.py b/official/vision/beta/serving/export_saved_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..51966e91dda8212c9fd016928c6a4dccbddcca0d
--- /dev/null
+++ b/official/vision/beta/serving/export_saved_model.py
@@ -0,0 +1,103 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+r"""Vision models export binary for serving/inference.
+
+To export a trained checkpoint in saved_model format (shell script):
+
+EXPERIMENT_TYPE = XX
+CHECKPOINT_PATH = XX
+EXPORT_DIR_PATH = XX
+export_saved_model --experiment=${EXPERIMENT_TYPE} \
+ --export_dir=${EXPORT_DIR_PATH}/ \
+ --checkpoint_path=${CHECKPOINT_PATH} \
+ --batch_size=2 \
+ --input_image_size=224,224
+
+To serve (python):
+
+export_dir_path = XX
+input_type = XX
+input_images = XX
+imported = tf.saved_model.load(export_dir_path)
+model_fn = imported.signatures['serving_default']
+output = model_fn(input_images)
+"""
+
+from absl import app
+from absl import flags
+
+from official.common import registry_imports # pylint: disable=unused-import
+from official.core import exp_factory
+from official.modeling import hyperparams
+from official.vision.beta.serving import export_saved_model_lib
+
+FLAGS = flags.FLAGS
+
+
+flags.DEFINE_string(
+ 'experiment', None, 'experiment type, e.g. retinanet_resnetfpn_coco')
+flags.DEFINE_string('export_dir', None, 'The export directory.')
+flags.DEFINE_string('checkpoint_path', None, 'Checkpoint path.')
+flags.DEFINE_multi_string(
+ 'config_file',
+ default=None,
+ help='YAML/JSON files which specifies overrides. The override order '
+ 'follows the order of args. Note that each file '
+ 'can be used as an override template to override the default parameters '
+ 'specified in Python. If the same parameter is specified in both '
+ '`--config_file` and `--params_override`, `config_file` will be used '
+ 'first, followed by params_override.')
+flags.DEFINE_string(
+ 'params_override', '',
+ 'The JSON/YAML file or string which specifies the parameter to be overriden'
+ ' on top of `config_file` template.')
+flags.DEFINE_integer(
+ 'batch_size', None, 'The batch size.')
+flags.DEFINE_string(
+ 'input_type', 'image_tensor',
+ 'One of `image_tensor`, `image_bytes`, `tf_example`.')
+flags.DEFINE_string(
+ 'input_image_size', '224,224',
+ 'The comma-separated string of two integers representing the height,width '
+ 'of the input to the model.')
+
+
+def main(_):
+
+ params = exp_factory.get_exp_config(FLAGS.experiment)
+ for config_file in FLAGS.config_file or []:
+ params = hyperparams.override_params_dict(
+ params, config_file, is_strict=True)
+ if FLAGS.params_override:
+ params = hyperparams.override_params_dict(
+ params, FLAGS.params_override, is_strict=True)
+
+ params.validate()
+ params.lock()
+
+ export_saved_model_lib.export_inference_graph(
+ input_type=FLAGS.input_type,
+ batch_size=FLAGS.batch_size,
+ input_image_size=[int(x) for x in FLAGS.input_image_size.split(',')],
+ params=params,
+ checkpoint_path=FLAGS.checkpoint_path,
+ export_dir=FLAGS.export_dir,
+ export_checkpoint_subdir='checkpoint',
+ export_saved_model_subdir='saved_model')
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/official/vision/beta/serving/export_saved_model_lib.py b/official/vision/beta/serving/export_saved_model_lib.py
new file mode 100644
index 0000000000000000000000000000000000000000..ec391846515618a46fb94fa85c450915b5c00f64
--- /dev/null
+++ b/official/vision/beta/serving/export_saved_model_lib.py
@@ -0,0 +1,113 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+r"""Vision models export utility function for serving/inference."""
+
+import os
+from typing import Optional, List
+
+import tensorflow as tf
+
+from official.core import config_definitions as cfg
+from official.core import export_base
+from official.core import train_utils
+from official.vision.beta import configs
+from official.vision.beta.serving import detection
+from official.vision.beta.serving import image_classification
+from official.vision.beta.serving import semantic_segmentation
+
+
+def export_inference_graph(
+ input_type: str,
+ batch_size: Optional[int],
+ input_image_size: List[int],
+ params: cfg.ExperimentConfig,
+ checkpoint_path: str,
+ export_dir: str,
+ num_channels: Optional[int] = 3,
+ export_module: Optional[export_base.ExportModule] = None,
+ export_checkpoint_subdir: Optional[str] = None,
+ export_saved_model_subdir: Optional[str] = None):
+ """Exports inference graph for the model specified in the exp config.
+
+ Saved model is stored at export_dir/saved_model, checkpoint is saved
+ at export_dir/checkpoint, and params is saved at export_dir/params.yaml.
+
+ Args:
+ input_type: One of `image_tensor`, `image_bytes`, `tf_example`.
+ batch_size: 'int', or None.
+ input_image_size: List or Tuple of height and width.
+ params: Experiment params.
+ checkpoint_path: Trained checkpoint path or directory.
+ export_dir: Export directory path.
+ num_channels: The number of input image channels.
+ export_module: Optional export module to be used instead of using params
+ to create one. If None, the params will be used to create an export
+ module.
+ export_checkpoint_subdir: Optional subdirectory under export_dir
+ to store checkpoint.
+ export_saved_model_subdir: Optional subdirectory under export_dir
+ to store saved model.
+ """
+
+ if export_checkpoint_subdir:
+ output_checkpoint_directory = os.path.join(
+ export_dir, export_checkpoint_subdir)
+ else:
+ output_checkpoint_directory = export_dir
+
+ if export_saved_model_subdir:
+ output_saved_model_directory = os.path.join(
+ export_dir, export_saved_model_subdir)
+ else:
+ output_saved_model_directory = export_dir
+
+ # TODO(arashwan): Offers a direct path to use ExportModule with Task objects.
+ if not export_module:
+ if isinstance(params.task,
+ configs.image_classification.ImageClassificationTask):
+ export_module = image_classification.ClassificationModule(
+ params=params,
+ batch_size=batch_size,
+ input_image_size=input_image_size,
+ num_channels=num_channels)
+ elif isinstance(params.task, configs.retinanet.RetinaNetTask) or isinstance(
+ params.task, configs.maskrcnn.MaskRCNNTask):
+ export_module = detection.DetectionModule(
+ params=params,
+ batch_size=batch_size,
+ input_image_size=input_image_size,
+ num_channels=num_channels)
+ elif isinstance(params.task,
+ configs.semantic_segmentation.SemanticSegmentationTask):
+ export_module = semantic_segmentation.SegmentationModule(
+ params=params,
+ batch_size=batch_size,
+ input_image_size=input_image_size,
+ num_channels=num_channels)
+ else:
+ raise ValueError('Export module not implemented for {} task.'.format(
+ type(params.task)))
+
+ export_base.export(
+ export_module,
+ function_keys=[input_type],
+ export_savedmodel_dir=output_saved_model_directory,
+ checkpoint_path=checkpoint_path,
+ timestamped=False)
+
+ ckpt = tf.train.Checkpoint(model=export_module.model)
+ ckpt.save(os.path.join(output_checkpoint_directory, 'ckpt'))
+ train_utils.serialize_config(params, export_dir)
diff --git a/official/vision/beta/serving/export_tfhub.py b/official/vision/beta/serving/export_tfhub.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d8af0899034065f9750ea1ab51e23dea40bb915
--- /dev/null
+++ b/official/vision/beta/serving/export_tfhub.py
@@ -0,0 +1,105 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""A script to export the image classification as a TF-Hub SavedModel."""
+
+# Import libraries
+from absl import app
+from absl import flags
+
+import tensorflow as tf
+
+from official.common import registry_imports # pylint: disable=unused-import
+from official.core import exp_factory
+from official.modeling import hyperparams
+from official.vision.beta.modeling import factory
+
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+ 'experiment', None, 'experiment type, e.g. resnet_imagenet')
+flags.DEFINE_string(
+ 'checkpoint_path', None, 'Checkpoint path.')
+flags.DEFINE_string(
+ 'export_path', None, 'The export directory.')
+flags.DEFINE_multi_string(
+ 'config_file',
+ None,
+ 'A YAML/JSON files which specifies overrides. The override order '
+ 'follows the order of args. Note that each file '
+ 'can be used as an override template to override the default parameters '
+ 'specified in Python. If the same parameter is specified in both '
+ '`--config_file` and `--params_override`, `config_file` will be used '
+ 'first, followed by params_override.')
+flags.DEFINE_string(
+ 'params_override', '',
+ 'The JSON/YAML file or string which specifies the parameter to be overriden'
+ ' on top of `config_file` template.')
+flags.DEFINE_integer(
+ 'batch_size', None, 'The batch size.')
+flags.DEFINE_string(
+ 'input_image_size',
+ '224,224',
+ 'The comma-separated string of two integers representing the height,width '
+ 'of the input to the model.')
+flags.DEFINE_boolean(
+ 'skip_logits_layer',
+ False,
+ 'Whether to skip the prediction layer and only output the feature vector.')
+
+
+def export_model_to_tfhub(params,
+ batch_size,
+ input_image_size,
+ skip_logits_layer,
+ checkpoint_path,
+ export_path):
+ """Export an image classification model to TF-Hub."""
+ input_specs = tf.keras.layers.InputSpec(shape=[batch_size] +
+ input_image_size + [3])
+
+ model = factory.build_classification_model(
+ input_specs=input_specs,
+ model_config=params.task.model,
+ l2_regularizer=None,
+ skip_logits_layer=skip_logits_layer)
+ checkpoint = tf.train.Checkpoint(model=model)
+ checkpoint.restore(checkpoint_path).assert_existing_objects_matched()
+ model.save(export_path, include_optimizer=False, save_format='tf')
+
+
+def main(_):
+ params = exp_factory.get_exp_config(FLAGS.experiment)
+ for config_file in FLAGS.config_file or []:
+ params = hyperparams.override_params_dict(
+ params, config_file, is_strict=True)
+ if FLAGS.params_override:
+ params = hyperparams.override_params_dict(
+ params, FLAGS.params_override, is_strict=True)
+ params.validate()
+ params.lock()
+
+ export_model_to_tfhub(
+ params=params,
+ batch_size=FLAGS.batch_size,
+ input_image_size=[int(x) for x in FLAGS.input_image_size.split(',')],
+ skip_logits_layer=FLAGS.skip_logits_layer,
+ checkpoint_path=FLAGS.checkpoint_path,
+ export_path=FLAGS.export_path)
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/official/vision/beta/serving/image_classification.py b/official/vision/beta/serving/image_classification.py
new file mode 100644
index 0000000000000000000000000000000000000000..739e07ab28e4bcfb247e92c9622715bf6e849398
--- /dev/null
+++ b/official/vision/beta/serving/image_classification.py
@@ -0,0 +1,82 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Detection input and model functions for serving/inference."""
+
+import tensorflow as tf
+
+from official.vision.beta.modeling import factory
+from official.vision.beta.ops import preprocess_ops
+from official.vision.beta.serving import export_base
+
+
+MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255)
+STDDEV_RGB = (0.229 * 255, 0.224 * 255, 0.225 * 255)
+
+
+class ClassificationModule(export_base.ExportModule):
+ """classification Module."""
+
+ def _build_model(self):
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[self._batch_size] + self._input_image_size + [3])
+
+ return factory.build_classification_model(
+ input_specs=input_specs,
+ model_config=self.params.task.model,
+ l2_regularizer=None)
+
+ def _build_inputs(self, image):
+ """Builds classification model inputs for serving."""
+ # Center crops and resizes image.
+ image = preprocess_ops.center_crop_image(image)
+
+ image = tf.image.resize(
+ image, self._input_image_size, method=tf.image.ResizeMethod.BILINEAR)
+
+ image = tf.reshape(
+ image, [self._input_image_size[0], self._input_image_size[1], 3])
+
+ # Normalizes image with mean and std pixel values.
+ image = preprocess_ops.normalize_image(image,
+ offset=MEAN_RGB,
+ scale=STDDEV_RGB)
+ return image
+
+ def serve(self, images):
+ """Cast image to float and run inference.
+
+ Args:
+ images: uint8 Tensor of shape [batch_size, None, None, 3]
+ Returns:
+ Tensor holding classification output logits.
+ """
+ with tf.device('cpu:0'):
+ images = tf.cast(images, dtype=tf.float32)
+
+ images = tf.nest.map_structure(
+ tf.identity,
+ tf.map_fn(
+ self._build_inputs, elems=images,
+ fn_output_signature=tf.TensorSpec(
+ shape=self._input_image_size + [3], dtype=tf.float32),
+ parallel_iterations=32
+ )
+ )
+
+ logits = self.inference_step(images)
+ probs = tf.nn.softmax(logits)
+
+ return {'logits': logits, 'probs': probs}
diff --git a/official/vision/beta/serving/image_classification_test.py b/official/vision/beta/serving/image_classification_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..a88646f3a2a4ad6c92dc21607f83886271aef078
--- /dev/null
+++ b/official/vision/beta/serving/image_classification_test.py
@@ -0,0 +1,112 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Test for image classification export lib."""
+
+import io
+import os
+
+from absl.testing import parameterized
+import numpy as np
+from PIL import Image
+import tensorflow as tf
+
+from official.common import registry_imports # pylint: disable=unused-import
+from official.core import exp_factory
+from official.vision.beta.serving import image_classification
+
+
+class ImageClassificationExportTest(tf.test.TestCase, parameterized.TestCase):
+
+ def _get_classification_module(self):
+ params = exp_factory.get_exp_config('resnet_imagenet')
+ params.task.model.backbone.resnet.model_id = 18
+ classification_module = image_classification.ClassificationModule(
+ params, batch_size=1, input_image_size=[224, 224])
+ return classification_module
+
+ def _export_from_module(self, module, input_type, save_directory):
+ signatures = module.get_inference_signatures(
+ {input_type: 'serving_default'})
+ tf.saved_model.save(module,
+ save_directory,
+ signatures=signatures)
+
+ def _get_dummy_input(self, input_type):
+ """Get dummy input for the given input type."""
+
+ if input_type == 'image_tensor':
+ return tf.zeros((1, 224, 224, 3), dtype=np.uint8)
+ elif input_type == 'image_bytes':
+ image = Image.fromarray(np.zeros((224, 224, 3), dtype=np.uint8))
+ byte_io = io.BytesIO()
+ image.save(byte_io, 'PNG')
+ return [byte_io.getvalue()]
+ elif input_type == 'tf_example':
+ image_tensor = tf.zeros((224, 224, 3), dtype=tf.uint8)
+ encoded_jpeg = tf.image.encode_jpeg(tf.constant(image_tensor)).numpy()
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ tf.train.Feature(
+ bytes_list=tf.train.BytesList(value=[encoded_jpeg])),
+ })).SerializeToString()
+ return [example]
+
+ @parameterized.parameters(
+ {'input_type': 'image_tensor'},
+ {'input_type': 'image_bytes'},
+ {'input_type': 'tf_example'},
+ )
+ def test_export(self, input_type='image_tensor'):
+ tmp_dir = self.get_temp_dir()
+ module = self._get_classification_module()
+ # Test that the model restores any attrs that are trackable objects
+ # (eg: tables, resource variables, keras models/layers, tf.hub modules).
+ module.model.test_trackable = tf.keras.layers.InputLayer(input_shape=(4,))
+
+ self._export_from_module(module, input_type, tmp_dir)
+
+ self.assertTrue(os.path.exists(os.path.join(tmp_dir, 'saved_model.pb')))
+ self.assertTrue(os.path.exists(
+ os.path.join(tmp_dir, 'variables', 'variables.index')))
+ self.assertTrue(os.path.exists(
+ os.path.join(tmp_dir, 'variables', 'variables.data-00000-of-00001')))
+
+ imported = tf.saved_model.load(tmp_dir)
+ classification_fn = imported.signatures['serving_default']
+
+ images = self._get_dummy_input(input_type)
+ processed_images = tf.nest.map_structure(
+ tf.stop_gradient,
+ tf.map_fn(
+ module._build_inputs,
+ elems=tf.zeros((1, 224, 224, 3), dtype=tf.uint8),
+ fn_output_signature=tf.TensorSpec(
+ shape=[224, 224, 3], dtype=tf.float32)))
+ expected_logits = module.model(processed_images, training=False)
+ expected_prob = tf.nn.softmax(expected_logits)
+ out = classification_fn(tf.constant(images))
+
+ # The imported model should contain any trackable attrs that the original
+ # model had.
+ self.assertTrue(hasattr(imported.model, 'test_trackable'))
+ self.assertAllClose(out['logits'].numpy(), expected_logits.numpy())
+ self.assertAllClose(out['probs'].numpy(), expected_prob.numpy())
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/serving/semantic_segmentation.py b/official/vision/beta/serving/semantic_segmentation.py
new file mode 100644
index 0000000000000000000000000000000000000000..0650886c84513a2837b6c5d1aa0b2cffba5573ac
--- /dev/null
+++ b/official/vision/beta/serving/semantic_segmentation.py
@@ -0,0 +1,81 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Semantic segmentation input and model functions for serving/inference."""
+
+import tensorflow as tf
+
+from official.vision.beta.modeling import factory
+from official.vision.beta.ops import preprocess_ops
+from official.vision.beta.serving import export_base
+
+
+MEAN_RGB = (0.485 * 255, 0.456 * 255, 0.406 * 255)
+STDDEV_RGB = (0.229 * 255, 0.224 * 255, 0.225 * 255)
+
+
+class SegmentationModule(export_base.ExportModule):
+ """Segmentation Module."""
+
+ def _build_model(self):
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[self._batch_size] + self._input_image_size + [3])
+
+ return factory.build_segmentation_model(
+ input_specs=input_specs,
+ model_config=self.params.task.model,
+ l2_regularizer=None)
+
+ def _build_inputs(self, image):
+ """Builds classification model inputs for serving."""
+
+ # Normalizes image with mean and std pixel values.
+ image = preprocess_ops.normalize_image(image,
+ offset=MEAN_RGB,
+ scale=STDDEV_RGB)
+
+ image, _ = preprocess_ops.resize_and_crop_image(
+ image,
+ self._input_image_size,
+ padded_size=self._input_image_size,
+ aug_scale_min=1.0,
+ aug_scale_max=1.0)
+ return image
+
+ def serve(self, images):
+ """Cast image to float and run inference.
+
+ Args:
+ images: uint8 Tensor of shape [batch_size, None, None, 3]
+ Returns:
+ Tensor holding classification output logits.
+ """
+ with tf.device('cpu:0'):
+ images = tf.cast(images, dtype=tf.float32)
+
+ images = tf.nest.map_structure(
+ tf.identity,
+ tf.map_fn(
+ self._build_inputs, elems=images,
+ fn_output_signature=tf.TensorSpec(
+ shape=self._input_image_size + [3], dtype=tf.float32),
+ parallel_iterations=32
+ )
+ )
+
+ masks = self.inference_step(images)
+ masks = tf.image.resize(masks, self._input_image_size, method='bilinear')
+
+ return dict(predicted_masks=masks)
diff --git a/official/vision/beta/serving/semantic_segmentation_test.py b/official/vision/beta/serving/semantic_segmentation_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..f45802f90a93fcca58781896aa47a3dbcc6169fe
--- /dev/null
+++ b/official/vision/beta/serving/semantic_segmentation_test.py
@@ -0,0 +1,105 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Test for semantic segmentation export lib."""
+
+import io
+import os
+
+from absl.testing import parameterized
+import numpy as np
+from PIL import Image
+import tensorflow as tf
+
+from official.common import registry_imports # pylint: disable=unused-import
+from official.core import exp_factory
+from official.vision.beta.serving import semantic_segmentation
+
+
+class SemanticSegmentationExportTest(tf.test.TestCase, parameterized.TestCase):
+
+ def _get_segmentation_module(self):
+ params = exp_factory.get_exp_config('seg_deeplabv3_pascal')
+ params.task.model.backbone.dilated_resnet.model_id = 50
+ segmentation_module = semantic_segmentation.SegmentationModule(
+ params, batch_size=1, input_image_size=[112, 112])
+ return segmentation_module
+
+ def _export_from_module(self, module, input_type, save_directory):
+ signatures = module.get_inference_signatures(
+ {input_type: 'serving_default'})
+ tf.saved_model.save(module, save_directory, signatures=signatures)
+
+ def _get_dummy_input(self, input_type):
+ """Get dummy input for the given input type."""
+
+ if input_type == 'image_tensor':
+ return tf.zeros((1, 112, 112, 3), dtype=np.uint8)
+ elif input_type == 'image_bytes':
+ image = Image.fromarray(np.zeros((112, 112, 3), dtype=np.uint8))
+ byte_io = io.BytesIO()
+ image.save(byte_io, 'PNG')
+ return [byte_io.getvalue()]
+ elif input_type == 'tf_example':
+ image_tensor = tf.zeros((112, 112, 3), dtype=tf.uint8)
+ encoded_jpeg = tf.image.encode_jpeg(tf.constant(image_tensor)).numpy()
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ tf.train.Feature(
+ bytes_list=tf.train.BytesList(value=[encoded_jpeg])),
+ })).SerializeToString()
+ return [example]
+
+ @parameterized.parameters(
+ {'input_type': 'image_tensor'},
+ {'input_type': 'image_bytes'},
+ {'input_type': 'tf_example'},
+ )
+ def test_export(self, input_type='image_tensor'):
+ tmp_dir = self.get_temp_dir()
+ module = self._get_segmentation_module()
+
+ self._export_from_module(module, input_type, tmp_dir)
+
+ self.assertTrue(os.path.exists(os.path.join(tmp_dir, 'saved_model.pb')))
+ self.assertTrue(
+ os.path.exists(os.path.join(tmp_dir, 'variables', 'variables.index')))
+ self.assertTrue(
+ os.path.exists(
+ os.path.join(tmp_dir, 'variables',
+ 'variables.data-00000-of-00001')))
+
+ imported = tf.saved_model.load(tmp_dir)
+ segmentation_fn = imported.signatures['serving_default']
+
+ images = self._get_dummy_input(input_type)
+ processed_images = tf.nest.map_structure(
+ tf.stop_gradient,
+ tf.map_fn(
+ module._build_inputs,
+ elems=tf.zeros((1, 112, 112, 3), dtype=tf.uint8),
+ fn_output_signature=tf.TensorSpec(
+ shape=[112, 112, 3], dtype=tf.float32)))
+ expected_output = tf.image.resize(
+ module.model(processed_images, training=False), [112, 112],
+ method='bilinear')
+ out = segmentation_fn(tf.constant(images))
+ self.assertAllClose(out['predicted_masks'].numpy(), expected_output.numpy())
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/beta/tasks/__init__.py b/official/vision/beta/tasks/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8410d0d5b44fad9fa2627a24773ebe02c5df19cb
--- /dev/null
+++ b/official/vision/beta/tasks/__init__.py
@@ -0,0 +1,22 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Tasks package definition."""
+
+from official.vision.beta.tasks import image_classification
+from official.vision.beta.tasks import maskrcnn
+from official.vision.beta.tasks import retinanet
+from official.vision.beta.tasks import semantic_segmentation
+from official.vision.beta.tasks import video_classification
diff --git a/official/vision/beta/tasks/image_classification.py b/official/vision/beta/tasks/image_classification.py
new file mode 100644
index 0000000000000000000000000000000000000000..0db37d711d3a4cb4d6f31a9aeb3bf699dff22961
--- /dev/null
+++ b/official/vision/beta/tasks/image_classification.py
@@ -0,0 +1,254 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Image classification task definition."""
+from typing import Any, Optional, List, Tuple
+from absl import logging
+import tensorflow as tf
+
+from official.common import dataset_fn
+from official.core import base_task
+from official.core import task_factory
+from official.modeling import tf_utils
+from official.vision.beta.configs import image_classification as exp_cfg
+from official.vision.beta.dataloaders import classification_input
+from official.vision.beta.dataloaders import input_reader_factory
+from official.vision.beta.dataloaders import tfds_classification_decoders
+from official.vision.beta.modeling import factory
+
+
+@task_factory.register_task_cls(exp_cfg.ImageClassificationTask)
+class ImageClassificationTask(base_task.Task):
+ """A task for image classification."""
+
+ def build_model(self):
+ """Builds classification model."""
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[None] + self.task_config.model.input_size)
+
+ l2_weight_decay = self.task_config.losses.l2_weight_decay
+ # Divide weight decay by 2.0 to match the implementation of tf.nn.l2_loss.
+ # (https://www.tensorflow.org/api_docs/python/tf/keras/regularizers/l2)
+ # (https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss)
+ l2_regularizer = (tf.keras.regularizers.l2(
+ l2_weight_decay / 2.0) if l2_weight_decay else None)
+
+ model = factory.build_classification_model(
+ input_specs=input_specs,
+ model_config=self.task_config.model,
+ l2_regularizer=l2_regularizer)
+ return model
+
+ def initialize(self, model: tf.keras.Model):
+ """Loads pretrained checkpoint."""
+ if not self.task_config.init_checkpoint:
+ return
+
+ ckpt_dir_or_file = self.task_config.init_checkpoint
+ if tf.io.gfile.isdir(ckpt_dir_or_file):
+ ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file)
+
+ # Restoring checkpoint.
+ if self.task_config.init_checkpoint_modules == 'all':
+ ckpt = tf.train.Checkpoint(**model.checkpoint_items)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.assert_consumed()
+ elif self.task_config.init_checkpoint_modules == 'backbone':
+ ckpt = tf.train.Checkpoint(backbone=model.backbone)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.expect_partial().assert_existing_objects_matched()
+ else:
+ raise ValueError(
+ "Only 'all' or 'backbone' can be used to initialize the model.")
+
+ logging.info('Finished loading pretrained checkpoint from %s',
+ ckpt_dir_or_file)
+
+ def build_inputs(self,
+ params: exp_cfg.DataConfig,
+ input_context: Optional[tf.distribute.InputContext] = None):
+ """Builds classification input."""
+
+ num_classes = self.task_config.model.num_classes
+ input_size = self.task_config.model.input_size
+ image_field_key = self.task_config.train_data.image_field_key
+ label_field_key = self.task_config.train_data.label_field_key
+
+ if params.tfds_name:
+ if params.tfds_name in tfds_classification_decoders.TFDS_ID_TO_DECODER_MAP:
+ decoder = tfds_classification_decoders.TFDS_ID_TO_DECODER_MAP[
+ params.tfds_name]()
+ else:
+ raise ValueError('TFDS {} is not supported'.format(params.tfds_name))
+ else:
+ decoder = classification_input.Decoder(
+ image_field_key=image_field_key, label_field_key=label_field_key)
+
+ parser = classification_input.Parser(
+ output_size=input_size[:2],
+ num_classes=num_classes,
+ image_field_key=image_field_key,
+ label_field_key=label_field_key,
+ aug_rand_hflip=params.aug_rand_hflip,
+ aug_type=params.aug_type,
+ dtype=params.dtype)
+
+ reader = input_reader_factory.input_reader_generator(
+ params,
+ dataset_fn=dataset_fn.pick_dataset_fn(params.file_type),
+ decoder_fn=decoder.decode,
+ parser_fn=parser.parse_fn(params.is_training))
+
+ dataset = reader.read(input_context=input_context)
+
+ return dataset
+
+ def build_losses(self,
+ labels: tf.Tensor,
+ model_outputs: tf.Tensor,
+ aux_losses: Optional[Any] = None):
+ """Builds sparse categorical cross entropy loss.
+
+ Args:
+ labels: Input groundtruth labels.
+ model_outputs: Output logits of the classifier.
+ aux_losses: The auxiliarly loss tensors, i.e. `losses` in tf.keras.Model.
+
+ Returns:
+ The total loss tensor.
+ """
+ losses_config = self.task_config.losses
+ if losses_config.one_hot:
+ total_loss = tf.keras.losses.categorical_crossentropy(
+ labels,
+ model_outputs,
+ from_logits=True,
+ label_smoothing=losses_config.label_smoothing)
+ else:
+ total_loss = tf.keras.losses.sparse_categorical_crossentropy(
+ labels, model_outputs, from_logits=True)
+
+ total_loss = tf_utils.safe_mean(total_loss)
+ if aux_losses:
+ total_loss += tf.add_n(aux_losses)
+
+ return total_loss
+
+ def build_metrics(self, training: bool = True):
+ """Gets streaming metrics for training/validation."""
+ k = self.task_config.evaluation.top_k
+ if self.task_config.losses.one_hot:
+ metrics = [
+ tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
+ tf.keras.metrics.TopKCategoricalAccuracy(
+ k=k, name='top_{}_accuracy'.format(k))]
+ else:
+ metrics = [
+ tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
+ tf.keras.metrics.SparseTopKCategoricalAccuracy(
+ k=k, name='top_{}_accuracy'.format(k))]
+ return metrics
+
+ def train_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ optimizer: tf.keras.optimizers.Optimizer,
+ metrics: Optional[List[Any]] = None):
+ """Does forward and backward.
+
+ Args:
+ inputs: A tuple of of input tensors of (features, labels).
+ model: A tf.keras.Model instance.
+ optimizer: The optimizer for this training step.
+ metrics: A nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+ if self.task_config.losses.one_hot:
+ labels = tf.one_hot(labels, self.task_config.model.num_classes)
+
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(features, training=True)
+ # Casting output layer as float32 is necessary when mixed_precision is
+ # mixed_float16 or mixed_bfloat16 to ensure output is casted as float32.
+ outputs = tf.nest.map_structure(
+ lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss.
+ loss = self.build_losses(
+ model_outputs=outputs, labels=labels, aux_losses=model.losses)
+ # Scales loss as the default gradients allreduce performs sum inside the
+ # optimizer.
+ scaled_loss = loss / num_replicas
+
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(
+ optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient before apply_gradients when LossScaleOptimizer is
+ # used.
+ if isinstance(
+ optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = {self.loss: loss}
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ elif model.compiled_metrics:
+ self.process_compiled_metrics(model.compiled_metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in model.metrics})
+ return logs
+
+ def validation_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ metrics: Optional[List[Any]] = None):
+ """Runs validatation step.
+
+ Args:
+ inputs: A tuple of of input tensors of (features, labels).
+ model: A tf.keras.Model instance.
+ metrics: A nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+ if self.task_config.losses.one_hot:
+ labels = tf.one_hot(labels, self.task_config.model.num_classes)
+
+ outputs = self.inference_step(features, model)
+ outputs = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), outputs)
+ loss = self.build_losses(model_outputs=outputs, labels=labels,
+ aux_losses=model.losses)
+
+ logs = {self.loss: loss}
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ elif model.compiled_metrics:
+ self.process_compiled_metrics(model.compiled_metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in model.metrics})
+ return logs
+
+ def inference_step(self, inputs: tf.Tensor, model: tf.keras.Model):
+ """Performs the forward step."""
+ return model(inputs, training=False)
diff --git a/official/vision/beta/tasks/maskrcnn.py b/official/vision/beta/tasks/maskrcnn.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e290dac652b321826c0337cd5508882dcaec53f
--- /dev/null
+++ b/official/vision/beta/tasks/maskrcnn.py
@@ -0,0 +1,375 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""RetinaNet task definition."""
+from typing import Any, Optional, List, Tuple, Mapping
+
+from absl import logging
+import tensorflow as tf
+from official.common import dataset_fn
+from official.core import base_task
+from official.core import task_factory
+from official.vision.beta.configs import maskrcnn as exp_cfg
+from official.vision.beta.dataloaders import input_reader_factory
+from official.vision.beta.dataloaders import maskrcnn_input
+from official.vision.beta.dataloaders import tf_example_decoder
+from official.vision.beta.dataloaders import tf_example_label_map_decoder
+from official.vision.beta.evaluation import coco_evaluator
+from official.vision.beta.losses import maskrcnn_losses
+from official.vision.beta.modeling import factory
+
+
+def zero_out_disallowed_class_ids(batch_class_ids: tf.Tensor,
+ allowed_class_ids: List[int]):
+ """Zero out IDs of classes not in allowed_class_ids.
+
+ Args:
+ batch_class_ids: A [batch_size, num_instances] int tensor of input
+ class IDs.
+ allowed_class_ids: A python list of class IDs which we want to allow.
+
+ Returns:
+ filtered_class_ids: A [batch_size, num_instances] int tensor with any
+ class ID not in allowed_class_ids set to 0.
+ """
+
+ allowed_class_ids = tf.constant(allowed_class_ids,
+ dtype=batch_class_ids.dtype)
+
+ match_ids = (batch_class_ids[:, :, tf.newaxis] ==
+ allowed_class_ids[tf.newaxis, tf.newaxis, :])
+
+ match_ids = tf.reduce_any(match_ids, axis=2)
+ return tf.where(match_ids, batch_class_ids, tf.zeros_like(batch_class_ids))
+
+
+@task_factory.register_task_cls(exp_cfg.MaskRCNNTask)
+class MaskRCNNTask(base_task.Task):
+ """A single-replica view of training procedure.
+
+ Mask R-CNN task provides artifacts for training/evalution procedures,
+ including loading/iterating over Datasets, initializing the model, calculating
+ the loss, post-processing, and customized metrics with reduction.
+ """
+
+ def build_model(self):
+ """Build Mask R-CNN model."""
+
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[None] + self.task_config.model.input_size)
+
+ l2_weight_decay = self.task_config.losses.l2_weight_decay
+ # Divide weight decay by 2.0 to match the implementation of tf.nn.l2_loss.
+ # (https://www.tensorflow.org/api_docs/python/tf/keras/regularizers/l2)
+ # (https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss)
+ l2_regularizer = (tf.keras.regularizers.l2(
+ l2_weight_decay / 2.0) if l2_weight_decay else None)
+
+ model = factory.build_maskrcnn(
+ input_specs=input_specs,
+ model_config=self.task_config.model,
+ l2_regularizer=l2_regularizer)
+ return model
+
+ def initialize(self, model: tf.keras.Model):
+ """Loading pretrained checkpoint."""
+ if not self.task_config.init_checkpoint:
+ return
+
+ ckpt_dir_or_file = self.task_config.init_checkpoint
+ if tf.io.gfile.isdir(ckpt_dir_or_file):
+ ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file)
+
+ # Restoring checkpoint.
+ if self.task_config.init_checkpoint_modules == 'all':
+ ckpt = tf.train.Checkpoint(**model.checkpoint_items)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.assert_consumed()
+ elif self.task_config.init_checkpoint_modules == 'backbone':
+ ckpt = tf.train.Checkpoint(backbone=model.backbone)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.expect_partial().assert_existing_objects_matched()
+ else:
+ raise ValueError(
+ "Only 'all' or 'backbone' can be used to initialize the model.")
+
+ logging.info('Finished loading pretrained checkpoint from %s',
+ ckpt_dir_or_file)
+
+ def build_inputs(self,
+ params: exp_cfg.DataConfig,
+ input_context: Optional[tf.distribute.InputContext] = None):
+ """Build input dataset."""
+ decoder_cfg = params.decoder.get()
+ if params.decoder.type == 'simple_decoder':
+ decoder = tf_example_decoder.TfExampleDecoder(
+ include_mask=self._task_config.model.include_mask,
+ regenerate_source_id=decoder_cfg.regenerate_source_id,
+ mask_binarize_threshold=decoder_cfg.mask_binarize_threshold)
+ elif params.decoder.type == 'label_map_decoder':
+ decoder = tf_example_label_map_decoder.TfExampleDecoderLabelMap(
+ label_map=decoder_cfg.label_map,
+ include_mask=self._task_config.model.include_mask,
+ regenerate_source_id=decoder_cfg.regenerate_source_id,
+ mask_binarize_threshold=decoder_cfg.mask_binarize_threshold)
+ else:
+ raise ValueError('Unknown decoder type: {}!'.format(params.decoder.type))
+
+ parser = maskrcnn_input.Parser(
+ output_size=self.task_config.model.input_size[:2],
+ min_level=self.task_config.model.min_level,
+ max_level=self.task_config.model.max_level,
+ num_scales=self.task_config.model.anchor.num_scales,
+ aspect_ratios=self.task_config.model.anchor.aspect_ratios,
+ anchor_size=self.task_config.model.anchor.anchor_size,
+ dtype=params.dtype,
+ rpn_match_threshold=params.parser.rpn_match_threshold,
+ rpn_unmatched_threshold=params.parser.rpn_unmatched_threshold,
+ rpn_batch_size_per_im=params.parser.rpn_batch_size_per_im,
+ rpn_fg_fraction=params.parser.rpn_fg_fraction,
+ aug_rand_hflip=params.parser.aug_rand_hflip,
+ aug_scale_min=params.parser.aug_scale_min,
+ aug_scale_max=params.parser.aug_scale_max,
+ skip_crowd_during_training=params.parser.skip_crowd_during_training,
+ max_num_instances=params.parser.max_num_instances,
+ include_mask=self._task_config.model.include_mask,
+ mask_crop_size=params.parser.mask_crop_size)
+
+ reader = input_reader_factory.input_reader_generator(
+ params,
+ dataset_fn=dataset_fn.pick_dataset_fn(params.file_type),
+ decoder_fn=decoder.decode,
+ parser_fn=parser.parse_fn(params.is_training))
+ dataset = reader.read(input_context=input_context)
+
+ return dataset
+
+ def build_losses(self,
+ outputs: Mapping[str, Any],
+ labels: Mapping[str, Any],
+ aux_losses: Optional[Any] = None):
+ """Build Mask R-CNN losses."""
+ params = self.task_config
+ cascade_ious = params.model.roi_sampler.cascade_iou_thresholds
+
+ rpn_score_loss_fn = maskrcnn_losses.RpnScoreLoss(
+ tf.shape(outputs['box_outputs'])[1])
+ rpn_box_loss_fn = maskrcnn_losses.RpnBoxLoss(
+ params.losses.rpn_huber_loss_delta)
+ rpn_score_loss = tf.reduce_mean(
+ rpn_score_loss_fn(
+ outputs['rpn_scores'], labels['rpn_score_targets']))
+ rpn_box_loss = tf.reduce_mean(
+ rpn_box_loss_fn(
+ outputs['rpn_boxes'], labels['rpn_box_targets']))
+
+ frcnn_cls_loss_fn = maskrcnn_losses.FastrcnnClassLoss()
+ frcnn_box_loss_fn = maskrcnn_losses.FastrcnnBoxLoss(
+ params.losses.frcnn_huber_loss_delta,
+ params.model.detection_head.class_agnostic_bbox_pred)
+
+ # Final cls/box losses are computed as an average of all detection heads.
+ frcnn_cls_loss = 0.0
+ frcnn_box_loss = 0.0
+ num_det_heads = 1 if cascade_ious is None else 1 + len(cascade_ious)
+ for cas_num in range(num_det_heads):
+ frcnn_cls_loss_i = tf.reduce_mean(
+ frcnn_cls_loss_fn(
+ outputs['class_outputs_{}'
+ .format(cas_num) if cas_num else 'class_outputs'],
+ outputs['class_targets_{}'
+ .format(cas_num) if cas_num else 'class_targets']))
+ frcnn_box_loss_i = tf.reduce_mean(
+ frcnn_box_loss_fn(
+ outputs['box_outputs_{}'.format(cas_num
+ ) if cas_num else 'box_outputs'],
+ outputs['class_targets_{}'
+ .format(cas_num) if cas_num else 'class_targets'],
+ outputs['box_targets_{}'.format(cas_num
+ ) if cas_num else 'box_targets']))
+ frcnn_cls_loss += frcnn_cls_loss_i
+ frcnn_box_loss += frcnn_box_loss_i
+ frcnn_cls_loss /= num_det_heads
+ frcnn_box_loss /= num_det_heads
+
+ if params.model.include_mask:
+ mask_loss_fn = maskrcnn_losses.MaskrcnnLoss()
+ mask_class_targets = outputs['mask_class_targets']
+ if self._task_config.allowed_mask_class_ids is not None:
+ # Classes with ID=0 are ignored by mask_loss_fn in loss computation.
+ mask_class_targets = zero_out_disallowed_class_ids(
+ mask_class_targets, self._task_config.allowed_mask_class_ids)
+
+ mask_loss = tf.reduce_mean(
+ mask_loss_fn(
+ outputs['mask_outputs'],
+ outputs['mask_targets'],
+ mask_class_targets))
+ else:
+ mask_loss = 0.0
+
+ model_loss = (
+ params.losses.rpn_score_weight * rpn_score_loss +
+ params.losses.rpn_box_weight * rpn_box_loss +
+ params.losses.frcnn_class_weight * frcnn_cls_loss +
+ params.losses.frcnn_box_weight * frcnn_box_loss +
+ params.losses.mask_weight * mask_loss)
+
+ total_loss = model_loss
+ if aux_losses:
+ reg_loss = tf.reduce_sum(aux_losses)
+ total_loss = model_loss + reg_loss
+
+ losses = {
+ 'total_loss': total_loss,
+ 'rpn_score_loss': rpn_score_loss,
+ 'rpn_box_loss': rpn_box_loss,
+ 'frcnn_cls_loss': frcnn_cls_loss,
+ 'frcnn_box_loss': frcnn_box_loss,
+ 'mask_loss': mask_loss,
+ 'model_loss': model_loss,
+ }
+ return losses
+
+ def build_metrics(self, training: bool = True):
+ """Build detection metrics."""
+ metrics = []
+ if training:
+ metric_names = [
+ 'total_loss',
+ 'rpn_score_loss',
+ 'rpn_box_loss',
+ 'frcnn_cls_loss',
+ 'frcnn_box_loss',
+ 'mask_loss',
+ 'model_loss'
+ ]
+ for name in metric_names:
+ metrics.append(tf.keras.metrics.Mean(name, dtype=tf.float32))
+
+ else:
+ self.coco_metric = coco_evaluator.COCOEvaluator(
+ annotation_file=self._task_config.annotation_file,
+ include_mask=self._task_config.model.include_mask,
+ per_category_metrics=self._task_config.per_category_metrics)
+
+ return metrics
+
+ def train_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ optimizer: tf.keras.optimizers.Optimizer,
+ metrics: Optional[List[Any]] = None):
+ """Does forward and backward.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the model, forward pass definition.
+ optimizer: the optimizer for this training step.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ images, labels = inputs
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(
+ images,
+ image_shape=labels['image_info'][:, 1, :],
+ anchor_boxes=labels['anchor_boxes'],
+ gt_boxes=labels['gt_boxes'],
+ gt_classes=labels['gt_classes'],
+ gt_masks=(labels['gt_masks'] if self.task_config.model.include_mask
+ else None),
+ training=True)
+ outputs = tf.nest.map_structure(
+ lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss.
+ losses = self.build_losses(
+ outputs=outputs, labels=labels, aux_losses=model.losses)
+ scaled_loss = losses['total_loss'] / num_replicas
+
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient when LossScaleOptimizer is used.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = {self.loss: losses['total_loss']}
+
+ if metrics:
+ for m in metrics:
+ m.update_state(losses[m.name])
+
+ return logs
+
+ def validation_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ metrics: Optional[List[Any]] = None):
+ """Validatation step.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the keras.Model.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ images, labels = inputs
+
+ outputs = model(
+ images,
+ anchor_boxes=labels['anchor_boxes'],
+ image_shape=labels['image_info'][:, 1, :],
+ training=False)
+
+ logs = {self.loss: 0}
+ coco_model_outputs = {
+ 'detection_boxes': outputs['detection_boxes'],
+ 'detection_scores': outputs['detection_scores'],
+ 'detection_classes': outputs['detection_classes'],
+ 'num_detections': outputs['num_detections'],
+ 'source_id': labels['groundtruths']['source_id'],
+ 'image_info': labels['image_info']
+ }
+ if self.task_config.model.include_mask:
+ coco_model_outputs.update({
+ 'detection_masks': outputs['detection_masks'],
+ })
+ logs.update({
+ self.coco_metric.name: (labels['groundtruths'], coco_model_outputs)
+ })
+ return logs
+
+ def aggregate_logs(self, state=None, step_outputs=None):
+ if state is None:
+ self.coco_metric.reset_states()
+ state = self.coco_metric
+ self.coco_metric.update_state(
+ step_outputs[self.coco_metric.name][0],
+ step_outputs[self.coco_metric.name][1])
+ return state
+
+ def reduce_aggregated_logs(self, aggregated_logs, global_step=None):
+ return self.coco_metric.result()
diff --git a/official/vision/beta/tasks/retinanet.py b/official/vision/beta/tasks/retinanet.py
new file mode 100644
index 0000000000000000000000000000000000000000..a1dca4205c5b83ac8b1d9cde2a7299ea55b6e14f
--- /dev/null
+++ b/official/vision/beta/tasks/retinanet.py
@@ -0,0 +1,308 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""RetinaNet task definition."""
+from typing import Any, Optional, List, Tuple, Mapping
+
+from absl import logging
+import tensorflow as tf
+from official.common import dataset_fn
+from official.core import base_task
+from official.core import task_factory
+from official.vision import keras_cv
+from official.vision.beta.configs import retinanet as exp_cfg
+from official.vision.beta.dataloaders import input_reader_factory
+from official.vision.beta.dataloaders import retinanet_input
+from official.vision.beta.dataloaders import tf_example_decoder
+from official.vision.beta.dataloaders import tfds_detection_decoders
+from official.vision.beta.dataloaders import tf_example_label_map_decoder
+from official.vision.beta.evaluation import coco_evaluator
+from official.vision.beta.modeling import factory
+
+
+@task_factory.register_task_cls(exp_cfg.RetinaNetTask)
+class RetinaNetTask(base_task.Task):
+ """A single-replica view of training procedure.
+
+ RetinaNet task provides artifacts for training/evalution procedures, including
+ loading/iterating over Datasets, initializing the model, calculating the loss,
+ post-processing, and customized metrics with reduction.
+ """
+
+ def build_model(self):
+ """Build RetinaNet model."""
+
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[None] + self.task_config.model.input_size)
+
+ l2_weight_decay = self.task_config.losses.l2_weight_decay
+ # Divide weight decay by 2.0 to match the implementation of tf.nn.l2_loss.
+ # (https://www.tensorflow.org/api_docs/python/tf/keras/regularizers/l2)
+ # (https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss)
+ l2_regularizer = (tf.keras.regularizers.l2(
+ l2_weight_decay / 2.0) if l2_weight_decay else None)
+
+ model = factory.build_retinanet(
+ input_specs=input_specs,
+ model_config=self.task_config.model,
+ l2_regularizer=l2_regularizer)
+ return model
+
+ def initialize(self, model: tf.keras.Model):
+ """Loading pretrained checkpoint."""
+ if not self.task_config.init_checkpoint:
+ return
+
+ ckpt_dir_or_file = self.task_config.init_checkpoint
+ if tf.io.gfile.isdir(ckpt_dir_or_file):
+ ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file)
+
+ # Restoring checkpoint.
+ if self.task_config.init_checkpoint_modules == 'all':
+ ckpt = tf.train.Checkpoint(**model.checkpoint_items)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.assert_consumed()
+ elif self.task_config.init_checkpoint_modules == 'backbone':
+ ckpt = tf.train.Checkpoint(backbone=model.backbone)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.expect_partial().assert_existing_objects_matched()
+ else:
+ raise ValueError(
+ "Only 'all' or 'backbone' can be used to initialize the model.")
+
+ logging.info('Finished loading pretrained checkpoint from %s',
+ ckpt_dir_or_file)
+
+ def build_inputs(self,
+ params: exp_cfg.DataConfig,
+ input_context: Optional[tf.distribute.InputContext] = None):
+ """Build input dataset."""
+
+ if params.tfds_name:
+ if params.tfds_name in tfds_detection_decoders.TFDS_ID_TO_DECODER_MAP:
+ decoder = tfds_detection_decoders.TFDS_ID_TO_DECODER_MAP[
+ params.tfds_name]()
+ else:
+ raise ValueError('TFDS {} is not supported'.format(params.tfds_name))
+ else:
+ decoder_cfg = params.decoder.get()
+ if params.decoder.type == 'simple_decoder':
+ decoder = tf_example_decoder.TfExampleDecoder(
+ regenerate_source_id=decoder_cfg.regenerate_source_id)
+ elif params.decoder.type == 'label_map_decoder':
+ decoder = tf_example_label_map_decoder.TfExampleDecoderLabelMap(
+ label_map=decoder_cfg.label_map,
+ regenerate_source_id=decoder_cfg.regenerate_source_id)
+ else:
+ raise ValueError('Unknown decoder type: {}!'.format(
+ params.decoder.type))
+
+ parser = retinanet_input.Parser(
+ output_size=self.task_config.model.input_size[:2],
+ min_level=self.task_config.model.min_level,
+ max_level=self.task_config.model.max_level,
+ num_scales=self.task_config.model.anchor.num_scales,
+ aspect_ratios=self.task_config.model.anchor.aspect_ratios,
+ anchor_size=self.task_config.model.anchor.anchor_size,
+ dtype=params.dtype,
+ match_threshold=params.parser.match_threshold,
+ unmatched_threshold=params.parser.unmatched_threshold,
+ aug_rand_hflip=params.parser.aug_rand_hflip,
+ aug_scale_min=params.parser.aug_scale_min,
+ aug_scale_max=params.parser.aug_scale_max,
+ skip_crowd_during_training=params.parser.skip_crowd_during_training,
+ max_num_instances=params.parser.max_num_instances)
+
+ reader = input_reader_factory.input_reader_generator(
+ params,
+ dataset_fn=dataset_fn.pick_dataset_fn(params.file_type),
+ decoder_fn=decoder.decode,
+ parser_fn=parser.parse_fn(params.is_training))
+ dataset = reader.read(input_context=input_context)
+
+ return dataset
+
+ def build_losses(self,
+ outputs: Mapping[str, Any],
+ labels: Mapping[str, Any],
+ aux_losses: Optional[Any] = None):
+ """Build RetinaNet losses."""
+ params = self.task_config
+ cls_loss_fn = keras_cv.losses.FocalLoss(
+ alpha=params.losses.focal_loss_alpha,
+ gamma=params.losses.focal_loss_gamma,
+ reduction=tf.keras.losses.Reduction.SUM)
+ box_loss_fn = tf.keras.losses.Huber(
+ params.losses.huber_loss_delta, reduction=tf.keras.losses.Reduction.SUM)
+
+ # Sums all positives in a batch for normalization and avoids zero
+ # num_positives_sum, which would lead to inf loss during training
+ cls_sample_weight = labels['cls_weights']
+ box_sample_weight = labels['box_weights']
+ num_positives = tf.reduce_sum(box_sample_weight) + 1.0
+ cls_sample_weight = cls_sample_weight / num_positives
+ box_sample_weight = box_sample_weight / num_positives
+ y_true_cls = keras_cv.losses.multi_level_flatten(
+ labels['cls_targets'], last_dim=None)
+ y_true_cls = tf.one_hot(y_true_cls, params.model.num_classes)
+ y_pred_cls = keras_cv.losses.multi_level_flatten(
+ outputs['cls_outputs'], last_dim=params.model.num_classes)
+ y_true_box = keras_cv.losses.multi_level_flatten(
+ labels['box_targets'], last_dim=4)
+ y_pred_box = keras_cv.losses.multi_level_flatten(
+ outputs['box_outputs'], last_dim=4)
+
+ cls_loss = cls_loss_fn(
+ y_true=y_true_cls, y_pred=y_pred_cls, sample_weight=cls_sample_weight)
+ box_loss = box_loss_fn(
+ y_true=y_true_box, y_pred=y_pred_box, sample_weight=box_sample_weight)
+
+ model_loss = cls_loss + params.losses.box_loss_weight * box_loss
+
+ total_loss = model_loss
+ if aux_losses:
+ reg_loss = tf.reduce_sum(aux_losses)
+ total_loss = model_loss + reg_loss
+
+ return total_loss, cls_loss, box_loss, model_loss
+
+ def build_metrics(self, training: bool = True):
+ """Build detection metrics."""
+ metrics = []
+ metric_names = ['total_loss', 'cls_loss', 'box_loss', 'model_loss']
+ for name in metric_names:
+ metrics.append(tf.keras.metrics.Mean(name, dtype=tf.float32))
+
+ if not training:
+ if self.task_config.validation_data.tfds_name and self.task_config.annotation_file:
+ raise ValueError(
+ "Can't evaluate using annotation file when TFDS is used.")
+ self.coco_metric = coco_evaluator.COCOEvaluator(
+ annotation_file=self.task_config.annotation_file,
+ include_mask=False,
+ per_category_metrics=self.task_config.per_category_metrics)
+
+ return metrics
+
+ def train_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ optimizer: tf.keras.optimizers.Optimizer,
+ metrics: Optional[List[Any]] = None):
+ """Does forward and backward.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the model, forward pass definition.
+ optimizer: the optimizer for this training step.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(features, training=True)
+ outputs = tf.nest.map_structure(
+ lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss.
+ loss, cls_loss, box_loss, model_loss = self.build_losses(
+ outputs=outputs, labels=labels, aux_losses=model.losses)
+ scaled_loss = loss / num_replicas
+
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient when LossScaleOptimizer is used.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = {self.loss: loss}
+
+ all_losses = {
+ 'total_loss': loss,
+ 'cls_loss': cls_loss,
+ 'box_loss': box_loss,
+ 'model_loss': model_loss,
+ }
+ if metrics:
+ for m in metrics:
+ m.update_state(all_losses[m.name])
+ logs.update({m.name: m.result()})
+
+ return logs
+
+ def validation_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ metrics: Optional[List[Any]] = None):
+ """Validatation step.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the keras.Model.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+
+ outputs = model(features, anchor_boxes=labels['anchor_boxes'],
+ image_shape=labels['image_info'][:, 1, :],
+ training=False)
+ loss, cls_loss, box_loss, model_loss = self.build_losses(
+ outputs=outputs, labels=labels, aux_losses=model.losses)
+ logs = {self.loss: loss}
+
+ all_losses = {
+ 'total_loss': loss,
+ 'cls_loss': cls_loss,
+ 'box_loss': box_loss,
+ 'model_loss': model_loss,
+ }
+
+ coco_model_outputs = {
+ 'detection_boxes': outputs['detection_boxes'],
+ 'detection_scores': outputs['detection_scores'],
+ 'detection_classes': outputs['detection_classes'],
+ 'num_detections': outputs['num_detections'],
+ 'source_id': labels['groundtruths']['source_id'],
+ 'image_info': labels['image_info']
+ }
+ logs.update({self.coco_metric.name: (labels['groundtruths'],
+ coco_model_outputs)})
+ if metrics:
+ for m in metrics:
+ m.update_state(all_losses[m.name])
+ logs.update({m.name: m.result()})
+ return logs
+
+ def aggregate_logs(self, state=None, step_outputs=None):
+ if state is None:
+ self.coco_metric.reset_states()
+ state = self.coco_metric
+ self.coco_metric.update_state(step_outputs[self.coco_metric.name][0],
+ step_outputs[self.coco_metric.name][1])
+ return state
+
+ def reduce_aggregated_logs(self, aggregated_logs, global_step=None):
+ return self.coco_metric.result()
diff --git a/official/vision/beta/tasks/semantic_segmentation.py b/official/vision/beta/tasks/semantic_segmentation.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd01f78a6db40351a816aaae03b2268b3803cc56
--- /dev/null
+++ b/official/vision/beta/tasks/semantic_segmentation.py
@@ -0,0 +1,287 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Image segmentation task definition."""
+from typing import Any, Optional, List, Tuple, Mapping, Union
+
+from absl import logging
+import tensorflow as tf
+from official.common import dataset_fn
+from official.core import base_task
+from official.core import task_factory
+from official.vision.beta.configs import semantic_segmentation as exp_cfg
+from official.vision.beta.dataloaders import input_reader_factory
+from official.vision.beta.dataloaders import segmentation_input
+from official.vision.beta.dataloaders import tfds_segmentation_decoders
+from official.vision.beta.evaluation import segmentation_metrics
+from official.vision.beta.losses import segmentation_losses
+from official.vision.beta.modeling import factory
+
+
+@task_factory.register_task_cls(exp_cfg.SemanticSegmentationTask)
+class SemanticSegmentationTask(base_task.Task):
+ """A task for semantic segmentation."""
+
+ def build_model(self):
+ """Builds segmentation model."""
+ input_specs = tf.keras.layers.InputSpec(
+ shape=[None] + self.task_config.model.input_size)
+
+ l2_weight_decay = self.task_config.losses.l2_weight_decay
+ # Divide weight decay by 2.0 to match the implementation of tf.nn.l2_loss.
+ # (https://www.tensorflow.org/api_docs/python/tf/keras/regularizers/l2)
+ # (https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss)
+ l2_regularizer = (tf.keras.regularizers.l2(
+ l2_weight_decay / 2.0) if l2_weight_decay else None)
+
+ model = factory.build_segmentation_model(
+ input_specs=input_specs,
+ model_config=self.task_config.model,
+ l2_regularizer=l2_regularizer)
+ return model
+
+ def initialize(self, model: tf.keras.Model):
+ """Loads pretrained checkpoint."""
+ if not self.task_config.init_checkpoint:
+ return
+
+ ckpt_dir_or_file = self.task_config.init_checkpoint
+ if tf.io.gfile.isdir(ckpt_dir_or_file):
+ ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file)
+
+ # Restoring checkpoint.
+ if 'all' in self.task_config.init_checkpoint_modules:
+ ckpt = tf.train.Checkpoint(**model.checkpoint_items)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.assert_consumed()
+ else:
+ ckpt_items = {}
+ if 'backbone' in self.task_config.init_checkpoint_modules:
+ ckpt_items.update(backbone=model.backbone)
+ if 'decoder' in self.task_config.init_checkpoint_modules:
+ ckpt_items.update(decoder=model.decoder)
+
+ ckpt = tf.train.Checkpoint(**ckpt_items)
+ status = ckpt.restore(ckpt_dir_or_file)
+ status.expect_partial().assert_existing_objects_matched()
+
+ logging.info('Finished loading pretrained checkpoint from %s',
+ ckpt_dir_or_file)
+
+ def build_inputs(self,
+ params: exp_cfg.DataConfig,
+ input_context: Optional[tf.distribute.InputContext] = None):
+ """Builds classification input."""
+
+ ignore_label = self.task_config.losses.ignore_label
+
+ if params.tfds_name:
+ if params.tfds_name in tfds_segmentation_decoders.TFDS_ID_TO_DECODER_MAP:
+ decoder = tfds_segmentation_decoders.TFDS_ID_TO_DECODER_MAP[
+ params.tfds_name]()
+ else:
+ raise ValueError('TFDS {} is not supported'.format(params.tfds_name))
+ else:
+ decoder = segmentation_input.Decoder()
+
+ parser = segmentation_input.Parser(
+ output_size=params.output_size,
+ crop_size=params.crop_size,
+ ignore_label=ignore_label,
+ resize_eval_groundtruth=params.resize_eval_groundtruth,
+ groundtruth_padded_size=params.groundtruth_padded_size,
+ aug_scale_min=params.aug_scale_min,
+ aug_scale_max=params.aug_scale_max,
+ aug_rand_hflip=params.aug_rand_hflip,
+ dtype=params.dtype)
+
+ reader = input_reader_factory.input_reader_generator(
+ params,
+ dataset_fn=dataset_fn.pick_dataset_fn(params.file_type),
+ decoder_fn=decoder.decode,
+ parser_fn=parser.parse_fn(params.is_training))
+
+ dataset = reader.read(input_context=input_context)
+
+ return dataset
+
+ def build_losses(self,
+ labels: Mapping[str, tf.Tensor],
+ model_outputs: Union[Mapping[str, tf.Tensor], tf.Tensor],
+ aux_losses: Optional[Any] = None):
+ """Segmentation loss.
+
+ Args:
+ labels: labels.
+ model_outputs: Output logits of the classifier.
+ aux_losses: auxiliarly loss tensors, i.e. `losses` in keras.Model.
+
+ Returns:
+ The total loss tensor.
+ """
+ loss_params = self._task_config.losses
+ segmentation_loss_fn = segmentation_losses.SegmentationLoss(
+ loss_params.label_smoothing,
+ loss_params.class_weights,
+ loss_params.ignore_label,
+ use_groundtruth_dimension=loss_params.use_groundtruth_dimension,
+ top_k_percent_pixels=loss_params.top_k_percent_pixels)
+
+ total_loss = segmentation_loss_fn(model_outputs, labels['masks'])
+
+ if aux_losses:
+ total_loss += tf.add_n(aux_losses)
+
+ return total_loss
+
+ def build_metrics(self, training: bool = True):
+ """Gets streaming metrics for training/validation."""
+ metrics = []
+ if training and self.task_config.evaluation.report_train_mean_iou:
+ metrics.append(segmentation_metrics.MeanIoU(
+ name='mean_iou',
+ num_classes=self.task_config.model.num_classes,
+ rescale_predictions=False,
+ dtype=tf.float32))
+ else:
+ self.iou_metric = segmentation_metrics.PerClassIoU(
+ name='per_class_iou',
+ num_classes=self.task_config.model.num_classes,
+ rescale_predictions=not self.task_config.validation_data
+ .resize_eval_groundtruth,
+ dtype=tf.float32)
+
+ return metrics
+
+ def train_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ optimizer: tf.keras.optimizers.Optimizer,
+ metrics: Optional[List[Any]] = None):
+ """Does forward and backward.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the model, forward pass definition.
+ optimizer: the optimizer for this training step.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+
+ input_partition_dims = self.task_config.train_input_partition_dims
+ if input_partition_dims:
+ strategy = tf.distribute.get_strategy()
+ features = strategy.experimental_split_to_logical_devices(
+ features, input_partition_dims)
+
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(features, training=True)
+ # Casting output layer as float32 is necessary when mixed_precision is
+ # mixed_float16 or mixed_bfloat16 to ensure output is casted as float32.
+ outputs = tf.nest.map_structure(
+ lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss.
+ loss = self.build_losses(
+ model_outputs=outputs, labels=labels, aux_losses=model.losses)
+ # Scales loss as the default gradients allreduce performs sum inside the
+ # optimizer.
+ scaled_loss = loss / num_replicas
+
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient before apply_gradients when LossScaleOptimizer is
+ # used.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = {self.loss: loss}
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in metrics})
+
+ return logs
+
+ def validation_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ metrics: Optional[List[Any]] = None):
+ """Validatation step.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the keras.Model.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+
+ input_partition_dims = self.task_config.eval_input_partition_dims
+ if input_partition_dims:
+ strategy = tf.distribute.get_strategy()
+ features = strategy.experimental_split_to_logical_devices(
+ features, input_partition_dims)
+
+ outputs = self.inference_step(features, model)
+ outputs = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), outputs)
+
+ if self.task_config.validation_data.resize_eval_groundtruth:
+ loss = self.build_losses(model_outputs=outputs, labels=labels,
+ aux_losses=model.losses)
+ else:
+ loss = 0
+
+ logs = {self.loss: loss}
+ logs.update({self.iou_metric.name: (labels, outputs)})
+
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in metrics})
+
+ return logs
+
+ def inference_step(self, inputs: tf.Tensor, model: tf.keras.Model):
+ """Performs the forward step."""
+ return model(inputs, training=False)
+
+ def aggregate_logs(self, state=None, step_outputs=None):
+ if state is None:
+ self.iou_metric.reset_states()
+ state = self.iou_metric
+ self.iou_metric.update_state(step_outputs[self.iou_metric.name][0],
+ step_outputs[self.iou_metric.name][1])
+ return state
+
+ def reduce_aggregated_logs(self, aggregated_logs, global_step=None):
+ result = {}
+ ious = self.iou_metric.result()
+ # TODO(arashwan): support loading class name from a label map file.
+ if self.task_config.evaluation.report_per_class_iou:
+ for i, value in enumerate(ious.numpy()):
+ result.update({'iou/{}'.format(i): value})
+ # Computes mean IoU
+ result.update({'mean_iou': tf.reduce_mean(ious).numpy()})
+ return result
diff --git a/official/vision/beta/tasks/video_classification.py b/official/vision/beta/tasks/video_classification.py
new file mode 100644
index 0000000000000000000000000000000000000000..125d5ba4bd71ea6972be0ddf9c85aaeb567ee720
--- /dev/null
+++ b/official/vision/beta/tasks/video_classification.py
@@ -0,0 +1,301 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Video classification task definition."""
+from typing import Any, Optional, List, Tuple
+
+from absl import logging
+import tensorflow as tf
+from official.core import base_task
+from official.core import task_factory
+from official.modeling import tf_utils
+from official.vision.beta.configs import video_classification as exp_cfg
+from official.vision.beta.dataloaders import input_reader_factory
+from official.vision.beta.dataloaders import video_input
+from official.vision.beta.modeling import factory_3d
+
+
+@task_factory.register_task_cls(exp_cfg.VideoClassificationTask)
+class VideoClassificationTask(base_task.Task):
+ """A task for video classification."""
+
+ def build_model(self):
+ """Builds video classification model."""
+ common_input_shape = [
+ d1 if d1 == d2 else None
+ for d1, d2 in zip(self.task_config.train_data.feature_shape,
+ self.task_config.validation_data.feature_shape)
+ ]
+ input_specs = tf.keras.layers.InputSpec(shape=[None] + common_input_shape)
+ logging.info('Build model input %r', common_input_shape)
+
+ l2_weight_decay = self.task_config.losses.l2_weight_decay
+ # Divide weight decay by 2.0 to match the implementation of tf.nn.l2_loss.
+ # (https://www.tensorflow.org/api_docs/python/tf/keras/regularizers/l2)
+ # (https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss)
+ l2_regularizer = (tf.keras.regularizers.l2(
+ l2_weight_decay / 2.0) if l2_weight_decay else None)
+
+ model = factory_3d.build_model(
+ self.task_config.model.model_type,
+ input_specs=input_specs,
+ model_config=self.task_config.model,
+ num_classes=self.task_config.train_data.num_classes,
+ l2_regularizer=l2_regularizer)
+ return model
+
+ def _get_dataset_fn(self, params):
+ if params.file_type == 'tfrecord':
+ return tf.data.TFRecordDataset
+ else:
+ raise ValueError('Unknown input file type {!r}'.format(params.file_type))
+
+ def _get_decoder_fn(self, params):
+ decoder = video_input.Decoder(
+ image_key=params.image_field_key, label_key=params.label_field_key)
+ if self.task_config.train_data.output_audio:
+ assert self.task_config.train_data.audio_feature, 'audio feature is empty'
+ decoder.add_feature(self.task_config.train_data.audio_feature,
+ tf.io.VarLenFeature(dtype=tf.float32))
+ return decoder.decode
+
+ def build_inputs(self,
+ params: exp_cfg.DataConfig,
+ input_context: Optional[tf.distribute.InputContext] = None):
+ """Builds classification input."""
+
+ parser = video_input.Parser(
+ input_params=params,
+ image_key=params.image_field_key,
+ label_key=params.label_field_key)
+ postprocess_fn = video_input.PostBatchProcessor(params)
+
+ reader = input_reader_factory.input_reader_generator(
+ params,
+ dataset_fn=self._get_dataset_fn(params),
+ decoder_fn=self._get_decoder_fn(params),
+ parser_fn=parser.parse_fn(params.is_training),
+ postprocess_fn=postprocess_fn)
+
+ dataset = reader.read(input_context=input_context)
+
+ return dataset
+
+ def build_losses(self,
+ labels: Any,
+ model_outputs: Any,
+ aux_losses: Optional[Any] = None):
+ """Sparse categorical cross entropy loss.
+
+ Args:
+ labels: labels.
+ model_outputs: Output logits of the classifier.
+ aux_losses: auxiliarly loss tensors, i.e. `losses` in keras.Model.
+
+ Returns:
+ The total loss tensor.
+ """
+ all_losses = {}
+ losses_config = self.task_config.losses
+ total_loss = None
+ if self.task_config.train_data.is_multilabel:
+ entropy = -tf.reduce_mean(
+ tf.reduce_sum(model_outputs * tf.math.log(model_outputs + 1e-8), -1))
+ total_loss = tf.keras.losses.binary_crossentropy(
+ labels, model_outputs, from_logits=False)
+ all_losses.update({
+ 'class_loss': total_loss,
+ 'entropy': entropy,
+ })
+ else:
+ if losses_config.one_hot:
+ total_loss = tf.keras.losses.categorical_crossentropy(
+ labels,
+ model_outputs,
+ from_logits=False,
+ label_smoothing=losses_config.label_smoothing)
+ else:
+ total_loss = tf.keras.losses.sparse_categorical_crossentropy(
+ labels, model_outputs, from_logits=False)
+
+ total_loss = tf_utils.safe_mean(total_loss)
+ all_losses.update({
+ 'class_loss': total_loss,
+ })
+ if aux_losses:
+ all_losses.update({
+ 'reg_loss': aux_losses,
+ })
+ total_loss += tf.add_n(aux_losses)
+ all_losses[self.loss] = total_loss
+
+ return all_losses
+
+ def build_metrics(self, training: bool = True):
+ """Gets streaming metrics for training/validation."""
+ if self.task_config.losses.one_hot:
+ metrics = [
+ tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
+ tf.keras.metrics.TopKCategoricalAccuracy(k=1, name='top_1_accuracy'),
+ tf.keras.metrics.TopKCategoricalAccuracy(k=5, name='top_5_accuracy')
+ ]
+ if self.task_config.train_data.is_multilabel:
+ metrics.append(
+ tf.keras.metrics.AUC(
+ curve='ROC',
+ multi_label=self.task_config.train_data.is_multilabel,
+ name='ROC-AUC'))
+ metrics.append(
+ tf.keras.metrics.RecallAtPrecision(
+ 0.95, name='RecallAtPrecision95'))
+ metrics.append(
+ tf.keras.metrics.AUC(
+ curve='PR',
+ multi_label=self.task_config.train_data.is_multilabel,
+ name='PR-AUC'))
+ if self.task_config.metrics.use_per_class_recall:
+ for i in range(self.task_config.train_data.num_classes):
+ metrics.append(
+ tf.keras.metrics.Recall(class_id=i, name=f'recall-{i}'))
+ else:
+ metrics = [
+ tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
+ tf.keras.metrics.SparseTopKCategoricalAccuracy(
+ k=1, name='top_1_accuracy'),
+ tf.keras.metrics.SparseTopKCategoricalAccuracy(
+ k=5, name='top_5_accuracy')
+ ]
+ return metrics
+
+ def process_metrics(self, metrics: List[Any], labels: Any,
+ model_outputs: Any):
+ """Process and update metrics.
+
+ Called when using custom training loop API.
+
+ Args:
+ metrics: a nested structure of metrics objects. The return of function
+ self.build_metrics.
+ labels: a tensor or a nested structure of tensors.
+ model_outputs: a tensor or a nested structure of tensors. For example,
+ output of the keras model built by self.build_model.
+ """
+ for metric in metrics:
+ metric.update_state(labels, model_outputs)
+
+ def train_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ optimizer: tf.keras.optimizers.Optimizer,
+ metrics: Optional[List[Any]] = None):
+ """Does forward and backward.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the model, forward pass definition.
+ optimizer: the optimizer for this training step.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+
+ num_replicas = tf.distribute.get_strategy().num_replicas_in_sync
+ with tf.GradientTape() as tape:
+ outputs = model(features, training=True)
+ # Casting output layer as float32 is necessary when mixed_precision is
+ # mixed_float16 or mixed_bfloat16 to ensure output is casted as float32.
+ outputs = tf.nest.map_structure(
+ lambda x: tf.cast(x, tf.float32), outputs)
+
+ # Computes per-replica loss.
+ if self.task_config.train_data.is_multilabel:
+ outputs = tf.math.sigmoid(outputs)
+ else:
+ outputs = tf.math.softmax(outputs)
+ all_losses = self.build_losses(
+ model_outputs=outputs, labels=labels, aux_losses=model.losses)
+ loss = all_losses[self.loss]
+ # Scales loss as the default gradients allreduce performs sum inside the
+ # optimizer.
+ scaled_loss = loss / num_replicas
+
+ # For mixed_precision policy, when LossScaleOptimizer is used, loss is
+ # scaled for numerical stability.
+ if isinstance(
+ optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ scaled_loss = optimizer.get_scaled_loss(scaled_loss)
+
+ tvars = model.trainable_variables
+ grads = tape.gradient(scaled_loss, tvars)
+ # Scales back gradient before apply_gradients when LossScaleOptimizer is
+ # used.
+ if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer):
+ grads = optimizer.get_unscaled_gradients(grads)
+ optimizer.apply_gradients(list(zip(grads, tvars)))
+
+ logs = all_losses
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in metrics})
+ elif model.compiled_metrics:
+ self.process_compiled_metrics(model.compiled_metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in model.metrics})
+ return logs
+
+ def validation_step(self,
+ inputs: Tuple[Any, Any],
+ model: tf.keras.Model,
+ metrics: Optional[List[Any]] = None):
+ """Validatation step.
+
+ Args:
+ inputs: a dictionary of input tensors.
+ model: the keras.Model.
+ metrics: a nested structure of metrics objects.
+
+ Returns:
+ A dictionary of logs.
+ """
+ features, labels = inputs
+
+ outputs = self.inference_step(features, model)
+ outputs = tf.nest.map_structure(lambda x: tf.cast(x, tf.float32), outputs)
+ logs = self.build_losses(model_outputs=outputs, labels=labels,
+ aux_losses=model.losses)
+
+ if metrics:
+ self.process_metrics(metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in metrics})
+ elif model.compiled_metrics:
+ self.process_compiled_metrics(model.compiled_metrics, labels, outputs)
+ logs.update({m.name: m.result() for m in model.metrics})
+ return logs
+
+ def inference_step(self, features: tf.Tensor, model: tf.keras.Model):
+ """Performs the forward step."""
+ outputs = model(features, training=False)
+ if self.task_config.train_data.is_multilabel:
+ outputs = tf.math.sigmoid(outputs)
+ else:
+ outputs = tf.math.softmax(outputs)
+ num_test_clips = self.task_config.validation_data.num_test_clips
+ num_test_crops = self.task_config.validation_data.num_test_crops
+ num_test_views = num_test_clips * num_test_crops
+ if num_test_views > 1:
+ # Averaging output probabilities across multiples views.
+ outputs = tf.reshape(outputs, [-1, num_test_views, outputs.shape[-1]])
+ outputs = tf.reduce_mean(outputs, axis=1)
+ return outputs
diff --git a/official/vision/beta/train.py b/official/vision/beta/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..d922972e4269f08e7ec181f32388c8afd772bc0f
--- /dev/null
+++ b/official/vision/beta/train.py
@@ -0,0 +1,69 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""TensorFlow Model Garden Vision training driver."""
+
+from absl import app
+from absl import flags
+import gin
+
+# pylint: disable=unused-import
+from official.common import registry_imports
+# pylint: enable=unused-import
+from official.common import distribute_utils
+from official.common import flags as tfm_flags
+from official.core import task_factory
+from official.core import train_lib
+from official.core import train_utils
+from official.modeling import performance
+
+FLAGS = flags.FLAGS
+
+
+def main(_):
+ gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params)
+ params = train_utils.parse_configuration(FLAGS)
+ model_dir = FLAGS.model_dir
+ if 'train' in FLAGS.mode:
+ # Pure eval modes do not output yaml files. Otherwise continuous eval job
+ # may race against the train job for writing the same file.
+ train_utils.serialize_config(params, model_dir)
+
+ # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16'
+ # can have significant impact on model speeds by utilizing float16 in case of
+ # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when
+ # dtype is float16
+ if params.runtime.mixed_precision_dtype:
+ performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype)
+ distribution_strategy = distribute_utils.get_distribution_strategy(
+ distribution_strategy=params.runtime.distribution_strategy,
+ all_reduce_alg=params.runtime.all_reduce_alg,
+ num_gpus=params.runtime.num_gpus,
+ tpu_address=params.runtime.tpu)
+ with distribution_strategy.scope():
+ task = task_factory.get_task(params.task, logging_dir=model_dir)
+
+ train_lib.run_experiment(
+ distribution_strategy=distribution_strategy,
+ task=task,
+ mode=FLAGS.mode,
+ params=params,
+ model_dir=model_dir)
+
+ train_utils.save_gin_config(FLAGS.mode, model_dir)
+
+if __name__ == '__main__':
+ tfm_flags.define_flags()
+ app.run(main)
diff --git a/official/vision/beta/train_spatial_partitioning.py b/official/vision/beta/train_spatial_partitioning.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ccd562f40e8633a54a0d1954594be14ffd41bea
--- /dev/null
+++ b/official/vision/beta/train_spatial_partitioning.py
@@ -0,0 +1,136 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""TensorFlow Model Garden Vision training driver with spatial partitioning."""
+
+from absl import app
+from absl import flags
+import gin
+import numpy as np
+import tensorflow as tf
+
+from official.common import registry_imports # pylint: disable=unused-import
+from official.common import distribute_utils
+from official.common import flags as tfm_flags
+from official.core import task_factory
+from official.core import train_lib
+from official.core import train_utils
+from official.modeling import performance
+
+
+FLAGS = flags.FLAGS
+
+
+def get_computation_shape_for_model_parallelism(input_partition_dims):
+ """Return computation shape to be used for TPUStrategy spatial partition."""
+ num_logical_devices = np.prod(input_partition_dims)
+ if num_logical_devices == 1:
+ return [1, 1, 1, 1]
+ if num_logical_devices == 2:
+ return [1, 1, 1, 2]
+ if num_logical_devices == 4:
+ return [1, 2, 1, 2]
+ if num_logical_devices == 8:
+ return [2, 2, 1, 2]
+ if num_logical_devices == 16:
+ return [4, 2, 1, 2]
+
+
+def create_distribution_strategy(distribution_strategy,
+ tpu_address,
+ input_partition_dims=None,
+ num_gpus=None):
+ """Creates distribution strategy to use for computation."""
+
+ if input_partition_dims is not None:
+ if distribution_strategy != 'tpu':
+ raise ValueError('Spatial partitioning is only supported '
+ 'for TPUStrategy.')
+
+ # When `input_partition_dims` is specified create custom TPUStrategy
+ # instance with computation shape for model parallelism.
+ resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
+ tpu=tpu_address)
+ if tpu_address not in ('', 'local'):
+ tf.config.experimental_connect_to_cluster(resolver)
+
+ topology = tf.tpu.experimental.initialize_tpu_system(resolver)
+ num_replicas = resolver.get_tpu_system_metadata().num_cores // np.prod(
+ input_partition_dims)
+ device_assignment = tf.tpu.experimental.DeviceAssignment.build(
+ topology,
+ num_replicas=num_replicas,
+ computation_shape=input_partition_dims)
+ return tf.distribute.TPUStrategy(
+ resolver, experimental_device_assignment=device_assignment)
+
+ return distribute_utils.get_distribution_strategy(
+ distribution_strategy=distribution_strategy,
+ tpu_address=tpu_address,
+ num_gpus=num_gpus)
+
+
+def main(_):
+ gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params)
+ params = train_utils.parse_configuration(FLAGS)
+ model_dir = FLAGS.model_dir
+ if 'train' in FLAGS.mode:
+ # Pure eval modes do not output yaml files. Otherwise continuous eval job
+ # may race against the train job for writing the same file.
+ train_utils.serialize_config(params, model_dir)
+
+ # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16'
+ # can have significant impact on model speeds by utilizing float16 in case of
+ # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when
+ # dtype is float16
+ if params.runtime.mixed_precision_dtype:
+ performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype)
+
+ input_partition_dims = None
+ if FLAGS.mode == 'train_and_eval':
+ if np.prod(params.task.train_input_partition_dims) != np.prod(
+ params.task.eval_input_partition_dims):
+ raise ValueError('Train and eval input partition dims can not be'
+ 'partitioned on the same node')
+ else:
+ input_partition_dims = get_computation_shape_for_model_parallelism(
+ params.task.train_input_partition_dims)
+ elif FLAGS.mode == 'train':
+ if params.task.train_input_partition_dims:
+ input_partition_dims = get_computation_shape_for_model_parallelism(
+ params.task.train_input_partition_dims)
+ elif FLAGS.mode == 'eval' or FLAGS.mode == 'continuous_eval':
+ if params.task.eval_input_partition_dims:
+ input_partition_dims = get_computation_shape_for_model_parallelism(
+ params.task.eval_input_partition_dims)
+
+ distribution_strategy = create_distribution_strategy(
+ distribution_strategy=params.runtime.distribution_strategy,
+ num_gpus=params.runtime.num_gpus,
+ input_partition_dims=input_partition_dims,
+ tpu_address=params.runtime.tpu)
+ with distribution_strategy.scope():
+ task = task_factory.get_task(params.task, logging_dir=model_dir)
+
+ train_lib.run_experiment(
+ distribution_strategy=distribution_strategy,
+ task=task,
+ mode=FLAGS.mode,
+ params=params,
+ model_dir=model_dir)
+
+if __name__ == '__main__':
+ tfm_flags.define_flags()
+ app.run(main)
diff --git a/official/vision/detection/README.md b/official/vision/detection/README.md
index d6cb5d4645f11681c450ab8c4ed33b38eba74b6b..2633f86d5dc4feed71ba170b92e9ffb66021652d 100644
--- a/official/vision/detection/README.md
+++ b/official/vision/detection/README.md
@@ -1,7 +1,7 @@
# Object Detection Models on TensorFlow 2
-**Note**: This repository is still under construction.
-More features and instructions will be added soon.
+**WARNING**: This repository will be deprecated and replaced by the solid
+implementations inside vision/beta/.
## Prerequsite
To get started, download the code from TensorFlow models GitHub repository or
diff --git a/official/vision/detection/__init__.py b/official/vision/detection/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/__init__.py
+++ b/official/vision/detection/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/official/vision/detection/configs/__init__.py b/official/vision/detection/configs/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/configs/__init__.py
+++ b/official/vision/detection/configs/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/configs/base_config.py b/official/vision/detection/configs/base_config.py
index 4505da517bb500a6fe32f5849971fc92e2d726e7..32b8bcc1be551c249cafeab6706ae3bc58cc2d08 100644
--- a/official/vision/detection/configs/base_config.py
+++ b/official/vision/detection/configs/base_config.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Base config template."""
diff --git a/official/vision/detection/configs/factory.py b/official/vision/detection/configs/factory.py
index d60ea1e01133fdfffd76ad54daf4ee20ed1e46e0..58530518b7f6cbbb33e244c8f746fead7822add4 100644
--- a/official/vision/detection/configs/factory.py
+++ b/official/vision/detection/configs/factory.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,11 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Factory to provide model configs."""
from official.modeling.hyperparams import params_dict
from official.vision.detection.configs import maskrcnn_config
+from official.vision.detection.configs import olnmask_config
from official.vision.detection.configs import retinanet_config
from official.vision.detection.configs import shapemask_config
@@ -28,6 +29,9 @@ def config_generator(model):
elif model == 'mask_rcnn':
default_config = maskrcnn_config.MASKRCNN_CFG
restrictions = maskrcnn_config.MASKRCNN_RESTRICTIONS
+ elif model == 'olnmask':
+ default_config = olnmask_config.OLNMASK_CFG
+ restrictions = olnmask_config.OLNMASK_RESTRICTIONS
elif model == 'shapemask':
default_config = shapemask_config.SHAPEMASK_CFG
restrictions = shapemask_config.SHAPEMASK_RESTRICTIONS
diff --git a/official/vision/detection/configs/maskrcnn_config.py b/official/vision/detection/configs/maskrcnn_config.py
index 70c9b31448d3d83754c439c87ce9f0d0a04f88c9..e421fb4e7174b09c576423efe0cdbd5622d82304 100644
--- a/official/vision/detection/configs/maskrcnn_config.py
+++ b/official/vision/detection/configs/maskrcnn_config.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Config template to train Mask R-CNN."""
from official.modeling.hyperparams import params_dict
@@ -52,7 +52,6 @@ MASKRCNN_CFG.override({
'anchor_size': 8,
},
'rpn_head': {
- 'anchors_per_location': 3,
'num_convs': 2,
'num_filters': 256,
'use_separable_conv': False,
diff --git a/official/vision/detection/configs/olnmask_config.py b/official/vision/detection/configs/olnmask_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..2888cc2ad87a6b6fbf2c0364337ace8d9ac5a30a
--- /dev/null
+++ b/official/vision/detection/configs/olnmask_config.py
@@ -0,0 +1,143 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Config template to train Object Localization Network (OLN)."""
+
+from official.modeling.hyperparams import params_dict
+from official.vision.detection.configs import base_config
+
+
+# pylint: disable=line-too-long
+OLNMASK_CFG = params_dict.ParamsDict(base_config.BASE_CFG)
+OLNMASK_CFG.override({
+ 'type': 'olnmask',
+ 'eval': {
+ 'type': 'oln_xclass_box',
+ 'use_category': False,
+ 'seen_class': 'voc',
+ 'num_images_to_visualize': 0,
+ },
+ 'architecture': {
+ 'parser': 'olnmask_parser',
+ 'min_level': 2,
+ 'max_level': 6,
+ 'include_rpn_class': False,
+ 'include_frcnn_class': False,
+ 'include_frcnn_box': True,
+ 'include_mask': False,
+ 'mask_target_size': 28,
+ 'num_classes': 2,
+ },
+ 'olnmask_parser': {
+ 'output_size': [640, 640],
+ 'num_channels': 3,
+ 'rpn_match_threshold': 0.7,
+ 'rpn_unmatched_threshold': 0.3,
+ 'rpn_batch_size_per_im': 256,
+ 'rpn_fg_fraction': 0.5,
+ 'aug_rand_hflip': True,
+ 'aug_scale_min': 0.5,
+ 'aug_scale_max': 2.0,
+ 'skip_crowd_during_training': True,
+ 'max_num_instances': 100,
+ 'mask_crop_size': 112,
+ # centerness targets.
+ 'has_centerness': True,
+ 'rpn_center_match_iou_threshold': 0.3,
+ 'rpn_center_unmatched_iou_threshold': 0.1,
+ 'rpn_num_center_samples_per_im': 256,
+ # class manipulation.
+ 'class_agnostic': True,
+ 'train_class': 'voc',
+ },
+ 'anchor': {
+ 'num_scales': 1,
+ 'aspect_ratios': [1.0],
+ 'anchor_size': 8,
+ },
+ 'rpn_head': {
+ 'num_convs': 2,
+ 'num_filters': 256,
+ 'use_separable_conv': False,
+ 'use_batch_norm': False,
+ # RPN-Centerness learning {
+ 'has_centerness': True, # }
+ },
+ 'frcnn_head': {
+ 'num_convs': 0,
+ 'num_filters': 256,
+ 'use_separable_conv': False,
+ 'num_fcs': 2,
+ 'fc_dims': 1024,
+ 'use_batch_norm': False,
+ 'has_scoring': True,
+ },
+ 'mrcnn_head': {
+ 'num_convs': 4,
+ 'num_filters': 256,
+ 'use_separable_conv': False,
+ 'use_batch_norm': False,
+ 'has_scoring': False,
+ },
+ 'rpn_score_loss': {
+ 'rpn_batch_size_per_im': 256,
+ },
+ 'rpn_box_loss': {
+ 'huber_loss_delta': 1.0 / 9.0,
+ },
+ 'frcnn_box_loss': {
+ 'huber_loss_delta': 1.0,
+ },
+ 'frcnn_box_score_loss': {
+ 'ignore_threshold': 0.3,
+ },
+ 'roi_proposal': {
+ 'rpn_pre_nms_top_k': 2000,
+ 'rpn_post_nms_top_k': 2000,
+ 'rpn_nms_threshold': 0.7,
+ 'rpn_score_threshold': 0.0,
+ 'rpn_min_size_threshold': 0.0,
+ 'test_rpn_pre_nms_top_k': 2000,
+ 'test_rpn_post_nms_top_k': 2000,
+ 'test_rpn_nms_threshold': 0.7,
+ 'test_rpn_score_threshold': 0.0,
+ 'test_rpn_min_size_threshold': 0.0,
+ 'use_batched_nms': False,
+ },
+ 'roi_sampling': {
+ 'num_samples_per_image': 512,
+ 'fg_fraction': 0.25,
+ 'fg_iou_thresh': 0.5,
+ 'bg_iou_thresh_hi': 0.5,
+ 'bg_iou_thresh_lo': 0.0,
+ 'mix_gt_boxes': True,
+ },
+ 'mask_sampling': {
+ 'num_mask_samples_per_image': 128, # Typically = `num_samples_per_image` * `fg_fraction`.
+ },
+ 'postprocess': {
+ 'use_batched_nms': False,
+ 'max_total_size': 100,
+ 'nms_iou_threshold': 0.5,
+ 'score_threshold': 0.00,
+ 'pre_nms_num_boxes': 2000,
+ },
+}, is_strict=False)
+
+
+OLNMASK_RESTRICTIONS = [
+ # 'anchor.aspect_ratios == [1.0]',
+ # 'anchor.scales == 1',
+]
+# pylint: enable=line-too-long
diff --git a/official/vision/detection/configs/retinanet_config.py b/official/vision/detection/configs/retinanet_config.py
index 579e30d083aacf138a2f9baffe1be7713ad21583..fb55a8a3bbedd1a0f54719c2385087edf2733853 100644
--- a/official/vision/detection/configs/retinanet_config.py
+++ b/official/vision/detection/configs/retinanet_config.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Config template to train Retinanet."""
from official.modeling.hyperparams import params_dict
@@ -39,7 +39,6 @@ RETINANET_CFG.override({
'max_num_instances': 100,
},
'retinanet_head': {
- 'anchors_per_location': 9,
'num_convs': 4,
'num_filters': 256,
'use_separable_conv': False,
diff --git a/official/vision/detection/configs/shapemask_config.py b/official/vision/detection/configs/shapemask_config.py
index 0914c492e15f65e5ba66701f27ca0d88d13698ff..aef823275c17f12b91b9c30f3cb3ca5b45b4e2cf 100644
--- a/official/vision/detection/configs/shapemask_config.py
+++ b/official/vision/detection/configs/shapemask_config.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,13 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Config to train shapemask on COCO."""
from official.modeling.hyperparams import params_dict
from official.vision.detection.configs import base_config
-SHAPEMASK_RESNET_FROZEN_VAR_PREFIX = r'(resnet\d+/)conv2d(|_([1-9]|10))\/'
+SHAPEMASK_RESNET_FROZEN_VAR_PREFIX = r'(conv2d(|_([1-9]|10))|batch_normalization(|_([1-9]|10)))\/'
SHAPEMASK_CFG = params_dict.ParamsDict(base_config.BASE_CFG)
SHAPEMASK_CFG.override({
@@ -62,7 +62,6 @@ SHAPEMASK_CFG.override({
'upsample_factor': 4,
},
'retinanet_head': {
- 'anchors_per_location': 9,
'num_convs': 4,
'num_filters': 256,
'use_separable_conv': False,
diff --git a/official/vision/detection/dataloader/__init__.py b/official/vision/detection/dataloader/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/dataloader/__init__.py
+++ b/official/vision/detection/dataloader/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/dataloader/anchor.py b/official/vision/detection/dataloader/anchor.py
index f46f7480062e75cec55d48ff683dcad8301e4994..c0abaadb476286ffaef9d88bd1d3fde7358d77f4 100644
--- a/official/vision/detection/dataloader/anchor.py
+++ b/official/vision/detection/dataloader/anchor.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Anchor box and labeler definition."""
from __future__ import absolute_import
@@ -19,42 +19,39 @@ from __future__ import division
from __future__ import print_function
import collections
+
import tensorflow as tf
+from official.vision import keras_cv
+from official.vision.detection.utils import box_utils
from official.vision.detection.utils.object_detection import argmax_matcher
from official.vision.detection.utils.object_detection import balanced_positive_negative_sampler
from official.vision.detection.utils.object_detection import box_list
from official.vision.detection.utils.object_detection import faster_rcnn_box_coder
-from official.vision.detection.utils.object_detection import region_similarity_calculator
from official.vision.detection.utils.object_detection import target_assigner
class Anchor(object):
"""Anchor class for anchor-based object detectors."""
- def __init__(self,
- min_level,
- max_level,
- num_scales,
- aspect_ratios,
- anchor_size,
- image_size):
+ def __init__(self, min_level, max_level, num_scales, aspect_ratios,
+ anchor_size, image_size):
"""Constructs multiscale anchors.
Args:
min_level: integer number of minimum level of the output feature pyramid.
max_level: integer number of maximum level of the output feature pyramid.
- num_scales: integer number representing intermediate scales added
- on each level. For instances, num_scales=2 adds one additional
- intermediate anchor scales [2^0, 2^0.5] on each level.
- aspect_ratios: list of float numbers representing the aspect raito anchors
+ num_scales: integer number representing intermediate scales added on each
+ level. For instances, num_scales=2 adds one additional intermediate
+ anchor scales [2^0, 2^0.5] on each level.
+ aspect_ratios: list of float numbers representing the aspect ratio anchors
added on each level. The number indicates the ratio of width to height.
For instances, aspect_ratios=[1.0, 2.0, 0.5] adds three anchors on each
scale level.
anchor_size: float number representing the scale of size of the base
anchor to the feature stride 2^level.
- image_size: a list of integer numbers or Tensors representing
- [height, width] of the input image size.The image_size should be divided
- by the largest feature stride 2^max_level.
+ image_size: a list of integer numbers or Tensors representing [height,
+ width] of the input image size.The image_size should be divisible by the
+ largest feature stride 2^max_level.
"""
self.min_level = min_level
self.max_level = max_level
@@ -76,11 +73,11 @@ class Anchor(object):
boxes_l = []
for scale in range(self.num_scales):
for aspect_ratio in self.aspect_ratios:
- stride = 2 ** level
- intermidate_scale = 2 ** (scale / float(self.num_scales))
- base_anchor_size = self.anchor_size * stride * intermidate_scale
- aspect_x = aspect_ratio ** 0.5
- aspect_y = aspect_ratio ** -0.5
+ stride = 2**level
+ intermediate_scale = 2**(scale / float(self.num_scales))
+ base_anchor_size = self.anchor_size * stride * intermediate_scale
+ aspect_x = aspect_ratio**0.5
+ aspect_y = aspect_ratio**-0.5
half_anchor_size_x = base_anchor_size * aspect_x / 2.0
half_anchor_size_y = base_anchor_size * aspect_y / 2.0
x = tf.range(stride / 2, self.image_size[1], stride)
@@ -89,8 +86,10 @@ class Anchor(object):
xv = tf.cast(tf.reshape(xv, [-1]), dtype=tf.float32)
yv = tf.cast(tf.reshape(yv, [-1]), dtype=tf.float32)
# Tensor shape Nx4.
- boxes = tf.stack([yv - half_anchor_size_y, xv - half_anchor_size_x,
- yv + half_anchor_size_y, xv + half_anchor_size_x],
+ boxes = tf.stack([
+ yv - half_anchor_size_y, xv - half_anchor_size_x,
+ yv + half_anchor_size_y, xv + half_anchor_size_x
+ ],
axis=1)
boxes_l.append(boxes)
# Concat anchors on the same level to tensor shape NxAx4.
@@ -104,11 +103,11 @@ class Anchor(object):
unpacked_labels = collections.OrderedDict()
count = 0
for level in range(self.min_level, self.max_level + 1):
- feat_size_y = tf.cast(self.image_size[0] / 2 ** level, tf.int32)
- feat_size_x = tf.cast(self.image_size[1] / 2 ** level, tf.int32)
+ feat_size_y = tf.cast(self.image_size[0] / 2**level, tf.int32)
+ feat_size_x = tf.cast(self.image_size[1] / 2**level, tf.int32)
steps = feat_size_y * feat_size_x * self.anchors_per_location
- unpacked_labels[level] = tf.reshape(
- labels[count:count + steps], [feat_size_y, feat_size_x, -1])
+ unpacked_labels[level] = tf.reshape(labels[count:count + steps],
+ [feat_size_y, feat_size_x, -1])
count += steps
return unpacked_labels
@@ -124,10 +123,7 @@ class Anchor(object):
class AnchorLabeler(object):
"""Labeler for dense object detector."""
- def __init__(self,
- anchor,
- match_threshold=0.5,
- unmatched_threshold=0.5):
+ def __init__(self, anchor, match_threshold=0.5, unmatched_threshold=0.5):
"""Constructs anchor labeler to assign labels to anchors.
Args:
@@ -139,7 +135,7 @@ class AnchorLabeler(object):
upper-bound threshold to assign negative labels for anchors. An anchor
with a score below the threshold is labeled negative.
"""
- similarity_calc = region_similarity_calculator.IouSimilarity()
+ similarity_calc = keras_cv.ops.IouSimilarity()
matcher = argmax_matcher.ArgMaxMatcher(
match_threshold,
unmatched_threshold=unmatched_threshold,
@@ -161,6 +157,7 @@ class AnchorLabeler(object):
For each row, it stores [y0, x0, y1, x1] for four corners of a box.
gt_labels: A integer tensor with shape [N, 1] representing groundtruth
classes.
+
Returns:
cls_targets_dict: ordered dictionary with keys
[min_level, min_level+1, ..., max_level]. The values are tensor with
@@ -205,11 +202,14 @@ class AnchorLabeler(object):
class RpnAnchorLabeler(AnchorLabeler):
"""Labeler for Region Proposal Network."""
- def __init__(self, anchor, match_threshold=0.7,
- unmatched_threshold=0.3, rpn_batch_size_per_im=256,
+ def __init__(self,
+ anchor,
+ match_threshold=0.7,
+ unmatched_threshold=0.3,
+ rpn_batch_size_per_im=256,
rpn_fg_fraction=0.5):
- AnchorLabeler.__init__(self, anchor, match_threshold=0.7,
- unmatched_threshold=0.3)
+ AnchorLabeler.__init__(
+ self, anchor, match_threshold=0.7, unmatched_threshold=0.3)
self._rpn_batch_size_per_im = rpn_batch_size_per_im
self._rpn_fg_fraction = rpn_fg_fraction
@@ -219,11 +219,12 @@ class RpnAnchorLabeler(AnchorLabeler):
This function performs subsampling for foreground (fg) and background (bg)
anchors.
Args:
- match_results: A integer tensor with shape [N] representing the
- matching results of anchors. (1) match_results[i]>=0,
- meaning that column i is matched with row match_results[i].
- (2) match_results[i]=-1, meaning that column i is not matched.
- (3) match_results[i]=-2, meaning that column i is ignored.
+ match_results: A integer tensor with shape [N] representing the matching
+ results of anchors. (1) match_results[i]>=0, meaning that column i is
+ matched with row match_results[i]. (2) match_results[i]=-1, meaning that
+ column i is not matched. (3) match_results[i]=-2, meaning that column i
+ is ignored.
+
Returns:
score_targets: a integer tensor with the a shape of [N].
(1) score_targets[i]=1, the anchor is a positive sample.
@@ -241,8 +242,7 @@ class RpnAnchorLabeler(AnchorLabeler):
indicator = tf.greater(match_results, -2)
labels = tf.greater(match_results, -1)
- samples = sampler.subsample(
- indicator, self._rpn_batch_size_per_im, labels)
+ samples = sampler.subsample(indicator, self._rpn_batch_size_per_im, labels)
positive_labels = tf.where(
tf.logical_and(samples, labels),
tf.constant(2, dtype=tf.int32, shape=match_results.shape),
@@ -253,8 +253,8 @@ class RpnAnchorLabeler(AnchorLabeler):
tf.constant(0, dtype=tf.int32, shape=match_results.shape))
ignore_labels = tf.fill(match_results.shape, -1)
- return (ignore_labels + positive_labels + negative_labels,
- positive_labels, negative_labels)
+ return (ignore_labels + positive_labels + negative_labels, positive_labels,
+ negative_labels)
def label_anchors(self, gt_boxes, gt_labels):
"""Labels anchors with ground truth inputs.
@@ -264,6 +264,7 @@ class RpnAnchorLabeler(AnchorLabeler):
For each row, it stores [y0, x0, y1, x1] for four corners of a box.
gt_labels: A integer tensor with shape [N, 1] representing groundtruth
classes.
+
Returns:
score_targets_dict: ordered dictionary with keys
[min_level, min_level+1, ..., max_level]. The values are tensor with
@@ -290,3 +291,168 @@ class RpnAnchorLabeler(AnchorLabeler):
box_targets_dict = self._anchor.unpack_labels(box_targets)
return score_targets_dict, box_targets_dict
+
+
+class OlnAnchorLabeler(RpnAnchorLabeler):
+ """Labeler for Region Proposal Network."""
+
+ def __init__(self,
+ anchor,
+ match_threshold=0.7,
+ unmatched_threshold=0.3,
+ rpn_batch_size_per_im=256,
+ rpn_fg_fraction=0.5,
+ has_centerness=False,
+ center_match_iou_threshold=0.3,
+ center_unmatched_iou_threshold=0.1,
+ num_center_samples_per_im=256):
+ """Constructs rpn anchor labeler to assign labels and centerness to anchors.
+
+ Args:
+ anchor: an instance of class Anchors.
+ match_threshold: a float number between 0 and 1 representing the
+ lower-bound threshold to assign positive labels for anchors. An anchor
+ with a score over the threshold is labeled positive.
+ unmatched_threshold: a float number between 0 and 1 representing the
+ upper-bound threshold to assign negative labels for anchors. An anchor
+ with a score below the threshold is labeled negative.
+ rpn_batch_size_per_im: number of anchors that are sampled per image.
+ rpn_fg_fraction:
+ has_centerness: whether to include centerness target creation. An anchor
+ is paired with one centerness score.
+ center_match_iou_threshold: a float number between 0 and 1 representing
+ the lower-bound threshold to sample foreground anchors for centerness
+ regression. An anchor with a score over the threshold is sampled as
+ foreground sample for centerness regression. We sample mostly from the
+ foreground region (255 out of 256 samples). That is, we sample 255 vs 1
+ (foreground vs background) anchor points to learn centerness regression.
+ center_unmatched_iou_threshold: a float number between 0 and 1
+ representing the lower-bound threshold to sample background anchors for
+ centerness regression. An anchor with a score over the threshold is
+ sampled as foreground sample for centerness regression. We sample very
+ sparsely from the background region (1 out of 256 samples). That is, we
+ sample 255 vs 1 (foreground vs background) anchor points to learn
+ centerness regression.
+ num_center_samples_per_im: number of anchor points per image that are
+ sampled as centerness targets.
+ """
+ super(OlnAnchorLabeler, self).__init__(
+ anchor, match_threshold=match_threshold,
+ unmatched_threshold=unmatched_threshold,
+ rpn_batch_size_per_im=rpn_batch_size_per_im,
+ rpn_fg_fraction=rpn_fg_fraction)
+ similarity_calc = keras_cv.ops.IouSimilarity()
+ matcher = argmax_matcher.ArgMaxMatcher(
+ match_threshold,
+ unmatched_threshold=unmatched_threshold,
+ negatives_lower_than_unmatched=True,
+ force_match_for_each_row=True)
+ box_coder = faster_rcnn_box_coder.FasterRcnnBoxCoder()
+ if has_centerness:
+ center_matcher = argmax_matcher.ArgMaxMatcher(
+ center_match_iou_threshold,
+ unmatched_threshold=center_match_iou_threshold,
+ negatives_lower_than_unmatched=True,
+ force_match_for_each_row=True,)
+ else:
+ center_matcher = None
+
+ self._target_assigner = target_assigner.OlnTargetAssigner(
+ similarity_calc, matcher, box_coder,
+ center_matcher=center_matcher)
+ self._num_center_samples_per_im = num_center_samples_per_im
+ self._center_unmatched_iou_threshold = center_unmatched_iou_threshold
+ self._rpn_batch_size_per_im = rpn_batch_size_per_im
+ self._rpn_fg_fraction = rpn_fg_fraction
+
+ def label_anchors_lrtb(self, gt_boxes, gt_labels):
+ """Labels anchors with ground truth inputs.
+
+ Args:
+ gt_boxes: A float tensor with shape [N, 4] representing groundtruth boxes.
+ For each row, it stores [y0, x0, y1, x1] for four corners of a box.
+ gt_labels: A integer tensor with shape [N, 1] representing groundtruth
+ classes.
+
+ Returns:
+ score_targets_dict: ordered dictionary with keys
+ [min_level, min_level+1, ..., max_level]. The values are tensor with
+ shape [height_l, width_l, num_anchors]. The height_l and width_l
+ represent the dimension of class logits at l-th level.
+ box_targets_dict: ordered dictionary with keys
+ [min_level, min_level+1, ..., max_level]. The values are tensor with
+ shape [height_l, width_l, num_anchors * 4]. The height_l and
+ width_l represent the dimension of bounding box regression output at
+ l-th level.
+ lrtb_targets_dict: Same strucure to box_target_dict, except the regression
+ targets are converted from xyhw to lrtb format. Ordered dictionary with
+ keys [min_level, min_level+1, ..., max_level]. The values are tensor
+ with shape [height_l, width_l, num_anchors * 4]. The height_l and
+ width_l represent the dimension of bounding box regression output at
+ l-th level.
+ center_targets_dict: Same structure to score_tragets_dict, except the
+ scores are centerness values ranging from 0 to 1. Ordered dictionary
+ with keys [min_level, min_level+1, ..., max_level]. The values are
+ tensor with shape [height_l, width_l, num_anchors]. The height_l and
+ width_l represent the dimension of class logits at l-th level.
+ """
+ gt_box_list = box_list.BoxList(gt_boxes)
+ anchor_box_list = box_list.BoxList(self._anchor.boxes)
+
+ # cls_targets, cls_weights, box_weights are not used.
+ (_, _, box_targets, _, matches,
+ matched_gt_box_list, matched_anchors_mask,
+ center_matched_gt_box_list, center_matched_anchors_mask,
+ matched_ious) = self._target_assigner.assign(
+ anchor_box_list, gt_box_list, gt_labels)
+ # Box lrtb_targets.
+ lrtb_targets, _ = box_utils.encode_boxes_lrtb(
+ matched_gt_box_list.data['boxes'],
+ anchor_box_list.data['boxes'],
+ weights=[1.0, 1.0, 1.0, 1.0])
+ lrtb_sanity = tf.logical_and(
+ tf.greater(tf.reduce_min(lrtb_targets, -1), 0.),
+ matched_anchors_mask)
+ # To broadcast lrtb_sanity to the same shape as lrtb_targets.
+ lrtb_sanity = tf.tile(tf.expand_dims(lrtb_sanity, 1),
+ [1, tf.shape(lrtb_targets)[1]])
+ lrtb_targets = tf.where(lrtb_sanity,
+ lrtb_targets,
+ tf.zeros_like(lrtb_targets))
+ # RPN anchor-gtbox iou values.
+ iou_targets = tf.where(tf.greater(matched_ious, 0.0),
+ matched_ious,
+ tf.zeros_like(matched_ious))
+ # Centerness_targets.
+ _, center_targets = box_utils.encode_boxes_lrtb(
+ center_matched_gt_box_list.data['boxes'],
+ anchor_box_list.data['boxes'],
+ weights=[1.0, 1.0, 1.0, 1.0])
+ # Positive-negative centerness sampler.
+ num_center_samples_per_im = self._num_center_samples_per_im
+ center_pos_neg_sampler = (
+ balanced_positive_negative_sampler.BalancedPositiveNegativeSampler(
+ positive_fraction=(1.- 1./num_center_samples_per_im),
+ is_static=False))
+ center_pos_neg_indicator = tf.logical_or(
+ center_matched_anchors_mask,
+ tf.less(iou_targets, self._center_unmatched_iou_threshold))
+ center_pos_labels = center_matched_anchors_mask
+ center_samples = center_pos_neg_sampler.subsample(
+ center_pos_neg_indicator, num_center_samples_per_im, center_pos_labels)
+ is_valid = center_samples
+ center_targets = tf.where(is_valid,
+ center_targets,
+ (-1) * tf.ones_like(center_targets))
+
+ # score_targets contains the subsampled positive and negative anchors.
+ score_targets, _, _ = self._get_rpn_samples(matches.match_results)
+
+ # Unpacks labels.
+ score_targets_dict = self._anchor.unpack_labels(score_targets)
+ box_targets_dict = self._anchor.unpack_labels(box_targets)
+ lrtb_targets_dict = self._anchor.unpack_labels(lrtb_targets)
+ center_targets_dict = self._anchor.unpack_labels(center_targets)
+
+ return (score_targets_dict, box_targets_dict,
+ lrtb_targets_dict, center_targets_dict)
diff --git a/official/vision/detection/dataloader/factory.py b/official/vision/detection/dataloader/factory.py
index 1e13aec222f529d97ee9c502d408648b9d091e5b..3d1e8574a497747463dadd792efc2ccadba7e3ed 100644
--- a/official/vision/detection/dataloader/factory.py
+++ b/official/vision/detection/dataloader/factory.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Model architecture factory."""
from __future__ import absolute_import
@@ -19,6 +19,7 @@ from __future__ import division
from __future__ import print_function
from official.vision.detection.dataloader import maskrcnn_parser
+from official.vision.detection.dataloader import olnmask_parser
from official.vision.detection.dataloader import retinanet_parser
from official.vision.detection.dataloader import shapemask_parser
@@ -69,6 +70,38 @@ def parser_generator(params, mode):
mask_crop_size=parser_params.mask_crop_size,
use_bfloat16=params.architecture.use_bfloat16,
mode=mode)
+ elif params.architecture.parser == 'olnmask_parser':
+ anchor_params = params.anchor
+ parser_params = params.olnmask_parser
+ parser_fn = olnmask_parser.Parser(
+ output_size=parser_params.output_size,
+ min_level=params.architecture.min_level,
+ max_level=params.architecture.max_level,
+ num_scales=anchor_params.num_scales,
+ aspect_ratios=anchor_params.aspect_ratios,
+ anchor_size=anchor_params.anchor_size,
+ rpn_match_threshold=parser_params.rpn_match_threshold,
+ rpn_unmatched_threshold=parser_params.rpn_unmatched_threshold,
+ rpn_batch_size_per_im=parser_params.rpn_batch_size_per_im,
+ rpn_fg_fraction=parser_params.rpn_fg_fraction,
+ aug_rand_hflip=parser_params.aug_rand_hflip,
+ aug_scale_min=parser_params.aug_scale_min,
+ aug_scale_max=parser_params.aug_scale_max,
+ skip_crowd_during_training=parser_params.skip_crowd_during_training,
+ max_num_instances=parser_params.max_num_instances,
+ include_mask=params.architecture.include_mask,
+ mask_crop_size=parser_params.mask_crop_size,
+ use_bfloat16=params.architecture.use_bfloat16,
+ mode=mode,
+ has_centerness=parser_params.has_centerness,
+ rpn_center_match_iou_threshold=(
+ parser_params.rpn_center_match_iou_threshold),
+ rpn_center_unmatched_iou_threshold=(
+ parser_params.rpn_center_unmatched_iou_threshold),
+ rpn_num_center_samples_per_im=(
+ parser_params.rpn_num_center_samples_per_im),
+ class_agnostic=parser_params.class_agnostic,
+ train_class=parser_params.train_class,)
elif params.architecture.parser == 'shapemask_parser':
anchor_params = params.anchor
parser_params = params.shapemask_parser
diff --git a/official/vision/detection/dataloader/input_reader.py b/official/vision/detection/dataloader/input_reader.py
index 6e65243f6863ccadb45704b3ed487aec3b8ab21a..d0f9b7abefb5a49bec7bd8ed65b005c56acdc68f 100644
--- a/official/vision/detection/dataloader/input_reader.py
+++ b/official/vision/detection/dataloader/input_reader.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Data loader and input processing."""
from __future__ import absolute_import
@@ -91,7 +91,8 @@ class InputFn(object):
dataset = dataset.repeat()
dataset = dataset.interleave(
- map_func=self._dataset_fn, cycle_length=32,
+ map_func=self._dataset_fn,
+ cycle_length=32,
num_parallel_calls=tf.data.experimental.AUTOTUNE)
if self._is_training:
diff --git a/official/vision/detection/dataloader/maskrcnn_parser.py b/official/vision/detection/dataloader/maskrcnn_parser.py
index 35db6f1478236d347839625d397fc918478694c4..7df1d547cbab068f68fc750927c945eefddec48e 100644
--- a/official/vision/detection/dataloader/maskrcnn_parser.py
+++ b/official/vision/detection/dataloader/maskrcnn_parser.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Data parser and processing for Mask R-CNN."""
import tensorflow as tf
diff --git a/official/vision/detection/dataloader/mode_keys.py b/official/vision/detection/dataloader/mode_keys.py
index 020382b2486ca25a41f0c3eb88b1f2038c538e7e..d6fdd9008bd4491ebec171d25c14d517ca3647c6 100644
--- a/official/vision/detection/dataloader/mode_keys.py
+++ b/official/vision/detection/dataloader/mode_keys.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Standard names for input dataloader modes.
The following standard keys are defined:
diff --git a/official/vision/detection/dataloader/olnmask_parser.py b/official/vision/detection/dataloader/olnmask_parser.py
new file mode 100644
index 0000000000000000000000000000000000000000..5f05f7387c7eca91ba4b42ed87bbd5a3a3926a73
--- /dev/null
+++ b/official/vision/detection/dataloader/olnmask_parser.py
@@ -0,0 +1,327 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Data parser and processing for Mask R-CNN."""
+
+import tensorflow as tf
+
+from official.vision.detection.dataloader import anchor
+from official.vision.detection.dataloader.maskrcnn_parser import Parser as MaskrcnnParser
+from official.vision.detection.utils import box_utils
+from official.vision.detection.utils import class_utils
+from official.vision.detection.utils import input_utils
+
+
+class Parser(MaskrcnnParser):
+ """Parser to parse an image and its annotations into a dictionary of tensors."""
+
+ def __init__(self,
+ output_size,
+ min_level,
+ max_level,
+ num_scales,
+ aspect_ratios,
+ anchor_size,
+ rpn_match_threshold=0.7,
+ rpn_unmatched_threshold=0.3,
+ rpn_batch_size_per_im=256,
+ rpn_fg_fraction=0.5,
+ aug_rand_hflip=False,
+ aug_scale_min=1.0,
+ aug_scale_max=1.0,
+ skip_crowd_during_training=True,
+ max_num_instances=100,
+ include_mask=False,
+ mask_crop_size=112,
+ use_bfloat16=True,
+ mode=None,
+ # for centerness learning.
+ has_centerness=False,
+ rpn_center_match_iou_threshold=0.3,
+ rpn_center_unmatched_iou_threshold=0.1,
+ rpn_num_center_samples_per_im=256,
+ # for class manipulation.
+ class_agnostic=False,
+ train_class='all',
+ ):
+ """Initializes parameters for parsing annotations in the dataset.
+
+ Args:
+ output_size: `Tensor` or `list` for [height, width] of output image. The
+ output_size should be divided by the largest feature stride 2^max_level.
+ min_level: `int` number of minimum level of the output feature pyramid.
+ max_level: `int` number of maximum level of the output feature pyramid.
+ num_scales: `int` number representing intermediate scales added
+ on each level. For instances, num_scales=2 adds one additional
+ intermediate anchor scales [2^0, 2^0.5] on each level.
+ aspect_ratios: `list` of float numbers representing the aspect raito
+ anchors added on each level. The number indicates the ratio of width to
+ height. For instances, aspect_ratios=[1.0, 2.0, 0.5] adds three anchors
+ on each scale level.
+ anchor_size: `float` number representing the scale of size of the base
+ anchor to the feature stride 2^level.
+ rpn_match_threshold:
+ rpn_unmatched_threshold:
+ rpn_batch_size_per_im:
+ rpn_fg_fraction:
+ aug_rand_hflip: `bool`, if True, augment training with random
+ horizontal flip.
+ aug_scale_min: `float`, the minimum scale applied to `output_size` for
+ data augmentation during training.
+ aug_scale_max: `float`, the maximum scale applied to `output_size` for
+ data augmentation during training.
+ skip_crowd_during_training: `bool`, if True, skip annotations labeled with
+ `is_crowd` equals to 1.
+ max_num_instances: `int` number of maximum number of instances in an
+ image. The groundtruth data will be padded to `max_num_instances`.
+ include_mask: a bool to indicate whether parse mask groundtruth.
+ mask_crop_size: the size which groundtruth mask is cropped to.
+ use_bfloat16: `bool`, if True, cast output image to tf.bfloat16.
+ mode: a ModeKeys. Specifies if this is training, evaluation, prediction
+ or prediction with groundtruths in the outputs.
+ has_centerness: whether to create centerness targets
+ rpn_center_match_iou_threshold: iou threshold for valid centerness samples
+ ,set to 0.3 by default.
+ rpn_center_unmatched_iou_threshold: iou threshold for invalid centerness
+ samples, set to 0.1 by default.
+ rpn_num_center_samples_per_im: number of centerness samples per image,
+ 256 by default.
+ class_agnostic: whether to merge class ids into one foreground(=1) class,
+ False by default.
+ train_class: 'all' or 'voc' or 'nonvoc', 'all' by default.
+ """
+ super(Parser, self).__init__(
+ output_size=output_size,
+ min_level=min_level,
+ max_level=max_level,
+ num_scales=num_scales,
+ aspect_ratios=aspect_ratios,
+ anchor_size=anchor_size,
+ rpn_match_threshold=rpn_match_threshold,
+ rpn_unmatched_threshold=rpn_unmatched_threshold,
+ rpn_batch_size_per_im=rpn_batch_size_per_im,
+ rpn_fg_fraction=rpn_fg_fraction,
+ aug_rand_hflip=aug_rand_hflip,
+ aug_scale_min=aug_scale_min,
+ aug_scale_max=aug_scale_max,
+ skip_crowd_during_training=skip_crowd_during_training,
+ max_num_instances=max_num_instances,
+ include_mask=include_mask,
+ mask_crop_size=mask_crop_size,
+ use_bfloat16=use_bfloat16,
+ mode=mode,)
+
+ # Centerness target assigning.
+ self._has_centerness = has_centerness
+ self._rpn_center_match_iou_threshold = rpn_center_match_iou_threshold
+ self._rpn_center_unmatched_iou_threshold = (
+ rpn_center_unmatched_iou_threshold)
+ self._rpn_num_center_samples_per_im = rpn_num_center_samples_per_im
+
+ # Class manipulation.
+ self._class_agnostic = class_agnostic
+ self._train_class = train_class
+
+ def _parse_train_data(self, data):
+ """Parses data for training.
+
+ Args:
+ data: the decoded tensor dictionary from TfExampleDecoder.
+
+ Returns:
+ image: image tensor that is preproessed to have normalized value and
+ dimension [output_size[0], output_size[1], 3]
+ labels: a dictionary of tensors used for training. The following describes
+ {key: value} pairs in the dictionary.
+ image_info: a 2D `Tensor` that encodes the information of the image and
+ the applied preprocessing. It is in the format of
+ [[original_height, original_width], [scaled_height, scaled_width],
+ anchor_boxes: ordered dictionary with keys
+ [min_level, min_level+1, ..., max_level]. The values are tensor with
+ shape [height_l, width_l, 4] representing anchor boxes at each level.
+ rpn_score_targets: ordered dictionary with keys
+ [min_level, min_level+1, ..., max_level]. The values are tensor with
+ shape [height_l, width_l, anchors_per_location]. The height_l and
+ width_l represent the dimension of class logits at l-th level.
+ rpn_box_targets: ordered dictionary with keys
+ [min_level, min_level+1, ..., max_level]. The values are tensor with
+ shape [height_l, width_l, anchors_per_location * 4]. The height_l and
+ width_l represent the dimension of bounding box regression output at
+ l-th level.
+ gt_boxes: Groundtruth bounding box annotations. The box is represented
+ in [y1, x1, y2, x2] format. The coordinates are w.r.t the scaled
+ image that is fed to the network. The tennsor is padded with -1 to
+ the fixed dimension [self._max_num_instances, 4].
+ gt_classes: Groundtruth classes annotations. The tennsor is padded
+ with -1 to the fixed dimension [self._max_num_instances].
+ gt_masks: groundtrugh masks cropped by the bounding box and
+ resized to a fixed size determined by mask_crop_size.
+ """
+ classes = data['groundtruth_classes']
+ boxes = data['groundtruth_boxes']
+ if self._include_mask:
+ masks = data['groundtruth_instance_masks']
+
+ is_crowds = data['groundtruth_is_crowd']
+ # Skips annotations with `is_crowd` = True.
+ if self._skip_crowd_during_training and self._is_training:
+ num_groundtruths = tf.shape(classes)[0]
+ with tf.control_dependencies([num_groundtruths, is_crowds]):
+ indices = tf.cond(
+ tf.greater(tf.size(is_crowds), 0),
+ lambda: tf.where(tf.logical_not(is_crowds))[:, 0],
+ lambda: tf.cast(tf.range(num_groundtruths), tf.int64))
+ classes = tf.gather(classes, indices)
+ boxes = tf.gather(boxes, indices)
+ if self._include_mask:
+ masks = tf.gather(masks, indices)
+
+ # Gets original image and its size.
+ image = data['image']
+ image_shape = tf.shape(image)[0:2]
+
+ # Normalizes image with mean and std pixel values.
+ image = input_utils.normalize_image(image)
+
+ # Flips image randomly during training.
+ if self._aug_rand_hflip:
+ if self._include_mask:
+ image, boxes, masks = input_utils.random_horizontal_flip(
+ image, boxes, masks)
+ else:
+ image, boxes = input_utils.random_horizontal_flip(
+ image, boxes)
+
+ # Converts boxes from normalized coordinates to pixel coordinates.
+ # Now the coordinates of boxes are w.r.t. the original image.
+ boxes = box_utils.denormalize_boxes(boxes, image_shape)
+
+ # Resizes and crops image.
+ image, image_info = input_utils.resize_and_crop_image(
+ image,
+ self._output_size,
+ padded_size=input_utils.compute_padded_size(
+ self._output_size, 2 ** self._max_level),
+ aug_scale_min=self._aug_scale_min,
+ aug_scale_max=self._aug_scale_max)
+ image_height, image_width, _ = image.get_shape().as_list()
+
+ # Resizes and crops boxes.
+ # Now the coordinates of boxes are w.r.t the scaled image.
+ image_scale = image_info[2, :]
+ offset = image_info[3, :]
+ boxes = input_utils.resize_and_crop_boxes(
+ boxes, image_scale, image_info[1, :], offset)
+
+ # Filters out ground truth boxes that are all zeros.
+ indices = box_utils.get_non_empty_box_indices(boxes)
+ boxes = tf.gather(boxes, indices)
+ classes = tf.gather(classes, indices)
+ if self._include_mask:
+ masks = tf.gather(masks, indices)
+ # Transfer boxes to the original image space and do normalization.
+ cropped_boxes = boxes + tf.tile(tf.expand_dims(offset, axis=0), [1, 2])
+ cropped_boxes /= tf.tile(tf.expand_dims(image_scale, axis=0), [1, 2])
+ cropped_boxes = box_utils.normalize_boxes(cropped_boxes, image_shape)
+ num_masks = tf.shape(masks)[0]
+ masks = tf.image.crop_and_resize(
+ tf.expand_dims(masks, axis=-1),
+ cropped_boxes,
+ box_indices=tf.range(num_masks, dtype=tf.int32),
+ crop_size=[self._mask_crop_size, self._mask_crop_size],
+ method='bilinear')
+ masks = tf.squeeze(masks, axis=-1)
+
+ # Class manipulation.
+ # Filter out novel split classes from training.
+ if self._train_class != 'all':
+ valid_classes = tf.cast(
+ class_utils.coco_split_class_ids(self._train_class),
+ dtype=classes.dtype)
+ match = tf.reduce_any(tf.equal(
+ tf.expand_dims(valid_classes, 1),
+ tf.expand_dims(classes, 0)), 0)
+ # kill novel split classes and boxes.
+ boxes = tf.gather(boxes, tf.where(match)[:, 0])
+ classes = tf.gather(classes, tf.where(match)[:, 0])
+ if self._include_mask:
+ masks = tf.gather(masks, tf.where(match)[:, 0])
+
+ # Assigns anchor targets.
+ # Note that after the target assignment, box targets are absolute pixel
+ # offsets w.r.t. the scaled image.
+ input_anchor = anchor.Anchor(
+ self._min_level,
+ self._max_level,
+ self._num_scales,
+ self._aspect_ratios,
+ self._anchor_size,
+ (image_height, image_width))
+ anchor_labeler = anchor.OlnAnchorLabeler(
+ input_anchor,
+ self._rpn_match_threshold,
+ self._rpn_unmatched_threshold,
+ self._rpn_batch_size_per_im,
+ self._rpn_fg_fraction,
+ # for centerness target.
+ self._has_centerness,
+ self._rpn_center_match_iou_threshold,
+ self._rpn_center_unmatched_iou_threshold,
+ self._rpn_num_center_samples_per_im,)
+
+ if self._has_centerness:
+ rpn_score_targets, _, rpn_lrtb_targets, rpn_center_targets = (
+ anchor_labeler.label_anchors_lrtb(
+ gt_boxes=boxes,
+ gt_labels=tf.cast(
+ tf.expand_dims(classes, axis=-1), dtype=tf.float32)))
+ else:
+ rpn_score_targets, rpn_box_targets = anchor_labeler.label_anchors(
+ boxes, tf.cast(tf.expand_dims(classes, axis=-1), dtype=tf.float32))
+ # For base rpn, dummy placeholder for centerness target.
+ rpn_center_targets = rpn_score_targets.copy()
+
+ # If bfloat16 is used, casts input image to tf.bfloat16.
+ if self._use_bfloat16:
+ image = tf.cast(image, dtype=tf.bfloat16)
+
+ inputs = {
+ 'image': image,
+ 'image_info': image_info,
+ }
+ # Packs labels for model_fn outputs.
+ labels = {
+ 'anchor_boxes': input_anchor.multilevel_boxes,
+ 'image_info': image_info,
+ 'rpn_score_targets': rpn_score_targets,
+ 'rpn_box_targets': (rpn_lrtb_targets if self._has_centerness
+ else rpn_box_targets),
+ 'rpn_center_targets': rpn_center_targets,
+ }
+ # If class_agnostic, convert to binary classes.
+ if self._class_agnostic:
+ classes = tf.where(tf.greater(classes, 0),
+ tf.ones_like(classes),
+ tf.zeros_like(classes))
+
+ inputs['gt_boxes'] = input_utils.pad_to_fixed_size(boxes,
+ self._max_num_instances,
+ -1)
+ inputs['gt_classes'] = input_utils.pad_to_fixed_size(
+ classes, self._max_num_instances, -1)
+ if self._include_mask:
+ inputs['gt_masks'] = input_utils.pad_to_fixed_size(
+ masks, self._max_num_instances, -1)
+
+ return inputs, labels
diff --git a/official/vision/detection/dataloader/retinanet_parser.py b/official/vision/detection/dataloader/retinanet_parser.py
index d226a6da7e2fc2e650ad6ecdfb5a431d13df97a3..8e9c3397ed304ec505e2030ddd5eb825273d2ff0 100644
--- a/official/vision/detection/dataloader/retinanet_parser.py
+++ b/official/vision/detection/dataloader/retinanet_parser.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Data parser and processing.
Parse image and ground truths in a dataset to training targets and package them
@@ -79,9 +79,9 @@ class Parser(object):
output_size should be divided by the largest feature stride 2^max_level.
min_level: `int` number of minimum level of the output feature pyramid.
max_level: `int` number of maximum level of the output feature pyramid.
- num_scales: `int` number representing intermediate scales added
- on each level. For instances, num_scales=2 adds one additional
- intermediate anchor scales [2^0, 2^0.5] on each level.
+ num_scales: `int` number representing intermediate scales added on each
+ level. For instances, num_scales=2 adds one additional intermediate
+ anchor scales [2^0, 2^0.5] on each level.
aspect_ratios: `list` of float numbers representing the aspect raito
anchors added on each level. The number indicates the ratio of width to
height. For instances, aspect_ratios=[1.0, 2.0, 0.5] adds three anchors
@@ -94,8 +94,8 @@ class Parser(object):
unmatched_threshold: `float` number between 0 and 1 representing the
upper-bound threshold to assign negative labels for anchors. An anchor
with a score below the threshold is labeled negative.
- aug_rand_hflip: `bool`, if True, augment training with random
- horizontal flip.
+ aug_rand_hflip: `bool`, if True, augment training with random horizontal
+ flip.
aug_scale_min: `float`, the minimum scale applied to `output_size` for
data augmentation during training.
aug_scale_max: `float`, the maximum scale applied to `output_size` for
@@ -109,8 +109,8 @@ class Parser(object):
max_num_instances: `int` number of maximum number of instances in an
image. The groundtruth data will be padded to `max_num_instances`.
use_bfloat16: `bool`, if True, cast output image to tf.bfloat16.
- mode: a ModeKeys. Specifies if this is training, evaluation, prediction
- or prediction with groundtruths in the outputs.
+ mode: a ModeKeys. Specifies if this is training, evaluation, prediction or
+ prediction with groundtruths in the outputs.
"""
self._mode = mode
self._max_num_instances = max_num_instances
@@ -232,8 +232,8 @@ class Parser(object):
image, image_info = input_utils.resize_and_crop_image(
image,
self._output_size,
- padded_size=input_utils.compute_padded_size(
- self._output_size, 2 ** self._max_level),
+ padded_size=input_utils.compute_padded_size(self._output_size,
+ 2**self._max_level),
aug_scale_min=self._aug_scale_min,
aug_scale_max=self._aug_scale_max)
image_height, image_width, _ = image.get_shape().as_list()
@@ -241,22 +241,21 @@ class Parser(object):
# Resizes and crops boxes.
image_scale = image_info[2, :]
offset = image_info[3, :]
- boxes = input_utils.resize_and_crop_boxes(
- boxes, image_scale, image_info[1, :], offset)
+ boxes = input_utils.resize_and_crop_boxes(boxes, image_scale,
+ image_info[1, :], offset)
# Filters out ground truth boxes that are all zeros.
indices = box_utils.get_non_empty_box_indices(boxes)
boxes = tf.gather(boxes, indices)
classes = tf.gather(classes, indices)
# Assigns anchors.
- input_anchor = anchor.Anchor(
- self._min_level, self._max_level, self._num_scales,
- self._aspect_ratios, self._anchor_size, (image_height, image_width))
- anchor_labeler = anchor.AnchorLabeler(
- input_anchor, self._match_threshold, self._unmatched_threshold)
+ input_anchor = anchor.Anchor(self._min_level, self._max_level,
+ self._num_scales, self._aspect_ratios,
+ self._anchor_size, (image_height, image_width))
+ anchor_labeler = anchor.AnchorLabeler(input_anchor, self._match_threshold,
+ self._unmatched_threshold)
(cls_targets, box_targets, num_positives) = anchor_labeler.label_anchors(
- boxes,
- tf.cast(tf.expand_dims(classes, axis=1), tf.float32))
+ boxes, tf.cast(tf.expand_dims(classes, axis=1), tf.float32))
# If bfloat16 is used, casts input image to tf.bfloat16.
if self._use_bfloat16:
@@ -292,8 +291,8 @@ class Parser(object):
image, image_info = input_utils.resize_and_crop_image(
image,
self._output_size,
- padded_size=input_utils.compute_padded_size(
- self._output_size, 2 ** self._max_level),
+ padded_size=input_utils.compute_padded_size(self._output_size,
+ 2**self._max_level),
aug_scale_min=1.0,
aug_scale_max=1.0)
image_height, image_width, _ = image.get_shape().as_list()
@@ -301,22 +300,21 @@ class Parser(object):
# Resizes and crops boxes.
image_scale = image_info[2, :]
offset = image_info[3, :]
- boxes = input_utils.resize_and_crop_boxes(
- boxes, image_scale, image_info[1, :], offset)
+ boxes = input_utils.resize_and_crop_boxes(boxes, image_scale,
+ image_info[1, :], offset)
# Filters out ground truth boxes that are all zeros.
indices = box_utils.get_non_empty_box_indices(boxes)
boxes = tf.gather(boxes, indices)
classes = tf.gather(classes, indices)
# Assigns anchors.
- input_anchor = anchor.Anchor(
- self._min_level, self._max_level, self._num_scales,
- self._aspect_ratios, self._anchor_size, (image_height, image_width))
- anchor_labeler = anchor.AnchorLabeler(
- input_anchor, self._match_threshold, self._unmatched_threshold)
+ input_anchor = anchor.Anchor(self._min_level, self._max_level,
+ self._num_scales, self._aspect_ratios,
+ self._anchor_size, (image_height, image_width))
+ anchor_labeler = anchor.AnchorLabeler(input_anchor, self._match_threshold,
+ self._unmatched_threshold)
(cls_targets, box_targets, num_positives) = anchor_labeler.label_anchors(
- boxes,
- tf.cast(tf.expand_dims(classes, axis=1), tf.float32))
+ boxes, tf.cast(tf.expand_dims(classes, axis=1), tf.float32))
# If bfloat16 is used, casts input image to tf.bfloat16.
if self._use_bfloat16:
@@ -324,18 +322,24 @@ class Parser(object):
# Sets up groundtruth data for evaluation.
groundtruths = {
- 'source_id': data['source_id'],
- 'num_groundtrtuhs': tf.shape(data['groundtruth_classes']),
- 'image_info': image_info,
- 'boxes': box_utils.denormalize_boxes(
- data['groundtruth_boxes'], image_shape),
- 'classes': data['groundtruth_classes'],
- 'areas': data['groundtruth_area'],
- 'is_crowds': tf.cast(data['groundtruth_is_crowd'], tf.int32),
+ 'source_id':
+ data['source_id'],
+ 'num_groundtrtuhs':
+ tf.shape(data['groundtruth_classes']),
+ 'image_info':
+ image_info,
+ 'boxes':
+ box_utils.denormalize_boxes(data['groundtruth_boxes'], image_shape),
+ 'classes':
+ data['groundtruth_classes'],
+ 'areas':
+ data['groundtruth_area'],
+ 'is_crowds':
+ tf.cast(data['groundtruth_is_crowd'], tf.int32),
}
groundtruths['source_id'] = process_source_id(groundtruths['source_id'])
- groundtruths = pad_groundtruths_to_fixed_size(
- groundtruths, self._max_num_instances)
+ groundtruths = pad_groundtruths_to_fixed_size(groundtruths,
+ self._max_num_instances)
# Packs labels for model_fn outputs.
labels = {
@@ -361,8 +365,8 @@ class Parser(object):
image, image_info = input_utils.resize_and_crop_image(
image,
self._output_size,
- padded_size=input_utils.compute_padded_size(
- self._output_size, 2 ** self._max_level),
+ padded_size=input_utils.compute_padded_size(self._output_size,
+ 2**self._max_level),
aug_scale_min=1.0,
aug_scale_max=1.0)
image_height, image_width, _ = image.get_shape().as_list()
@@ -372,9 +376,9 @@ class Parser(object):
image = tf.cast(image, dtype=tf.bfloat16)
# Compute Anchor boxes.
- input_anchor = anchor.Anchor(
- self._min_level, self._max_level, self._num_scales,
- self._aspect_ratios, self._anchor_size, (image_height, image_width))
+ input_anchor = anchor.Anchor(self._min_level, self._max_level,
+ self._num_scales, self._aspect_ratios,
+ self._anchor_size, (image_height, image_width))
labels = {
'anchor_boxes': input_anchor.multilevel_boxes,
@@ -384,8 +388,8 @@ class Parser(object):
# in labels.
if self._mode == ModeKeys.PREDICT_WITH_GT:
# Converts boxes from normalized coordinates to pixel coordinates.
- boxes = box_utils.denormalize_boxes(
- data['groundtruth_boxes'], image_shape)
+ boxes = box_utils.denormalize_boxes(data['groundtruth_boxes'],
+ image_shape)
groundtruths = {
'source_id': data['source_id'],
'num_detections': tf.shape(data['groundtruth_classes']),
@@ -395,8 +399,8 @@ class Parser(object):
'is_crowds': tf.cast(data['groundtruth_is_crowd'], tf.int32),
}
groundtruths['source_id'] = process_source_id(groundtruths['source_id'])
- groundtruths = pad_groundtruths_to_fixed_size(
- groundtruths, self._max_num_instances)
+ groundtruths = pad_groundtruths_to_fixed_size(groundtruths,
+ self._max_num_instances)
labels['groundtruths'] = groundtruths
# Computes training objective for evaluation loss.
@@ -404,18 +408,17 @@ class Parser(object):
image_scale = image_info[2, :]
offset = image_info[3, :]
- boxes = input_utils.resize_and_crop_boxes(
- boxes, image_scale, image_info[1, :], offset)
+ boxes = input_utils.resize_and_crop_boxes(boxes, image_scale,
+ image_info[1, :], offset)
# Filters out ground truth boxes that are all zeros.
indices = box_utils.get_non_empty_box_indices(boxes)
boxes = tf.gather(boxes, indices)
# Assigns anchors.
- anchor_labeler = anchor.AnchorLabeler(
- input_anchor, self._match_threshold, self._unmatched_threshold)
+ anchor_labeler = anchor.AnchorLabeler(input_anchor, self._match_threshold,
+ self._unmatched_threshold)
(cls_targets, box_targets, num_positives) = anchor_labeler.label_anchors(
- boxes,
- tf.cast(tf.expand_dims(classes, axis=1), tf.float32))
+ boxes, tf.cast(tf.expand_dims(classes, axis=1), tf.float32))
labels['cls_targets'] = cls_targets
labels['box_targets'] = box_targets
labels['num_positives'] = num_positives
diff --git a/official/vision/detection/dataloader/shapemask_parser.py b/official/vision/detection/dataloader/shapemask_parser.py
index 3bc368c0ef290291405157b772ed523f3725e0a3..c0e79e071692adcb8f1328563ccb9bd40734df80 100644
--- a/official/vision/detection/dataloader/shapemask_parser.py
+++ b/official/vision/detection/dataloader/shapemask_parser.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Data parser and processing.
Parse image and ground truths in a dataset to training targets and package them
@@ -21,7 +21,6 @@ Weicheng Kuo, Anelia Angelova, Jitendra Malik, Tsung-Yi Lin
ShapeMask: Learning to Segment Novel Objects by Refining Shape Priors.
arXiv:1904.03239.
"""
-
import tensorflow as tf
from official.vision.detection.dataloader import anchor
diff --git a/official/vision/detection/dataloader/tf_example_decoder.py b/official/vision/detection/dataloader/tf_example_decoder.py
index f719a9168a4d3106600fffcc47c14cc90f3cadc7..e6472a36b9a31a8e8a98cecf10a6abf8ccb03985 100644
--- a/official/vision/detection/dataloader/tf_example_decoder.py
+++ b/official/vision/detection/dataloader/tf_example_decoder.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Tensorflow Example proto decoder for object detection.
diff --git a/official/vision/detection/evaluation/__init__.py b/official/vision/detection/evaluation/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/evaluation/__init__.py
+++ b/official/vision/detection/evaluation/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/evaluation/coco_evaluator.py b/official/vision/detection/evaluation/coco_evaluator.py
index dc56a9332784dd66d5393bbf0d4c996fe5141c6d..108290bb7bef6633c4be579b8ca8c929b34213cb 100644
--- a/official/vision/detection/evaluation/coco_evaluator.py
+++ b/official/vision/detection/evaluation/coco_evaluator.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""The COCO-style evaluator.
The following snippet demonstrates the use of interfaces:
@@ -31,9 +31,11 @@ from __future__ import division
from __future__ import print_function
import atexit
+import copy
import tempfile
-import numpy as np
+
from absl import logging
+import numpy as np
from pycocotools import cocoeval
import six
import tensorflow as tf
@@ -197,22 +199,21 @@ class COCOEvaluator(object):
"""Update and aggregate detection results and groundtruth data.
Args:
- predictions: a dictionary of numpy arrays including the fields below.
- See different parsers under `../dataloader` for more details.
+ predictions: a dictionary of numpy arrays including the fields below. See
+ different parsers under `../dataloader` for more details.
Required fields:
- source_id: a numpy array of int or string of shape [batch_size].
- image_info [if `need_rescale_bboxes` is True]: a numpy array of
float of shape [batch_size, 4, 2].
- - num_detections: a numpy array of
- int of shape [batch_size].
+ - num_detections: a numpy array of int of shape [batch_size].
- detection_boxes: a numpy array of float of shape [batch_size, K, 4].
- detection_classes: a numpy array of int of shape [batch_size, K].
- detection_scores: a numpy array of float of shape [batch_size, K].
Optional fields:
- - detection_masks: a numpy array of float of shape
- [batch_size, K, mask_height, mask_width].
- groundtruths: a dictionary of numpy arrays including the fields below.
- See also different parsers under `../dataloader` for more details.
+ - detection_masks: a numpy array of float of shape [batch_size, K,
+ mask_height, mask_width].
+ groundtruths: a dictionary of numpy arrays including the fields below. See
+ also different parsers under `../dataloader` for more details.
Required fields:
- source_id: a numpy array of int or string of shape [batch_size].
- height: a numpy array of int of shape [batch_size].
@@ -222,12 +223,12 @@ class COCOEvaluator(object):
- classes: a numpy array of int of shape [batch_size, K].
Optional fields:
- is_crowds: a numpy array of int of shape [batch_size, K]. If the
- field is absent, it is assumed that this instance is not crowd.
- - areas: a numy array of float of shape [batch_size, K]. If the
- field is absent, the area is calculated using either boxes or
- masks depending on which one is available.
- - masks: a numpy array of float of shape
- [batch_size, K, mask_height, mask_width],
+ field is absent, it is assumed that this instance is not crowd.
+ - areas: a numy array of float of shape [batch_size, K]. If the field
+ is absent, the area is calculated using either boxes or masks
+ depending on which one is available.
+ - masks: a numpy array of float of shape [batch_size, K, mask_height,
+ mask_width],
Raises:
ValueError: if the required prediction or groundtruth fields are not
@@ -258,6 +259,278 @@ class COCOEvaluator(object):
self._groundtruths[k].append(v)
+class OlnXclassEvaluator(COCOEvaluator):
+ """COCO evaluation metric class."""
+
+ def __init__(self, annotation_file, include_mask, need_rescale_bboxes=True,
+ use_category=True, seen_class='all'):
+ """Constructs COCO evaluation class.
+
+ The class provides the interface to metrics_fn in TPUEstimator. The
+ _update_op() takes detections from each image and push them to
+ self.detections. The _evaluate() loads a JSON file in COCO annotation format
+ as the groundtruths and runs COCO evaluation.
+
+ Args:
+ annotation_file: a JSON file that stores annotations of the eval dataset.
+ If `annotation_file` is None, groundtruth annotations will be loaded
+ from the dataloader.
+ include_mask: a boolean to indicate whether or not to include the mask
+ eval.
+ need_rescale_bboxes: If true bboxes in `predictions` will be rescaled back
+ to absolute values (`image_info` is needed in this case).
+ use_category: if `False`, treat all object in all classes in one
+ foreground category.
+ seen_class: 'all' or 'voc' or 'nonvoc'
+ """
+ super(OlnXclassEvaluator, self).__init__(
+ annotation_file=annotation_file,
+ include_mask=include_mask,
+ need_rescale_bboxes=need_rescale_bboxes)
+ self._use_category = use_category
+ self._seen_class = seen_class
+ self._seen_class_ids = class_utils.coco_split_class_ids(seen_class)
+ self._metric_names = [
+ 'AP', 'AP50', 'AP75',
+ 'APs', 'APm', 'APl',
+ 'ARmax10', 'ARmax20', 'ARmax50', 'ARmax100', 'ARmax200',
+ 'ARmax10s', 'ARmax10m', 'ARmax10l'
+ ]
+ if self._seen_class != 'all':
+ self._metric_names.extend([
+ 'AP_seen', 'AP50_seen', 'AP75_seen',
+ 'APs_seen', 'APm_seen', 'APl_seen',
+ 'ARmax10_seen', 'ARmax20_seen', 'ARmax50_seen',
+ 'ARmax100_seen', 'ARmax200_seen',
+ 'ARmax10s_seen', 'ARmax10m_seen', 'ARmax10l_seen',
+
+ 'AP_novel', 'AP50_novel', 'AP75_novel',
+ 'APs_novel', 'APm_novel', 'APl_novel',
+ 'ARmax10_novel', 'ARmax20_novel', 'ARmax50_novel',
+ 'ARmax100_novel', 'ARmax200_novel',
+ 'ARmax10s_novel', 'ARmax10m_novel', 'ARmax10l_novel',
+ ])
+ if self._include_mask:
+ mask_metric_names = ['mask_' + x for x in self._metric_names]
+ self._metric_names.extend(mask_metric_names)
+ self._required_prediction_fields.extend(['detection_masks'])
+ self._required_groundtruth_fields.extend(['masks'])
+
+ self.reset()
+
+ def evaluate(self):
+ """Evaluates with detections from all images with COCO API.
+
+ Returns:
+ coco_metric: float numpy array with shape [24] representing the
+ coco-style evaluation metrics (box and mask).
+ """
+ if not self._annotation_file:
+ logging.info('Thre is no annotation_file in COCOEvaluator.')
+ gt_dataset = coco_utils.convert_groundtruths_to_coco_dataset(
+ self._groundtruths)
+ coco_gt = coco_utils.COCOWrapper(
+ eval_type=('mask' if self._include_mask else 'box'),
+ gt_dataset=gt_dataset)
+ else:
+ logging.info('Using annotation file: %s', self._annotation_file)
+ coco_gt = self._coco_gt
+
+ coco_predictions = coco_utils.convert_predictions_to_coco_annotations(
+ self._predictions)
+ coco_dt = coco_gt.loadRes(predictions=coco_predictions)
+ image_ids = [ann['image_id'] for ann in coco_predictions]
+ # Class manipulation: 'all' split samples -> ignored_split = 0.
+ for idx, ann in enumerate(coco_gt.dataset['annotations']):
+ coco_gt.dataset['annotations'][idx]['ignored_split'] = 0
+ coco_eval = cocoeval.OlnCOCOevalXclassWrapper(
+ coco_gt, coco_dt, iou_type='bbox')
+ coco_eval.params.maxDets = [10, 20, 50, 100, 200]
+ coco_eval.params.imgIds = image_ids
+ coco_eval.params.useCats = 0 if not self._use_category else 1
+ coco_eval.evaluate()
+ coco_eval.accumulate()
+ coco_eval.summarize()
+ coco_metrics = coco_eval.stats
+
+ if self._include_mask:
+ mcoco_eval = cocoeval.OlnCOCOevalXclassWrapper(
+ coco_gt, coco_dt, iou_type='segm')
+ mcoco_eval.params.maxDets = [10, 20, 50, 100, 200]
+ mcoco_eval.params.imgIds = image_ids
+ mcoco_eval.params.useCats = 0 if not self._use_category else 1
+ mcoco_eval.evaluate()
+ mcoco_eval.accumulate()
+ mcoco_eval.summarize()
+ mask_coco_metrics = mcoco_eval.stats
+
+ if self._include_mask:
+ metrics = np.hstack((coco_metrics, mask_coco_metrics))
+ else:
+ metrics = coco_metrics
+
+ if self._seen_class != 'all':
+ # for seen class eval, samples of novel_class are ignored.
+ coco_gt_seen = copy.deepcopy(coco_gt)
+ for idx, ann in enumerate(coco_gt.dataset['annotations']):
+ if ann['category_id'] in self._seen_class_ids:
+ coco_gt_seen.dataset['annotations'][idx]['ignored_split'] = 0
+ else:
+ coco_gt_seen.dataset['annotations'][idx]['ignored_split'] = 1
+ coco_eval_seen = cocoeval.OlnCOCOevalXclassWrapper(
+ coco_gt_seen, coco_dt, iou_type='bbox')
+ coco_eval_seen.params.maxDets = [10, 20, 50, 100, 200]
+ coco_eval_seen.params.imgIds = image_ids
+ coco_eval_seen.params.useCats = 0 if not self._use_category else 1
+ coco_eval_seen.evaluate()
+ coco_eval_seen.accumulate()
+ coco_eval_seen.summarize()
+ coco_metrics_seen = coco_eval_seen.stats
+ if self._include_mask:
+ mcoco_eval_seen = cocoeval.OlnCOCOevalXclassWrapper(
+ coco_gt_seen, coco_dt, iou_type='segm')
+ mcoco_eval_seen.params.maxDets = [10, 20, 50, 100, 200]
+ mcoco_eval_seen.params.imgIds = image_ids
+ mcoco_eval_seen.params.useCats = 0 if not self._use_category else 1
+ mcoco_eval_seen.evaluate()
+ mcoco_eval_seen.accumulate()
+ mcoco_eval_seen.summarize()
+ mask_coco_metrics_seen = mcoco_eval_seen.stats
+
+ # for novel class eval, samples of seen_class are ignored.
+ coco_gt_novel = copy.deepcopy(coco_gt)
+ for idx, ann in enumerate(coco_gt.dataset['annotations']):
+ if ann['category_id'] in self._seen_class_ids:
+ coco_gt_novel.dataset['annotations'][idx]['ignored_split'] = 1
+ else:
+ coco_gt_novel.dataset['annotations'][idx]['ignored_split'] = 0
+ coco_eval_novel = cocoeval.OlnCOCOevalXclassWrapper(
+ coco_gt_novel, coco_dt, iou_type='bbox')
+ coco_eval_novel.params.maxDets = [10, 20, 50, 100, 200]
+ coco_eval_novel.params.imgIds = image_ids
+ coco_eval_novel.params.useCats = 0 if not self._use_category else 1
+ coco_eval_novel.evaluate()
+ coco_eval_novel.accumulate()
+ coco_eval_novel.summarize()
+ coco_metrics_novel = coco_eval_novel.stats
+ if self._include_mask:
+ mcoco_eval_novel = cocoeval.OlnCOCOevalXclassWrapper(
+ coco_gt_novel, coco_dt, iou_type='segm')
+ mcoco_eval_novel.params.maxDets = [10, 20, 50, 100, 200]
+ mcoco_eval_novel.params.imgIds = image_ids
+ mcoco_eval_novel.params.useCats = 0 if not self._use_category else 1
+ mcoco_eval_novel.evaluate()
+ mcoco_eval_novel.accumulate()
+ mcoco_eval_novel.summarize()
+ mask_coco_metrics_novel = mcoco_eval_novel.stats
+
+ # Combine all splits.
+ if self._include_mask:
+ metrics = np.hstack((
+ coco_metrics, coco_metrics_seen, coco_metrics_novel,
+ mask_coco_metrics, mask_coco_metrics_seen, mask_coco_metrics_novel))
+ else:
+ metrics = np.hstack((
+ coco_metrics, coco_metrics_seen, coco_metrics_novel))
+
+ # Cleans up the internal variables in order for a fresh eval next time.
+ self.reset()
+
+ metrics_dict = {}
+ for i, name in enumerate(self._metric_names):
+ metrics_dict[name] = metrics[i].astype(np.float32)
+ return metrics_dict
+
+
+class OlnXdataEvaluator(OlnXclassEvaluator):
+ """COCO evaluation metric class."""
+
+ def __init__(self, annotation_file, include_mask, need_rescale_bboxes=True,
+ use_category=True, seen_class='all'):
+ """Constructs COCO evaluation class.
+
+ The class provides the interface to metrics_fn in TPUEstimator. The
+ _update_op() takes detections from each image and push them to
+ self.detections. The _evaluate() loads a JSON file in COCO annotation format
+ as the groundtruths and runs COCO evaluation.
+
+ Args:
+ annotation_file: a JSON file that stores annotations of the eval dataset.
+ If `annotation_file` is None, groundtruth annotations will be loaded
+ from the dataloader.
+ include_mask: a boolean to indicate whether or not to include the mask
+ eval.
+ need_rescale_bboxes: If true bboxes in `predictions` will be rescaled back
+ to absolute values (`image_info` is needed in this case).
+ use_category: if `False`, treat all object in all classes in one
+ foreground category.
+ seen_class: 'all' or 'voc' or 'nonvoc'
+ """
+ super(OlnXdataEvaluator, self).__init__(
+ annotation_file=annotation_file,
+ include_mask=include_mask,
+ need_rescale_bboxes=need_rescale_bboxes,
+ use_category=False,
+ seen_class='all')
+
+ def evaluate(self):
+ """Evaluates with detections from all images with COCO API.
+
+ Returns:
+ coco_metric: float numpy array with shape [24] representing the
+ coco-style evaluation metrics (box and mask).
+ """
+ if not self._annotation_file:
+ logging.info('Thre is no annotation_file in COCOEvaluator.')
+ gt_dataset = coco_utils.convert_groundtruths_to_coco_dataset(
+ self._groundtruths)
+ coco_gt = coco_utils.COCOWrapper(
+ eval_type=('mask' if self._include_mask else 'box'),
+ gt_dataset=gt_dataset)
+ else:
+ logging.info('Using annotation file: %s', self._annotation_file)
+ coco_gt = self._coco_gt
+ coco_predictions = coco_utils.convert_predictions_to_coco_annotations(
+ self._predictions)
+ coco_dt = coco_gt.loadRes(predictions=coco_predictions)
+ image_ids = [ann['image_id'] for ann in coco_predictions]
+ # Class manipulation: 'all' split samples -> ignored_split = 0.
+ for idx, _ in enumerate(coco_gt.dataset['annotations']):
+ coco_gt.dataset['annotations'][idx]['ignored_split'] = 0
+ coco_eval = cocoeval.OlnCOCOevalWrapper(coco_gt, coco_dt, iou_type='bbox')
+ coco_eval.params.maxDets = [10, 20, 50, 100, 200]
+ coco_eval.params.imgIds = image_ids
+ coco_eval.params.useCats = 0 if not self._use_category else 1
+ coco_eval.evaluate()
+ coco_eval.accumulate()
+ coco_eval.summarize()
+ coco_metrics = coco_eval.stats
+
+ if self._include_mask:
+ mcoco_eval = cocoeval.OlnCOCOevalWrapper(coco_gt, coco_dt,
+ iou_type='segm')
+ mcoco_eval.params.maxDets = [10, 20, 50, 100, 200]
+ mcoco_eval.params.imgIds = image_ids
+ mcoco_eval.params.useCats = 0 if not self._use_category else 1
+ mcoco_eval.evaluate()
+ mcoco_eval.accumulate()
+ mcoco_eval.summarize()
+ mask_coco_metrics = mcoco_eval.stats
+
+ if self._include_mask:
+ metrics = np.hstack((coco_metrics, mask_coco_metrics))
+ else:
+ metrics = coco_metrics
+
+ # Cleans up the internal variables in order for a fresh eval next time.
+ self.reset()
+
+ metrics_dict = {}
+ for i, name in enumerate(self._metric_names):
+ metrics_dict[name] = metrics[i].astype(np.float32)
+ return metrics_dict
+
+
class ShapeMaskCOCOEvaluator(COCOEvaluator):
"""COCO evaluation metric class for ShapeMask."""
@@ -318,8 +591,7 @@ class ShapeMaskCOCOEvaluator(COCOEvaluator):
metrics = np.hstack((coco_metrics, mcoco_eval.stats))
else:
mask_coco_metrics = mcoco_eval.category_stats
- val_catg_idx = np.isin(mcoco_eval.params.catIds,
- self._eval_categories)
+ val_catg_idx = np.isin(mcoco_eval.params.catIds, self._eval_categories)
# Gather the valid evaluation of the eval categories.
if np.any(val_catg_idx):
mean_val_metrics = []
diff --git a/official/vision/detection/evaluation/coco_utils.py b/official/vision/detection/evaluation/coco_utils.py
index 8155d1fbb89ac143eb7cc03457a6645a5b5ab505..0f289d354bacfc97e48c2b7d5af9fb6f72feae77 100644
--- a/official/vision/detection/evaluation/coco_utils.py
+++ b/official/vision/detection/evaluation/coco_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Util functions related to pycocotools and COCO eval."""
from __future__ import absolute_import
@@ -237,7 +237,7 @@ def convert_groundtruths_to_coco_dataset(groundtruths, label_map=None):
(boxes[j, k, 3] - boxes[j, k, 1]) *
(boxes[j, k, 2] - boxes[j, k, 0]))
if 'masks' in groundtruths:
- mask = Image.open(six.StringIO(groundtruths['masks'][i][j, k]))
+ mask = Image.open(six.BytesIO(groundtruths['masks'][i][j, k]))
width, height = mask.size
np_mask = (
np.array(mask.getdata()).reshape(height, width).astype(np.uint8))
diff --git a/official/vision/detection/evaluation/factory.py b/official/vision/detection/evaluation/factory.py
index 4d44bf177071a97b663b41410a05d59d59f04456..fcc543bfd00b72c08540088f74d89e410569d020 100644
--- a/official/vision/detection/evaluation/factory.py
+++ b/official/vision/detection/evaluation/factory.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Evaluator factory."""
from __future__ import absolute_import
@@ -29,6 +29,18 @@ def evaluator_generator(params):
elif params.type == 'box_and_mask':
evaluator = coco_evaluator.COCOEvaluator(
annotation_file=params.val_json_file, include_mask=True)
+ elif params.type == 'oln_xclass_box':
+ evaluator = coco_evaluator.OlnXclassEvaluator(
+ annotation_file=params.val_json_file, include_mask=False,
+ use_category=False, seen_class=params.seen_class,)
+ elif params.type == 'oln_xclass_box_and_mask':
+ evaluator = coco_evaluator.OlnXclassEvaluator(
+ annotation_file=params.val_json_file, include_mask=True,
+ use_category=False, seen_class=params.seen_class,)
+ elif params.type == 'oln_xdata_box':
+ evaluator = coco_evaluator.OlnXdataEvaluator(
+ annotation_file=params.val_json_file, include_mask=False,
+ use_category=False, seen_class='all',)
elif params.type == 'shapemask_box_and_mask':
evaluator = coco_evaluator.ShapeMaskCOCOEvaluator(
mask_eval_class=params.mask_eval_class,
diff --git a/official/vision/detection/executor/__init__.py b/official/vision/detection/executor/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/executor/__init__.py
+++ b/official/vision/detection/executor/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/executor/detection_executor.py b/official/vision/detection/executor/detection_executor.py
index 26ff028cf67d6df37e5a0af31bc2e54844231fcd..91c557b6c37eb8de06706bf76fd896aee4683725 100644
--- a/official/vision/detection/executor/detection_executor.py
+++ b/official/vision/detection/executor/detection_executor.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""An executor class for running model on TensorFlow 2.0."""
from __future__ import absolute_import
@@ -22,7 +22,7 @@ from __future__ import print_function
from absl import logging
import tensorflow as tf
-from official.modeling.training import distributed_executor as executor
+from official.vision.detection.executor import distributed_executor as executor
from official.vision.detection.utils.object_detection import visualization_utils
@@ -63,10 +63,9 @@ class DetectionDistributedExecutor(executor.DistributedExecutor):
trainable_variables)
logging.info('Filter trainable variables from %d to %d',
len(model.trainable_variables), len(trainable_variables))
- _update_state = lambda labels, outputs: None
+ update_state_fn = lambda labels, outputs: None
if isinstance(metric, tf.keras.metrics.Metric):
- _update_state = lambda labels, outputs: metric.update_state(
- labels, outputs)
+ update_state_fn = metric.update_state
else:
logging.error('Detection: train metric is not an instance of '
'tf.keras.metrics.Metric.')
@@ -82,10 +81,11 @@ class DetectionDistributedExecutor(executor.DistributedExecutor):
for k, v in all_losses.items():
losses[k] = tf.reduce_mean(v)
per_replica_loss = losses['total_loss'] / strategy.num_replicas_in_sync
- _update_state(labels, outputs)
+ update_state_fn(labels, outputs)
grads = tape.gradient(per_replica_loss, trainable_variables)
- optimizer.apply_gradients(zip(grads, trainable_variables))
+ clipped_grads, _ = tf.clip_by_global_norm(grads, clip_norm=1.0)
+ optimizer.apply_gradients(zip(clipped_grads, trainable_variables))
return losses
return _replicated_step
diff --git a/official/modeling/training/distributed_executor.py b/official/vision/detection/executor/distributed_executor.py
similarity index 93%
rename from official/modeling/training/distributed_executor.py
rename to official/vision/detection/executor/distributed_executor.py
index 11451260cdca52a9c9f4019010123c4d2b40e99e..8f8c861c99f2094756ca90052769bb2a8f2e1fe6 100644
--- a/official/modeling/training/distributed_executor.py
+++ b/official/vision/detection/executor/distributed_executor.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Custom training loop for running TensorFlow 2.0 models."""
from __future__ import absolute_import
@@ -31,7 +31,7 @@ import tensorflow as tf
from typing import Optional, Dict, List, Text, Callable, Union, Iterator, Any
from official.modeling.hyperparams import params_dict
from official.utils import hyperparams_flags
-from official.utils.misc import distribution_utils
+from official.common import distribute_utils
from official.utils.misc import keras_utils
FLAGS = flags.FLAGS
@@ -63,8 +63,8 @@ def metrics_as_dict(metric):
"""Puts input metric(s) into a list.
Args:
- metric: metric(s) to be put into the list. `metric` could be a object, a
- list or a dict of tf.keras.metrics.Metric or has the `required_method`.
+ metric: metric(s) to be put into the list. `metric` could be an object, a
+ list, or a dict of tf.keras.metrics.Metric or has the `required_method`.
Returns:
A dictionary of valid metrics.
@@ -108,7 +108,7 @@ class SummaryWriter(object):
def __init__(self, model_dir: Text, name: Text):
"""Inits SummaryWriter with paths.
- Arguments:
+ Args:
model_dir: the model folder path.
name: the summary subfolder name.
"""
@@ -133,15 +133,9 @@ class SummaryWriter(object):
class DistributedExecutor(object):
- """Interface to train and eval models with tf.distribute.Strategy.
- """
+ """Interface to train and eval models with tf.distribute.Strategy."""
- def __init__(self,
- strategy,
- params,
- model_fn,
- loss_fn,
- is_multi_host=False):
+ def __init__(self, strategy, params, model_fn, loss_fn, is_multi_host=False):
"""Constructor.
Args:
@@ -213,8 +207,7 @@ class DistributedExecutor(object):
# across workers. Since Dataset instance cannot be cloned in eager mode,
# we instead pass callable that returns a dataset.
if self._is_multi_host:
- return iter(
- strategy.experimental_distribute_datasets_from_function(input_fn))
+ return iter(strategy.distribute_datasets_from_function(input_fn))
else:
input_data = input_fn()
return iter(strategy.experimental_distribute_dataset(input_data))
@@ -293,8 +286,7 @@ class DistributedExecutor(object):
raise ValueError('steps should be an Tensor. Python object may cause '
'retracing.')
- per_replica_losses = strategy.run(
- replicated_step, args=(next(iterator),))
+ per_replica_losses = strategy.run(replicated_step, args=(next(iterator),))
for _ in tf.range(num_steps - 1):
per_replica_losses = strategy.run(
replicated_step, args=(next(iterator),))
@@ -351,7 +343,8 @@ class DistributedExecutor(object):
train_input_fn: (params: dict) -> tf.data.Dataset training data input
function.
eval_input_fn: (Optional) same type as train_input_fn. If not None, will
- trigger evaluting metric on eval data. If None, will not run eval step.
+ trigger evaluating metric on eval data. If None, will not run the eval
+ step.
model_dir: the folder path for model checkpoints.
total_steps: total training steps.
iterations_per_loop: train steps per loop. After each loop, this job will
@@ -367,6 +360,7 @@ class DistributedExecutor(object):
available checkpoints. If `False`, will do the evaluation once after the
final step.
save_config: bool. Whether to save params to model_dir.
+
Returns:
The training loss and eval metrics.
"""
@@ -476,16 +470,15 @@ class DistributedExecutor(object):
# Step-0 operations
if current_step == 0 and not latest_checkpoint_file:
- _save_checkpoint(
- checkpoint, model_dir, checkpoint_name.format(step=current_step))
+ _save_checkpoint(checkpoint, model_dir,
+ checkpoint_name.format(step=current_step))
if test_step:
eval_iterator = self._get_input_iterator(eval_input_fn, strategy)
- eval_metric_result = self._run_evaluation(
- test_step, current_step, eval_metric, eval_iterator)
- logging.info(
- 'Step: %s evalation metric = %s.', current_step, eval_metric_result)
- test_summary_writer(
- metrics=eval_metric_result, step=optimizer.iterations)
+ eval_metric_result = self._run_evaluation(test_step, current_step,
+ eval_metric, eval_iterator)
+ logging.info('Step: %s evalation metric = %s.', current_step,
+ eval_metric_result)
+ test_summary_writer(metrics=eval_metric_result, step=optimizer.iterations)
reset_states(eval_metric)
logging.info('Training started')
@@ -518,8 +511,7 @@ class DistributedExecutor(object):
else:
train_metric_result.update({'learning_rate': optimizer.lr.numpy()})
logging.info('Train Step: %d/%d / loss = %s / training metric = %s',
- current_step, total_steps, train_loss,
- train_metric_result)
+ current_step, total_steps, train_loss, train_metric_result)
train_summary_writer(
metrics=train_metric_result, step=optimizer.iterations)
@@ -560,8 +552,7 @@ class DistributedExecutor(object):
eval_metric_result = self._run_evaluation(test_step, current_step,
eval_metric, eval_iterator)
logging.info('Final evaluation metric = %s.', eval_metric_result)
- test_summary_writer(
- metrics=eval_metric_result, step=optimizer.iterations)
+ test_summary_writer(metrics=eval_metric_result, step=optimizer.iterations)
self.train_summary_writer.close()
self.eval_summary_writer.close()
@@ -672,7 +663,7 @@ class DistributedExecutor(object):
raise ValueError('if `eval_metric_fn` is specified, '
'eval_metric_fn must be a callable.')
- old_phrase = tf.keras.backend.learning_phase()
+ old_phase = tf.keras.backend.learning_phase()
tf.keras.backend.set_learning_phase(0)
params = self._params
strategy = self._strategy
@@ -695,10 +686,10 @@ class DistributedExecutor(object):
reader = tf.compat.v1.train.NewCheckpointReader(checkpoint_path)
current_step = reader.get_tensor(
'optimizer/iter/.ATTRIBUTES/VARIABLE_VALUE')
- logging.info(
- 'Checkpoint file %s found and restoring from '
- 'checkpoint', checkpoint_path)
- checkpoint.restore(checkpoint_path)
+ logging.info('Checkpoint file %s found and restoring from '
+ 'checkpoint', checkpoint_path)
+ status = checkpoint.restore(checkpoint_path)
+ status.expect_partial().assert_existing_objects_matched()
self.global_train_step = model.optimizer.iterations
eval_iterator = self._get_input_iterator(eval_input_fn, strategy)
@@ -709,7 +700,7 @@ class DistributedExecutor(object):
summary_writer(metrics=eval_metric_result, step=current_step)
reset_states(eval_metric)
- tf.keras.backend.set_learning_phase(old_phrase)
+ tf.keras.backend.set_learning_phase(old_phase)
return eval_metric_result, current_step
def predict(self):
@@ -753,18 +744,18 @@ class ExecutorBuilder(object):
"""
def __init__(self, strategy_type=None, strategy_config=None):
- _ = distribution_utils.configure_cluster(
- strategy_config.worker_hosts, strategy_config.task_index)
+ _ = distribute_utils.configure_cluster(strategy_config.worker_hosts,
+ strategy_config.task_index)
"""Constructor.
Args:
strategy_type: string. One of 'tpu', 'mirrored', 'multi_worker_mirrored'.
- If None. User is responsible to set the strategy before calling
+ If None, the user is responsible to set the strategy before calling
build_executor(...).
strategy_config: necessary config for constructing the proper Strategy.
Check strategy_flags_dict() for examples of the structure.
"""
- self._strategy = distribution_utils.get_distribution_strategy(
+ self._strategy = distribute_utils.get_distribution_strategy(
distribution_strategy=strategy_type,
num_gpus=strategy_config.num_gpus,
all_reduce_alg=strategy_config.all_reduce_alg,
diff --git a/official/vision/detection/main.py b/official/vision/detection/main.py
index 542be3a1dcc73f82719af2d60dc9abd210787931..6bfdd2906ca67b95bc4086d542066929f0539c85 100644
--- a/official/vision/detection/main.py
+++ b/official/vision/detection/main.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,34 +11,26 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""Main function to train various object detection models."""
-from __future__ import absolute_import
-from __future__ import division
-# from __future__ import google_type_annotations
-from __future__ import print_function
+"""Main function to train various object detection models."""
import functools
import pprint
-# pylint: disable=g-bad-import-order
-import tensorflow as tf
-
from absl import app
from absl import flags
from absl import logging
-# pylint: enable=g-bad-import-order
+import tensorflow as tf
+from official.common import distribute_utils
from official.modeling.hyperparams import params_dict
-from official.modeling.training import distributed_executor as executor
from official.utils import hyperparams_flags
from official.utils.flags import core as flags_core
-from official.utils.misc import distribution_utils
from official.utils.misc import keras_utils
from official.vision.detection.configs import factory as config_factory
from official.vision.detection.dataloader import input_reader
from official.vision.detection.dataloader import mode_keys as ModeKeys
+from official.vision.detection.executor import distributed_executor as executor
from official.vision.detection.executor.detection_executor import DetectionDistributedExecutor
from official.vision.detection.modeling import factory as model_factory
@@ -48,7 +40,9 @@ flags_core.define_log_steps()
flags.DEFINE_bool('enable_xla', default=False, help='Enable XLA for GPU')
flags.DEFINE_string(
- 'mode', default='train', help='Mode to run: `train` or `eval`.')
+ 'mode',
+ default='train',
+ help='Mode to run: `train`, `eval` or `eval_once`.')
flags.DEFINE_string(
'model', default='retinanet',
@@ -76,9 +70,7 @@ def run_executor(params,
"""Runs the object detection model on distribution strategy defined by the user."""
if params.architecture.use_bfloat16:
- policy = tf.compat.v2.keras.mixed_precision.experimental.Policy(
- 'mixed_bfloat16')
- tf.compat.v2.keras.mixed_precision.experimental.set_policy(policy)
+ tf.compat.v2.keras.mixed_precision.set_global_policy('mixed_bfloat16')
model_builder = model_factory.model_generator(params)
@@ -86,9 +78,9 @@ def run_executor(params,
strategy = prebuilt_strategy
else:
strategy_config = params.strategy_config
- distribution_utils.configure_cluster(strategy_config.worker_hosts,
- strategy_config.task_index)
- strategy = distribution_utils.get_distribution_strategy(
+ distribute_utils.configure_cluster(strategy_config.worker_hosts,
+ strategy_config.task_index)
+ strategy = distribute_utils.get_distribution_strategy(
distribution_strategy=params.strategy_type,
num_gpus=strategy_config.num_gpus,
all_reduce_alg=strategy_config.all_reduce_alg,
diff --git a/official/vision/detection/modeling/__init__.py b/official/vision/detection/modeling/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/modeling/__init__.py
+++ b/official/vision/detection/modeling/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/modeling/architecture/__init__.py b/official/vision/detection/modeling/architecture/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/modeling/architecture/__init__.py
+++ b/official/vision/detection/modeling/architecture/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/modeling/architecture/factory.py b/official/vision/detection/modeling/architecture/factory.py
index 403d815eaafd91feb999d13b034c864d99804963..f39949d26ffd0f1ba3ac195b4c059744b6c99579 100644
--- a/official/vision/detection/modeling/architecture/factory.py
+++ b/official/vision/detection/modeling/architecture/factory.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Model architecture factory."""
from __future__ import absolute_import
@@ -77,11 +77,13 @@ def multilevel_features_generator(params):
def retinanet_head_generator(params):
"""Generator function for RetinaNet head architecture."""
head_params = params.retinanet_head
+ anchors_per_location = params.anchor.num_scales * len(
+ params.anchor.aspect_ratios)
return heads.RetinanetHead(
params.architecture.min_level,
params.architecture.max_level,
params.architecture.num_classes,
- head_params.anchors_per_location,
+ anchors_per_location,
head_params.num_convs,
head_params.num_filters,
head_params.use_separable_conv,
@@ -91,10 +93,29 @@ def retinanet_head_generator(params):
def rpn_head_generator(params):
"""Generator function for RPN head architecture."""
head_params = params.rpn_head
+ anchors_per_location = params.anchor.num_scales * len(
+ params.anchor.aspect_ratios)
return heads.RpnHead(
params.architecture.min_level,
params.architecture.max_level,
- head_params.anchors_per_location,
+ anchors_per_location,
+ head_params.num_convs,
+ head_params.num_filters,
+ head_params.use_separable_conv,
+ params.norm_activation.activation,
+ head_params.use_batch_norm,
+ norm_activation=norm_activation_generator(params.norm_activation))
+
+
+def oln_rpn_head_generator(params):
+ """Generator function for OLN-proposal (OLN-RPN) head architecture."""
+ head_params = params.rpn_head
+ anchors_per_location = params.anchor.num_scales * len(
+ params.anchor.aspect_ratios)
+ return heads.OlnRpnHead(
+ params.architecture.min_level,
+ params.architecture.max_level,
+ anchors_per_location,
head_params.num_convs,
head_params.num_filters,
head_params.use_separable_conv,
@@ -118,6 +139,21 @@ def fast_rcnn_head_generator(params):
norm_activation=norm_activation_generator(params.norm_activation))
+def oln_box_score_head_generator(params):
+ """Generator function for Scoring Fast R-CNN head architecture."""
+ head_params = params.frcnn_head
+ return heads.OlnBoxScoreHead(
+ params.architecture.num_classes,
+ head_params.num_convs,
+ head_params.num_filters,
+ head_params.use_separable_conv,
+ head_params.num_fcs,
+ head_params.fc_dims,
+ params.norm_activation.activation,
+ head_params.use_batch_norm,
+ norm_activation=norm_activation_generator(params.norm_activation))
+
+
def mask_rcnn_head_generator(params):
"""Generator function for Mask R-CNN head architecture."""
head_params = params.mrcnn_head
@@ -132,6 +168,20 @@ def mask_rcnn_head_generator(params):
norm_activation=norm_activation_generator(params.norm_activation))
+def oln_mask_score_head_generator(params):
+ """Generator function for Scoring Mask R-CNN head architecture."""
+ head_params = params.mrcnn_head
+ return heads.OlnMaskScoreHead(
+ params.architecture.num_classes,
+ params.architecture.mask_target_size,
+ head_params.num_convs,
+ head_params.num_filters,
+ head_params.use_separable_conv,
+ params.norm_activation.activation,
+ head_params.use_batch_norm,
+ norm_activation=norm_activation_generator(params.norm_activation))
+
+
def shapeprior_head_generator(params):
"""Generator function for shape prior head architecture."""
head_params = params.shapemask_head
diff --git a/official/vision/detection/modeling/architecture/fpn.py b/official/vision/detection/modeling/architecture/fpn.py
index b968dc2e152eb66e2df7ca7673b506c123b59d0f..3cfb56dbdec6c6b09e4cc7f6bbd70b054f6cbc10 100644
--- a/official/vision/detection/modeling/architecture/fpn.py
+++ b/official/vision/detection/modeling/architecture/fpn.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Feature Pyramid Networks.
Feature Pyramid Networks were proposed in:
@@ -28,7 +28,6 @@ import functools
import tensorflow as tf
-from tensorflow.python.keras import backend
from official.vision.detection.modeling.architecture import nn_ops
from official.vision.detection.ops import spatial_transform_ops
@@ -120,7 +119,7 @@ class Fpn(object):
'The minimum backbone level %d should be '%(min(input_levels)) +
'less or equal to FPN minimum level %d.:'%(self._min_level))
backbone_max_level = min(max(input_levels), self._max_level)
- with backend.get_graph().as_default(), tf.name_scope('fpn'):
+ with tf.name_scope('fpn'):
# Adds lateral connections.
feats_lateral = {}
for level in range(self._min_level, backbone_max_level + 1):
diff --git a/official/vision/detection/modeling/architecture/heads.py b/official/vision/detection/modeling/architecture/heads.py
index 7f6954aecbbef8e8807345e643555ba222b0e1b9..8eb89892d67bd33541c6586cb035cdffbdc31ad8 100644
--- a/official/vision/detection/modeling/architecture/heads.py
+++ b/official/vision/detection/modeling/architecture/heads.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Classes to build various prediction heads in all supported models."""
from __future__ import absolute_import
@@ -22,7 +22,7 @@ import functools
import numpy as np
import tensorflow as tf
-from tensorflow.python.keras import backend
+
from official.vision.detection.modeling.architecture import nn_ops
from official.vision.detection.ops import spatial_transform_ops
@@ -30,17 +30,17 @@ from official.vision.detection.ops import spatial_transform_ops
class RpnHead(tf.keras.layers.Layer):
"""Region Proposal Network head."""
- def __init__(self,
- min_level,
- max_level,
- anchors_per_location,
- num_convs=2,
- num_filters=256,
- use_separable_conv=False,
- activation='relu',
- use_batch_norm=True,
- norm_activation=nn_ops.norm_activation_builder(
- activation='relu')):
+ def __init__(
+ self,
+ min_level,
+ max_level,
+ anchors_per_location,
+ num_convs=2,
+ num_filters=256,
+ use_separable_conv=False,
+ activation='relu',
+ use_batch_norm=True,
+ norm_activation=nn_ops.norm_activation_builder(activation='relu')):
"""Initialize params to build Region Proposal Network head.
Args:
@@ -56,9 +56,11 @@ class RpnHead(tf.keras.layers.Layer):
is used.
activation: activation function. Support 'relu' and 'swish'.
use_batch_norm: 'bool', indicating whether batchnorm layers are added.
- norm_activation: an operation that includes a normalization layer
- followed by an optional activation layer.
+ norm_activation: an operation that includes a normalization layer followed
+ by an optional activation layer.
"""
+ super().__init__(autocast=False)
+
self._min_level = min_level
self._max_level = max_level
self._anchors_per_location = anchors_per_location
@@ -122,12 +124,12 @@ class RpnHead(tf.keras.layers.Layer):
return scores, bboxes
- def __call__(self, features, is_training=None):
+ def call(self, features, is_training=None):
scores_outputs = {}
box_outputs = {}
- with backend.get_graph().as_default(), tf.name_scope('rpn_head'):
+ with tf.name_scope('rpn_head'):
for level in range(self._min_level, self._max_level + 1):
scores_output, box_output = self._shared_rpn_heads(
features[level], self._anchors_per_location, level, is_training)
@@ -136,20 +138,144 @@ class RpnHead(tf.keras.layers.Layer):
return scores_outputs, box_outputs
+class OlnRpnHead(tf.keras.layers.Layer):
+ """Region Proposal Network for Object Localization Network (OLN)."""
+
+ def __init__(
+ self,
+ min_level,
+ max_level,
+ anchors_per_location,
+ num_convs=2,
+ num_filters=256,
+ use_separable_conv=False,
+ activation='relu',
+ use_batch_norm=True,
+ norm_activation=nn_ops.norm_activation_builder(activation='relu')):
+ """Initialize params to build Region Proposal Network head.
+
+ Args:
+ min_level: `int` number of minimum feature level.
+ max_level: `int` number of maximum feature level.
+ anchors_per_location: `int` number of number of anchors per pixel
+ location.
+ num_convs: `int` number that represents the number of the intermediate
+ conv layers before the prediction.
+ num_filters: `int` number that represents the number of filters of the
+ intermediate conv layers.
+ use_separable_conv: `bool`, indicating whether the separable conv layers
+ is used.
+ activation: activation function. Support 'relu' and 'swish'.
+ use_batch_norm: 'bool', indicating whether batchnorm layers are added.
+ norm_activation: an operation that includes a normalization layer followed
+ by an optional activation layer.
+ """
+ self._min_level = min_level
+ self._max_level = max_level
+ self._anchors_per_location = anchors_per_location
+ if activation == 'relu':
+ self._activation_op = tf.nn.relu
+ elif activation == 'swish':
+ self._activation_op = tf.nn.swish
+ else:
+ raise ValueError('Unsupported activation `{}`.'.format(activation))
+ self._use_batch_norm = use_batch_norm
+
+ if use_separable_conv:
+ self._conv2d_op = functools.partial(
+ tf.keras.layers.SeparableConv2D,
+ depth_multiplier=1,
+ bias_initializer=tf.zeros_initializer())
+ else:
+ self._conv2d_op = functools.partial(
+ tf.keras.layers.Conv2D,
+ kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01),
+ bias_initializer=tf.zeros_initializer())
+
+ self._rpn_conv = self._conv2d_op(
+ num_filters,
+ kernel_size=(3, 3),
+ strides=(1, 1),
+ activation=(None if self._use_batch_norm else self._activation_op),
+ padding='same',
+ name='rpn')
+ self._rpn_class_conv = self._conv2d_op(
+ anchors_per_location,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ padding='valid',
+ name='rpn-class')
+ self._rpn_box_conv = self._conv2d_op(
+ 4 * anchors_per_location,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ padding='valid',
+ name='rpn-box-lrtb')
+ self._rpn_center_conv = self._conv2d_op(
+ anchors_per_location,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ padding='valid',
+ name='rpn-centerness')
+
+ self._norm_activations = {}
+ if self._use_batch_norm:
+ for level in range(self._min_level, self._max_level + 1):
+ self._norm_activations[level] = norm_activation(name='rpn-l%d-bn' %
+ level)
+
+ def _shared_rpn_heads(self, features, anchors_per_location, level,
+ is_training):
+ """Shared RPN heads."""
+ features = self._rpn_conv(features)
+ if self._use_batch_norm:
+ # The batch normalization layers are not shared between levels.
+ features = self._norm_activations[level](
+ features, is_training=is_training)
+ # Feature L2 normalization for training stability
+ features = tf.math.l2_normalize(
+ features,
+ axis=-1,
+ name='rpn-norm',)
+ # Proposal classification scores
+ scores = self._rpn_class_conv(features)
+ # Proposal bbox regression deltas
+ bboxes = self._rpn_box_conv(features)
+ # Proposal centerness scores
+ centers = self._rpn_center_conv(features)
+
+ return scores, bboxes, centers
+
+ def __call__(self, features, is_training=None):
+
+ scores_outputs = {}
+ box_outputs = {}
+ center_outputs = {}
+
+ with tf.name_scope('rpn_head'):
+ for level in range(self._min_level, self._max_level + 1):
+ scores_output, box_output, center_output = self._shared_rpn_heads(
+ features[level], self._anchors_per_location, level, is_training)
+ scores_outputs[level] = scores_output
+ box_outputs[level] = box_output
+ center_outputs[level] = center_output
+ return scores_outputs, box_outputs, center_outputs
+
+
class FastrcnnHead(tf.keras.layers.Layer):
"""Fast R-CNN box head."""
- def __init__(self,
- num_classes,
- num_convs=0,
- num_filters=256,
- use_separable_conv=False,
- num_fcs=2,
- fc_dims=1024,
- activation='relu',
- use_batch_norm=True,
- norm_activation=nn_ops.norm_activation_builder(
- activation='relu')):
+ def __init__(
+ self,
+ num_classes,
+ num_convs=0,
+ num_filters=256,
+ use_separable_conv=False,
+ num_fcs=2,
+ fc_dims=1024,
+ activation='relu',
+ use_batch_norm=True,
+ norm_activation=nn_ops.norm_activation_builder(activation='relu')):
"""Initialize params to build Fast R-CNN box head.
Args:
@@ -166,9 +292,11 @@ class FastrcnnHead(tf.keras.layers.Layer):
layers.
activation: activation function. Support 'relu' and 'swish'.
use_batch_norm: 'bool', indicating whether batchnorm layers are added.
- norm_activation: an operation that includes a normalization layer
- followed by an optional activation layer.
+ norm_activation: an operation that includes a normalization layer followed
+ by an optional activation layer.
"""
+ super(FastrcnnHead, self).__init__(autocast=False)
+
self._num_classes = num_classes
self._num_convs = num_convs
@@ -206,7 +334,8 @@ class FastrcnnHead(tf.keras.layers.Layer):
strides=(1, 1),
padding='same',
dilation_rate=(1, 1),
- activation=(None if self._use_batch_norm else self._activation_op),
+ activation=(None
+ if self._use_batch_norm else self._activation_op),
name='conv_{}'.format(i)))
if self._use_batch_norm:
self._conv_bn_ops.append(self._norm_activation())
@@ -217,7 +346,8 @@ class FastrcnnHead(tf.keras.layers.Layer):
self._fc_ops.append(
tf.keras.layers.Dense(
units=self._fc_dims,
- activation=(None if self._use_batch_norm else self._activation_op),
+ activation=(None
+ if self._use_batch_norm else self._activation_op),
name='fc{}'.format(i)))
if self._use_batch_norm:
self._fc_bn_ops.append(self._norm_activation(fused=False))
@@ -233,12 +363,12 @@ class FastrcnnHead(tf.keras.layers.Layer):
bias_initializer=tf.zeros_initializer(),
name='box-predict')
- def __call__(self, roi_features, is_training=None):
+ def call(self, roi_features, is_training=None):
"""Box and class branches for the Mask-RCNN model.
Args:
- roi_features: A ROI feature tensor of shape
- [batch_size, num_rois, height_l, width_l, num_filters].
+ roi_features: A ROI feature tensor of shape [batch_size, num_rois,
+ height_l, width_l, num_filters].
is_training: `boolean`, if True if model is in training mode.
Returns:
@@ -249,7 +379,8 @@ class FastrcnnHead(tf.keras.layers.Layer):
predictions.
"""
- with backend.get_graph().as_default(), tf.name_scope('fast_rcnn_head'):
+ with tf.name_scope(
+ 'fast_rcnn_head'):
# reshape inputs beofre FC.
_, num_rois, height, width, filters = roi_features.get_shape().as_list()
@@ -272,19 +403,163 @@ class FastrcnnHead(tf.keras.layers.Layer):
return class_outputs, box_outputs
+class OlnBoxScoreHead(tf.keras.layers.Layer):
+ """Box head of Object Localization Network (OLN)."""
+
+ def __init__(
+ self,
+ num_classes,
+ num_convs=0,
+ num_filters=256,
+ use_separable_conv=False,
+ num_fcs=2,
+ fc_dims=1024,
+ activation='relu',
+ use_batch_norm=True,
+ norm_activation=nn_ops.norm_activation_builder(activation='relu')):
+ """Initialize params to build OLN box head.
+
+ Args:
+ num_classes: a integer for the number of classes.
+ num_convs: `int` number that represents the number of the intermediate
+ conv layers before the FC layers.
+ num_filters: `int` number that represents the number of filters of the
+ intermediate conv layers.
+ use_separable_conv: `bool`, indicating whether the separable conv layers
+ is used.
+ num_fcs: `int` number that represents the number of FC layers before the
+ predictions.
+ fc_dims: `int` number that represents the number of dimension of the FC
+ layers.
+ activation: activation function. Support 'relu' and 'swish'.
+ use_batch_norm: 'bool', indicating whether batchnorm layers are added.
+ norm_activation: an operation that includes a normalization layer followed
+ by an optional activation layer.
+ """
+ self._num_classes = num_classes
+
+ self._num_convs = num_convs
+ self._num_filters = num_filters
+ if use_separable_conv:
+ self._conv2d_op = functools.partial(
+ tf.keras.layers.SeparableConv2D,
+ depth_multiplier=1,
+ bias_initializer=tf.zeros_initializer())
+ else:
+ self._conv2d_op = functools.partial(
+ tf.keras.layers.Conv2D,
+ kernel_initializer=tf.keras.initializers.VarianceScaling(
+ scale=2, mode='fan_out', distribution='untruncated_normal'),
+ bias_initializer=tf.zeros_initializer())
+
+ self._num_fcs = num_fcs
+ self._fc_dims = fc_dims
+ if activation == 'relu':
+ self._activation_op = tf.nn.relu
+ elif activation == 'swish':
+ self._activation_op = tf.nn.swish
+ else:
+ raise ValueError('Unsupported activation `{}`.'.format(activation))
+ self._use_batch_norm = use_batch_norm
+ self._norm_activation = norm_activation
+
+ self._conv_ops = []
+ self._conv_bn_ops = []
+ for i in range(self._num_convs):
+ self._conv_ops.append(
+ self._conv2d_op(
+ self._num_filters,
+ kernel_size=(3, 3),
+ strides=(1, 1),
+ padding='same',
+ dilation_rate=(1, 1),
+ activation=(None
+ if self._use_batch_norm else self._activation_op),
+ name='conv_{}'.format(i)))
+ if self._use_batch_norm:
+ self._conv_bn_ops.append(self._norm_activation())
+
+ self._fc_ops = []
+ self._fc_bn_ops = []
+ for i in range(self._num_fcs):
+ self._fc_ops.append(
+ tf.keras.layers.Dense(
+ units=self._fc_dims,
+ activation=(None
+ if self._use_batch_norm else self._activation_op),
+ name='fc{}'.format(i)))
+ if self._use_batch_norm:
+ self._fc_bn_ops.append(self._norm_activation(fused=False))
+
+ self._class_predict = tf.keras.layers.Dense(
+ self._num_classes,
+ kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01),
+ bias_initializer=tf.zeros_initializer(),
+ name='class-predict')
+ self._box_predict = tf.keras.layers.Dense(
+ self._num_classes * 4,
+ kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.001),
+ bias_initializer=tf.zeros_initializer(),
+ name='box-predict')
+ self._score_predict = tf.keras.layers.Dense(
+ 1,
+ kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01),
+ bias_initializer=tf.zeros_initializer(),
+ name='score-predict')
+
+ def __call__(self, roi_features, is_training=None):
+ """Box and class branches for the Mask-RCNN model.
+
+ Args:
+ roi_features: A ROI feature tensor of shape [batch_size, num_rois,
+ height_l, width_l, num_filters].
+ is_training: `boolean`, if True if model is in training mode.
+
+ Returns:
+ class_outputs: a tensor with a shape of
+ [batch_size, num_rois, num_classes], representing the class predictions.
+ box_outputs: a tensor with a shape of
+ [batch_size, num_rois, num_classes * 4], representing the box
+ predictions.
+ """
+
+ with tf.name_scope('fast_rcnn_head'):
+ # reshape inputs beofre FC.
+ _, num_rois, height, width, filters = roi_features.get_shape().as_list()
+
+ net = tf.reshape(roi_features, [-1, height, width, filters])
+ for i in range(self._num_convs):
+ net = self._conv_ops[i](net)
+ if self._use_batch_norm:
+ net = self._conv_bn_ops[i](net, is_training=is_training)
+
+ filters = self._num_filters if self._num_convs > 0 else filters
+ net = tf.reshape(net, [-1, num_rois, height * width * filters])
+
+ for i in range(self._num_fcs):
+ net = self._fc_ops[i](net)
+ if self._use_batch_norm:
+ net = self._fc_bn_ops[i](net, is_training=is_training)
+
+ class_outputs = self._class_predict(net)
+ box_outputs = self._box_predict(net)
+ score_outputs = self._score_predict(net)
+ return class_outputs, box_outputs, score_outputs
+
+
class MaskrcnnHead(tf.keras.layers.Layer):
"""Mask R-CNN head."""
- def __init__(self,
- num_classes,
- mask_target_size,
- num_convs=4,
- num_filters=256,
- use_separable_conv=False,
- activation='relu',
- use_batch_norm=True,
- norm_activation=nn_ops.norm_activation_builder(
- activation='relu')):
+ def __init__(
+ self,
+ num_classes,
+ mask_target_size,
+ num_convs=4,
+ num_filters=256,
+ use_separable_conv=False,
+ activation='relu',
+ use_batch_norm=True,
+ norm_activation=nn_ops.norm_activation_builder(activation='relu')):
"""Initialize params to build Fast R-CNN head.
Args:
@@ -298,9 +573,10 @@ class MaskrcnnHead(tf.keras.layers.Layer):
is used.
activation: activation function. Support 'relu' and 'swish'.
use_batch_norm: 'bool', indicating whether batchnorm layers are added.
- norm_activation: an operation that includes a normalization layer
- followed by an optional activation layer.
+ norm_activation: an operation that includes a normalization layer followed
+ by an optional activation layer.
"""
+ super(MaskrcnnHead, self).__init__(autocast=False)
self._num_classes = num_classes
self._mask_target_size = mask_target_size
@@ -334,7 +610,8 @@ class MaskrcnnHead(tf.keras.layers.Layer):
strides=(1, 1),
padding='same',
dilation_rate=(1, 1),
- activation=(None if self._use_batch_norm else self._activation_op),
+ activation=(None
+ if self._use_batch_norm else self._activation_op),
name='mask-conv-l%d' % i))
self._mask_conv_transpose = tf.keras.layers.Conv2DTranspose(
self._num_filters,
@@ -347,14 +624,22 @@ class MaskrcnnHead(tf.keras.layers.Layer):
bias_initializer=tf.zeros_initializer(),
name='conv5-mask')
- def __call__(self, roi_features, class_indices, is_training=None):
+ with tf.name_scope('mask_head'):
+ self._mask_conv2d_op = self._conv2d_op(
+ self._num_classes,
+ kernel_size=(1, 1),
+ strides=(1, 1),
+ padding='valid',
+ name='mask_fcn_logits')
+
+ def call(self, roi_features, class_indices, is_training=None):
"""Mask branch for the Mask-RCNN model.
Args:
- roi_features: A ROI feature tensor of shape
- [batch_size, num_rois, height_l, width_l, num_filters].
- class_indices: a Tensor of shape [batch_size, num_rois], indicating
- which class the ROI is.
+ roi_features: A ROI feature tensor of shape [batch_size, num_rois,
+ height_l, width_l, num_filters].
+ class_indices: a Tensor of shape [batch_size, num_rois], indicating which
+ class the ROI is.
is_training: `boolean`, if True if model is in training mode.
Returns:
@@ -368,61 +653,54 @@ class MaskrcnnHead(tf.keras.layers.Layer):
boxes is not 4.
"""
- with backend.get_graph().as_default():
- with tf.name_scope('mask_head'):
- _, num_rois, height, width, filters = roi_features.get_shape().as_list()
- net = tf.reshape(roi_features, [-1, height, width, filters])
-
- for i in range(self._num_convs):
- net = self._conv2d_ops[i](net)
- if self._use_batch_norm:
- net = self._norm_activation()(net, is_training=is_training)
+ with tf.name_scope('mask_head'):
+ _, num_rois, height, width, filters = roi_features.get_shape().as_list()
+ net = tf.reshape(roi_features, [-1, height, width, filters])
- net = self._mask_conv_transpose(net)
+ for i in range(self._num_convs):
+ net = self._conv2d_ops[i](net)
if self._use_batch_norm:
net = self._norm_activation()(net, is_training=is_training)
- mask_outputs = self._conv2d_op(
- self._num_classes,
- kernel_size=(1, 1),
- strides=(1, 1),
- padding='valid',
- name='mask_fcn_logits')(
- net)
- mask_outputs = tf.reshape(mask_outputs, [
- -1, num_rois, self._mask_target_size, self._mask_target_size,
- self._num_classes
- ])
-
- with tf.name_scope('masks_post_processing'):
- # TODO(pengchong): Figure out the way not to use the static inferred
- # batch size.
- batch_size, num_masks = class_indices.get_shape().as_list()
- mask_outputs = tf.transpose(a=mask_outputs, perm=[0, 1, 4, 2, 3])
- # Contructs indices for gather.
- batch_indices = tf.tile(
- tf.expand_dims(tf.range(batch_size), axis=1), [1, num_masks])
- mask_indices = tf.tile(
- tf.expand_dims(tf.range(num_masks), axis=0), [batch_size, 1])
- gather_indices = tf.stack(
- [batch_indices, mask_indices, class_indices], axis=2)
- mask_outputs = tf.gather_nd(mask_outputs, gather_indices)
+ net = self._mask_conv_transpose(net)
+ if self._use_batch_norm:
+ net = self._norm_activation()(net, is_training=is_training)
+
+ mask_outputs = self._mask_conv2d_op(net)
+ mask_outputs = tf.reshape(mask_outputs, [
+ -1, num_rois, self._mask_target_size, self._mask_target_size,
+ self._num_classes
+ ])
+
+ with tf.name_scope('masks_post_processing'):
+ # TODO(pengchong): Figure out the way not to use the static inferred
+ # batch size.
+ batch_size, num_masks = class_indices.get_shape().as_list()
+ mask_outputs = tf.transpose(a=mask_outputs, perm=[0, 1, 4, 2, 3])
+ # Constructs indices for gather.
+ batch_indices = tf.tile(
+ tf.expand_dims(tf.range(batch_size), axis=1), [1, num_masks])
+ mask_indices = tf.tile(
+ tf.expand_dims(tf.range(num_masks), axis=0), [batch_size, 1])
+ gather_indices = tf.stack(
+ [batch_indices, mask_indices, class_indices], axis=2)
+ mask_outputs = tf.gather_nd(mask_outputs, gather_indices)
return mask_outputs
class RetinanetHead(object):
"""RetinaNet head."""
- def __init__(self,
- min_level,
- max_level,
- num_classes,
- anchors_per_location,
- num_convs=4,
- num_filters=256,
- use_separable_conv=False,
- norm_activation=nn_ops.norm_activation_builder(
- activation='relu')):
+ def __init__(
+ self,
+ min_level,
+ max_level,
+ num_classes,
+ anchors_per_location,
+ num_convs=4,
+ num_filters=256,
+ use_separable_conv=False,
+ norm_activation=nn_ops.norm_activation_builder(activation='relu')):
"""Initialize params to build RetinaNet head.
Args:
@@ -435,8 +713,8 @@ class RetinanetHead(object):
num_filters: `int` number of filters used in the head architecture.
use_separable_conv: `bool` to indicate whether to use separable
convoluation.
- norm_activation: an operation that includes a normalization layer
- followed by an optional activation layer.
+ norm_activation: an operation that includes a normalization layer followed
+ by an optional activation layer.
"""
self._min_level = min_level
self._max_level = max_level
@@ -552,7 +830,7 @@ class RetinanetHead(object):
"""Returns outputs of RetinaNet head."""
class_outputs = {}
box_outputs = {}
- with backend.get_graph().as_default(), tf.name_scope('retinanet_head'):
+ with tf.name_scope('retinanet_head'):
for level in range(self._min_level, self._max_level + 1):
features = fpn_features[level]
@@ -597,12 +875,8 @@ class RetinanetHead(object):
class ShapemaskPriorHead(object):
"""ShapeMask Prior head."""
- def __init__(self,
- num_classes,
- num_downsample_channels,
- mask_crop_size,
- use_category_for_mask,
- shape_prior_path):
+ def __init__(self, num_classes, num_downsample_channels, mask_crop_size,
+ use_category_for_mask, shape_prior_path):
"""Initialize params to build RetinaNet head.
Args:
@@ -629,12 +903,12 @@ class ShapemaskPriorHead(object):
Args:
fpn_features: a dictionary of FPN features.
- boxes: a float tensor of shape [batch_size, num_instances, 4]
- representing the tight gt boxes from dataloader/detection.
+ boxes: a float tensor of shape [batch_size, num_instances, 4] representing
+ the tight gt boxes from dataloader/detection.
outer_boxes: a float tensor of shape [batch_size, num_instances, 4]
representing the loose gt boxes from dataloader/detection.
- classes: a int Tensor of shape [batch_size, num_instances]
- of instance classes.
+ classes: a int Tensor of shape [batch_size, num_instances] of instance
+ classes.
is_training: training mode or not.
Returns:
@@ -644,7 +918,7 @@ class ShapemaskPriorHead(object):
detection_priors: A float Tensor of shape [batch_size * num_instances,
mask_size, mask_size, 1].
"""
- with backend.get_graph().as_default(), tf.name_scope('prior_mask'):
+ with tf.name_scope('prior_mask'):
batch_size, num_instances, _ = boxes.get_shape().as_list()
outer_boxes = tf.cast(outer_boxes, tf.float32)
boxes = tf.cast(boxes, tf.float32)
@@ -655,8 +929,9 @@ class ShapemaskPriorHead(object):
shape_priors = self._get_priors()
# Get uniform priors for each outer box.
- uniform_priors = tf.ones([batch_size, num_instances, self._mask_crop_size,
- self._mask_crop_size])
+ uniform_priors = tf.ones([
+ batch_size, num_instances, self._mask_crop_size, self._mask_crop_size
+ ])
uniform_priors = spatial_transform_ops.crop_mask_in_target_box(
uniform_priors, boxes, outer_boxes, self._mask_crop_size)
@@ -665,8 +940,9 @@ class ShapemaskPriorHead(object):
tf.cast(instance_features, tf.float32), uniform_priors, classes)
instance_priors = tf.gather(shape_priors, classes)
- instance_priors *= tf.expand_dims(tf.expand_dims(
- tf.cast(prior_distribution, tf.float32), axis=-1), axis=-1)
+ instance_priors *= tf.expand_dims(
+ tf.expand_dims(tf.cast(prior_distribution, tf.float32), axis=-1),
+ axis=-1)
instance_priors = tf.reduce_sum(instance_priors, axis=2)
detection_priors = spatial_transform_ops.crop_mask_in_target_box(
instance_priors, boxes, outer_boxes, self._mask_crop_size)
@@ -685,8 +961,10 @@ class ShapemaskPriorHead(object):
# If prior path does not exist, do not use priors, i.e., pirors equal to
# uniform empty 32x32 patch.
self._num_clusters = 1
- priors = tf.zeros([self._mask_num_classes, self._num_clusters,
- self._mask_crop_size, self._mask_crop_size])
+ priors = tf.zeros([
+ self._mask_num_classes, self._num_clusters, self._mask_crop_size,
+ self._mask_crop_size
+ ])
return priors
def _classify_shape_priors(self, features, uniform_priors, classes):
@@ -696,12 +974,12 @@ class ShapemaskPriorHead(object):
category.
Args:
- features: A float Tensor of shape [batch_size, num_instances,
- mask_size, mask_size, num_channels].
+ features: A float Tensor of shape [batch_size, num_instances, mask_size,
+ mask_size, num_channels].
uniform_priors: A float Tensor of shape [batch_size, num_instances,
mask_size, mask_size] representing the uniform detection priors.
- classes: A int Tensor of shape [batch_size, num_instances]
- of detection class ids.
+ classes: A int Tensor of shape [batch_size, num_instances] of detection
+ class ids.
Returns:
prior_distribution: A float Tensor of shape
@@ -716,10 +994,11 @@ class ShapemaskPriorHead(object):
features = tf.reduce_mean(features, axis=(2, 3))
logits = tf.keras.layers.Dense(
self._mask_num_classes * self._num_clusters,
- kernel_initializer=tf.random_normal_initializer(stddev=0.01))(features)
- logits = tf.reshape(logits,
- [batch_size, num_instances,
- self._mask_num_classes, self._num_clusters])
+ kernel_initializer=tf.random_normal_initializer(stddev=0.01),
+ name='classify-shape-prior-fc')(features)
+ logits = tf.reshape(
+ logits,
+ [batch_size, num_instances, self._mask_num_classes, self._num_clusters])
if self._use_category_for_mask:
logits = tf.gather(logits, tf.expand_dims(classes, axis=-1), batch_dims=2)
logits = tf.squeeze(logits, axis=2)
@@ -749,8 +1028,8 @@ class ShapemaskCoarsemaskHead(object):
use_category_for_mask: use class information in mask branch.
num_convs: `int` number of stacked convolution before the last prediction
layer.
- norm_activation: an operation that includes a normalization layer
- followed by an optional activation layer.
+ norm_activation: an operation that includes a normalization layer followed
+ by an optional activation layer.
"""
self._mask_num_classes = num_classes if use_category_for_mask else 1
self._use_category_for_mask = use_category_for_mask
@@ -766,13 +1045,15 @@ class ShapemaskCoarsemaskHead(object):
self._class_norm_activation = []
for i in range(self._num_convs):
- self._class_conv.append(tf.keras.layers.Conv2D(
- self._num_downsample_channels,
- kernel_size=(3, 3),
- bias_initializer=tf.zeros_initializer(),
- kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01),
- padding='same',
- name='coarse-mask-class-%d' % i))
+ self._class_conv.append(
+ tf.keras.layers.Conv2D(
+ self._num_downsample_channels,
+ kernel_size=(3, 3),
+ bias_initializer=tf.zeros_initializer(),
+ kernel_initializer=tf.keras.initializers.RandomNormal(
+ stddev=0.01),
+ padding='same',
+ name='coarse-mask-class-%d' % i))
self._class_norm_activation.append(
norm_activation(name='coarse-mask-class-%d-bn' % i))
@@ -797,17 +1078,17 @@ class ShapemaskCoarsemaskHead(object):
mask_crop_size, mask_crop_size, num_downsample_channels]. This is the
instance feature crop.
detection_priors: a float Tensor of shape [batch_size, num_instances,
- mask_crop_size, mask_crop_size, 1]. This is the detection prior for
- the instance.
- classes: a int Tensor of shape [batch_size, num_instances]
- of instance classes.
+ mask_crop_size, mask_crop_size, 1]. This is the detection prior for the
+ instance.
+ classes: a int Tensor of shape [batch_size, num_instances] of instance
+ classes.
is_training: a bool indicating whether in training mode.
Returns:
mask_outputs: instance mask prediction as a float Tensor of shape
[batch_size, num_instances, mask_size, mask_size].
"""
- with backend.get_graph().as_default(), tf.name_scope('coarse_mask'):
+ with tf.name_scope('coarse_mask'):
# Transform detection priors to have the same dimension as features.
detection_priors = tf.expand_dims(detection_priors, axis=-1)
detection_priors = self._coarse_mask_fc(detection_priors)
@@ -817,8 +1098,8 @@ class ShapemaskCoarsemaskHead(object):
# Gather the logits with right input class.
if self._use_category_for_mask:
mask_logits = tf.transpose(mask_logits, [0, 1, 4, 2, 3])
- mask_logits = tf.gather(mask_logits, tf.expand_dims(classes, -1),
- batch_dims=2)
+ mask_logits = tf.gather(
+ mask_logits, tf.expand_dims(classes, -1), batch_dims=2)
mask_logits = tf.squeeze(mask_logits, axis=2)
else:
mask_logits = mask_logits[..., 0]
@@ -838,16 +1119,17 @@ class ShapemaskCoarsemaskHead(object):
"""
(batch_size, num_instances, height, width,
num_channels) = features.get_shape().as_list()
- features = tf.reshape(features, [batch_size * num_instances, height, width,
- num_channels])
+ features = tf.reshape(
+ features, [batch_size * num_instances, height, width, num_channels])
for i in range(self._num_convs):
features = self._class_conv[i](features)
- features = self._class_norm_activation[i](features,
- is_training=is_training)
+ features = self._class_norm_activation[i](
+ features, is_training=is_training)
mask_logits = self._class_predict(features)
- mask_logits = tf.reshape(mask_logits, [batch_size, num_instances, height,
- width, self._mask_num_classes])
+ mask_logits = tf.reshape(
+ mask_logits,
+ [batch_size, num_instances, height, width, self._mask_num_classes])
return mask_logits
@@ -904,8 +1186,8 @@ class ShapemaskFinemaskHead(object):
activation=None,
padding='same',
name='fine-mask-class-%d' % i))
- self._fine_class_bn.append(norm_activation(
- name='fine-mask-class-%d-bn' % i))
+ self._fine_class_bn.append(
+ norm_activation(name='fine-mask-class-%d-bn' % i))
self._class_predict_conv = tf.keras.layers.Conv2D(
self._mask_num_classes,
@@ -923,14 +1205,13 @@ class ShapemaskFinemaskHead(object):
https://arxiv.org/pdf/1904.03239.pdf
Args:
- features: a float Tensor of shape
- [batch_size, num_instances, mask_crop_size, mask_crop_size,
- num_downsample_channels]. This is the instance feature crop.
- mask_logits: a float Tensor of shape
- [batch_size, num_instances, mask_crop_size, mask_crop_size] indicating
- predicted mask logits.
- classes: a int Tensor of shape [batch_size, num_instances]
- of instance classes.
+ features: a float Tensor of shape [batch_size, num_instances,
+ mask_crop_size, mask_crop_size, num_downsample_channels]. This is the
+ instance feature crop.
+ mask_logits: a float Tensor of shape [batch_size, num_instances,
+ mask_crop_size, mask_crop_size] indicating predicted mask logits.
+ classes: a int Tensor of shape [batch_size, num_instances] of instance
+ classes.
is_training: a bool indicating whether in training mode.
Returns:
@@ -939,7 +1220,7 @@ class ShapemaskFinemaskHead(object):
"""
# Extract the foreground mean features
# with tf.variable_scope('fine_mask', reuse=tf.AUTO_REUSE):
- with backend.get_graph().as_default(), tf.name_scope('fine_mask'):
+ with tf.name_scope('fine_mask'):
mask_probs = tf.nn.sigmoid(mask_logits)
# Compute instance embedding for hard average.
binary_mask = tf.cast(tf.greater(mask_probs, 0.5), features.dtype)
@@ -957,8 +1238,8 @@ class ShapemaskFinemaskHead(object):
mask_logits = self.decoder_net(features, is_training)
if self._use_category_for_mask:
mask_logits = tf.transpose(mask_logits, [0, 1, 4, 2, 3])
- mask_logits = tf.gather(mask_logits,
- tf.expand_dims(classes, -1), batch_dims=2)
+ mask_logits = tf.gather(
+ mask_logits, tf.expand_dims(classes, -1), batch_dims=2)
mask_logits = tf.squeeze(mask_logits, axis=2)
else:
mask_logits = mask_logits[..., 0]
@@ -979,8 +1260,8 @@ class ShapemaskFinemaskHead(object):
"""
(batch_size, num_instances, height, width,
num_channels) = features.get_shape().as_list()
- features = tf.reshape(features, [batch_size * num_instances, height, width,
- num_channels])
+ features = tf.reshape(
+ features, [batch_size * num_instances, height, width, num_channels])
for i in range(self._num_convs):
features = self._fine_class_conv[i](features)
features = self._fine_class_bn[i](features, is_training=is_training)
@@ -991,9 +1272,8 @@ class ShapemaskFinemaskHead(object):
# Predict per-class instance masks.
mask_logits = self._class_predict_conv(features)
- mask_logits = tf.reshape(mask_logits,
- [batch_size, num_instances,
- height * self.up_sample_factor,
- width * self.up_sample_factor,
- self._mask_num_classes])
+ mask_logits = tf.reshape(mask_logits, [
+ batch_size, num_instances, height * self.up_sample_factor,
+ width * self.up_sample_factor, self._mask_num_classes
+ ])
return mask_logits
diff --git a/official/vision/detection/modeling/architecture/identity.py b/official/vision/detection/modeling/architecture/identity.py
index acc90c4d5efddcac50eb95b1229c3c5500917445..778297f8919f8a90875c69ce1f11ef5dfd9fc95f 100644
--- a/official/vision/detection/modeling/architecture/identity.py
+++ b/official/vision/detection/modeling/architecture/identity.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Identity Fn that forwards the input features."""
from __future__ import absolute_import
diff --git a/official/vision/detection/modeling/architecture/nn_blocks.py b/official/vision/detection/modeling/architecture/nn_blocks.py
index c94a079f9a4ce4081c478bc373d381070ddfaf96..7a59bfc68a32c50814b11445c85a9add01048229 100644
--- a/official/vision/detection/modeling/architecture/nn_blocks.py
+++ b/official/vision/detection/modeling/architecture/nn_blocks.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Contains common building blocks for neural networks."""
from __future__ import absolute_import
diff --git a/official/vision/detection/modeling/architecture/nn_ops.py b/official/vision/detection/modeling/architecture/nn_ops.py
index 8b3617d6c5b23dd31a9f891985dcf8361ff1e177..76a33d98d0037361e1607d95ed275043fa41d364 100644
--- a/official/vision/detection/modeling/architecture/nn_ops.py
+++ b/official/vision/detection/modeling/architecture/nn_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Neural network operations commonly shared by the architectures."""
from __future__ import absolute_import
@@ -19,6 +19,7 @@ from __future__ import division
from __future__ import print_function
import functools
+
import tensorflow as tf
@@ -43,7 +44,7 @@ class NormActivation(tf.keras.layers.Layer):
GraphKeys.TRAINABLE_VARIABLES. If False, freeze batch normalization
layer.
init_zero: `bool` if True, initializes scale parameter of batch
- normalization with 0. If False, initialize it with 1.
+ normalization with 0. If False, initialize it with 1.
fused: `bool` fused option in batch normalziation.
use_actiation: `bool`, whether to add the optional activation layer after
the batch normalization layer.
diff --git a/official/vision/detection/modeling/architecture/resnet.py b/official/vision/detection/modeling/architecture/resnet.py
index abbc7213ea971f0cb014d770e7e0c1707855fb08..6f76e880ed701e17795454c252d97d9a876d6d16 100644
--- a/official/vision/detection/modeling/architecture/resnet.py
+++ b/official/vision/detection/modeling/architecture/resnet.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Contains definitions for the post-activation form of Residual Networks.
Residual networks (ResNets) were proposed in:
@@ -23,27 +23,26 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
-from absl import logging
import tensorflow as tf
-from tensorflow.python.keras import backend
from official.vision.detection.modeling.architecture import nn_ops
+
# TODO(b/140112644): Refactor the code with Keras style, i.e. build and call.
class Resnet(object):
"""Class to build ResNet family model."""
- def __init__(self,
- resnet_depth,
- activation='relu',
- norm_activation=nn_ops.norm_activation_builder(
- activation='relu'),
- data_format='channels_last'):
+ def __init__(
+ self,
+ resnet_depth,
+ activation='relu',
+ norm_activation=nn_ops.norm_activation_builder(activation='relu'),
+ data_format='channels_last'):
"""ResNet initialization function.
Args:
resnet_depth: `int` depth of ResNet backbone model.
- norm_activation: an operation that includes a normalization layer
- followed by an optional activation layer.
+ norm_activation: an operation that includes a normalization layer followed
+ by an optional activation layer.
data_format: `str` either "channels_first" for `[batch, channels, height,
width]` or "channels_last for `[batch, height, width, channels]`.
"""
@@ -58,24 +57,45 @@ class Resnet(object):
self._data_format = data_format
model_params = {
- 10: {'block': self.residual_block, 'layers': [1, 1, 1, 1]},
- 18: {'block': self.residual_block, 'layers': [2, 2, 2, 2]},
- 34: {'block': self.residual_block, 'layers': [3, 4, 6, 3]},
- 50: {'block': self.bottleneck_block, 'layers': [3, 4, 6, 3]},
- 101: {'block': self.bottleneck_block, 'layers': [3, 4, 23, 3]},
- 152: {'block': self.bottleneck_block, 'layers': [3, 8, 36, 3]},
- 200: {'block': self.bottleneck_block, 'layers': [3, 24, 36, 3]}
+ 10: {
+ 'block': self.residual_block,
+ 'layers': [1, 1, 1, 1]
+ },
+ 18: {
+ 'block': self.residual_block,
+ 'layers': [2, 2, 2, 2]
+ },
+ 34: {
+ 'block': self.residual_block,
+ 'layers': [3, 4, 6, 3]
+ },
+ 50: {
+ 'block': self.bottleneck_block,
+ 'layers': [3, 4, 6, 3]
+ },
+ 101: {
+ 'block': self.bottleneck_block,
+ 'layers': [3, 4, 23, 3]
+ },
+ 152: {
+ 'block': self.bottleneck_block,
+ 'layers': [3, 8, 36, 3]
+ },
+ 200: {
+ 'block': self.bottleneck_block,
+ 'layers': [3, 24, 36, 3]
+ }
}
if resnet_depth not in model_params:
valid_resnet_depths = ', '.join(
[str(depth) for depth in sorted(model_params.keys())])
raise ValueError(
- 'The resnet_depth should be in [%s]. Not a valid resnet_depth:'%(
- valid_resnet_depths), self._resnet_depth)
+ 'The resnet_depth should be in [%s]. Not a valid resnet_depth:' %
+ (valid_resnet_depths), self._resnet_depth)
params = model_params[resnet_depth]
- self._resnet_fn = self.resnet_v1_generator(
- params['block'], params['layers'])
+ self._resnet_fn = self.resnet_v1_generator(params['block'],
+ params['layers'])
def __call__(self, inputs, is_training=None):
"""Returns the ResNet model for a given size and number of output classes.
@@ -90,18 +110,17 @@ class Resnet(object):
The values are corresponding feature hierarchy in ResNet with shape
[batch_size, height_l, width_l, num_filters].
"""
- with backend.get_graph().as_default():
- with tf.name_scope('resnet%s' % self._resnet_depth):
- return self._resnet_fn(inputs, is_training)
+ with tf.name_scope('resnet%s' % self._resnet_depth):
+ return self._resnet_fn(inputs, is_training)
def fixed_padding(self, inputs, kernel_size):
"""Pads the input along the spatial dimensions independently of input size.
Args:
- inputs: `Tensor` of size `[batch, channels, height, width]` or
- `[batch, height, width, channels]` depending on `data_format`.
+ inputs: `Tensor` of size `[batch, channels, height, width]` or `[batch,
+ height, width, channels]` depending on `data_format`.
kernel_size: `int` kernel size to be used for `conv2d` or max_pool2d`
- operations. Should be a positive integer.
+ operations. Should be a positive integer.
Returns:
A padded `Tensor` of the same `data_format` with size either intact
@@ -160,14 +179,15 @@ class Resnet(object):
Args:
inputs: `Tensor` of size `[batch, channels, height, width]`.
filters: `int` number of filters for the first two convolutions. Note that
- the third and final convolution will use 4 times as many filters.
+ the third and final convolution will use 4 times as many filters.
strides: `int` block stride. If greater than 1, this block will ultimately
- downsample the input.
+ downsample the input.
use_projection: `bool` for whether this block should use a projection
- shortcut (versus the default identity shortcut). This is usually
- `True` for the first block of a block group, which may change the
- number of filters and the resolution.
+ shortcut (versus the default identity shortcut). This is usually `True`
+ for the first block of a block group, which may change the number of
+ filters and the resolution.
is_training: `bool` if True, the model is in training mode.
+
Returns:
The output `Tensor` of the block.
"""
@@ -185,8 +205,9 @@ class Resnet(object):
inputs = self.conv2d_fixed_padding(
inputs=inputs, filters=filters, kernel_size=3, strides=1)
- inputs = self._norm_activation(use_activation=False, init_zero=True)(
- inputs, is_training=is_training)
+ inputs = self._norm_activation(
+ use_activation=False, init_zero=True)(
+ inputs, is_training=is_training)
return self._activation_op(inputs + shortcut)
@@ -201,13 +222,13 @@ class Resnet(object):
Args:
inputs: `Tensor` of size `[batch, channels, height, width]`.
filters: `int` number of filters for the first two convolutions. Note that
- the third and final convolution will use 4 times as many filters.
+ the third and final convolution will use 4 times as many filters.
strides: `int` block stride. If greater than 1, this block will ultimately
- downsample the input.
+ downsample the input.
use_projection: `bool` for whether this block should use a projection
- shortcut (versus the default identity shortcut). This is usually
- `True` for the first block of a block group, which may change the
- number of filters and the resolution.
+ shortcut (versus the default identity shortcut). This is usually `True`
+ for the first block of a block group, which may change the number of
+ filters and the resolution.
is_training: `bool` if True, the model is in training mode.
Returns:
@@ -233,8 +254,9 @@ class Resnet(object):
inputs = self.conv2d_fixed_padding(
inputs=inputs, filters=4 * filters, kernel_size=1, strides=1)
- inputs = self._norm_activation(use_activation=False, init_zero=True)(
- inputs, is_training=is_training)
+ inputs = self._norm_activation(
+ use_activation=False, init_zero=True)(
+ inputs, is_training=is_training)
return self._activation_op(inputs + shortcut)
@@ -248,7 +270,7 @@ class Resnet(object):
block_fn: `function` for the block to use within the model
blocks: `int` number of blocks contained in the layer.
strides: `int` stride to use for the first convolution of the layer. If
- greater than 1, this layer will downsample the input.
+ greater than 1, this layer will downsample the input.
name: `str`name for the Tensor output of the block layer.
is_training: `bool` if True, the model is in training mode.
@@ -256,8 +278,8 @@ class Resnet(object):
The output `Tensor` of the block layer.
"""
# Only the first block per block_group uses projection shortcut and strides.
- inputs = block_fn(inputs, filters, strides, use_projection=True,
- is_training=is_training)
+ inputs = block_fn(
+ inputs, filters, strides, use_projection=True, is_training=is_training)
for _ in range(1, blocks):
inputs = block_fn(inputs, filters, 1, is_training=is_training)
@@ -269,7 +291,7 @@ class Resnet(object):
Args:
block_fn: `function` for the block to use within the model. Either
- `residual_block` or `bottleneck_block`.
+ `residual_block` or `bottleneck_block`.
layers: list of 4 `int`s denoting the number of blocks to include in each
of the 4 block groups. Each group consists of blocks that take inputs of
the same resolution.
@@ -293,17 +315,37 @@ class Resnet(object):
inputs = tf.identity(inputs, 'initial_max_pool')
c2 = self.block_group(
- inputs=inputs, filters=64, block_fn=block_fn, blocks=layers[0],
- strides=1, name='block_group1', is_training=is_training)
+ inputs=inputs,
+ filters=64,
+ block_fn=block_fn,
+ blocks=layers[0],
+ strides=1,
+ name='block_group1',
+ is_training=is_training)
c3 = self.block_group(
- inputs=c2, filters=128, block_fn=block_fn, blocks=layers[1],
- strides=2, name='block_group2', is_training=is_training)
+ inputs=c2,
+ filters=128,
+ block_fn=block_fn,
+ blocks=layers[1],
+ strides=2,
+ name='block_group2',
+ is_training=is_training)
c4 = self.block_group(
- inputs=c3, filters=256, block_fn=block_fn, blocks=layers[2],
- strides=2, name='block_group3', is_training=is_training)
+ inputs=c3,
+ filters=256,
+ block_fn=block_fn,
+ blocks=layers[2],
+ strides=2,
+ name='block_group3',
+ is_training=is_training)
c5 = self.block_group(
- inputs=c4, filters=512, block_fn=block_fn, blocks=layers[3],
- strides=2, name='block_group4', is_training=is_training)
+ inputs=c4,
+ filters=512,
+ block_fn=block_fn,
+ blocks=layers[3],
+ strides=2,
+ name='block_group4',
+ is_training=is_training)
return {2: c2, 3: c3, 4: c4, 5: c5}
return model
diff --git a/official/vision/detection/modeling/architecture/spinenet.py b/official/vision/detection/modeling/architecture/spinenet.py
index a11d11d5b52b1c754037558c4c5030ff7b9b4417..291203d9c25c3b5d4d4ee07e993ec44a611ef508 100644
--- a/official/vision/detection/modeling/architecture/spinenet.py
+++ b/official/vision/detection/modeling/architecture/spinenet.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,6 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+# Lint as: python3
# ==============================================================================
"""Implementation of SpineNet model.
@@ -24,7 +25,6 @@ import math
from absl import logging
import tensorflow as tf
-from tensorflow.python.keras import backend
from official.modeling import tf_utils
from official.vision.detection.modeling.architecture import nn_blocks
@@ -486,21 +486,20 @@ class SpineNetBuilder(object):
self._norm_epsilon = norm_epsilon
def __call__(self, inputs, is_training=None):
- with backend.get_graph().as_default():
- model = SpineNet(
- input_specs=self._input_specs,
- min_level=self._min_level,
- max_level=self._max_level,
- block_specs=self._block_specs,
- endpoints_num_filters=self._endpoints_num_filters,
- resample_alpha=self._resample_alpha,
- block_repeats=self._block_repeats,
- filter_size_scale=self._filter_size_scale,
- kernel_initializer=self._kernel_initializer,
- kernel_regularizer=self._kernel_regularizer,
- bias_regularizer=self._bias_regularizer,
- activation=self._activation,
- use_sync_bn=self._use_sync_bn,
- norm_momentum=self._norm_momentum,
- norm_epsilon=self._norm_epsilon)
- return model(inputs)
+ model = SpineNet(
+ input_specs=self._input_specs,
+ min_level=self._min_level,
+ max_level=self._max_level,
+ block_specs=self._block_specs,
+ endpoints_num_filters=self._endpoints_num_filters,
+ resample_alpha=self._resample_alpha,
+ block_repeats=self._block_repeats,
+ filter_size_scale=self._filter_size_scale,
+ kernel_initializer=self._kernel_initializer,
+ kernel_regularizer=self._kernel_regularizer,
+ bias_regularizer=self._bias_regularizer,
+ activation=self._activation,
+ use_sync_bn=self._use_sync_bn,
+ norm_momentum=self._norm_momentum,
+ norm_epsilon=self._norm_epsilon)
+ return model(inputs)
diff --git a/official/vision/detection/modeling/base_model.py b/official/vision/detection/modeling/base_model.py
index 8d18f12f5b7c52ca02334c4c685b70d353de83c5..0558e1db5530f3b85dd8ce9acf5d4c3b23146bc6 100644
--- a/official/vision/detection/modeling/base_model.py
+++ b/official/vision/detection/modeling/base_model.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Base Model definition."""
from __future__ import absolute_import
@@ -21,6 +21,7 @@ from __future__ import print_function
import abc
import functools
import re
+
import tensorflow as tf
from official.vision.detection.modeling import checkpoint_utils
from official.vision.detection.modeling import learning_rates
@@ -42,8 +43,7 @@ def _make_filter_trainable_variables_fn(frozen_variable_prefix):
# frozen_variable_prefix: a regex string specifing the prefix pattern of
# the frozen variables' names.
filtered_variables = [
- v for v in variables
- if not frozen_variable_prefix or
+ v for v in variables if not frozen_variable_prefix or
not re.match(frozen_variable_prefix, v.name)
]
return filtered_variables
@@ -60,9 +60,7 @@ class Model(object):
self._use_bfloat16 = params.architecture.use_bfloat16
if params.architecture.use_bfloat16:
- policy = tf.compat.v2.keras.mixed_precision.experimental.Policy(
- 'mixed_bfloat16')
- tf.compat.v2.keras.mixed_precision.experimental.set_policy(policy)
+ tf.compat.v2.keras.mixed_precision.set_global_policy('mixed_bfloat16')
# Optimization.
self._optimizer_fn = optimizers.OptimizerFactory(params.train.optimizer)
@@ -115,8 +113,8 @@ class Model(object):
def weight_decay_loss(self, trainable_variables):
reg_variables = [
v for v in trainable_variables
- if self._regularization_var_regex is None
- or re.match(self._regularization_var_regex, v.name)
+ if self._regularization_var_regex is None or
+ re.match(self._regularization_var_regex, v.name)
]
return self._l2_weight_decay * tf.add_n(
diff --git a/official/vision/detection/modeling/checkpoint_utils.py b/official/vision/detection/modeling/checkpoint_utils.py
index 1bb798396a714cbbc1a36309c99ceaa636a30354..fc0c09b7fcfee32b84db139b60880c018503d84a 100644
--- a/official/vision/detection/modeling/checkpoint_utils.py
+++ b/official/vision/detection/modeling/checkpoint_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,8 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""Util functions for loading checkpoints. Especially for loading Tensorflow 1.x
+
+"""Util functions for loading checkpoints.
+
+Especially for loading Tensorflow 1.x
checkpoint to Tensorflow 2.x (keras) model.
"""
@@ -20,18 +22,19 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
-
import re
+
from absl import logging
import tensorflow as tf
def _build_assignment_map(keras_model,
- prefix='',
- skip_variables_regex=None,
- var_to_shape_map=None):
+ prefix='',
+ skip_variables_regex=None,
+ var_to_shape_map=None):
"""Compute an assignment mapping for loading older checkpoints into a Keras
+
model. Variable names are remapped from the original TPUEstimator model to
the new Keras name.
@@ -48,12 +51,15 @@ def _build_assignment_map(keras_model,
"""
assignment_map = {}
-
- checkpoint_names = None
+ checkpoint_names = []
if var_to_shape_map:
- checkpoint_names = list(filter(
- lambda x: not x.endswith('Momentum') and not x.endswith(
- 'global_step'), var_to_shape_map.keys()))
+ checkpoint_names = list(
+ filter(
+ lambda x: not x.endswith('Momentum') and not x.endswith(
+ 'global_step'), var_to_shape_map.keys()))
+
+ logging.info('Number of variables in the checkpoint %d',
+ len(checkpoint_names))
for var in keras_model.variables:
var_name = var.name
@@ -84,7 +90,7 @@ def _build_assignment_map(keras_model,
logging.info('Error removing the match_name: %s', match_names)
logging.info('Exception: %s', e)
raise
- logging.info('Found variable in checkpoint: %d', len(assignment_map))
+ logging.info('Found matching variable in checkpoint: %d', len(assignment_map))
return assignment_map
@@ -95,14 +101,15 @@ def _get_checkpoint_map(checkpoint_path):
def make_restore_checkpoint_fn(checkpoint_path, prefix='', skip_regex=None):
"""Returns scaffold function to restore parameters from v1 checkpoint.
+
Args:
checkpoint_path: path of the checkpoint folder or file.
Example 1: '/path/to/model_dir/'
Example 2: '/path/to/model.ckpt-22500'
prefix: prefix in the variable name to be remove for alignment with names in
the checkpoint.
- skip_regex: regular expression to math the names of variables that
- do not need to be assign.
+ skip_regex: regular expression to math the names of variables that do not
+ need to be assign.
Returns:
Callable[tf.kears.Model] -> void. Fn to load v1 checkpoint to keras model.
@@ -125,7 +132,6 @@ def make_restore_checkpoint_fn(checkpoint_path, prefix='', skip_regex=None):
var_to_shape_map=var_to_shape_map)
if not vars_to_load:
raise ValueError('Variables to load is empty.')
- tf.compat.v1.train.init_from_checkpoint(checkpoint_path,
- vars_to_load)
+ tf.compat.v1.train.init_from_checkpoint(checkpoint_path, vars_to_load)
return _restore_checkpoint_fn
diff --git a/official/vision/detection/modeling/factory.py b/official/vision/detection/modeling/factory.py
index b140416dfdba90420f99a8bcb3b07cc04a63cc3e..c1393bcce047bce63cfae6cd9c963b783816d355 100644
--- a/official/vision/detection/modeling/factory.py
+++ b/official/vision/detection/modeling/factory.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,11 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Factory to build detection model."""
from official.vision.detection.modeling import maskrcnn_model
+from official.vision.detection.modeling import olnmask_model
from official.vision.detection.modeling import retinanet_model
from official.vision.detection.modeling import shapemask_model
@@ -26,6 +27,8 @@ def model_generator(params):
model_fn = retinanet_model.RetinanetModel(params)
elif params.type == 'mask_rcnn':
model_fn = maskrcnn_model.MaskrcnnModel(params)
+ elif params.type == 'olnmask':
+ model_fn = olnmask_model.OlnMaskModel(params)
elif params.type == 'shapemask':
model_fn = shapemask_model.ShapeMaskModel(params)
else:
diff --git a/official/vision/detection/modeling/learning_rates.py b/official/vision/detection/modeling/learning_rates.py
index ecc24ffadb073c79f71725b1adcb61cbd83127cd..7c1cc147942af63064ae174baeeb0d5ead3a5d3e 100644
--- a/official/vision/detection/modeling/learning_rates.py
+++ b/official/vision/detection/modeling/learning_rates.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Learning rate schedule."""
from __future__ import absolute_import
@@ -25,7 +25,8 @@ import tensorflow as tf
from official.modeling.hyperparams import params_dict
-class StepLearningRateWithLinearWarmup(tf.keras.optimizers.schedules.LearningRateSchedule):
+class StepLearningRateWithLinearWarmup(
+ tf.keras.optimizers.schedules.LearningRateSchedule):
"""Class to generate learning rate tensor."""
def __init__(self, total_steps, params):
@@ -57,7 +58,8 @@ class StepLearningRateWithLinearWarmup(tf.keras.optimizers.schedules.LearningRat
return {'_params': self._params.as_dict()}
-class CosineLearningRateWithLinearWarmup(tf.keras.optimizers.schedules.LearningRateSchedule):
+class CosineLearningRateWithLinearWarmup(
+ tf.keras.optimizers.schedules.LearningRateSchedule):
"""Class to generate learning rate tensor."""
def __init__(self, total_steps, params):
diff --git a/official/vision/detection/modeling/losses.py b/official/vision/detection/modeling/losses.py
index 4b993061b3c51c9ae6456d84a79f7fea5d74c77e..02e2632ae60c9da49f58c1239964d2f1104b52f8 100644
--- a/official/vision/detection/modeling/losses.py
+++ b/official/vision/detection/modeling/losses.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Losses used for detection models."""
from __future__ import absolute_import
@@ -195,6 +195,149 @@ class RpnBoxLoss(object):
return box_loss
+class OlnRpnCenterLoss(object):
+ """Object Localization Network RPN centerness regression loss function."""
+
+ def __init__(self):
+ self._l1_loss = tf.keras.losses.MeanAbsoluteError(
+ reduction=tf.keras.losses.Reduction.SUM)
+
+ def __call__(self, center_outputs, labels):
+ """Computes total RPN centerness regression loss.
+
+ Computes total RPN centerness score regression loss from all levels.
+
+ Args:
+ center_outputs: an OrderDict with keys representing levels and values
+ representing anchor centerness regression targets in
+ [batch_size, height, width, num_anchors * 4].
+ labels: the dictionary that returned from dataloader that includes
+ groundturth targets.
+
+ Returns:
+ rpn_center_loss: a scalar tensor representing total centerness regression
+ loss.
+ """
+ with tf.name_scope('rpn_loss'):
+ # Normalizer.
+ levels = sorted(center_outputs.keys())
+ num_valid = 0
+ # 00, neg=0, ign=-1.
+ mask_ = tf.cast(tf.logical_and(
+ tf.greater(center_targets[level][..., 0], 0.0),
+ tf.greater(tf.reduce_min(labels[level], -1), 0.0)), tf.float32)
+ normalizer += tf.reduce_sum(mask_)
+ normalizer += 1e-8
+ # iou_loss over multi levels.
+ iou_losses = []
+ for level in levels:
+ iou_losses.append(
+ self._rpn_iou_loss(
+ box_outputs[level], labels[level],
+ center_weight=center_targets[level][..., 0],
+ normalizer=normalizer))
+ # Sum per level losses to total loss.
+ return tf.add_n(iou_losses)
+
+ def _rpn_iou_loss(self, box_outputs, box_targets,
+ center_weight=None, normalizer=1.0):
+ """Computes box regression loss."""
+ # for instances, the regression targets of 512x512 input with 6 anchors on
+ # P2-P6 pyramid is about [0.1, 0.1, 0.2, 0.2].
+ with tf.name_scope('rpn_iou_loss'):
+ mask = tf.logical_and(
+ tf.greater(center_weight, 0.0),
+ tf.greater(tf.reduce_min(box_targets, -1), 0.0))
+
+ pred_left = box_outputs[..., 0]
+ pred_right = box_outputs[..., 1]
+ pred_top = box_outputs[..., 2]
+ pred_bottom = box_outputs[..., 3]
+
+ gt_left = box_targets[..., 0]
+ gt_right = box_targets[..., 1]
+ gt_top = box_targets[..., 2]
+ gt_bottom = box_targets[..., 3]
+
+ inter_width = (tf.minimum(pred_left, gt_left) +
+ tf.minimum(pred_right, gt_right))
+ inter_height = (tf.minimum(pred_top, gt_top) +
+ tf.minimum(pred_bottom, gt_bottom))
+ inter_area = inter_width * inter_height
+ union_area = ((pred_left + pred_right) * (pred_top + pred_bottom) +
+ (gt_left + gt_right) * (gt_top + gt_bottom) -
+ inter_area)
+ iou = inter_area / (union_area + 1e-8)
+ mask_ = tf.cast(mask, tf.float32)
+ iou = tf.clip_by_value(iou, clip_value_min=1e-8, clip_value_max=1.0)
+ neg_log_iou = -tf.math.log(iou)
+ iou_loss = tf.reduce_sum(neg_log_iou * mask_)
+ iou_loss /= normalizer
+ return iou_loss
+
+
class FastrcnnClassLoss(object):
"""Fast R-CNN classification loss function."""
@@ -317,6 +460,47 @@ class FastrcnnBoxLoss(object):
return box_loss
+class OlnBoxScoreLoss(object):
+ """Object Localization Network Box-Iou scoring function."""
+
+ def __init__(self, params):
+ self._ignore_threshold = params.ignore_threshold
+ self._l1_loss = tf.keras.losses.MeanAbsoluteError(
+ reduction=tf.keras.losses.Reduction.SUM)
+
+ def __call__(self, score_outputs, score_targets):
+ """Computes the class loss (Fast-RCNN branch) of Mask-RCNN.
+
+ This function implements the classification loss of the Fast-RCNN.
+
+ The classification loss is softmax on all RoIs.
+ Reference: https://github.com/facebookresearch/Detectron/blob/master/detectron/modeling/fast_rcnn_heads.py # pylint: disable=line-too-long
+
+ Args:
+ score_outputs: a float tensor representing the class prediction for each box
+ with a shape of [batch_size, num_boxes, num_classes].
+ score_targets: a float tensor representing the class label for each box
+ with a shape of [batch_size, num_boxes].
+
+ Returns:
+ a scalar tensor representing total score loss.
+ """
+ with tf.name_scope('fast_rcnn_loss'):
+ score_outputs = tf.squeeze(score_outputs, -1)
+
+ mask = tf.greater(score_targets, self._ignore_threshold)
+ num_valid = tf.reduce_sum(tf.cast(mask, tf.float32))
+ score_targets = tf.maximum(score_targets, tf.zeros_like(score_targets))
+ score_outputs = tf.sigmoid(score_outputs)
+ score_targets = tf.expand_dims(score_targets, -1)
+ score_outputs = tf.expand_dims(score_outputs, -1)
+ mask = tf.cast(mask, dtype=tf.float32)
+ score_loss = self._l1_loss(score_targets, score_outputs,
+ sample_weight=mask)
+ score_loss /= (num_valid + 1e-10)
+ return score_loss
+
+
class MaskrcnnLoss(object):
"""Mask R-CNN instance segmentation mask loss function."""
@@ -449,7 +633,7 @@ class RetinanetBoxLoss(object):
num_positives: number of positive examples in the minibatch.
Returns:
- an integar tensor representing total box regression loss.
+ an integer tensor representing total box regression loss.
"""
# Sums all positives in a batch for normalization and avoids zero
# num_positives_sum, which would lead to inf loss during training
@@ -457,7 +641,6 @@ class RetinanetBoxLoss(object):
box_losses = []
for level in box_outputs.keys():
- # Onehot encoding for classification labels.
box_targets_l = labels[level]
box_losses.append(
self.box_loss(box_outputs[level], box_targets_l, num_positives_sum))
diff --git a/official/vision/detection/modeling/maskrcnn_model.py b/official/vision/detection/modeling/maskrcnn_model.py
index e5cbe7d56ba7d82836ef58df201aa74779cb2f69..e9e6bb2697d7f78d4d01c9dceb8e0997376aecab 100644
--- a/official/vision/detection/modeling/maskrcnn_model.py
+++ b/official/vision/detection/modeling/maskrcnn_model.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Model defination for the Mask R-CNN Model."""
from __future__ import absolute_import
@@ -20,7 +20,6 @@ from __future__ import print_function
import tensorflow as tf
-from tensorflow.python.keras import backend
from official.vision.detection.dataloader import anchor
from official.vision.detection.dataloader import mode_keys
from official.vision.detection.evaluation import factory as eval_factory
@@ -118,9 +117,7 @@ class MaskrcnnModel(base_model.Model):
box_targets = tf.where(
tf.tile(
tf.expand_dims(tf.equal(matched_gt_classes, 0), axis=-1),
- [1, 1, 4]),
- tf.zeros_like(box_targets),
- box_targets)
+ [1, 1, 4]), tf.zeros_like(box_targets), box_targets)
model_outputs.update({
'class_targets': matched_gt_classes,
'box_targets': box_targets,
@@ -183,9 +180,7 @@ class MaskrcnnModel(base_model.Model):
mask_outputs),
})
else:
- model_outputs.update({
- 'detection_masks': tf.nn.sigmoid(mask_outputs)
- })
+ model_outputs.update({'detection_masks': tf.nn.sigmoid(mask_outputs)})
return model_outputs
@@ -297,14 +292,13 @@ class MaskrcnnModel(base_model.Model):
def build_model(self, params, mode):
if self._keras_model is None:
input_layers = self.build_input_layers(self._params, mode)
- with backend.get_graph().as_default():
- outputs = self.model_outputs(input_layers, mode)
+ outputs = self.model_outputs(input_layers, mode)
- model = tf.keras.models.Model(
- inputs=input_layers, outputs=outputs, name='maskrcnn')
- assert model is not None, 'Fail to build tf.keras.Model.'
- model.optimizer = self.build_optimizer()
- self._keras_model = model
+ model = tf.keras.models.Model(
+ inputs=input_layers, outputs=outputs, name='maskrcnn')
+ assert model is not None, 'Fail to build tf.keras.Model.'
+ model.optimizer = self.build_optimizer()
+ self._keras_model = model
return self._keras_model
@@ -312,8 +306,8 @@ class MaskrcnnModel(base_model.Model):
required_output_fields = ['class_outputs', 'box_outputs']
for field in required_output_fields:
if field not in outputs:
- raise ValueError('"%s" is missing in outputs, requried %s found %s'
- %(field, required_output_fields, outputs.keys()))
+ raise ValueError('"%s" is missing in outputs, requried %s found %s' %
+ (field, required_output_fields, outputs.keys()))
predictions = {
'image_info': labels['image_info'],
'num_detections': outputs['num_detections'],
diff --git a/official/vision/detection/modeling/olnmask_model.py b/official/vision/detection/modeling/olnmask_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..60d59c1bd12bd25f8c2ed0e30bc32c7dad4cbdcf
--- /dev/null
+++ b/official/vision/detection/modeling/olnmask_model.py
@@ -0,0 +1,432 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Model defination for the Object Localization Network (OLN) Model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+
+from official.vision.detection.dataloader import anchor
+from official.vision.detection.dataloader import mode_keys
+from official.vision.detection.modeling import losses
+from official.vision.detection.modeling.architecture import factory
+from official.vision.detection.modeling.maskrcnn_model import MaskrcnnModel
+from official.vision.detection.ops import postprocess_ops
+from official.vision.detection.ops import roi_ops
+from official.vision.detection.ops import spatial_transform_ops
+from official.vision.detection.ops import target_ops
+from official.vision.detection.utils import box_utils
+
+
+class OlnMaskModel(MaskrcnnModel):
+ """OLN-Mask model function."""
+
+ def __init__(self, params):
+ super(OlnMaskModel, self).__init__(params)
+
+ self._params = params
+
+ # Different heads and layers.
+ self._include_rpn_class = params.architecture.include_rpn_class
+ self._include_mask = params.architecture.include_mask
+ self._include_frcnn_class = params.architecture.include_frcnn_class
+ self._include_frcnn_box = params.architecture.include_frcnn_box
+ self._include_centerness = params.rpn_head.has_centerness
+ self._include_box_score = (params.frcnn_head.has_scoring and
+ params.architecture.include_frcnn_box)
+ self._include_mask_score = (params.mrcnn_head.has_scoring and
+ params.architecture.include_mask)
+
+ # Architecture generators.
+ self._backbone_fn = factory.backbone_generator(params)
+ self._fpn_fn = factory.multilevel_features_generator(params)
+ self._rpn_head_fn = factory.rpn_head_generator(params)
+ if self._include_centerness:
+ self._rpn_head_fn = factory.oln_rpn_head_generator(params)
+ else:
+ self._rpn_head_fn = factory.rpn_head_generator(params)
+ self._generate_rois_fn = roi_ops.OlnROIGenerator(params.roi_proposal)
+ self._sample_rois_fn = target_ops.ROIScoreSampler(params.roi_sampling)
+ self._sample_masks_fn = target_ops.MaskSampler(
+ params.architecture.mask_target_size,
+ params.mask_sampling.num_mask_samples_per_image)
+
+ if self._include_box_score:
+ self._frcnn_head_fn = factory.oln_box_score_head_generator(params)
+ else:
+ self._frcnn_head_fn = factory.fast_rcnn_head_generator(params)
+
+ if self._include_mask:
+ if self._include_mask_score:
+ self._mrcnn_head_fn = factory.oln_mask_score_head_generator(params)
+ else:
+ self._mrcnn_head_fn = factory.mask_rcnn_head_generator(params)
+
+ # Loss function.
+ self._rpn_score_loss_fn = losses.RpnScoreLoss(params.rpn_score_loss)
+ self._rpn_box_loss_fn = losses.RpnBoxLoss(params.rpn_box_loss)
+ if self._include_centerness:
+ self._rpn_iou_loss_fn = losses.OlnRpnIoULoss()
+ self._rpn_center_loss_fn = losses.OlnRpnCenterLoss()
+ self._frcnn_class_loss_fn = losses.FastrcnnClassLoss()
+ self._frcnn_box_loss_fn = losses.FastrcnnBoxLoss(params.frcnn_box_loss)
+ if self._include_box_score:
+ self._frcnn_box_score_loss_fn = losses.OlnBoxScoreLoss(
+ params.frcnn_box_score_loss)
+ if self._include_mask:
+ self._mask_loss_fn = losses.MaskrcnnLoss()
+
+ self._generate_detections_fn = postprocess_ops.OlnDetectionGenerator(
+ params.postprocess)
+
+ self._transpose_input = params.train.transpose_input
+ assert not self._transpose_input, 'Transpose input is not supportted.'
+
+ def build_outputs(self, inputs, mode):
+ is_training = mode == mode_keys.TRAIN
+ model_outputs = {}
+
+ image = inputs['image']
+ _, image_height, image_width, _ = image.get_shape().as_list()
+ backbone_features = self._backbone_fn(image, is_training)
+ fpn_features = self._fpn_fn(backbone_features, is_training)
+
+ # rpn_centerness.
+ if self._include_centerness:
+ rpn_score_outputs, rpn_box_outputs, rpn_center_outputs = (
+ self._rpn_head_fn(fpn_features, is_training))
+ model_outputs.update({
+ 'rpn_center_outputs':
+ tf.nest.map_structure(lambda x: tf.cast(x, tf.float32),
+ rpn_center_outputs),
+ })
+ object_scores = rpn_center_outputs
+ else:
+ rpn_score_outputs, rpn_box_outputs = self._rpn_head_fn(
+ fpn_features, is_training)
+ object_scores = None
+ model_outputs.update({
+ 'rpn_score_outputs':
+ tf.nest.map_structure(lambda x: tf.cast(x, tf.float32),
+ rpn_score_outputs),
+ 'rpn_box_outputs':
+ tf.nest.map_structure(lambda x: tf.cast(x, tf.float32),
+ rpn_box_outputs),
+ })
+ input_anchor = anchor.Anchor(self._params.architecture.min_level,
+ self._params.architecture.max_level,
+ self._params.anchor.num_scales,
+ self._params.anchor.aspect_ratios,
+ self._params.anchor.anchor_size,
+ (image_height, image_width))
+ rpn_rois, rpn_roi_scores = self._generate_rois_fn(
+ rpn_box_outputs,
+ rpn_score_outputs,
+ input_anchor.multilevel_boxes,
+ inputs['image_info'][:, 1, :],
+ is_training,
+ is_box_lrtb=self._include_centerness,
+ object_scores=object_scores,
+ )
+ if (not self._include_frcnn_class and
+ not self._include_frcnn_box and
+ not self._include_mask):
+ # if not is_training:
+ # For direct RPN detection,
+ # use dummy box_outputs = (dy,dx,dh,dw = 0,0,0,0)
+ box_outputs = tf.zeros_like(rpn_rois)
+ box_outputs = tf.concat([box_outputs, box_outputs], -1)
+ boxes, scores, classes, valid_detections = self._generate_detections_fn(
+ box_outputs, rpn_roi_scores, rpn_rois,
+ inputs['image_info'][:, 1:2, :],
+ is_single_fg_score=True, # if no_background, no softmax is applied.
+ keep_nms=True)
+ model_outputs.update({
+ 'num_detections': valid_detections,
+ 'detection_boxes': boxes,
+ 'detection_classes': classes,
+ 'detection_scores': scores,
+ })
+ return model_outputs
+
+ # ---- OLN-Proposal finishes here. ----
+
+ if is_training:
+ rpn_rois = tf.stop_gradient(rpn_rois)
+ rpn_roi_scores = tf.stop_gradient(rpn_roi_scores)
+
+ # Sample proposals.
+ (rpn_rois, rpn_roi_scores, matched_gt_boxes, matched_gt_classes,
+ matched_gt_indices) = (
+ self._sample_rois_fn(rpn_rois, rpn_roi_scores, inputs['gt_boxes'],
+ inputs['gt_classes']))
+ # Create bounding box training targets.
+ box_targets = box_utils.encode_boxes(
+ matched_gt_boxes, rpn_rois, weights=[10.0, 10.0, 5.0, 5.0])
+ # If the target is background, the box target is set to all 0s.
+ box_targets = tf.where(
+ tf.tile(
+ tf.expand_dims(tf.equal(matched_gt_classes, 0), axis=-1),
+ [1, 1, 4]), tf.zeros_like(box_targets), box_targets)
+ model_outputs.update({
+ 'class_targets': matched_gt_classes,
+ 'box_targets': box_targets,
+ })
+ # Create Box-IoU targets. {
+ box_ious = box_utils.bbox_overlap(
+ rpn_rois, inputs['gt_boxes'])
+ matched_box_ious = tf.reduce_max(box_ious, 2)
+ model_outputs.update({
+ 'box_iou_targets': matched_box_ious,}) # }
+
+ roi_features = spatial_transform_ops.multilevel_crop_and_resize(
+ fpn_features, rpn_rois, output_size=7)
+
+ if not self._include_box_score:
+ class_outputs, box_outputs = self._frcnn_head_fn(
+ roi_features, is_training)
+ else:
+ class_outputs, box_outputs, score_outputs = self._frcnn_head_fn(
+ roi_features, is_training)
+ model_outputs.update({
+ 'box_score_outputs':
+ tf.nest.map_structure(lambda x: tf.cast(x, tf.float32),
+ score_outputs),})
+ model_outputs.update({
+ 'class_outputs':
+ tf.nest.map_structure(lambda x: tf.cast(x, tf.float32),
+ class_outputs),
+ 'box_outputs':
+ tf.nest.map_structure(lambda x: tf.cast(x, tf.float32),
+ box_outputs),
+ })
+
+ # Add this output to train to make the checkpoint loadable in predict mode.
+ # If we skip it in train mode, the heads will be out-of-order and checkpoint
+ # loading will fail.
+ if not self._include_frcnn_box:
+ box_outputs = tf.zeros_like(box_outputs) # dummy zeros.
+
+ if self._include_box_score:
+ score_outputs = tf.cast(tf.squeeze(score_outputs, -1),
+ rpn_roi_scores.dtype)
+
+ # box-score = (rpn-centerness * box-iou)^(1/2)
+ # TR: rpn_roi_scores: b,1000, score_outputs: b,512
+ # TS: rpn_roi_scores: b,1000, score_outputs: b,1000
+ box_scores = tf.pow(
+ rpn_roi_scores * tf.sigmoid(score_outputs), 1/2.)
+
+ if not self._include_frcnn_class:
+ boxes, scores, classes, valid_detections = self._generate_detections_fn(
+ box_outputs,
+ box_scores,
+ rpn_rois,
+ inputs['image_info'][:, 1:2, :],
+ is_single_fg_score=True,
+ keep_nms=True,)
+ else:
+ boxes, scores, classes, valid_detections = self._generate_detections_fn(
+ box_outputs, class_outputs, rpn_rois,
+ inputs['image_info'][:, 1:2, :],
+ keep_nms=True,)
+ model_outputs.update({
+ 'num_detections': valid_detections,
+ 'detection_boxes': boxes,
+ 'detection_classes': classes,
+ 'detection_scores': scores,
+ })
+
+ # ---- OLN-Box finishes here. ----
+
+ if not self._include_mask:
+ return model_outputs
+
+ if is_training:
+ rpn_rois, classes, mask_targets = self._sample_masks_fn(
+ rpn_rois, matched_gt_boxes, matched_gt_classes, matched_gt_indices,
+ inputs['gt_masks'])
+ mask_targets = tf.stop_gradient(mask_targets)
+
+ classes = tf.cast(classes, dtype=tf.int32)
+
+ model_outputs.update({
+ 'mask_targets': mask_targets,
+ 'sampled_class_targets': classes,
+ })
+ else:
+ rpn_rois = boxes
+ classes = tf.cast(classes, dtype=tf.int32)
+
+ mask_roi_features = spatial_transform_ops.multilevel_crop_and_resize(
+ fpn_features, rpn_rois, output_size=14)
+
+ mask_outputs = self._mrcnn_head_fn(mask_roi_features, classes, is_training)
+
+ if is_training:
+ model_outputs.update({
+ 'mask_outputs':
+ tf.nest.map_structure(lambda x: tf.cast(x, tf.float32),
+ mask_outputs),
+ })
+ else:
+ model_outputs.update({'detection_masks': tf.nn.sigmoid(mask_outputs)})
+
+ return model_outputs
+
+ def build_loss_fn(self):
+ if self._keras_model is None:
+ raise ValueError('build_loss_fn() must be called after build_model().')
+
+ filter_fn = self.make_filter_trainable_variables_fn()
+ trainable_variables = filter_fn(self._keras_model.trainable_variables)
+
+ def _total_loss_fn(labels, outputs):
+ if self._include_rpn_class:
+ rpn_score_loss = self._rpn_score_loss_fn(outputs['rpn_score_outputs'],
+ labels['rpn_score_targets'])
+ else:
+ rpn_score_loss = 0.0
+ if self._include_centerness:
+ rpn_center_loss = self._rpn_center_loss_fn(
+ outputs['rpn_center_outputs'], labels['rpn_center_targets'])
+ rpn_box_loss = self._rpn_iou_loss_fn(
+ outputs['rpn_box_outputs'], labels['rpn_box_targets'],
+ labels['rpn_center_targets'])
+ else:
+ rpn_center_loss = 0.0
+ rpn_box_loss = self._rpn_box_loss_fn(
+ outputs['rpn_box_outputs'], labels['rpn_box_targets'])
+
+ if self._include_frcnn_class:
+ frcnn_class_loss = self._frcnn_class_loss_fn(
+ outputs['class_outputs'], outputs['class_targets'])
+ else:
+ frcnn_class_loss = 0.0
+ if self._include_frcnn_box:
+ frcnn_box_loss = self._frcnn_box_loss_fn(
+ outputs['box_outputs'], outputs['class_targets'],
+ outputs['box_targets'])
+ else:
+ frcnn_box_loss = 0.0
+ if self._include_box_score:
+ box_score_loss = self._frcnn_box_score_loss_fn(
+ outputs['box_score_outputs'], outputs['box_iou_targets'])
+ else:
+ box_score_loss = 0.0
+
+ if self._include_mask:
+ mask_loss = self._mask_loss_fn(outputs['mask_outputs'],
+ outputs['mask_targets'],
+ outputs['sampled_class_targets'])
+ else:
+ mask_loss = 0.0
+
+ model_loss = (
+ rpn_score_loss + rpn_box_loss + rpn_center_loss +
+ frcnn_class_loss + frcnn_box_loss + box_score_loss +
+ mask_loss)
+
+ l2_regularization_loss = self.weight_decay_loss(trainable_variables)
+ total_loss = model_loss + l2_regularization_loss
+ return {
+ 'total_loss': total_loss,
+ 'loss': total_loss,
+ 'fast_rcnn_class_loss': frcnn_class_loss,
+ 'fast_rcnn_box_loss': frcnn_box_loss,
+ 'fast_rcnn_box_score_loss': box_score_loss,
+ 'mask_loss': mask_loss,
+ 'model_loss': model_loss,
+ 'l2_regularization_loss': l2_regularization_loss,
+ 'rpn_score_loss': rpn_score_loss,
+ 'rpn_box_loss': rpn_box_loss,
+ 'rpn_center_loss': rpn_center_loss,
+ }
+
+ return _total_loss_fn
+
+ def build_input_layers(self, params, mode):
+ is_training = mode == mode_keys.TRAIN
+ input_shape = (
+ params.olnmask_parser.output_size +
+ [params.olnmask_parser.num_channels])
+ if is_training:
+ batch_size = params.train.batch_size
+ input_layer = {
+ 'image':
+ tf.keras.layers.Input(
+ shape=input_shape,
+ batch_size=batch_size,
+ name='image',
+ dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32),
+ 'image_info':
+ tf.keras.layers.Input(
+ shape=[4, 2],
+ batch_size=batch_size,
+ name='image_info',
+ ),
+ 'gt_boxes':
+ tf.keras.layers.Input(
+ shape=[params.olnmask_parser.max_num_instances, 4],
+ batch_size=batch_size,
+ name='gt_boxes'),
+ 'gt_classes':
+ tf.keras.layers.Input(
+ shape=[params.olnmask_parser.max_num_instances],
+ batch_size=batch_size,
+ name='gt_classes',
+ dtype=tf.int64),
+ }
+ if self._include_mask:
+ input_layer['gt_masks'] = tf.keras.layers.Input(
+ shape=[
+ params.olnmask_parser.max_num_instances,
+ params.olnmask_parser.mask_crop_size,
+ params.olnmask_parser.mask_crop_size
+ ],
+ batch_size=batch_size,
+ name='gt_masks')
+ else:
+ batch_size = params.eval.batch_size
+ input_layer = {
+ 'image':
+ tf.keras.layers.Input(
+ shape=input_shape,
+ batch_size=batch_size,
+ name='image',
+ dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32),
+ 'image_info':
+ tf.keras.layers.Input(
+ shape=[4, 2],
+ batch_size=batch_size,
+ name='image_info',
+ ),
+ }
+ return input_layer
+
+ def build_model(self, params, mode):
+ if self._keras_model is None:
+ input_layers = self.build_input_layers(self._params, mode)
+ outputs = self.model_outputs(input_layers, mode)
+
+ model = tf.keras.models.Model(
+ inputs=input_layers, outputs=outputs, name='olnmask')
+ assert model is not None, 'Fail to build tf.keras.Model.'
+ model.optimizer = self.build_optimizer()
+ self._keras_model = model
+
+ return self._keras_model
diff --git a/official/vision/detection/modeling/optimizers.py b/official/vision/detection/modeling/optimizers.py
index fd51bb59f579b3de027cba26ef3bee0e67d0c74f..8b098c9f6456f77e720af387ec3a31ddb4ff2947 100644
--- a/official/vision/detection/modeling/optimizers.py
+++ b/official/vision/detection/modeling/optimizers.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Optimizers."""
from __future__ import absolute_import
diff --git a/official/vision/detection/modeling/retinanet_model.py b/official/vision/detection/modeling/retinanet_model.py
index 270cd14d27f16d5d0c4f5f0e8c902091a964f7d5..7a0a307c27ceb035b4f4c752ccbb1cd4ead4da29 100644
--- a/official/vision/detection/modeling/retinanet_model.py
+++ b/official/vision/detection/modeling/retinanet_model.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Model defination for the RetinaNet Model."""
from __future__ import absolute_import
@@ -20,7 +20,6 @@ from __future__ import print_function
import tensorflow as tf
-from tensorflow.python.keras import backend
from official.vision.detection.dataloader import mode_keys
from official.vision.detection.evaluation import factory as eval_factory
from official.vision.detection.modeling import base_model
@@ -52,18 +51,15 @@ class RetinanetModel(base_model.Model):
# Predict function.
self._generate_detections_fn = postprocess_ops.MultilevelDetectionGenerator(
- params.architecture.min_level,
- params.architecture.max_level,
+ params.architecture.min_level, params.architecture.max_level,
params.postprocess)
self._transpose_input = params.train.transpose_input
assert not self._transpose_input, 'Transpose input is not supported.'
# Input layer.
- input_shape = (
- params.retinanet_parser.output_size +
- [params.retinanet_parser.num_channels])
self._input_layer = tf.keras.layers.Input(
- shape=input_shape, name='',
+ shape=(None, None, params.retinanet_parser.num_channels),
+ name='',
dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32)
def build_outputs(self, inputs, mode):
@@ -120,14 +116,13 @@ class RetinanetModel(base_model.Model):
def build_model(self, params, mode=None):
if self._keras_model is None:
- with backend.get_graph().as_default():
- outputs = self.model_outputs(self._input_layer, mode)
+ outputs = self.model_outputs(self._input_layer, mode)
- model = tf.keras.models.Model(
- inputs=self._input_layer, outputs=outputs, name='retinanet')
- assert model is not None, 'Fail to build tf.keras.Model.'
- model.optimizer = self.build_optimizer()
- self._keras_model = model
+ model = tf.keras.models.Model(
+ inputs=self._input_layer, outputs=outputs, name='retinanet')
+ assert model is not None, 'Fail to build tf.keras.Model.'
+ model.optimizer = self.build_optimizer()
+ self._keras_model = model
return self._keras_model
@@ -144,8 +139,8 @@ class RetinanetModel(base_model.Model):
raise ValueError('"%s" is missing in outputs, requried %s found %s',
field, required_label_fields, labels.keys())
boxes, scores, classes, valid_detections = self._generate_detections_fn(
- outputs['box_outputs'], outputs['cls_outputs'],
- labels['anchor_boxes'], labels['image_info'][:, 1:2, :])
+ outputs['box_outputs'], outputs['cls_outputs'], labels['anchor_boxes'],
+ labels['image_info'][:, 1:2, :])
# Discards the old output tensors to save memory. The `cls_outputs` and
# `box_outputs` are pretty big and could potentiall lead to memory issue.
outputs = {
diff --git a/official/vision/detection/modeling/shapemask_model.py b/official/vision/detection/modeling/shapemask_model.py
index 174187ed02ae7a7617f259974d64b1906a3d16e0..d197ec2fa38c167f616c0e60c5951bfb12ff94fb 100644
--- a/official/vision/detection/modeling/shapemask_model.py
+++ b/official/vision/detection/modeling/shapemask_model.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Model definition for the ShapeMask Model."""
from __future__ import absolute_import
@@ -20,7 +20,6 @@ from __future__ import print_function
import tensorflow as tf
-from tensorflow.python.keras import backend
from official.vision.detection.dataloader import anchor
from official.vision.detection.dataloader import mode_keys
from official.vision.detection.evaluation import factory as eval_factory
@@ -61,13 +60,11 @@ class ShapeMaskModel(base_model.Model):
params.shapemask_loss.shape_prior_loss_weight)
self._coarse_mask_loss_weight = (
params.shapemask_loss.coarse_mask_loss_weight)
- self._fine_mask_loss_weight = (
- params.shapemask_loss.fine_mask_loss_weight)
+ self._fine_mask_loss_weight = (params.shapemask_loss.fine_mask_loss_weight)
# Predict function.
self._generate_detections_fn = postprocess_ops.MultilevelDetectionGenerator(
- params.architecture.min_level,
- params.architecture.max_level,
+ params.architecture.min_level, params.architecture.max_level,
params.postprocess)
def build_outputs(self, inputs, mode):
@@ -79,10 +76,8 @@ class ShapeMaskModel(base_model.Model):
else:
anchor_boxes = anchor.Anchor(
self._params.architecture.min_level,
- self._params.architecture.max_level,
- self._params.anchor.num_scales,
- self._params.anchor.aspect_ratios,
- self._params.anchor.anchor_size,
+ self._params.architecture.max_level, self._params.anchor.num_scales,
+ self._params.anchor.aspect_ratios, self._params.anchor.anchor_size,
images.get_shape().as_list()[1:3]).multilevel_boxes
batch_size = tf.shape(images)[0]
@@ -96,8 +91,7 @@ class ShapeMaskModel(base_model.Model):
fpn_features, is_training=is_training)
valid_boxes, valid_scores, valid_classes, valid_detections = (
- self._generate_detections_fn(box_outputs, cls_outputs,
- anchor_boxes,
+ self._generate_detections_fn(box_outputs, cls_outputs, anchor_boxes,
inputs['image_info'][:, 1:2, :]))
image_size = images.get_shape().as_list()[1:3]
@@ -124,22 +118,18 @@ class ShapeMaskModel(base_model.Model):
return boxes, classes, outer_boxes
boxes, classes, outer_boxes = SampledBoxesLayer()(
- inputs, valid_boxes, valid_classes,
- valid_outer_boxes, training=is_training)
-
- instance_features, prior_masks = self._shape_prior_head_fn(fpn_features,
- boxes,
- outer_boxes,
- classes,
- is_training)
- coarse_mask_logits = self._coarse_mask_fn(instance_features,
- prior_masks,
- classes,
- is_training)
- fine_mask_logits = self._fine_mask_fn(instance_features,
- coarse_mask_logits,
- classes,
- is_training)
+ inputs,
+ valid_boxes,
+ valid_classes,
+ valid_outer_boxes,
+ training=is_training)
+
+ instance_features, prior_masks = self._shape_prior_head_fn(
+ fpn_features, boxes, outer_boxes, classes, is_training)
+ coarse_mask_logits = self._coarse_mask_fn(instance_features, prior_masks,
+ classes, is_training)
+ fine_mask_logits = self._fine_mask_fn(instance_features, coarse_mask_logits,
+ classes, is_training)
model_outputs = {
'cls_outputs': cls_outputs,
@@ -177,18 +167,15 @@ class ShapeMaskModel(base_model.Model):
labels['num_positives'])
# Adds Shapemask model losses.
- shape_prior_loss = self._shapemask_prior_loss_fn(
- outputs['prior_masks'],
- labels['mask_targets'],
- labels['mask_is_valid'])
- coarse_mask_loss = self._shapemask_loss_fn(
- outputs['coarse_mask_logits'],
- labels['mask_targets'],
- labels['mask_is_valid'])
- fine_mask_loss = self._shapemask_loss_fn(
- outputs['fine_mask_logits'],
- labels['fine_mask_targets'],
- labels['mask_is_valid'])
+ shape_prior_loss = self._shapemask_prior_loss_fn(outputs['prior_masks'],
+ labels['mask_targets'],
+ labels['mask_is_valid'])
+ coarse_mask_loss = self._shapemask_loss_fn(outputs['coarse_mask_logits'],
+ labels['mask_targets'],
+ labels['mask_is_valid'])
+ fine_mask_loss = self._shapemask_loss_fn(outputs['fine_mask_logits'],
+ labels['fine_mask_targets'],
+ labels['mask_is_valid'])
model_loss = (
cls_loss + self._box_loss_weight * box_loss +
@@ -222,64 +209,67 @@ class ShapeMaskModel(base_model.Model):
if is_training:
batch_size = params.train.batch_size
input_layer = {
- 'image': tf.keras.layers.Input(
- shape=input_shape,
- batch_size=batch_size,
- name='image',
- dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32),
- 'image_info': tf.keras.layers.Input(
- shape=[4, 2],
- batch_size=batch_size,
- name='image_info'),
- 'mask_classes': tf.keras.layers.Input(
- shape=[params.shapemask_parser.num_sampled_masks],
- batch_size=batch_size,
- name='mask_classes',
- dtype=tf.int64),
- 'mask_outer_boxes': tf.keras.layers.Input(
- shape=[params.shapemask_parser.num_sampled_masks, 4],
- batch_size=batch_size,
- name='mask_outer_boxes',
- dtype=tf.float32),
- 'mask_boxes': tf.keras.layers.Input(
- shape=[params.shapemask_parser.num_sampled_masks, 4],
- batch_size=batch_size,
- name='mask_boxes',
- dtype=tf.float32),
+ 'image':
+ tf.keras.layers.Input(
+ shape=input_shape,
+ batch_size=batch_size,
+ name='image',
+ dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32),
+ 'image_info':
+ tf.keras.layers.Input(
+ shape=[4, 2], batch_size=batch_size, name='image_info'),
+ 'mask_classes':
+ tf.keras.layers.Input(
+ shape=[params.shapemask_parser.num_sampled_masks],
+ batch_size=batch_size,
+ name='mask_classes',
+ dtype=tf.int64),
+ 'mask_outer_boxes':
+ tf.keras.layers.Input(
+ shape=[params.shapemask_parser.num_sampled_masks, 4],
+ batch_size=batch_size,
+ name='mask_outer_boxes',
+ dtype=tf.float32),
+ 'mask_boxes':
+ tf.keras.layers.Input(
+ shape=[params.shapemask_parser.num_sampled_masks, 4],
+ batch_size=batch_size,
+ name='mask_boxes',
+ dtype=tf.float32),
}
else:
batch_size = params.eval.batch_size
input_layer = {
- 'image': tf.keras.layers.Input(
- shape=input_shape,
- batch_size=batch_size,
- name='image',
- dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32),
- 'image_info': tf.keras.layers.Input(
- shape=[4, 2],
- batch_size=batch_size,
- name='image_info'),
+ 'image':
+ tf.keras.layers.Input(
+ shape=input_shape,
+ batch_size=batch_size,
+ name='image',
+ dtype=tf.bfloat16 if self._use_bfloat16 else tf.float32),
+ 'image_info':
+ tf.keras.layers.Input(
+ shape=[4, 2], batch_size=batch_size, name='image_info'),
}
return input_layer
def build_model(self, params, mode):
if self._keras_model is None:
input_layers = self.build_input_layers(self._params, mode)
- with backend.get_graph().as_default():
- outputs = self.model_outputs(input_layers, mode)
+ outputs = self.model_outputs(input_layers, mode)
- model = tf.keras.models.Model(
- inputs=input_layers, outputs=outputs, name='shapemask')
- assert model is not None, 'Fail to build tf.keras.Model.'
- model.optimizer = self.build_optimizer()
- self._keras_model = model
+ model = tf.keras.models.Model(
+ inputs=input_layers, outputs=outputs, name='shapemask')
+ assert model is not None, 'Fail to build tf.keras.Model.'
+ model.optimizer = self.build_optimizer()
+ self._keras_model = model
return self._keras_model
def post_processing(self, labels, outputs):
- required_output_fields = ['num_detections', 'detection_boxes',
- 'detection_classes', 'detection_masks',
- 'detection_scores']
+ required_output_fields = [
+ 'num_detections', 'detection_boxes', 'detection_classes',
+ 'detection_masks', 'detection_scores'
+ ]
for field in required_output_fields:
if field not in outputs:
diff --git a/official/vision/detection/ops/__init__.py b/official/vision/detection/ops/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/ops/__init__.py
+++ b/official/vision/detection/ops/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/ops/nms.py b/official/vision/detection/ops/nms.py
index bc516e5991a824b1d2f8e0261750cde2481fda2f..a81ff1e8fcd44ddd35dcf1a3bf7a9dad1831c76f 100644
--- a/official/vision/detection/ops/nms.py
+++ b/official/vision/detection/ops/nms.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Tensorflow implementation of non max suppression."""
from __future__ import absolute_import
@@ -22,7 +22,6 @@ import tensorflow as tf
from official.vision.detection.utils import box_utils
-
NMS_TILE_SIZE = 512
@@ -106,9 +105,7 @@ def _suppression_loop_body(boxes, iou_threshold, output_size, idx):
return boxes, iou_threshold, output_size, idx + 1
-def sorted_non_max_suppression_padded(scores,
- boxes,
- max_output_size,
+def sorted_non_max_suppression_padded(scores, boxes, max_output_size,
iou_threshold):
"""A wrapper that handles non-maximum suppression.
@@ -177,19 +174,18 @@ def sorted_non_max_suppression_padded(scores,
idx < num_boxes // NMS_TILE_SIZE)
selected_boxes, _, output_size, _ = tf.while_loop(
- _loop_cond, _suppression_loop_body, [
- boxes, iou_threshold,
- tf.zeros([batch_size], tf.int32),
- tf.constant(0)
- ])
+ _loop_cond, _suppression_loop_body,
+ [boxes, iou_threshold,
+ tf.zeros([batch_size], tf.int32),
+ tf.constant(0)])
idx = num_boxes - tf.cast(
tf.nn.top_k(
tf.cast(tf.reduce_any(selected_boxes > 0, [2]), tf.int32) *
tf.expand_dims(tf.range(num_boxes, 0, -1), 0), max_output_size)[0],
tf.int32)
idx = tf.minimum(idx, num_boxes - 1)
- idx = tf.reshape(
- idx + tf.reshape(tf.range(batch_size) * num_boxes, [-1, 1]), [-1])
+ idx = tf.reshape(idx + tf.reshape(tf.range(batch_size) * num_boxes, [-1, 1]),
+ [-1])
boxes = tf.reshape(
tf.gather(tf.reshape(boxes, [-1, 4]), idx),
[batch_size, max_output_size, 4])
diff --git a/official/vision/detection/ops/postprocess_ops.py b/official/vision/detection/ops/postprocess_ops.py
index 2cb06c34ab114d171f30cb52e69d8dc73996e302..ba0f3c40664381c4fa76e4d617721edccbaab7d7 100644
--- a/official/vision/detection/ops/postprocess_ops.py
+++ b/official/vision/detection/ops/postprocess_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Post-processing model outputs to generate detection."""
from __future__ import absolute_import
@@ -19,6 +19,7 @@ from __future__ import division
from __future__ import print_function
import functools
+
import tensorflow as tf
from official.vision.detection.ops import nms
@@ -202,15 +203,14 @@ def _generate_detections_per_image(boxes,
scores_i, k=tf.minimum(tf.shape(input=scores_i)[-1], pre_nms_num_boxes))
boxes_i = tf.gather(boxes_i, indices)
- (nmsed_indices_i,
- nmsed_num_valid_i) = tf.image.non_max_suppression_padded(
- tf.cast(boxes_i, tf.float32),
- tf.cast(scores_i, tf.float32),
- max_total_size,
- iou_threshold=nms_iou_threshold,
- score_threshold=score_threshold,
- pad_to_max_output_size=True,
- name='nms_detections_' + str(i))
+ (nmsed_indices_i, nmsed_num_valid_i) = tf.image.non_max_suppression_padded(
+ tf.cast(boxes_i, tf.float32),
+ tf.cast(scores_i, tf.float32),
+ max_total_size,
+ iou_threshold=nms_iou_threshold,
+ score_threshold=score_threshold,
+ pad_to_max_output_size=True,
+ name='nms_detections_' + str(i))
nmsed_boxes_i = tf.gather(boxes_i, nmsed_indices_i)
nmsed_scores_i = tf.gather(scores_i, nmsed_indices_i)
# Sets scores of invalid boxes to -1.
@@ -235,11 +235,8 @@ def _generate_detections_per_image(boxes,
return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections
-def _generate_detections_batched(boxes,
- scores,
- max_total_size,
- nms_iou_threshold,
- score_threshold):
+def _generate_detections_batched(boxes, scores, max_total_size,
+ nms_iou_threshold, score_threshold):
"""Generates detected boxes with scores and classes for one-stage detector.
The function takes output of multi-level ConvNets and anchor boxes and
@@ -247,19 +244,20 @@ def _generate_detections_batched(boxes,
supported on TPU currently.
Args:
- boxes: a tensor with shape [batch_size, N, num_classes, 4] or
- [batch_size, N, 1, 4], which box predictions on all feature levels. The N
- is the number of total anchors on all levels.
- scores: a tensor with shape [batch_size, N, num_classes], which
- stacks class probability on all feature levels. The N is the number of
- total anchors on all levels. The num_classes is the number of classes
- predicted by the model. Note that the class_outputs here is the raw score.
+ boxes: a tensor with shape [batch_size, N, num_classes, 4] or [batch_size,
+ N, 1, 4], which box predictions on all feature levels. The N is the number
+ of total anchors on all levels.
+ scores: a tensor with shape [batch_size, N, num_classes], which stacks class
+ probability on all feature levels. The N is the number of total anchors on
+ all levels. The num_classes is the number of classes predicted by the
+ model. Note that the class_outputs here is the raw score.
max_total_size: a scalar representing maximum number of boxes retained over
all classes.
nms_iou_threshold: a float representing the threshold for deciding whether
boxes overlap too much with respect to IOU.
score_threshold: a float representing the threshold for deciding when to
remove boxes based on score.
+
Returns:
nms_boxes: `float` Tensor of shape [batch_size, max_total_size, 4]
representing top detected boxes in [y1, x1, y2, x2].
@@ -285,22 +283,24 @@ def _generate_detections_batched(boxes,
max_total_size=max_total_size,
iou_threshold=nms_iou_threshold,
score_threshold=score_threshold,
- pad_per_class=False,)
+ pad_per_class=False,
+ )
# De-normalizes box cooridinates.
nmsed_boxes *= normalizer
nmsed_classes = tf.cast(nmsed_classes, tf.int32)
return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections
-class MultilevelDetectionGenerator(object):
+class MultilevelDetectionGenerator(tf.keras.layers.Layer):
"""Generates detected boxes with scores and classes for one-stage detector."""
def __init__(self, min_level, max_level, params):
self._min_level = min_level
self._max_level = max_level
self._generate_detections = generate_detections_factory(params)
+ super(MultilevelDetectionGenerator, self).__init__(autocast=False)
- def __call__(self, box_outputs, class_outputs, anchor_boxes, image_shape):
+ def call(self, box_outputs, class_outputs, anchor_boxes, image_shape):
# Collects outputs from all levels into a list.
boxes = []
scores = []
@@ -338,13 +338,14 @@ class MultilevelDetectionGenerator(object):
return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections
-class GenericDetectionGenerator(object):
+class GenericDetectionGenerator(tf.keras.layers.Layer):
"""Generates the final detected boxes with scores and classes."""
def __init__(self, params):
+ super(GenericDetectionGenerator, self).__init__(autocast=False)
self._generate_detections = generate_detections_factory(params)
- def __call__(self, box_outputs, class_outputs, anchor_boxes, image_shape):
+ def call(self, box_outputs, class_outputs, anchor_boxes, image_shape):
"""Generate final detections.
Args:
@@ -382,16 +383,13 @@ class GenericDetectionGenerator(object):
box_outputs = tf.reshape(
box_outputs,
tf.stack([batch_size, num_locations, num_classes, 4], axis=-1))
- box_outputs = tf.slice(
- box_outputs, [0, 0, 1, 0], [-1, -1, -1, -1])
+ box_outputs = tf.slice(box_outputs, [0, 0, 1, 0], [-1, -1, -1, -1])
anchor_boxes = tf.tile(
tf.expand_dims(anchor_boxes, axis=2), [1, 1, num_classes - 1, 1])
- box_outputs = tf.reshape(
- box_outputs,
- tf.stack([batch_size, num_detections, 4], axis=-1))
+ box_outputs = tf.reshape(box_outputs,
+ tf.stack([batch_size, num_detections, 4], axis=-1))
anchor_boxes = tf.reshape(
- anchor_boxes,
- tf.stack([batch_size, num_detections, 4], axis=-1))
+ anchor_boxes, tf.stack([batch_size, num_detections, 4], axis=-1))
# Box decoding.
decoded_boxes = box_utils.decode_boxes(
@@ -411,3 +409,89 @@ class GenericDetectionGenerator(object):
nmsed_classes += 1
return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections
+
+
+class OlnDetectionGenerator(GenericDetectionGenerator):
+ """Generates the final detected boxes with scores and classes."""
+
+ def __call__(self, box_outputs, class_outputs, anchor_boxes, image_shape,
+ is_single_fg_score=False, keep_nms=True):
+ """Generate final detections for Object Localization Network (OLN).
+
+ Args:
+ box_outputs: a tensor of shape of [batch_size, K, num_classes * 4]
+ representing the class-specific box coordinates relative to anchors.
+ class_outputs: a tensor of shape of [batch_size, K, num_classes]
+ representing the class logits before applying score activiation.
+ anchor_boxes: a tensor of shape of [batch_size, K, 4] representing the
+ corresponding anchor boxes w.r.t `box_outputs`.
+ image_shape: a tensor of shape of [batch_size, 2] storing the image height
+ and width w.r.t. the scaled image, i.e. the same image space as
+ `box_outputs` and `anchor_boxes`.
+ is_single_fg_score: a Bool indicator of whether class_outputs includes the
+ background scores concatenated or not. By default, class_outputs is a
+ concatenation of both scores for the foreground and background. That is,
+ scores_without_bg=False.
+ keep_nms: a Bool indicator of whether to perform NMS or not.
+
+ Returns:
+ nms_boxes: `float` Tensor of shape [batch_size, max_total_size, 4]
+ representing top detected boxes in [y1, x1, y2, x2].
+ nms_scores: `float` Tensor of shape [batch_size, max_total_size]
+ representing sorted confidence scores for detected boxes. The values are
+ between [0, 1].
+ nms_classes: `int` Tensor of shape [batch_size, max_total_size]
+ representing classes for detected boxes.
+ valid_detections: `int` Tensor of shape [batch_size] only the top
+ `valid_detections` boxes are valid detections.
+ """
+ if is_single_fg_score:
+ # Concatenates dummy background scores.
+ dummy_bg_scores = tf.zeros_like(class_outputs)
+ class_outputs = tf.stack([dummy_bg_scores, class_outputs], -1)
+ else:
+ class_outputs = tf.nn.softmax(class_outputs, axis=-1)
+
+ # Removes the background class.
+ class_outputs_shape = tf.shape(class_outputs)
+ batch_size = class_outputs_shape[0]
+ num_locations = class_outputs_shape[1]
+ num_classes = class_outputs_shape[-1]
+ num_detections = num_locations * (num_classes - 1)
+
+ class_outputs = tf.slice(class_outputs, [0, 0, 1], [-1, -1, -1])
+ box_outputs = tf.reshape(
+ box_outputs,
+ tf.stack([batch_size, num_locations, num_classes, 4], axis=-1))
+ box_outputs = tf.slice(box_outputs, [0, 0, 1, 0], [-1, -1, -1, -1])
+ anchor_boxes = tf.tile(
+ tf.expand_dims(anchor_boxes, axis=2), [1, 1, num_classes - 1, 1])
+ box_outputs = tf.reshape(box_outputs,
+ tf.stack([batch_size, num_detections, 4], axis=-1))
+ anchor_boxes = tf.reshape(
+ anchor_boxes, tf.stack([batch_size, num_detections, 4], axis=-1))
+
+ # Box decoding. For RPN outputs, box_outputs are all zeros.
+ decoded_boxes = box_utils.decode_boxes(
+ box_outputs, anchor_boxes, weights=[10.0, 10.0, 5.0, 5.0])
+
+ # Box clipping
+ decoded_boxes = box_utils.clip_boxes(decoded_boxes, image_shape)
+
+ decoded_boxes = tf.reshape(
+ decoded_boxes,
+ tf.stack([batch_size, num_locations, num_classes - 1, 4], axis=-1))
+
+ if keep_nms:
+ nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections = (
+ self._generate_detections(decoded_boxes, class_outputs))
+ # Adds 1 to offset the background class which has index 0.
+ nmsed_classes += 1
+ else:
+ nmsed_boxes = decoded_boxes[:, :, 0, :]
+ nmsed_scores = class_outputs[:, :, 0]
+ nmsed_classes = tf.cast(tf.ones_like(nmsed_scores), tf.int32)
+ valid_detections = tf.cast(
+ tf.reduce_sum(tf.ones_like(nmsed_scores), axis=-1), tf.int32)
+
+ return nmsed_boxes, nmsed_scores, nmsed_classes, valid_detections
diff --git a/official/vision/detection/ops/roi_ops.py b/official/vision/detection/ops/roi_ops.py
index a21bc7b2882de39b12bc76dacd37047fabac1766..a198d0ee204996e695f0e58bacada4bf3154ac98 100644
--- a/official/vision/detection/ops/roi_ops.py
+++ b/official/vision/detection/ops/roi_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""ROI-related ops."""
from __future__ import absolute_import
@@ -56,8 +56,8 @@ def multilevel_propose_rois(rpn_boxes,
rpn_scores: a dict with keys representing FPN levels and values representing
logit tensors of shape [batch_size, feature_h, feature_w, num_anchors].
anchor_boxes: a dict with keys representing FPN levels and values
- representing anchor box tensors of shape
- [batch_size, feature_h, feature_w, num_anchors * 4].
+ representing anchor box tensors of shape [batch_size, feature_h,
+ feature_w, num_anchors * 4].
image_shape: a tensor of shape [batch_size, 2] where the last dimension are
[height, width] of the scaled image.
rpn_pre_nms_top_k: an integer of top scoring RPN proposals *per level* to
@@ -112,17 +112,14 @@ def multilevel_propose_rois(rpn_boxes,
this_level_scores = tf.sigmoid(this_level_scores)
if decode_boxes:
- this_level_boxes = box_utils.decode_boxes(
- this_level_boxes, this_level_anchors)
+ this_level_boxes = box_utils.decode_boxes(this_level_boxes,
+ this_level_anchors)
if clip_boxes:
- this_level_boxes = box_utils.clip_boxes(
- this_level_boxes, image_shape)
+ this_level_boxes = box_utils.clip_boxes(this_level_boxes, image_shape)
if rpn_min_size_threshold > 0.0:
this_level_boxes, this_level_scores = box_utils.filter_boxes(
- this_level_boxes,
- this_level_scores,
- image_shape,
+ this_level_boxes, this_level_scores, image_shape,
rpn_min_size_threshold)
this_level_pre_nms_top_k = min(num_boxes, rpn_pre_nms_top_k)
@@ -142,8 +139,9 @@ def multilevel_propose_rois(rpn_boxes,
else:
if rpn_score_threshold > 0.0:
this_level_boxes, this_level_scores = (
- box_utils.filter_boxes_by_scores(
- this_level_boxes, this_level_scores, rpn_score_threshold))
+ box_utils.filter_boxes_by_scores(this_level_boxes,
+ this_level_scores,
+ rpn_score_threshold))
this_level_boxes, this_level_scores = box_utils.top_k_boxes(
this_level_boxes, this_level_scores, k=this_level_pre_nms_top_k)
this_level_roi_scores, this_level_rois = (
@@ -154,9 +152,7 @@ def multilevel_propose_rois(rpn_boxes,
iou_threshold=rpn_nms_threshold))
else:
this_level_rois, this_level_roi_scores = box_utils.top_k_boxes(
- this_level_rois,
- this_level_scores,
- k=this_level_post_nms_top_k)
+ this_level_rois, this_level_scores, k=this_level_post_nms_top_k)
rois.append(this_level_rois)
roi_scores.append(this_level_roi_scores)
@@ -174,7 +170,7 @@ def multilevel_propose_rois(rpn_boxes,
return selected_rois, selected_roi_scores
-class ROIGenerator(object):
+class ROIGenerator(tf.keras.layers.Layer):
"""Proposes RoIs for the second stage processing."""
def __init__(self, params):
@@ -189,8 +185,9 @@ class ROIGenerator(object):
self._test_rpn_score_threshold = params.test_rpn_score_threshold
self._test_rpn_min_size_threshold = params.test_rpn_min_size_threshold
self._use_batched_nms = params.use_batched_nms
+ super(ROIGenerator, self).__init__(autocast=False)
- def __call__(self, boxes, scores, anchor_boxes, image_shape, is_training):
+ def call(self, boxes, scores, anchor_boxes, image_shape, is_training):
"""Generates RoI proposals.
Args:
@@ -199,8 +196,8 @@ class ROIGenerator(object):
scores: a dict with keys representing FPN levels and values representing
logit tensors of shape [batch_size, feature_h, feature_w, num_anchors].
anchor_boxes: a dict with keys representing FPN levels and values
- representing anchor box tensors of shape
- [batch_size, feature_h, feature_w, num_anchors * 4].
+ representing anchor box tensors of shape [batch_size, feature_h,
+ feature_w, num_anchors * 4].
image_shape: a tensor of shape [batch_size, 2] where the last dimension
are [height, width] of the scaled image.
is_training: a bool indicating whether it is in training or inference
@@ -220,18 +217,252 @@ class ROIGenerator(object):
scores,
anchor_boxes,
image_shape,
- rpn_pre_nms_top_k=(self._rpn_pre_nms_top_k if is_training
- else self._test_rpn_pre_nms_top_k),
- rpn_post_nms_top_k=(self._rpn_post_nms_top_k if is_training
- else self._test_rpn_post_nms_top_k),
- rpn_nms_threshold=(self._rpn_nms_threshold if is_training
- else self._test_rpn_nms_threshold),
- rpn_score_threshold=(self._rpn_score_threshold if is_training
- else self._test_rpn_score_threshold),
- rpn_min_size_threshold=(self._rpn_min_size_threshold if is_training
- else self._test_rpn_min_size_threshold),
+ rpn_pre_nms_top_k=(self._rpn_pre_nms_top_k
+ if is_training else self._test_rpn_pre_nms_top_k),
+ rpn_post_nms_top_k=(self._rpn_post_nms_top_k
+ if is_training else self._test_rpn_post_nms_top_k),
+ rpn_nms_threshold=(self._rpn_nms_threshold
+ if is_training else self._test_rpn_nms_threshold),
+ rpn_score_threshold=(self._rpn_score_threshold if is_training else
+ self._test_rpn_score_threshold),
+ rpn_min_size_threshold=(self._rpn_min_size_threshold if is_training else
+ self._test_rpn_min_size_threshold),
decode_boxes=True,
clip_boxes=True,
use_batched_nms=self._use_batched_nms,
apply_sigmoid_to_score=True)
return proposed_rois, proposed_roi_scores
+
+
+class OlnROIGenerator(ROIGenerator):
+ """Proposes RoIs for the second stage processing."""
+
+ def __call__(self, boxes, scores, anchor_boxes, image_shape, is_training,
+ is_box_lrtb=False, object_scores=None):
+ """Generates RoI proposals.
+
+ Args:
+ boxes: a dict with keys representing FPN levels and values representing
+ box tenors of shape [batch_size, feature_h, feature_w, num_anchors * 4].
+ scores: a dict with keys representing FPN levels and values representing
+ logit tensors of shape [batch_size, feature_h, feature_w, num_anchors].
+ anchor_boxes: a dict with keys representing FPN levels and values
+ representing anchor box tensors of shape [batch_size, feature_h,
+ feature_w, num_anchors * 4].
+ image_shape: a tensor of shape [batch_size, 2] where the last dimension
+ are [height, width] of the scaled image.
+ is_training: a bool indicating whether it is in training or inference
+ mode.
+ is_box_lrtb: a bool indicating whether boxes are in lrtb (=left,right,top,
+ bottom) format.
+ object_scores: another objectness score (e.g., centerness). In OLN, we use
+ object_scores=centerness as a replacement of the scores at each level.
+ A dict with keys representing FPN levels and values representing logit
+ tensors of shape [batch_size, feature_h, feature_w, num_anchors].
+
+ Returns:
+ proposed_rois: a tensor of shape [batch_size, rpn_post_nms_top_k, 4],
+ representing the box coordinates of the proposed RoIs w.r.t. the
+ scaled image.
+ proposed_roi_scores: a tensor of shape
+ [batch_size, rpn_post_nms_top_k, 1], representing the scores of the
+ proposed RoIs.
+
+ """
+ proposed_rois, proposed_roi_scores = self.oln_multilevel_propose_rois(
+ boxes,
+ scores,
+ anchor_boxes,
+ image_shape,
+ rpn_pre_nms_top_k=(self._rpn_pre_nms_top_k
+ if is_training else self._test_rpn_pre_nms_top_k),
+ rpn_post_nms_top_k=(self._rpn_post_nms_top_k
+ if is_training else self._test_rpn_post_nms_top_k),
+ rpn_nms_threshold=(self._rpn_nms_threshold
+ if is_training else self._test_rpn_nms_threshold),
+ rpn_score_threshold=(self._rpn_score_threshold if is_training else
+ self._test_rpn_score_threshold),
+ rpn_min_size_threshold=(self._rpn_min_size_threshold if is_training else
+ self._test_rpn_min_size_threshold),
+ decode_boxes=True,
+ clip_boxes=True,
+ use_batched_nms=self._use_batched_nms,
+ apply_sigmoid_to_score=True,
+ is_box_lrtb=is_box_lrtb,
+ rpn_object_scores=object_scores,)
+ return proposed_rois, proposed_roi_scores
+
+ def oln_multilevel_propose_rois(self,
+ rpn_boxes,
+ rpn_scores,
+ anchor_boxes,
+ image_shape,
+ rpn_pre_nms_top_k=2000,
+ rpn_post_nms_top_k=1000,
+ rpn_nms_threshold=0.7,
+ rpn_score_threshold=0.0,
+ rpn_min_size_threshold=0.0,
+ decode_boxes=True,
+ clip_boxes=True,
+ use_batched_nms=False,
+ apply_sigmoid_to_score=True,
+ is_box_lrtb=False,
+ rpn_object_scores=None,):
+ """Proposes RoIs given a group of candidates from different FPN levels.
+
+ The following describes the steps:
+ 1. For each individual level:
+ a. Adjust scores for each level if specified by rpn_object_scores.
+ b. Apply sigmoid transform if specified.
+ c. Decode boxes (either of xyhw or left-right-top-bottom format) if
+ specified.
+ d. Clip boxes if specified.
+ e. Filter small boxes and those fall outside image if specified.
+ f. Apply pre-NMS filtering including pre-NMS top k and score
+ thresholding.
+ g. Apply NMS.
+ 2. Aggregate post-NMS boxes from each level.
+ 3. Apply an overall top k to generate the final selected RoIs.
+
+ Args:
+ rpn_boxes: a dict with keys representing FPN levels and values
+ representing box tenors of shape [batch_size, feature_h, feature_w,
+ num_anchors * 4].
+ rpn_scores: a dict with keys representing FPN levels and values
+ representing logit tensors of shape [batch_size, feature_h, feature_w,
+ num_anchors].
+ anchor_boxes: a dict with keys representing FPN levels and values
+ representing anchor box tensors of shape [batch_size, feature_h,
+ feature_w, num_anchors * 4].
+ image_shape: a tensor of shape [batch_size, 2] where the last dimension
+ are [height, width] of the scaled image.
+ rpn_pre_nms_top_k: an integer of top scoring RPN proposals *per level* to
+ keep before applying NMS. Default: 2000.
+ rpn_post_nms_top_k: an integer of top scoring RPN proposals *in total* to
+ keep after applying NMS. Default: 1000.
+ rpn_nms_threshold: a float between 0 and 1 representing the IoU threshold
+ used for NMS. If 0.0, no NMS is applied. Default: 0.7.
+ rpn_score_threshold: a float between 0 and 1 representing the minimal box
+ score to keep before applying NMS. This is often used as a pre-filtering
+ step for better performance. If 0, no filtering is applied. Default: 0.
+ rpn_min_size_threshold: a float representing the minimal box size in each
+ side (w.r.t. the scaled image) to keep before applying NMS. This is
+ often used as a pre-filtering step for better performance. If 0, no
+ filtering is applied. Default: 0.
+ decode_boxes: a boolean indicating whether `rpn_boxes` needs to be decoded
+ using `anchor_boxes`. If False, use `rpn_boxes` directly and ignore
+ `anchor_boxes`. Default: True.
+ clip_boxes: a boolean indicating whether boxes are first clipped to the
+ scaled image size before appliying NMS. If False, no clipping is applied
+ and `image_shape` is ignored. Default: True.
+ use_batched_nms: a boolean indicating whether NMS is applied in batch
+ using `tf.image.combined_non_max_suppression`. Currently only available
+ in CPU/GPU. Default: False.
+ apply_sigmoid_to_score: a boolean indicating whether apply sigmoid to
+ `rpn_scores` before applying NMS. Default: True.
+ is_box_lrtb: a bool indicating whether boxes are in lrtb (=left,right,top,
+ bottom) format.
+ rpn_object_scores: a predicted objectness score (e.g., centerness). In
+ OLN, we use object_scores=centerness as a replacement of the scores at
+ each level. A dict with keys representing FPN levels and values
+ representing logit tensors of shape [batch_size, feature_h, feature_w,
+ num_anchors].
+
+ Returns:
+ selected_rois: a tensor of shape [batch_size, rpn_post_nms_top_k, 4],
+ representing the box coordinates of the selected proposals w.r.t. the
+ scaled image.
+ selected_roi_scores: a tensor of shape [batch_size, rpn_post_nms_top_k,
+ 1],representing the scores of the selected proposals.
+ """
+ with tf.name_scope('multilevel_propose_rois'):
+ rois = []
+ roi_scores = []
+ image_shape = tf.expand_dims(image_shape, axis=1)
+ for level in sorted(rpn_scores.keys()):
+ with tf.name_scope('level_%d' % level):
+ _, feature_h, feature_w, num_anchors_per_location = (
+ rpn_scores[level].get_shape().as_list())
+
+ num_boxes = feature_h * feature_w * num_anchors_per_location
+ this_level_scores = tf.reshape(rpn_scores[level], [-1, num_boxes])
+ this_level_boxes = tf.reshape(rpn_boxes[level], [-1, num_boxes, 4])
+ this_level_anchors = tf.cast(
+ tf.reshape(anchor_boxes[level], [-1, num_boxes, 4]),
+ dtype=this_level_scores.dtype)
+
+ if rpn_object_scores:
+ this_level_object_scores = rpn_object_scores[level]
+ this_level_object_scores = tf.reshape(this_level_object_scores,
+ [-1, num_boxes])
+ this_level_object_scores = tf.cast(this_level_object_scores,
+ this_level_scores.dtype)
+ this_level_scores = this_level_object_scores
+
+ if apply_sigmoid_to_score:
+ this_level_scores = tf.sigmoid(this_level_scores)
+
+ if decode_boxes:
+ if is_box_lrtb: # Box in left-right-top-bottom format.
+ this_level_boxes = box_utils.decode_boxes_lrtb(
+ this_level_boxes, this_level_anchors)
+ else: # Box in standard x-y-h-w format.
+ this_level_boxes = box_utils.decode_boxes(
+ this_level_boxes, this_level_anchors)
+
+ if clip_boxes:
+ this_level_boxes = box_utils.clip_boxes(
+ this_level_boxes, image_shape)
+
+ if rpn_min_size_threshold > 0.0:
+ this_level_boxes, this_level_scores = box_utils.filter_boxes(
+ this_level_boxes, this_level_scores, image_shape,
+ rpn_min_size_threshold)
+
+ this_level_pre_nms_top_k = min(num_boxes, rpn_pre_nms_top_k)
+ this_level_post_nms_top_k = min(num_boxes, rpn_post_nms_top_k)
+ if rpn_nms_threshold > 0.0:
+ if use_batched_nms:
+ this_level_rois, this_level_roi_scores, _, _ = (
+ tf.image.combined_non_max_suppression(
+ tf.expand_dims(this_level_boxes, axis=2),
+ tf.expand_dims(this_level_scores, axis=-1),
+ max_output_size_per_class=this_level_pre_nms_top_k,
+ max_total_size=this_level_post_nms_top_k,
+ iou_threshold=rpn_nms_threshold,
+ score_threshold=rpn_score_threshold,
+ pad_per_class=False,
+ clip_boxes=False))
+ else:
+ if rpn_score_threshold > 0.0:
+ this_level_boxes, this_level_scores = (
+ box_utils.filter_boxes_by_scores(this_level_boxes,
+ this_level_scores,
+ rpn_score_threshold))
+ this_level_boxes, this_level_scores = box_utils.top_k_boxes(
+ this_level_boxes, this_level_scores,
+ k=this_level_pre_nms_top_k)
+ this_level_roi_scores, this_level_rois = (
+ nms.sorted_non_max_suppression_padded(
+ this_level_scores,
+ this_level_boxes,
+ max_output_size=this_level_post_nms_top_k,
+ iou_threshold=rpn_nms_threshold))
+ else:
+ this_level_rois, this_level_roi_scores = box_utils.top_k_boxes(
+ this_level_rois, this_level_scores, k=this_level_post_nms_top_k)
+
+ rois.append(this_level_rois)
+ roi_scores.append(this_level_roi_scores)
+
+ all_rois = tf.concat(rois, axis=1)
+ all_roi_scores = tf.concat(roi_scores, axis=1)
+
+ with tf.name_scope('top_k_rois'):
+ _, num_valid_rois = all_roi_scores.get_shape().as_list()
+ overall_top_k = min(num_valid_rois, rpn_post_nms_top_k)
+
+ selected_rois, selected_roi_scores = box_utils.top_k_boxes(
+ all_rois, all_roi_scores, k=overall_top_k)
+
+ return selected_rois, selected_roi_scores
diff --git a/official/vision/detection/ops/spatial_transform_ops.py b/official/vision/detection/ops/spatial_transform_ops.py
index ae60d20f0e8c8454bd7972e851c33b6dca56ed90..4b7d7ecde48ca8dd1eeb4f7356a1642583b1754d 100644
--- a/official/vision/detection/ops/spatial_transform_ops.py
+++ b/official/vision/detection/ops/spatial_transform_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Functions to performa spatial transformation for Tensor."""
from __future__ import absolute_import
@@ -20,7 +20,6 @@ from __future__ import print_function
import tensorflow as tf
-
_EPSILON = 1e-8
@@ -30,6 +29,7 @@ def nearest_upsampling(data, scale):
Args:
data: A tensor with a shape of [batch, height_in, width_in, channels].
scale: An integer multiple to scale resolution of input data.
+
Returns:
data_up: A tensor with a shape of
[batch, height_in*scale, width_in*scale, channels]. Same dtype as input
@@ -382,8 +382,7 @@ def multilevel_crop_and_resize(features, boxes, output_size=7):
areas_sqrt = tf.sqrt(box_height * box_width)
levels = tf.cast(
tf.math.floordiv(
- tf.math.log(tf.divide(areas_sqrt, 224.0)), tf.math.log(2.0)) +
- 4.0,
+ tf.math.log(tf.divide(areas_sqrt, 224.0)), tf.math.log(2.0)) + 4.0,
dtype=tf.int32)
# Maps levels between [min_level, max_level].
levels = tf.minimum(max_level, tf.maximum(levels, min_level))
@@ -395,9 +394,12 @@ def multilevel_crop_and_resize(features, boxes, output_size=7):
boxes /= tf.expand_dims(scale_to_level, axis=2)
box_width /= scale_to_level
box_height /= scale_to_level
- boxes = tf.concat([boxes[:, :, 0:2],
- tf.expand_dims(box_height, -1),
- tf.expand_dims(box_width, -1)], axis=-1)
+ boxes = tf.concat([
+ boxes[:, :, 0:2],
+ tf.expand_dims(box_height, -1),
+ tf.expand_dims(box_width, -1)
+ ],
+ axis=-1)
# Maps levels to [0, max_level-min_level].
levels -= min_level
@@ -464,12 +466,12 @@ def single_level_feature_crop(features, level_boxes, detection_prior_levels,
Args:
- features: a float tensor of shape [batch_size, num_levels,
- max_feature_size, max_feature_size, num_downsample_channels].
- level_boxes: a float Tensor of the level boxes to crop from.
- [batch_size, num_instances, 4].
+ features: a float tensor of shape [batch_size, num_levels, max_feature_size,
+ max_feature_size, num_downsample_channels].
+ level_boxes: a float Tensor of the level boxes to crop from. [batch_size,
+ num_instances, 4].
detection_prior_levels: an int Tensor of instance assigned level of shape
- [batch_size, num_instances].
+ [batch_size, num_instances].
min_mask_level: minimum FPN level to crop mask feature from.
mask_crop_size: an int of mask crop size.
@@ -478,8 +480,8 @@ def single_level_feature_crop(features, level_boxes, detection_prior_levels,
mask_crop_size, mask_crop_size, num_downsample_channels]. This is the
instance feature crop.
"""
- (batch_size, num_levels, max_feature_size,
- _, num_downsample_channels) = features.get_shape().as_list()
+ (batch_size, num_levels, max_feature_size, _,
+ num_downsample_channels) = features.get_shape().as_list()
_, num_of_instances, _ = level_boxes.get_shape().as_list()
level_boxes = tf.cast(level_boxes, tf.int32)
assert num_of_instances == detection_prior_levels.get_shape().as_list()[1]
@@ -503,32 +505,25 @@ def single_level_feature_crop(features, level_boxes, detection_prior_levels,
indices = tf.reshape(
tf.tile(
tf.reshape(
- tf.range(batch_size) * batch_dim_size,
- [batch_size, 1, 1, 1]),
- [1, num_of_instances,
- mask_crop_size, mask_crop_size]) +
- tf.tile(
- tf.reshape(levels * level_dim_size,
- [batch_size, num_of_instances, 1, 1]),
- [1, 1, mask_crop_size, mask_crop_size]) +
- tf.tile(
- tf.reshape(y_indices * height_dim_size,
- [batch_size, num_of_instances,
- mask_crop_size, 1]),
- [1, 1, 1, mask_crop_size]) +
+ tf.range(batch_size) * batch_dim_size, [batch_size, 1, 1, 1]),
+ [1, num_of_instances, mask_crop_size, mask_crop_size]) + tf.tile(
+ tf.reshape(levels * level_dim_size,
+ [batch_size, num_of_instances, 1, 1]),
+ [1, 1, mask_crop_size, mask_crop_size]) + tf.tile(
+ tf.reshape(y_indices * height_dim_size,
+ [batch_size, num_of_instances, mask_crop_size, 1]),
+ [1, 1, 1, mask_crop_size]) +
tf.tile(
tf.reshape(x_indices,
- [batch_size, num_of_instances,
- 1, mask_crop_size]),
+ [batch_size, num_of_instances, 1, mask_crop_size]),
[1, 1, mask_crop_size, 1]), [-1])
- features_r2 = tf.reshape(features,
- [-1, num_downsample_channels])
+ features_r2 = tf.reshape(features, [-1, num_downsample_channels])
crop_features = tf.reshape(
- tf.gather(features_r2, indices),
- [batch_size * num_of_instances,
- mask_crop_size, mask_crop_size,
- num_downsample_channels])
+ tf.gather(features_r2, indices), [
+ batch_size * num_of_instances, mask_crop_size, mask_crop_size,
+ num_downsample_channels
+ ])
return crop_features
@@ -546,9 +541,9 @@ def crop_mask_in_target_box(masks,
boxes: a float tensor representing box cooridnates that tightly enclose
masks with a shape of [batch_size, num_masks, 4] in un-normalized
coordinates. A box is represented by [ymin, xmin, ymax, xmax].
- target_boxes: a float tensor representing target box cooridnates for
- masks with a shape of [batch_size, num_masks, 4] in un-normalized
- coordinates. A box is represented by [ymin, xmin, ymax, xmax].
+ target_boxes: a float tensor representing target box cooridnates for masks
+ with a shape of [batch_size, num_masks, 4] in un-normalized coordinates. A
+ box is represented by [ymin, xmin, ymax, xmax].
output_size: A scalar to indicate the output crop size. It currently only
supports to output a square shape outputs.
sample_offset: a float number in [0, 1] indicates the subpixel sample offset
@@ -561,10 +556,10 @@ def crop_mask_in_target_box(masks,
"""
with tf.name_scope('crop_mask_in_target_box'):
batch_size, num_masks, height, width = masks.get_shape().as_list()
- masks = tf.reshape(masks, [batch_size*num_masks, height, width, 1])
+ masks = tf.reshape(masks, [batch_size * num_masks, height, width, 1])
# Pad zeros on the boundary of masks.
masks = tf.image.pad_to_bounding_box(masks, 2, 2, height + 4, width + 4)
- masks = tf.reshape(masks, [batch_size, num_masks, height+4, width+4, 1])
+ masks = tf.reshape(masks, [batch_size, num_masks, height + 4, width + 4, 1])
# Projects target box locations and sizes to corresponding cropped
# mask coordinates.
@@ -572,10 +567,10 @@ def crop_mask_in_target_box(masks,
value=boxes, num_or_size_splits=4, axis=2)
bb_y_min, bb_x_min, bb_y_max, bb_x_max = tf.split(
value=target_boxes, num_or_size_splits=4, axis=2)
- y_transform = (bb_y_min - gt_y_min) * height / (
- gt_y_max - gt_y_min + _EPSILON) + 2
- x_transform = (bb_x_min - gt_x_min) * height / (
- gt_x_max - gt_x_min + _EPSILON) + 2
+ y_transform = (bb_y_min - gt_y_min) * height / (gt_y_max - gt_y_min +
+ _EPSILON) + 2
+ x_transform = (bb_x_min - gt_x_min) * height / (gt_x_max - gt_x_min +
+ _EPSILON) + 2
h_transform = (bb_y_max - bb_y_min) * width / (
gt_y_max - gt_y_min + _EPSILON)
w_transform = (bb_x_max - bb_x_min) * width / (
@@ -592,8 +587,8 @@ def crop_mask_in_target_box(masks,
# Reshape tensors to have the right shape for selective_crop_and_resize.
trasnformed_boxes = tf.concat(
[y_transform, x_transform, h_transform, w_transform], -1)
- levels = tf.tile(tf.reshape(tf.range(num_masks), [1, num_masks]),
- [batch_size, 1])
+ levels = tf.tile(
+ tf.reshape(tf.range(num_masks), [1, num_masks]), [batch_size, 1])
cropped_masks = selective_crop_and_resize(
masks,
diff --git a/official/vision/detection/ops/target_ops.py b/official/vision/detection/ops/target_ops.py
index 2a7d6856511f846365041527f2532c8f2b376244..c129e853be0d1ef63ab474506d59024a08ab13ff 100644
--- a/official/vision/detection/ops/target_ops.py
+++ b/official/vision/detection/ops/target_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Target and sampling related ops."""
from __future__ import absolute_import
@@ -87,18 +87,16 @@ def box_matching(boxes, gt_boxes, gt_classes):
matched_gt_boxes)
matched_gt_classes = tf.gather_nd(gt_classes, gather_nd_indices)
- matched_gt_classes = tf.where(
- background_box_mask,
- tf.zeros_like(matched_gt_classes),
- matched_gt_classes)
+ matched_gt_classes = tf.where(background_box_mask,
+ tf.zeros_like(matched_gt_classes),
+ matched_gt_classes)
- matched_gt_indices = tf.where(
- background_box_mask,
- -tf.ones_like(argmax_iou_indices),
- argmax_iou_indices)
+ matched_gt_indices = tf.where(background_box_mask,
+ -tf.ones_like(argmax_iou_indices),
+ argmax_iou_indices)
- return (matched_gt_boxes, matched_gt_classes, matched_gt_indices,
- matched_iou, iou)
+ return (matched_gt_boxes, matched_gt_classes, matched_gt_indices, matched_iou,
+ iou)
def assign_and_sample_proposals(proposed_boxes,
@@ -121,22 +119,21 @@ def assign_and_sample_proposals(proposed_boxes,
returns box_targets, class_targets, and RoIs.
Args:
- proposed_boxes: a tensor of shape of [batch_size, N, 4]. N is the number
- of proposals before groundtruth assignment. The last dimension is the
- box coordinates w.r.t. the scaled images in [ymin, xmin, ymax, xmax]
- format.
- gt_boxes: a tensor of shape of [batch_size, MAX_NUM_INSTANCES, 4].
- The coordinates of gt_boxes are in the pixel coordinates of the scaled
- image. This tensor might have padding of values -1 indicating the invalid
- box coordinates.
+ proposed_boxes: a tensor of shape of [batch_size, N, 4]. N is the number of
+ proposals before groundtruth assignment. The last dimension is the box
+ coordinates w.r.t. the scaled images in [ymin, xmin, ymax, xmax] format.
+ gt_boxes: a tensor of shape of [batch_size, MAX_NUM_INSTANCES, 4]. The
+ coordinates of gt_boxes are in the pixel coordinates of the scaled image.
+ This tensor might have padding of values -1 indicating the invalid box
+ coordinates.
gt_classes: a tensor with a shape of [batch_size, MAX_NUM_INSTANCES]. This
tensor might have paddings with values of -1 indicating the invalid
classes.
num_samples_per_image: a integer represents RoI minibatch size per image.
mix_gt_boxes: a bool indicating whether to mix the groundtruth boxes before
sampling proposals.
- fg_fraction: a float represents the target fraction of RoI minibatch that
- is labeled foreground (i.e., class > 0).
+ fg_fraction: a float represents the target fraction of RoI minibatch that is
+ labeled foreground (i.e., class > 0).
fg_iou_thresh: a float represents the IoU overlap threshold for an RoI to be
considered foreground (if >= fg_iou_thresh).
bg_iou_thresh_hi: a float represents the IoU overlap threshold for an RoI to
@@ -163,8 +160,8 @@ def assign_and_sample_proposals(proposed_boxes,
else:
boxes = proposed_boxes
- (matched_gt_boxes, matched_gt_classes, matched_gt_indices,
- matched_iou, _) = box_matching(boxes, gt_boxes, gt_classes)
+ (matched_gt_boxes, matched_gt_classes, matched_gt_indices, matched_iou,
+ _) = box_matching(boxes, gt_boxes, gt_classes)
positive_match = tf.greater(matched_iou, fg_iou_thresh)
negative_match = tf.logical_and(
@@ -173,10 +170,12 @@ def assign_and_sample_proposals(proposed_boxes,
ignored_match = tf.less(matched_iou, 0.0)
# re-assign negatively matched boxes to the background class.
- matched_gt_classes = tf.where(
- negative_match, tf.zeros_like(matched_gt_classes), matched_gt_classes)
- matched_gt_indices = tf.where(
- negative_match, tf.zeros_like(matched_gt_indices), matched_gt_indices)
+ matched_gt_classes = tf.where(negative_match,
+ tf.zeros_like(matched_gt_classes),
+ matched_gt_classes)
+ matched_gt_indices = tf.where(negative_match,
+ tf.zeros_like(matched_gt_indices),
+ matched_gt_indices)
sample_candidates = tf.logical_and(
tf.logical_or(positive_match, negative_match),
@@ -189,8 +188,9 @@ def assign_and_sample_proposals(proposed_boxes,
batch_size, _ = sample_candidates.get_shape().as_list()
sampled_indicators = []
for i in range(batch_size):
- sampled_indicator = sampler.subsample(
- sample_candidates[i], num_samples_per_image, positive_match[i])
+ sampled_indicator = sampler.subsample(sample_candidates[i],
+ num_samples_per_image,
+ positive_match[i])
sampled_indicators.append(sampled_indicator)
sampled_indicators = tf.stack(sampled_indicators)
_, sampled_indices = tf.nn.top_k(
@@ -206,10 +206,8 @@ def assign_and_sample_proposals(proposed_boxes,
sampled_rois = tf.gather_nd(boxes, gather_nd_indices)
sampled_gt_boxes = tf.gather_nd(matched_gt_boxes, gather_nd_indices)
- sampled_gt_classes = tf.gather_nd(
- matched_gt_classes, gather_nd_indices)
- sampled_gt_indices = tf.gather_nd(
- matched_gt_indices, gather_nd_indices)
+ sampled_gt_classes = tf.gather_nd(matched_gt_classes, gather_nd_indices)
+ sampled_gt_indices = tf.gather_nd(matched_gt_indices, gather_nd_indices)
return (sampled_rois, sampled_gt_boxes, sampled_gt_classes,
sampled_gt_indices)
@@ -237,8 +235,8 @@ def sample_and_crop_foreground_masks(candidate_rois,
candidate_gt_indices: a tensor of shape [batch_size, N], storing the
corresponding groundtruth instance indices to the `candidate_gt_boxes`,
i.e. gt_boxes[candidate_gt_indices[:, i]] = candidate_gt_boxes[:, i] and
- gt_boxes which is of shape [batch_size, MAX_INSTANCES, 4], M >= N, is the
- superset of candidate_gt_boxes.
+ gt_boxes which is of shape [batch_size, MAX_INSTANCES, 4], M >= N, is
+ the superset of candidate_gt_boxes.
gt_masks: a tensor of [batch_size, MAX_INSTANCES, mask_height, mask_width]
containing all the groundtruth masks which sample masks are drawn from.
num_mask_samples_per_image: an integer which specifies the number of masks
@@ -266,33 +264,35 @@ def sample_and_crop_foreground_masks(candidate_rois,
tf.expand_dims(tf.range(fg_instance_indices_shape[0]), axis=-1) *
tf.ones([1, fg_instance_indices_shape[-1]], dtype=tf.int32))
- gather_nd_instance_indices = tf.stack(
- [batch_indices, fg_instance_indices], axis=-1)
- foreground_rois = tf.gather_nd(
- candidate_rois, gather_nd_instance_indices)
- foreground_boxes = tf.gather_nd(
- candidate_gt_boxes, gather_nd_instance_indices)
- foreground_classes = tf.gather_nd(
- candidate_gt_classes, gather_nd_instance_indices)
- foreground_gt_indices = tf.gather_nd(
- candidate_gt_indices, gather_nd_instance_indices)
+ gather_nd_instance_indices = tf.stack([batch_indices, fg_instance_indices],
+ axis=-1)
+ foreground_rois = tf.gather_nd(candidate_rois, gather_nd_instance_indices)
+ foreground_boxes = tf.gather_nd(candidate_gt_boxes,
+ gather_nd_instance_indices)
+ foreground_classes = tf.gather_nd(candidate_gt_classes,
+ gather_nd_instance_indices)
+ foreground_gt_indices = tf.gather_nd(candidate_gt_indices,
+ gather_nd_instance_indices)
foreground_gt_indices_shape = tf.shape(foreground_gt_indices)
batch_indices = (
tf.expand_dims(tf.range(foreground_gt_indices_shape[0]), axis=-1) *
tf.ones([1, foreground_gt_indices_shape[-1]], dtype=tf.int32))
- gather_nd_gt_indices = tf.stack(
- [batch_indices, foreground_gt_indices], axis=-1)
+ gather_nd_gt_indices = tf.stack([batch_indices, foreground_gt_indices],
+ axis=-1)
foreground_masks = tf.gather_nd(gt_masks, gather_nd_gt_indices)
cropped_foreground_masks = spatial_transform_ops.crop_mask_in_target_box(
- foreground_masks, foreground_boxes, foreground_rois, mask_target_size,
+ foreground_masks,
+ foreground_boxes,
+ foreground_rois,
+ mask_target_size,
sample_offset=0.5)
return foreground_rois, foreground_classes, cropped_foreground_masks
-class ROISampler(object):
+class ROISampler(tf.keras.layers.Layer):
"""Samples RoIs and creates training targets."""
def __init__(self, params):
@@ -302,17 +302,17 @@ class ROISampler(object):
self._bg_iou_thresh_hi = params.bg_iou_thresh_hi
self._bg_iou_thresh_lo = params.bg_iou_thresh_lo
self._mix_gt_boxes = params.mix_gt_boxes
+ super(ROISampler, self).__init__(autocast=False)
- def __call__(self, rois, gt_boxes, gt_classes):
+ def call(self, rois, gt_boxes, gt_classes):
"""Sample and assign RoIs for training.
Args:
- rois: a tensor of shape of [batch_size, N, 4]. N is the number
- of proposals before groundtruth assignment. The last dimension is the
- box coordinates w.r.t. the scaled images in [ymin, xmin, ymax, xmax]
- format.
- gt_boxes: a tensor of shape of [batch_size, MAX_NUM_INSTANCES, 4].
- The coordinates of gt_boxes are in the pixel coordinates of the scaled
+ rois: a tensor of shape of [batch_size, N, 4]. N is the number of
+ proposals before groundtruth assignment. The last dimension is the box
+ coordinates w.r.t. the scaled images in [ymin, xmin, ymax, xmax] format.
+ gt_boxes: a tensor of shape of [batch_size, MAX_NUM_INSTANCES, 4]. The
+ coordinates of gt_boxes are in the pixel coordinates of the scaled
image. This tensor might have padding of values -1 indicating the
invalid box coordinates.
gt_classes: a tensor with a shape of [batch_size, MAX_NUM_INSTANCES]. This
@@ -343,19 +343,194 @@ class ROISampler(object):
sampled_gt_indices)
-class MaskSampler(object):
+class ROIScoreSampler(ROISampler):
+ """Samples RoIs, RoI-scores and creates training targets."""
+
+ def __call__(self, rois, roi_scores, gt_boxes, gt_classes):
+ """Sample and assign RoIs for training.
+
+ Args:
+ rois: a tensor of shape of [batch_size, N, 4]. N is the number of
+ proposals before groundtruth assignment. The last dimension is the box
+ coordinates w.r.t. the scaled images in [ymin, xmin, ymax, xmax] format.
+ roi_scores:
+ gt_boxes: a tensor of shape of [batch_size, MAX_NUM_INSTANCES, 4]. The
+ coordinates of gt_boxes are in the pixel coordinates of the scaled
+ image. This tensor might have padding of values -1 indicating the
+ invalid box coordinates.
+ gt_classes: a tensor with a shape of [batch_size, MAX_NUM_INSTANCES]. This
+ tensor might have paddings with values of -1 indicating the invalid
+ classes.
+
+ Returns:
+ sampled_rois: a tensor of shape of [batch_size, K, 4], representing the
+ coordinates of the sampled RoIs, where K is the number of the sampled
+ RoIs, i.e. K = num_samples_per_image.
+ sampled_roi_scores:
+ sampled_gt_boxes: a tensor of shape of [batch_size, K, 4], storing the
+ box coordinates of the matched groundtruth boxes of the samples RoIs.
+ sampled_gt_classes: a tensor of shape of [batch_size, K], storing the
+ classes of the matched groundtruth boxes of the sampled RoIs.
+ """
+ (sampled_rois, sampled_roi_scores, sampled_gt_boxes, sampled_gt_classes,
+ sampled_gt_indices) = (
+ self.assign_and_sample_proposals_and_scores(
+ rois,
+ roi_scores,
+ gt_boxes,
+ gt_classes,
+ num_samples_per_image=self._num_samples_per_image,
+ mix_gt_boxes=self._mix_gt_boxes,
+ fg_fraction=self._fg_fraction,
+ fg_iou_thresh=self._fg_iou_thresh,
+ bg_iou_thresh_hi=self._bg_iou_thresh_hi,
+ bg_iou_thresh_lo=self._bg_iou_thresh_lo))
+ return (sampled_rois, sampled_roi_scores, sampled_gt_boxes,
+ sampled_gt_classes, sampled_gt_indices)
+
+ def assign_and_sample_proposals_and_scores(self,
+ proposed_boxes,
+ proposed_scores,
+ gt_boxes,
+ gt_classes,
+ num_samples_per_image=512,
+ mix_gt_boxes=True,
+ fg_fraction=0.25,
+ fg_iou_thresh=0.5,
+ bg_iou_thresh_hi=0.5,
+ bg_iou_thresh_lo=0.0):
+ """Assigns the proposals with groundtruth classes and performs subsmpling.
+
+ Given `proposed_boxes`, `gt_boxes`, and `gt_classes`, the function uses the
+ following algorithm to generate the final `num_samples_per_image` RoIs.
+ 1. Calculates the IoU between each proposal box and each gt_boxes.
+ 2. Assigns each proposed box with a groundtruth class and box by choosing
+ the largest IoU overlap.
+ 3. Samples `num_samples_per_image` boxes from all proposed boxes, and
+ returns box_targets, class_targets, and RoIs.
+
+ Args:
+ proposed_boxes: a tensor of shape of [batch_size, N, 4]. N is the number
+ of proposals before groundtruth assignment. The last dimension is the
+ box coordinates w.r.t. the scaled images in [ymin, xmin, ymax, xmax]
+ format.
+ proposed_scores: a tensor of shape of [batch_size, N]. N is the number of
+ proposals before groundtruth assignment. It is the rpn scores for all
+ proposed boxes which can be either their classification or centerness
+ scores.
+ gt_boxes: a tensor of shape of [batch_size, MAX_NUM_INSTANCES, 4]. The
+ coordinates of gt_boxes are in the pixel coordinates of the scaled
+ image. This tensor might have padding of values -1 indicating the
+ invalid box coordinates.
+ gt_classes: a tensor with a shape of [batch_size, MAX_NUM_INSTANCES]. This
+ tensor might have paddings with values of -1 indicating the invalid
+ classes.
+ num_samples_per_image: a integer represents RoI minibatch size per image.
+ mix_gt_boxes: a bool indicating whether to mix the groundtruth boxes
+ before sampling proposals.
+ fg_fraction: a float represents the target fraction of RoI minibatch that
+ is labeled foreground (i.e., class > 0).
+ fg_iou_thresh: a float represents the IoU overlap threshold for an RoI to
+ be considered foreground (if >= fg_iou_thresh).
+ bg_iou_thresh_hi: a float represents the IoU overlap threshold for an RoI
+ to be considered background (class = 0 if overlap in [LO, HI)).
+ bg_iou_thresh_lo: a float represents the IoU overlap threshold for an RoI
+ to be considered background (class = 0 if overlap in [LO, HI)).
+
+ Returns:
+ sampled_rois: a tensor of shape of [batch_size, K, 4], representing the
+ coordinates of the sampled RoIs, where K is the number of the sampled
+ RoIs, i.e. K = num_samples_per_image.
+ sampled_scores: a tensor of shape of [batch_size, K], representing the
+ confidence score of the sampled RoIs, where K is the number of the
+ sampled RoIs, i.e. K = num_samples_per_image.
+ sampled_gt_boxes: a tensor of shape of [batch_size, K, 4], storing the
+ box coordinates of the matched groundtruth boxes of the samples RoIs.
+ sampled_gt_classes: a tensor of shape of [batch_size, K], storing the
+ classes of the matched groundtruth boxes of the sampled RoIs.
+ sampled_gt_indices: a tensor of shape of [batch_size, K], storing the
+ indices of the sampled groudntruth boxes in the original `gt_boxes`
+ tensor, i.e. gt_boxes[sampled_gt_indices[:, i]] =
+ sampled_gt_boxes[:, i].
+ """
+
+ with tf.name_scope('sample_proposals_and_scores'):
+ if mix_gt_boxes:
+ boxes = tf.concat([proposed_boxes, gt_boxes], axis=1)
+ gt_scores = tf.ones_like(gt_boxes[:, :, 0])
+ scores = tf.concat([proposed_scores, gt_scores], axis=1)
+ else:
+ boxes = proposed_boxes
+ scores = proposed_scores
+
+ (matched_gt_boxes, matched_gt_classes, matched_gt_indices, matched_iou,
+ _) = box_matching(boxes, gt_boxes, gt_classes)
+
+ positive_match = tf.greater(matched_iou, fg_iou_thresh)
+ negative_match = tf.logical_and(
+ tf.greater_equal(matched_iou, bg_iou_thresh_lo),
+ tf.less(matched_iou, bg_iou_thresh_hi))
+ ignored_match = tf.less(matched_iou, 0.0)
+
+ # re-assign negatively matched boxes to the background class.
+ matched_gt_classes = tf.where(negative_match,
+ tf.zeros_like(matched_gt_classes),
+ matched_gt_classes)
+ matched_gt_indices = tf.where(negative_match,
+ tf.zeros_like(matched_gt_indices),
+ matched_gt_indices)
+
+ sample_candidates = tf.logical_and(
+ tf.logical_or(positive_match, negative_match),
+ tf.logical_not(ignored_match))
+
+ sampler = (
+ balanced_positive_negative_sampler.BalancedPositiveNegativeSampler(
+ positive_fraction=fg_fraction, is_static=True))
+
+ batch_size, _ = sample_candidates.get_shape().as_list()
+ sampled_indicators = []
+ for i in range(batch_size):
+ sampled_indicator = sampler.subsample(sample_candidates[i],
+ num_samples_per_image,
+ positive_match[i])
+ sampled_indicators.append(sampled_indicator)
+ sampled_indicators = tf.stack(sampled_indicators)
+ _, sampled_indices = tf.nn.top_k(
+ tf.cast(sampled_indicators, dtype=tf.int32),
+ k=num_samples_per_image,
+ sorted=True)
+
+ sampled_indices_shape = tf.shape(sampled_indices)
+ batch_indices = (
+ tf.expand_dims(tf.range(sampled_indices_shape[0]), axis=-1) *
+ tf.ones([1, sampled_indices_shape[-1]], dtype=tf.int32))
+ gather_nd_indices = tf.stack([batch_indices, sampled_indices], axis=-1)
+
+ sampled_rois = tf.gather_nd(boxes, gather_nd_indices)
+ sampled_roi_scores = tf.gather_nd(scores, gather_nd_indices)
+ sampled_gt_boxes = tf.gather_nd(matched_gt_boxes, gather_nd_indices)
+ sampled_gt_classes = tf.gather_nd(matched_gt_classes, gather_nd_indices)
+ sampled_gt_indices = tf.gather_nd(matched_gt_indices, gather_nd_indices)
+
+ return (sampled_rois, sampled_roi_scores, sampled_gt_boxes,
+ sampled_gt_classes, sampled_gt_indices)
+
+
+class MaskSampler(tf.keras.layers.Layer):
"""Samples and creates mask training targets."""
def __init__(self, mask_target_size, num_mask_samples_per_image):
self._mask_target_size = mask_target_size
self._num_mask_samples_per_image = num_mask_samples_per_image
-
- def __call__(self,
- candidate_rois,
- candidate_gt_boxes,
- candidate_gt_classes,
- candidate_gt_indices,
- gt_masks):
+ super(MaskSampler, self).__init__(autocast=False)
+
+ def call(self,
+ candidate_rois,
+ candidate_gt_boxes,
+ candidate_gt_classes,
+ candidate_gt_indices,
+ gt_masks):
"""Sample and create mask targets for training.
Args:
@@ -371,8 +546,8 @@ class MaskSampler(object):
candidate_gt_indices: a tensor of shape [batch_size, N], storing the
corresponding groundtruth instance indices to the `candidate_gt_boxes`,
i.e. gt_boxes[candidate_gt_indices[:, i]] = candidate_gt_boxes[:, i],
- where gt_boxes which is of shape [batch_size, MAX_INSTANCES, 4], M >= N,
- is the superset of candidate_gt_boxes.
+ where gt_boxes which is of shape [batch_size, MAX_INSTANCES, 4], M >=
+ N, is the superset of candidate_gt_boxes.
gt_masks: a tensor of [batch_size, MAX_INSTANCES, mask_height, mask_width]
containing all the groundtruth masks which sample masks are drawn from.
after sampling. The output masks are resized w.r.t the sampled RoIs.
@@ -388,12 +563,9 @@ class MaskSampler(object):
cropped foreground masks used for training.
"""
foreground_rois, foreground_classes, cropped_foreground_masks = (
- sample_and_crop_foreground_masks(
- candidate_rois,
- candidate_gt_boxes,
- candidate_gt_classes,
- candidate_gt_indices,
- gt_masks,
- self._num_mask_samples_per_image,
- self._mask_target_size))
+ sample_and_crop_foreground_masks(candidate_rois, candidate_gt_boxes,
+ candidate_gt_classes,
+ candidate_gt_indices, gt_masks,
+ self._num_mask_samples_per_image,
+ self._mask_target_size))
return foreground_rois, foreground_classes, cropped_foreground_masks
diff --git a/official/vision/detection/utils/__init__.py b/official/vision/detection/utils/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/utils/__init__.py
+++ b/official/vision/detection/utils/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/utils/box_utils.py b/official/vision/detection/utils/box_utils.py
index 4c2ebf5781f44363b090f3e272101d6014f2edd0..27f051c4d0a6ad6c6808f606f1fe5248e816a93e 100644
--- a/official/vision/detection/utils/box_utils.py
+++ b/official/vision/detection/utils/box_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Utility functions for bounding box processing."""
from __future__ import absolute_import
@@ -115,8 +115,8 @@ def normalize_boxes(boxes, image_shape):
"""Converts boxes to the normalized coordinates.
Args:
- boxes: a tensor whose last dimension is 4 representing the coordinates
- of boxes in ymin, xmin, ymax, xmax order.
+ boxes: a tensor whose last dimension is 4 representing the coordinates of
+ boxes in ymin, xmin, ymax, xmax order.
image_shape: a list of two integers, a two-element vector or a tensor such
that all but the last dimensions are `broadcastable` to `boxes`. The last
dimension is 2, which represents [height, width].
@@ -153,8 +153,8 @@ def denormalize_boxes(boxes, image_shape):
"""Converts boxes normalized by [height, width] to pixel coordinates.
Args:
- boxes: a tensor whose last dimension is 4 representing the coordinates
- of boxes in ymin, xmin, ymax, xmax order.
+ boxes: a tensor whose last dimension is 4 representing the coordinates of
+ boxes in ymin, xmin, ymax, xmax order.
image_shape: a list of two integers, a two-element vector or a tensor such
that all but the last dimensions are `broadcastable` to `boxes`. The last
dimension is 2, which represents [height, width].
@@ -187,8 +187,8 @@ def clip_boxes(boxes, image_shape):
"""Clips boxes to image boundaries.
Args:
- boxes: a tensor whose last dimension is 4 representing the coordinates
- of boxes in ymin, xmin, ymax, xmax order.
+ boxes: a tensor whose last dimension is 4 representing the coordinates of
+ boxes in ymin, xmin, ymax, xmax order.
image_shape: a list of two integers, a two-element vector or a tensor such
that all but the last dimensions are `broadcastable` to `boxes`. The last
dimension is 2, which represents [height, width].
@@ -255,8 +255,8 @@ def encode_boxes(boxes, anchors, weights=None):
"""Encode boxes to targets.
Args:
- boxes: a tensor whose last dimension is 4 representing the coordinates
- of boxes in ymin, xmin, ymax, xmax order.
+ boxes: a tensor whose last dimension is 4 representing the coordinates of
+ boxes in ymin, xmin, ymax, xmax order.
anchors: a tensor whose shape is the same as, or `broadcastable` to `boxes`,
representing the coordinates of anchors in ymin, xmin, ymax, xmax order.
weights: None or a list of four float numbers used to scale coordinates.
@@ -302,9 +302,8 @@ def encode_boxes(boxes, anchors, weights=None):
encoded_dh *= weights[2]
encoded_dw *= weights[3]
- encoded_boxes = tf.concat(
- [encoded_dy, encoded_dx, encoded_dh, encoded_dw],
- axis=-1)
+ encoded_boxes = tf.concat([encoded_dy, encoded_dx, encoded_dh, encoded_dw],
+ axis=-1)
return encoded_boxes
@@ -359,11 +358,162 @@ def decode_boxes(encoded_boxes, anchors, weights=None):
decoded_boxes_ymax = decoded_boxes_ymin + decoded_boxes_h - 1.0
decoded_boxes_xmax = decoded_boxes_xmin + decoded_boxes_w - 1.0
- decoded_boxes = tf.concat(
+ decoded_boxes = tf.concat([
+ decoded_boxes_ymin, decoded_boxes_xmin, decoded_boxes_ymax,
+ decoded_boxes_xmax
+ ],
+ axis=-1)
+ return decoded_boxes
+
+
+def encode_boxes_lrtb(boxes, anchors, weights=None):
+ """Encode boxes to targets on lrtb (=left,right,top,bottom) format.
+
+ Args:
+ boxes: a tensor whose last dimension is 4 representing the coordinates
+ of boxes in ymin, xmin, ymax, xmax order.
+ anchors: a tensor whose shape is the same as, or `broadcastable` to `boxes`,
+ representing the coordinates of anchors in ymin, xmin, ymax, xmax order.
+ weights: None or a list of four float numbers used to scale coordinates.
+
+ Returns:
+ encoded_boxes_lrtb: a tensor whose shape is the same as `boxes` representing
+ the encoded box targets. The box targets encode the left, right, top,
+ bottom distances from an anchor location to the four borders of the
+ matched groundtruth bounding box.
+ center_targets: centerness targets defined by the left, right, top, and
+ bottom distance targets. The centerness is defined as the deviation of the
+ anchor location from the groundtruth object center. Formally, centerness =
+ sqrt(min(left, right)/max(left, right)*min(top, bottom)/max(top, bottom)).
+
+ Raises:
+ ValueError: If the last dimension of boxes is not 4.
+ """
+ if boxes.shape[-1] != 4:
+ raise ValueError(
+ 'boxes.shape[-1] is {:d}, but must be 4.'.format(boxes.shape[-1]))
+
+ with tf.name_scope('encode_boxes_lrtb'):
+ boxes = tf.cast(boxes, dtype=anchors.dtype)
+ ymin = boxes[..., 0:1]
+ xmin = boxes[..., 1:2]
+ ymax = boxes[..., 2:3]
+ xmax = boxes[..., 3:4]
+ # box_h = ymax - ymin + 1.0
+ # box_w = xmax - xmin + 1.0
+ box_h = ymax - ymin
+ box_w = xmax - xmin
+
+ anchor_ymin = anchors[..., 0:1]
+ anchor_xmin = anchors[..., 1:2]
+ anchor_ymax = anchors[..., 2:3]
+ anchor_xmax = anchors[..., 3:4]
+ # anchor_h = anchor_ymax - anchor_ymin + 1.0
+ # anchor_w = anchor_xmax - anchor_xmin + 1.0
+ anchor_h = anchor_ymax - anchor_ymin
+ anchor_w = anchor_xmax - anchor_xmin
+ anchor_yc = anchor_ymin + 0.5 * anchor_h
+ anchor_xc = anchor_xmin + 0.5 * anchor_w
+
+ box_h += EPSILON
+ box_w += EPSILON
+ anchor_h += EPSILON
+ anchor_w += EPSILON
+
+ left = (anchor_xc - xmin) / anchor_w
+ right = (xmax - anchor_xc) / anchor_w
+ top = (anchor_yc - ymin) / anchor_h
+ bottom = (ymax - anchor_yc) / anchor_h
+
+ # Create centerness target. {
+ lrtb_targets = tf.concat([left, right, top, bottom], axis=-1)
+ valid_match = tf.greater(tf.reduce_min(lrtb_targets, -1), 0.0)
+
+ # Centerness score.
+ left_right = tf.concat([left, right], axis=-1)
+
+ left_right = tf.where(tf.stack([valid_match, valid_match], -1),
+ left_right, tf.zeros_like(left_right))
+ top_bottom = tf.concat([top, bottom], axis=-1)
+ top_bottom = tf.where(tf.stack([valid_match, valid_match], -1),
+ top_bottom, tf.zeros_like(top_bottom))
+ center_targets = tf.sqrt(
+ (tf.reduce_min(left_right, -1) /
+ (tf.reduce_max(left_right, -1) + EPSILON)) *
+ (tf.reduce_min(top_bottom, -1) /
+ (tf.reduce_max(top_bottom, -1) + EPSILON)))
+ center_targets = tf.where(valid_match,
+ center_targets,
+ tf.zeros_like(center_targets))
+ if weights:
+ left *= weights[0]
+ right *= weights[1]
+ top *= weights[2]
+ bottom *= weights[3]
+
+ encoded_boxes_lrtb = tf.concat(
+ [left, right, top, bottom],
+ axis=-1)
+
+ return encoded_boxes_lrtb, center_targets
+
+
+def decode_boxes_lrtb(encoded_boxes_lrtb, anchors, weights=None):
+ """Decode boxes.
+
+ Args:
+ encoded_boxes_lrtb: a tensor whose last dimension is 4 representing the
+ coordinates of encoded boxes in left, right, top, bottom order.
+ anchors: a tensor whose shape is the same as, or `broadcastable` to `boxes`,
+ representing the coordinates of anchors in ymin, xmin, ymax, xmax order.
+ weights: None or a list of four float numbers used to scale coordinates.
+
+ Returns:
+ decoded_boxes_lrtb: a tensor whose shape is the same as `boxes` representing
+ the decoded box targets in lrtb (=left,right,top,bottom) format. The box
+ decoded box coordinates represent the left, right, top, and bottom
+ distances from an anchor location to the four borders of the matched
+ groundtruth bounding box.
+ """
+ if encoded_boxes_lrtb.shape[-1] != 4:
+ raise ValueError(
+ 'encoded_boxes_lrtb.shape[-1] is {:d}, but must be 4.'
+ .format(encoded_boxes_lrtb.shape[-1]))
+
+ with tf.name_scope('decode_boxes_lrtb'):
+ encoded_boxes_lrtb = tf.cast(encoded_boxes_lrtb, dtype=anchors.dtype)
+ left = encoded_boxes_lrtb[..., 0:1]
+ right = encoded_boxes_lrtb[..., 1:2]
+ top = encoded_boxes_lrtb[..., 2:3]
+ bottom = encoded_boxes_lrtb[..., 3:4]
+ if weights:
+ left /= weights[0]
+ right /= weights[1]
+ top /= weights[2]
+ bottom /= weights[3]
+
+ anchor_ymin = anchors[..., 0:1]
+ anchor_xmin = anchors[..., 1:2]
+ anchor_ymax = anchors[..., 2:3]
+ anchor_xmax = anchors[..., 3:4]
+
+ anchor_h = anchor_ymax - anchor_ymin
+ anchor_w = anchor_xmax - anchor_xmin
+ anchor_yc = anchor_ymin + 0.5 * anchor_h
+ anchor_xc = anchor_xmin + 0.5 * anchor_w
+ anchor_h += EPSILON
+ anchor_w += EPSILON
+
+ decoded_boxes_ymin = anchor_yc - top * anchor_h
+ decoded_boxes_xmin = anchor_xc - left * anchor_w
+ decoded_boxes_ymax = anchor_yc + bottom * anchor_h
+ decoded_boxes_xmax = anchor_xc + right * anchor_w
+
+ decoded_boxes_lrtb = tf.concat(
[decoded_boxes_ymin, decoded_boxes_xmin,
decoded_boxes_ymax, decoded_boxes_xmax],
axis=-1)
- return decoded_boxes
+ return decoded_boxes_lrtb
def filter_boxes(boxes, scores, image_shape, min_size_threshold):
@@ -546,6 +696,6 @@ def get_non_empty_box_indices(boxes):
# Selects indices if box height or width is 0.
height = boxes[:, 2] - boxes[:, 0]
width = boxes[:, 3] - boxes[:, 1]
- indices = tf.where(tf.logical_and(tf.greater(height, 0),
- tf.greater(width, 0)))
+ indices = tf.where(
+ tf.logical_and(tf.greater(height, 0), tf.greater(width, 0)))
return indices[:, 0]
diff --git a/official/vision/detection/utils/class_utils.py b/official/vision/detection/utils/class_utils.py
index cce9cf982bbbce7b90ee44e67ebe65997b7a91da..cbf806f11070736c17de79dd63240e9a626808d9 100644
--- a/official/vision/detection/utils/class_utils.py
+++ b/official/vision/detection/utils/class_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Utility functions for handling dataset object categories."""
diff --git a/official/vision/detection/utils/dataloader_utils.py b/official/vision/detection/utils/dataloader_utils.py
index da82203511da50393a352bf75ee56f25c6626c05..9569d7713c2177d233bcdc21934edb6ffbe0fefd 100644
--- a/official/vision/detection/utils/dataloader_utils.py
+++ b/official/vision/detection/utils/dataloader_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Utility functions for dataloader."""
import tensorflow as tf
diff --git a/official/vision/detection/utils/input_utils.py b/official/vision/detection/utils/input_utils.py
index 6010dc8973f387318c4553d3014ccf495cf01fc6..7f5502eeefd1a05c05d3beb9d5b2bd5975e4c18d 100644
--- a/official/vision/detection/utils/input_utils.py
+++ b/official/vision/detection/utils/input_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,10 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Utility functions for input processing."""
import math
+
import tensorflow as tf
from official.vision.detection.utils import box_utils
@@ -91,12 +92,12 @@ def compute_padded_size(desired_size, stride):
[height, width] of the padded output image size.
"""
if isinstance(desired_size, list) or isinstance(desired_size, tuple):
- padded_size = [int(math.ceil(d * 1.0 / stride) * stride)
- for d in desired_size]
+ padded_size = [
+ int(math.ceil(d * 1.0 / stride) * stride) for d in desired_size
+ ]
else:
padded_size = tf.cast(
- tf.math.ceil(
- tf.cast(desired_size, dtype=tf.float32) / stride) * stride,
+ tf.math.ceil(tf.cast(desired_size, dtype=tf.float32) / stride) * stride,
tf.int32)
return padded_size
@@ -158,8 +159,8 @@ def resize_and_crop_image(image,
else:
scaled_size = desired_size
- scale = tf.minimum(
- scaled_size[0] / image_size[0], scaled_size[1] / image_size[1])
+ scale = tf.minimum(scaled_size[0] / image_size[0],
+ scaled_size[1] / image_size[1])
scaled_size = tf.round(image_size * scale)
# Computes 2D image_scale.
@@ -169,9 +170,8 @@ def resize_and_crop_image(image,
# desired_size.
if random_jittering:
max_offset = scaled_size - desired_size
- max_offset = tf.where(tf.less(max_offset, 0),
- tf.zeros_like(max_offset),
- max_offset)
+ max_offset = tf.where(
+ tf.less(max_offset, 0), tf.zeros_like(max_offset), max_offset)
offset = max_offset * tf.random.uniform([
2,
], 0, 1, seed=seed)
@@ -191,9 +191,9 @@ def resize_and_crop_image(image,
image_info = tf.stack([
image_size,
- tf.cast(desired_size, dtype=tf.float32),
- image_scale,
- tf.cast(offset, tf.float32)])
+ tf.cast(desired_size, dtype=tf.float32), image_scale,
+ tf.cast(offset, tf.float32)
+ ])
return output_image, image_info
@@ -288,25 +288,21 @@ def resize_and_crop_image_v2(image,
image, tf.cast(scaled_size, tf.int32), method=method)
if random_jittering:
- scaled_image = scaled_image[
- offset[0]:offset[0] + desired_size[0],
- offset[1]:offset[1] + desired_size[1], :]
+ scaled_image = scaled_image[offset[0]:offset[0] + desired_size[0],
+ offset[1]:offset[1] + desired_size[1], :]
- output_image = tf.image.pad_to_bounding_box(
- scaled_image, 0, 0, padded_size[0], padded_size[1])
+ output_image = tf.image.pad_to_bounding_box(scaled_image, 0, 0,
+ padded_size[0], padded_size[1])
image_info = tf.stack([
image_size,
- tf.cast(desired_size, dtype=tf.float32),
- image_scale,
- tf.cast(offset, tf.float32)])
+ tf.cast(desired_size, dtype=tf.float32), image_scale,
+ tf.cast(offset, tf.float32)
+ ])
return output_image, image_info
-def resize_and_crop_boxes(boxes,
- image_scale,
- output_size,
- offset):
+def resize_and_crop_boxes(boxes, image_scale, output_size, offset):
"""Resizes boxes to output size with scale and offset.
Args:
@@ -329,10 +325,7 @@ def resize_and_crop_boxes(boxes,
return boxes
-def resize_and_crop_masks(masks,
- image_scale,
- output_size,
- offset):
+def resize_and_crop_masks(masks, image_scale, output_size, offset):
"""Resizes boxes to output size with scale and offset.
Args:
diff --git a/official/vision/detection/utils/mask_utils.py b/official/vision/detection/utils/mask_utils.py
index 637d0484f4b48213c4b323be6e0c88f9fa19ebcc..926c829b81b35b11ca53a5a3d351d0ebca36205e 100644
--- a/official/vision/detection/utils/mask_utils.py
+++ b/official/vision/detection/utils/mask_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,21 +11,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Utility functions for segmentations."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import math
+
import numpy as np
import cv2
-def paste_instance_masks(masks,
- detected_boxes,
- image_height,
- image_width):
+def paste_instance_masks(masks, detected_boxes, image_height, image_width):
"""Paste instance masks to generate the image segmentation results.
Args:
@@ -95,10 +93,8 @@ def paste_instance_masks(masks,
y_0 = min(max(ref_box[1], 0), image_height)
y_1 = min(max(ref_box[3] + 1, 0), image_height)
- im_mask[y_0:y_1, x_0:x_1] = mask[
- (y_0 - ref_box[1]):(y_1 - ref_box[1]),
- (x_0 - ref_box[0]):(x_1 - ref_box[0])
- ]
+ im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - ref_box[1]):(y_1 - ref_box[1]),
+ (x_0 - ref_box[0]):(x_1 - ref_box[0])]
segms.append(im_mask)
segms = np.array(segms)
@@ -106,10 +102,7 @@ def paste_instance_masks(masks,
return segms
-def paste_instance_masks_v2(masks,
- detected_boxes,
- image_height,
- image_width):
+def paste_instance_masks_v2(masks, detected_boxes, image_height, image_width):
"""Paste instance masks to generate the image segmentation (v2).
Args:
@@ -146,34 +139,22 @@ def paste_instance_masks_v2(masks,
beta = box[3] / (1.0 * mask_height)
# pylint: disable=invalid-name
# Transformation from mask pixel indices to image coordinate.
- M_mask_to_image = np.array(
- [[alpha, 0, xmin],
- [0, beta, ymin],
- [0, 0, 1]],
- dtype=np.float32)
+ M_mask_to_image = np.array([[alpha, 0, xmin], [0, beta, ymin], [0, 0, 1]],
+ dtype=np.float32)
# Transformation from image to cropped mask coordinate.
M_image_to_crop = np.array(
- [[1, 0, -xmin_int],
- [0, 1, -ymin_int],
- [0, 0, 1]],
- dtype=np.float32)
+ [[1, 0, -xmin_int], [0, 1, -ymin_int], [0, 0, 1]], dtype=np.float32)
M = np.dot(M_image_to_crop, M_mask_to_image)
# Compensate the half pixel offset that OpenCV has in the
# warpPerspective implementation: the top-left pixel is sampled
# at (0,0), but we want it to be at (0.5, 0.5).
M = np.dot(
np.dot(
- np.array([[1, 0, -0.5],
- [0, 1, -0.5],
- [0, 0, 1]], np.float32),
- M),
- np.array([[1, 0, 0.5],
- [0, 1, 0.5],
- [0, 0, 1]], np.float32))
+ np.array([[1, 0, -0.5], [0, 1, -0.5], [0, 0, 1]], np.float32), M),
+ np.array([[1, 0, 0.5], [0, 1, 0.5], [0, 0, 1]], np.float32))
# pylint: enable=invalid-name
cropped_mask = cv2.warpPerspective(
- mask.astype(np.float32), M,
- (xmax_int - xmin_int, ymax_int - ymin_int))
+ mask.astype(np.float32), M, (xmax_int - xmin_int, ymax_int - ymin_int))
cropped_mask = np.array(cropped_mask > 0.5, dtype=np.uint8)
img_mask = np.zeros((image_height, image_width))
@@ -181,12 +162,10 @@ def paste_instance_masks_v2(masks,
x1 = max(min(xmax_int, image_width), 0)
y0 = max(min(ymin_int, image_height), 0)
y1 = max(min(ymax_int, image_height), 0)
- img_mask[y0:y1, x0:x1] = cropped_mask[
- (y0 - ymin_int):(y1 - ymin_int),
- (x0 - xmin_int):(x1 - xmin_int)]
+ img_mask[y0:y1, x0:x1] = cropped_mask[(y0 - ymin_int):(y1 - ymin_int),
+ (x0 - xmin_int):(x1 - xmin_int)]
segms.append(img_mask)
segms = np.array(segms)
return segms
-
diff --git a/official/vision/detection/utils/object_detection/__init__.py b/official/vision/detection/utils/object_detection/__init__.py
index 85c94f4b6bd7567796755895505a320405a40777..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/detection/utils/object_detection/__init__.py
+++ b/official/vision/detection/utils/object_detection/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/detection/utils/object_detection/argmax_matcher.py b/official/vision/detection/utils/object_detection/argmax_matcher.py
index 3f8b051bfb08a72846482c0da9c79d1b98418c38..c92b69edce2fb3734f4b90bdb83c75a30e69d43b 100644
--- a/official/vision/detection/utils/object_detection/argmax_matcher.py
+++ b/official/vision/detection/utils/object_detection/argmax_matcher.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Argmax matcher implementation.
diff --git a/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py b/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py
index f969182b05a29167649d5c022a667b3f768f0143..e71dd1727ed590d92b2248b7070561c4246d275a 100644
--- a/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py
+++ b/official/vision/detection/utils/object_detection/balanced_positive_negative_sampler.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Class to subsample minibatches by balancing positives and negatives.
@@ -92,10 +91,10 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
Args:
input_tensor: An int32 tensor of shape [N] to be sliced.
- num_start_samples: Number of examples to be sliced from the beginning
- of the input tensor.
- num_end_samples: Number of examples to be sliced from the end of the
- input tensor.
+ num_start_samples: Number of examples to be sliced from the beginning of
+ the input tensor.
+ num_end_samples: Number of examples to be sliced from the end of the input
+ tensor.
total_num_samples: Sum of is num_start_samples and num_end_samples. This
should be a scalar.
@@ -110,13 +109,16 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
tf.range(input_length), input_length - num_end_samples)
selected_positions = tf.logical_or(start_positions, end_positions)
selected_positions = tf.cast(selected_positions, tf.float32)
- indexed_positions = tf.multiply(tf.cumsum(selected_positions),
- selected_positions)
- one_hot_selector = tf.one_hot(tf.cast(indexed_positions, tf.int32) - 1,
- total_num_samples,
- dtype=tf.float32)
- return tf.cast(tf.tensordot(tf.cast(input_tensor, tf.float32),
- one_hot_selector, axes=[0, 0]), tf.int32)
+ indexed_positions = tf.multiply(
+ tf.cumsum(selected_positions), selected_positions)
+ one_hot_selector = tf.one_hot(
+ tf.cast(indexed_positions, tf.int32) - 1,
+ total_num_samples,
+ dtype=tf.float32)
+ return tf.cast(
+ tf.tensordot(
+ tf.cast(input_tensor, tf.float32), one_hot_selector, axes=[0, 0]),
+ tf.int32)
def _static_subsample(self, indicator, batch_size, labels):
"""Returns subsampled minibatch.
@@ -182,13 +184,12 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
sorted_signed_indicator_idx = tf.nn.top_k(
signed_indicator_idx, input_length, sorted=True).values
- [num_positive_samples,
- num_negative_samples] = self._get_num_pos_neg_samples(
- sorted_signed_indicator_idx, batch_size)
+ [num_positive_samples, num_negative_samples
+ ] = self._get_num_pos_neg_samples(sorted_signed_indicator_idx, batch_size)
sampled_idx = self._get_values_from_start_and_end(
- sorted_signed_indicator_idx, num_positive_samples,
- num_negative_samples, batch_size)
+ sorted_signed_indicator_idx, num_positive_samples, num_negative_samples,
+ batch_size)
# Shift the indices to start from 0 and remove any samples that are set as
# False.
@@ -203,11 +204,13 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
tf.bool)
# project back the order based on stored permutations
- reprojections = tf.one_hot(permutation, depth=input_length,
- dtype=tf.float32)
- return tf.cast(tf.tensordot(
- tf.cast(sampled_idx_indicator, tf.float32),
- reprojections, axes=[0, 0]), tf.bool)
+ reprojections = tf.one_hot(
+ permutation, depth=input_length, dtype=tf.float32)
+ return tf.cast(
+ tf.tensordot(
+ tf.cast(sampled_idx_indicator, tf.float32),
+ reprojections,
+ axes=[0, 0]), tf.bool)
def subsample(self, indicator, batch_size, labels, scope=None):
"""Returns subsampled minibatch.
@@ -218,7 +221,7 @@ class BalancedPositiveNegativeSampler(minibatch_sampler.MinibatchSampler):
randomly selects negative samples so that the positive sample fraction
matches self._positive_fraction. It cannot be None is is_static is True.
labels: boolean tensor of shape [N] denoting positive(=True) and negative
- (=False) examples.
+ (=False) examples.
scope: name scope.
Returns:
diff --git a/official/vision/detection/utils/object_detection/box_coder.py b/official/vision/detection/utils/object_detection/box_coder.py
index f20ac956dfbce1fa69d1b9e6f5b023b704e1ec8a..c58eead30d4912b8c947fc532cb5b71dc5138233 100644
--- a/official/vision/detection/utils/object_detection/box_coder.py
+++ b/official/vision/detection/utils/object_detection/box_coder.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Base box coder.
@@ -32,7 +31,6 @@ from abc import abstractproperty
import tensorflow as tf
-
# Box coder types.
FASTER_RCNN = 'faster_rcnn'
KEYPOINT = 'keypoint'
@@ -138,11 +136,11 @@ def batch_decode(encoded_boxes, box_coder, anchors):
"""
encoded_boxes.get_shape().assert_has_rank(3)
if encoded_boxes.get_shape()[1].value != anchors.num_boxes_static():
- raise ValueError('The number of anchors inferred from encoded_boxes'
- ' and anchors are inconsistent: shape[1] of encoded_boxes'
- ' %s should be equal to the number of anchors: %s.' %
- (encoded_boxes.get_shape()[1].value,
- anchors.num_boxes_static()))
+ raise ValueError(
+ 'The number of anchors inferred from encoded_boxes'
+ ' and anchors are inconsistent: shape[1] of encoded_boxes'
+ ' %s should be equal to the number of anchors: %s.' %
+ (encoded_boxes.get_shape()[1].value, anchors.num_boxes_static()))
decoded_boxes = tf.stack([
box_coder.decode(boxes, anchors).get()
diff --git a/official/vision/detection/utils/object_detection/box_list.py b/official/vision/detection/utils/object_detection/box_list.py
index 113fab8c197194f1cd0099d5a177cd9f1fb6e64c..f5d4443c81b22f1586f6691c5c4e309de3046f9c 100644
--- a/official/vision/detection/utils/object_detection/box_list.py
+++ b/official/vision/detection/utils/object_detection/box_list.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Bounding Box List definition.
@@ -126,8 +125,8 @@ class BoxList(object):
it returns the box coordinates.
Args:
- field: this optional string parameter can be used to specify
- a related field to be accessed.
+ field: this optional string parameter can be used to specify a related
+ field to be accessed.
Returns:
a tensor representing the box collection or an associated field.
@@ -192,8 +191,8 @@ class BoxList(object):
"""Retrieves specified fields as a dictionary of tensors.
Args:
- fields: (optional) list of fields to return in the dictionary.
- If None (default), all fields are returned.
+ fields: (optional) list of fields to return in the dictionary. If None
+ (default), all fields are returned.
Returns:
tensor_dict: A dictionary of tensors specified by fields.
diff --git a/official/vision/detection/utils/object_detection/box_list_ops.py b/official/vision/detection/utils/object_detection/box_list_ops.py
index 9f1b06e28d588eb05c9ea8596b44d08690481eae..ef2fcdc2d6b79d292949b70fa84bc1bcdcea58d7 100644
--- a/official/vision/detection/utils/object_detection/box_list_ops.py
+++ b/official/vision/detection/utils/object_detection/box_list_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Bounding Box List operations.
@@ -152,8 +151,8 @@ def prune_outside_window(boxlist, window, scope=None):
Args:
boxlist: a BoxList holding M_in boxes.
- window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
- of the window
+ window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax] of
+ the window
scope: name scope.
Returns:
@@ -166,8 +165,10 @@ def prune_outside_window(boxlist, window, scope=None):
value=boxlist.get(), num_or_size_splits=4, axis=1)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
coordinate_violations = tf.concat([
- tf.less(y_min, win_y_min), tf.less(x_min, win_x_min),
- tf.greater(y_max, win_y_max), tf.greater(x_max, win_x_max)
+ tf.less(y_min, win_y_min),
+ tf.less(x_min, win_x_min),
+ tf.greater(y_max, win_y_max),
+ tf.greater(x_max, win_x_max)
], 1)
valid_indices = tf.reshape(
tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1])
@@ -183,8 +184,8 @@ def prune_completely_outside_window(boxlist, window, scope=None):
Args:
boxlist: a BoxList holding M_in boxes.
- window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
- of the window
+ window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax] of
+ the window
scope: name scope.
Returns:
@@ -198,8 +199,10 @@ def prune_completely_outside_window(boxlist, window, scope=None):
value=boxlist.get(), num_or_size_splits=4, axis=1)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
coordinate_violations = tf.concat([
- tf.greater_equal(y_min, win_y_max), tf.greater_equal(x_min, win_x_max),
- tf.less_equal(y_max, win_y_min), tf.less_equal(x_max, win_x_min)
+ tf.greater_equal(y_min, win_y_max),
+ tf.greater_equal(x_min, win_x_max),
+ tf.less_equal(y_max, win_y_min),
+ tf.less_equal(x_max, win_x_min)
], 1)
valid_indices = tf.reshape(
tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1])
@@ -274,8 +277,8 @@ def iou(boxlist1, boxlist2, scope=None):
unions = (
tf.expand_dims(areas1, 1) + tf.expand_dims(areas2, 0) - intersections)
return tf.where(
- tf.equal(intersections, 0.0),
- tf.zeros_like(intersections), tf.truediv(intersections, unions))
+ tf.equal(intersections, 0.0), tf.zeros_like(intersections),
+ tf.truediv(intersections, unions))
def matched_iou(boxlist1, boxlist2, scope=None):
@@ -295,8 +298,8 @@ def matched_iou(boxlist1, boxlist2, scope=None):
areas2 = area(boxlist2)
unions = areas1 + areas2 - intersections
return tf.where(
- tf.equal(intersections, 0.0),
- tf.zeros_like(intersections), tf.truediv(intersections, unions))
+ tf.equal(intersections, 0.0), tf.zeros_like(intersections),
+ tf.truediv(intersections, unions))
def ioa(boxlist1, boxlist2, scope=None):
@@ -320,8 +323,10 @@ def ioa(boxlist1, boxlist2, scope=None):
return tf.truediv(intersections, areas)
-def prune_non_overlapping_boxes(
- boxlist1, boxlist2, min_overlap=0.0, scope=None):
+def prune_non_overlapping_boxes(boxlist1,
+ boxlist2,
+ min_overlap=0.0,
+ scope=None):
"""Prunes the boxes in boxlist1 that overlap less than thresh with boxlist2.
For each box in boxlist1, we want its IOA to be more than minoverlap with
@@ -331,7 +336,7 @@ def prune_non_overlapping_boxes(
boxlist1: BoxList holding N boxes.
boxlist2: BoxList holding M boxes.
min_overlap: Minimum required overlap between boxes, to count them as
- overlapping.
+ overlapping.
scope: name scope.
Returns:
@@ -361,8 +366,8 @@ def prune_small_boxes(boxlist, min_side, scope=None):
"""
with tf.name_scope(scope, 'PruneSmallBoxes'):
height, width = height_width(boxlist)
- is_valid = tf.logical_and(tf.greater_equal(width, min_side),
- tf.greater_equal(height, min_side))
+ is_valid = tf.logical_and(
+ tf.greater_equal(width, min_side), tf.greater_equal(height, min_side))
return gather(boxlist, tf.reshape(tf.where(is_valid), [-1]))
@@ -389,9 +394,10 @@ def change_coordinate_frame(boxlist, window, scope=None):
with tf.name_scope(scope, 'ChangeCoordinateFrame'):
win_height = window[2] - window[0]
win_width = window[3] - window[1]
- boxlist_new = scale(box_list.BoxList(
- boxlist.get() - [window[0], window[1], window[0], window[1]]),
- 1.0 / win_height, 1.0 / win_width)
+ boxlist_new = scale(
+ box_list.BoxList(boxlist.get() -
+ [window[0], window[1], window[0], window[1]]),
+ 1.0 / win_height, 1.0 / win_width)
boxlist_new = _copy_extra_fields(boxlist_new, boxlist)
return boxlist_new
@@ -420,13 +426,17 @@ def sq_dist(boxlist1, boxlist2, scope=None):
with tf.name_scope(scope, 'SqDist'):
sqnorm1 = tf.reduce_sum(tf.square(boxlist1.get()), 1, keep_dims=True)
sqnorm2 = tf.reduce_sum(tf.square(boxlist2.get()), 1, keep_dims=True)
- innerprod = tf.matmul(boxlist1.get(), boxlist2.get(),
- transpose_a=False, transpose_b=True)
+ innerprod = tf.matmul(
+ boxlist1.get(), boxlist2.get(), transpose_a=False, transpose_b=True)
return sqnorm1 + tf.transpose(sqnorm2) - 2.0 * innerprod
-def boolean_mask(boxlist, indicator, fields=None, scope=None,
- use_static_shapes=False, indicator_sum=None):
+def boolean_mask(boxlist,
+ indicator,
+ fields=None,
+ scope=None,
+ use_static_shapes=False,
+ indicator_sum=None):
"""Select boxes from BoxList according to indicator and return new BoxList.
`boolean_mask` returns the subset of boxes that are marked as "True" by the
@@ -463,8 +473,7 @@ def boolean_mask(boxlist, indicator, fields=None, scope=None,
raise ValueError('`indicator_sum` must be a of type int')
selected_positions = tf.cast(indicator, dtype=tf.float32)
indexed_positions = tf.cast(
- tf.multiply(
- tf.cumsum(selected_positions), selected_positions),
+ tf.multiply(tf.cumsum(selected_positions), selected_positions),
dtype=tf.int32)
one_hot_selector = tf.one_hot(
indexed_positions - 1, indicator_sum, dtype=tf.float32)
@@ -541,9 +550,8 @@ def concatenate(boxlists, fields=None, scope=None):
Args:
boxlists: list of BoxList objects
- fields: optional list of fields to also concatenate. By default, all
- fields from the first BoxList in the list are included in the
- concatenation.
+ fields: optional list of fields to also concatenate. By default, all fields
+ from the first BoxList in the list are included in the concatenation.
scope: name scope.
Returns:
@@ -637,8 +645,8 @@ def visualize_boxes_in_image(image, boxlist, normalized=False, scope=None):
Args:
image: an image tensor with shape [height, width, 3]
boxlist: a BoxList
- normalized: (boolean) specify whether corners are to be interpreted
- as absolute coordinates in image space or normalized with respect to the
+ normalized: (boolean) specify whether corners are to be interpreted as
+ absolute coordinates in image space or normalized with respect to the
image size.
scope: name scope.
@@ -648,8 +656,7 @@ def visualize_boxes_in_image(image, boxlist, normalized=False, scope=None):
with tf.name_scope(scope, 'VisualizeBoxesInImage'):
if not normalized:
height, width, _ = tf.unstack(tf.shape(image))
- boxlist = scale(boxlist,
- 1.0 / tf.cast(height, tf.float32),
+ boxlist = scale(boxlist, 1.0 / tf.cast(height, tf.float32),
1.0 / tf.cast(width, tf.float32))
corners = tf.expand_dims(boxlist.get(), 0)
image = tf.expand_dims(image, 0)
@@ -714,9 +721,8 @@ def filter_greater_than(boxlist, thresh, scope=None):
if len(scores.shape.as_list()) == 2 and scores.shape.as_list()[1] != 1:
raise ValueError('Scores should have rank 1 or have shape '
'consistent with [None, 1]')
- high_score_indices = tf.cast(tf.reshape(
- tf.where(tf.greater(scores, thresh)),
- [-1]), tf.int32)
+ high_score_indices = tf.cast(
+ tf.reshape(tf.where(tf.greater(scores, thresh)), [-1]), tf.int32)
return gather(boxlist, high_score_indices)
@@ -748,8 +754,10 @@ def non_max_suppression(boxlist, thresh, max_output_size, scope=None):
if not boxlist.has_field('scores'):
raise ValueError('input boxlist must have \'scores\' field')
selected_indices = tf.image.non_max_suppression(
- boxlist.get(), boxlist.get_field('scores'),
- max_output_size, iou_threshold=thresh)
+ boxlist.get(),
+ boxlist.get_field('scores'),
+ max_output_size,
+ iou_threshold=thresh)
return gather(boxlist, selected_indices)
@@ -768,8 +776,11 @@ def _copy_extra_fields(boxlist_to_copy_to, boxlist_to_copy_from):
return boxlist_to_copy_to
-def to_normalized_coordinates(boxlist, height, width,
- check_range=True, scope=None):
+def to_normalized_coordinates(boxlist,
+ height,
+ width,
+ check_range=True,
+ scope=None):
"""Converts absolute box coordinates to normalized coordinates in [0, 1].
Usually one uses the dynamic shape of the image or conv-layer tensor:
@@ -797,8 +808,9 @@ def to_normalized_coordinates(boxlist, height, width,
if check_range:
max_val = tf.reduce_max(boxlist.get())
- max_assert = tf.Assert(tf.greater(max_val, 1.01),
- ['max value is lower than 1.01: ', max_val])
+ max_assert = tf.Assert(
+ tf.greater(max_val, 1.01),
+ ['max value is lower than 1.01: ', max_val])
with tf.control_dependencies([max_assert]):
width = tf.identity(width)
@@ -822,8 +834,8 @@ def to_absolute_coordinates(boxlist,
height: Maximum value for height of absolute box coordinates.
width: Maximum value for width of absolute box coordinates.
check_range: If True, checks if the coordinates are normalized or not.
- maximum_normalized_coordinate: Maximum coordinate value to be considered
- as normalized, default to 1.1.
+ maximum_normalized_coordinate: Maximum coordinate value to be considered as
+ normalized, default to 1.1.
scope: name scope.
Returns:
@@ -838,9 +850,10 @@ def to_absolute_coordinates(boxlist,
if check_range:
box_maximum = tf.reduce_max(boxlist.get())
max_assert = tf.Assert(
- tf.greater_equal(maximum_normalized_coordinate, box_maximum),
- ['maximum box coordinate value is larger '
- 'than %f: ' % maximum_normalized_coordinate, box_maximum])
+ tf.greater_equal(maximum_normalized_coordinate, box_maximum), [
+ 'maximum box coordinate value is larger '
+ 'than %f: ' % maximum_normalized_coordinate, box_maximum
+ ])
with tf.control_dependencies([max_assert]):
width = tf.identity(width)
@@ -924,13 +937,15 @@ def refine_boxes(pool_boxes,
if not pool_boxes.has_field('scores'):
raise ValueError('pool_boxes must have a \'scores\' field')
- nms_boxes = non_max_suppression(
- pool_boxes, nms_iou_thresh, nms_max_detections)
+ nms_boxes = non_max_suppression(pool_boxes, nms_iou_thresh,
+ nms_max_detections)
return box_voting(nms_boxes, pool_boxes, voting_iou_thresh)
def box_voting(selected_boxes, pool_boxes, iou_thresh=0.5):
- """Performs box voting as described in S. Gidaris and N. Komodakis, ICCV 2015.
+ """Performs box voting as described in S. Gidaris and N.
+
+ Komodakis, ICCV 2015.
Performs box voting as described in 'Object detection via a multi-region &
semantic segmentation-aware CNN model', Gidaris and Komodakis, ICCV 2015. For
@@ -972,9 +987,10 @@ def box_voting(selected_boxes, pool_boxes, iou_thresh=0.5):
# match to any boxes in pool_boxes. For such boxes without any matches, we
# should return the original boxes without voting.
match_assert = tf.Assert(
- tf.reduce_all(tf.greater(num_matches, 0)),
- ['Each box in selected_boxes must match with at least one box '
- 'in pool_boxes.'])
+ tf.reduce_all(tf.greater(num_matches, 0)), [
+ 'Each box in selected_boxes must match with at least one box '
+ 'in pool_boxes.'
+ ])
scores = tf.expand_dims(pool_boxes.get_field('scores'), 1)
scores_assert = tf.Assert(
@@ -993,9 +1009,7 @@ def box_voting(selected_boxes, pool_boxes, iou_thresh=0.5):
return averaged_boxes
-def get_minimal_coverage_box(boxlist,
- default_box=None,
- scope=None):
+def get_minimal_coverage_box(boxlist, default_box=None, scope=None):
"""Creates a single bounding box which covers all boxes in the boxlist.
Args:
@@ -1045,9 +1059,9 @@ def sample_boxes_by_jittering(boxlist,
boxlist: A boxlist containing N boxes in normalized coordinates.
num_boxes_to_sample: A positive integer containing the number of boxes to
sample.
- stddev: Standard deviation. This is used to draw random offsets for the
- box corners from a normal distribution. The offset is multiplied by the
- box size so will be larger in terms of pixels for larger boxes.
+ stddev: Standard deviation. This is used to draw random offsets for the box
+ corners from a normal distribution. The offset is multiplied by the box
+ size so will be larger in terms of pixels for larger boxes.
scope: Name scope.
Returns:
@@ -1056,11 +1070,10 @@ def sample_boxes_by_jittering(boxlist,
"""
with tf.name_scope(scope, 'SampleBoxesByJittering'):
num_boxes = boxlist.num_boxes()
- box_indices = tf.random_uniform(
- [num_boxes_to_sample],
- minval=0,
- maxval=num_boxes,
- dtype=tf.int32)
+ box_indices = tf.random_uniform([num_boxes_to_sample],
+ minval=0,
+ maxval=num_boxes,
+ dtype=tf.int32)
sampled_boxes = tf.gather(boxlist.get(), box_indices)
sampled_boxes_height = sampled_boxes[:, 2] - sampled_boxes[:, 0]
sampled_boxes_width = sampled_boxes[:, 3] - sampled_boxes[:, 1]
diff --git a/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py b/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py
index 235df4ede474e89687a17413e81e60aa21772e23..0ce22d5ecf1fa5585127325a68ffa325e552b7d3 100644
--- a/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py
+++ b/official/vision/detection/utils/object_detection/faster_rcnn_box_coder.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Faster RCNN box coder.
@@ -43,9 +42,9 @@ class FasterRcnnBoxCoder(box_coder.BoxCoder):
"""Constructor for FasterRcnnBoxCoder.
Args:
- scale_factors: List of 4 positive scalars to scale ty, tx, th and tw.
- If set to None, does not perform scaling. For Faster RCNN,
- the open-source implementation recommends using [10.0, 10.0, 5.0, 5.0].
+ scale_factors: List of 4 positive scalars to scale ty, tx, th and tw. If
+ set to None, does not perform scaling. For Faster RCNN, the open-source
+ implementation recommends using [10.0, 10.0, 5.0, 5.0].
"""
if scale_factors:
assert len(scale_factors) == 4
diff --git a/official/vision/detection/utils/object_detection/matcher.py b/official/vision/detection/utils/object_detection/matcher.py
index 4a025d5e7118ee20f136c8a31b4c183de11f1e7f..1586830970437a19566bf430c18e8ca2b7e47a62 100644
--- a/official/vision/detection/utils/object_detection/matcher.py
+++ b/official/vision/detection/utils/object_detection/matcher.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Matcher interface and Match class.
@@ -49,9 +48,9 @@ class Match(object):
Args:
match_results: Integer tensor of shape [N] with (1) match_results[i]>=0,
- meaning that column i is matched with row match_results[i].
- (2) match_results[i]=-1, meaning that column i is not matched.
- (3) match_results[i]=-2, meaning that column i is ignored.
+ meaning that column i is matched with row match_results[i]. (2)
+ match_results[i]=-1, meaning that column i is not matched. (3)
+ match_results[i]=-2, meaning that column i is ignored.
Raises:
ValueError: if match_results does not have rank 1 or is not an
@@ -168,8 +167,7 @@ class Match(object):
def _reshape_and_cast(self, t):
return tf.cast(tf.reshape(t, [-1]), tf.int32)
- def gather_based_on_match(self, input_tensor, unmatched_value,
- ignored_value):
+ def gather_based_on_match(self, input_tensor, unmatched_value, ignored_value):
"""Gathers elements from `input_tensor` based on match results.
For columns that are matched to a row, gathered_tensor[col] is set to
@@ -190,16 +188,15 @@ class Match(object):
The shape of the gathered tensor is [match_results.shape[0]] +
input_tensor.shape[1:].
"""
- input_tensor = tf.concat([tf.stack([ignored_value, unmatched_value]),
- input_tensor], axis=0)
+ input_tensor = tf.concat(
+ [tf.stack([ignored_value, unmatched_value]), input_tensor], axis=0)
gather_indices = tf.maximum(self.match_results + 2, 0)
gathered_tensor = tf.gather(input_tensor, gather_indices)
return gathered_tensor
class Matcher(object):
- """Abstract base class for matcher.
- """
+ """Abstract base class for matcher."""
__metaclass__ = ABCMeta
def match(self, similarity_matrix, scope=None, **params):
@@ -212,8 +209,8 @@ class Matcher(object):
similarity_matrix: Float tensor of shape [N, M] with pairwise similarity
where higher value means more similar.
scope: Op scope name. Defaults to 'Match' if None.
- **params: Additional keyword arguments for specific implementations of
- the Matcher.
+ **params: Additional keyword arguments for specific implementations of the
+ Matcher.
Returns:
A Match object with the results of matching.
@@ -230,8 +227,8 @@ class Matcher(object):
Args:
similarity_matrix: Float tensor of shape [N, M] with pairwise similarity
where higher value means more similar.
- **params: Additional keyword arguments for specific implementations of
- the Matcher.
+ **params: Additional keyword arguments for specific implementations of the
+ Matcher.
Returns:
match_results: Integer tensor of shape [M]: match_results[i]>=0 means
diff --git a/official/vision/detection/utils/object_detection/minibatch_sampler.py b/official/vision/detection/utils/object_detection/minibatch_sampler.py
index b9f529ab5976ca56f014788c1263e5887fde0444..63e99d85237b203c11f91d6dab5e11c954d89422 100644
--- a/official/vision/detection/utils/object_detection/minibatch_sampler.py
+++ b/official/vision/detection/utils/object_detection/minibatch_sampler.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Base minibatch sampler module.
@@ -53,8 +52,8 @@ class MinibatchSampler(object):
Args:
indicator: boolean tensor of shape [N] whose True entries can be sampled.
batch_size: desired batch size.
- **params: additional keyword arguments for specific implementations of
- the MinibatchSampler.
+ **params: additional keyword arguments for specific implementations of the
+ MinibatchSampler.
Returns:
sample_indicator: boolean tensor of shape [N] whose True entries have been
@@ -72,8 +71,8 @@ class MinibatchSampler(object):
is returned.
Args:
- indicator: a 1-dimensional boolean tensor indicating which elements
- are allowed to be sampled and which are not.
+ indicator: a 1-dimensional boolean tensor indicating which elements are
+ allowed to be sampled and which are not.
num_samples: int32 scalar tensor
Returns:
diff --git a/official/vision/detection/utils/object_detection/ops.py b/official/vision/detection/utils/object_detection/ops.py
index bbfc1ae9353604986ad3f1f06a4f8e2e72bb5ca0..052e90a61009f3ada2ee77c0f1c8ad114143e5b7 100644
--- a/official/vision/detection/utils/object_detection/ops.py
+++ b/official/vision/detection/utils/object_detection/ops.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""A module for helper tensorflow ops.
@@ -37,7 +36,7 @@ def indices_to_dense_vector(indices,
Args:
indices: 1d Tensor with integer indices which are to be set to
- indices_values.
+ indices_values.
size: scalar with size (integer) of output Tensor.
indices_value: values of elements specified by indices in the output vector
default_value: values of other elements in the output vector.
@@ -61,10 +60,10 @@ def matmul_gather_on_zeroth_axis(params, indices, scope=None):
TODO(rathodv, jonathanhuang): enable sparse matmul option.
Args:
- params: A float32 Tensor. The tensor from which to gather values.
- Must be at least rank 1.
- indices: A Tensor. Must be one of the following types: int32, int64.
- Must be in range [0, params.shape[0])
+ params: A float32 Tensor. The tensor from which to gather values. Must be at
+ least rank 1.
+ indices: A Tensor. Must be one of the following types: int32, int64. Must be
+ in range [0, params.shape[0])
scope: A name for the operation (optional).
Returns:
diff --git a/official/vision/detection/utils/object_detection/preprocessor.py b/official/vision/detection/utils/object_detection/preprocessor.py
index 55da5d2dfafda816be7dcb2d334a3a0711e0b699..c6678bf28d61d4ec666d11cf0cac81d398241936 100644
--- a/official/vision/detection/utils/object_detection/preprocessor.py
+++ b/official/vision/detection/utils/object_detection/preprocessor.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Preprocess images and bounding boxes for detection.
We perform two sets of operations in preprocessing stage:
@@ -50,10 +50,9 @@ def _flip_boxes_left_right(boxes):
"""Left-right flip the boxes.
Args:
- boxes: rank 2 float32 tensor containing the bounding boxes -> [N, 4].
- Boxes are in normalized form meaning their coordinates vary
- between [0, 1].
- Each row is in the form of [ymin, xmin, ymax, xmax].
+ boxes: rank 2 float32 tensor containing the bounding boxes -> [N, 4]. Boxes
+ are in normalized form meaning their coordinates vary between [0, 1]. Each
+ row is in the form of [ymin, xmin, ymax, xmax].
Returns:
Flipped boxes.
@@ -69,8 +68,8 @@ def _flip_masks_left_right(masks):
"""Left-right flip masks.
Args:
- masks: rank 3 float32 tensor with shape
- [num_instances, height, width] representing instance masks.
+ masks: rank 3 float32 tensor with shape [num_instances, height, width]
+ representing instance masks.
Returns:
flipped masks: rank 3 float32 tensor with shape
@@ -79,7 +78,9 @@ def _flip_masks_left_right(masks):
return masks[:, :, ::-1]
-def keypoint_flip_horizontal(keypoints, flip_point, flip_permutation,
+def keypoint_flip_horizontal(keypoints,
+ flip_point,
+ flip_permutation,
scope=None):
"""Flips the keypoints horizontally around the flip_point.
@@ -91,9 +92,9 @@ def keypoint_flip_horizontal(keypoints, flip_point, flip_permutation,
flip_point: (float) scalar tensor representing the x coordinate to flip the
keypoints around.
flip_permutation: rank 1 int32 tensor containing the keypoint flip
- permutation. This specifies the mapping from original keypoint indices
- to the flipped keypoint indices. This is used primarily for keypoints
- that are not reflection invariant. E.g. Suppose there are 3 keypoints
+ permutation. This specifies the mapping from original keypoint indices to
+ the flipped keypoint indices. This is used primarily for keypoints that
+ are not reflection invariant. E.g. Suppose there are 3 keypoints
representing ['head', 'right_eye', 'left_eye'], then a logical choice for
flip_permutation might be [0, 2, 1] since we want to swap the 'left_eye'
and 'right_eye' after a horizontal flip.
@@ -190,19 +191,16 @@ def random_horizontal_flip(image,
Args:
image: rank 3 float32 tensor with shape [height, width, channels].
- boxes: (optional) rank 2 float32 tensor with shape [N, 4]
- containing the bounding boxes.
- Boxes are in normalized form meaning their coordinates vary
- between [0, 1].
- Each row is in the form of [ymin, xmin, ymax, xmax].
- masks: (optional) rank 3 float32 tensor with shape
- [num_instances, height, width] containing instance masks. The masks
- are of the same height, width as the input `image`.
- keypoints: (optional) rank 3 float32 tensor with shape
- [num_instances, num_keypoints, 2]. The keypoints are in y-x
- normalized coordinates.
+ boxes: (optional) rank 2 float32 tensor with shape [N, 4] containing the
+ bounding boxes. Boxes are in normalized form meaning their coordinates
+ vary between [0, 1]. Each row is in the form of [ymin, xmin, ymax, xmax].
+ masks: (optional) rank 3 float32 tensor with shape [num_instances, height,
+ width] containing instance masks. The masks are of the same height, width
+ as the input `image`.
+ keypoints: (optional) rank 3 float32 tensor with shape [num_instances,
+ num_keypoints, 2]. The keypoints are in y-x normalized coordinates.
keypoint_flip_permutation: rank 1 int32 tensor containing the keypoint flip
- permutation.
+ permutation.
seed: random seed
Returns:
@@ -369,20 +367,19 @@ def resize_to_range(image,
Args:
image: A 3D tensor of shape [height, width, channels]
- masks: (optional) rank 3 float32 tensor with shape
- [num_instances, height, width] containing instance masks.
+ masks: (optional) rank 3 float32 tensor with shape [num_instances, height,
+ width] containing instance masks.
min_dimension: (optional) (scalar) desired size of the smaller image
- dimension.
- max_dimension: (optional) (scalar) maximum allowed size
- of the larger image dimension.
+ dimension.
+ max_dimension: (optional) (scalar) maximum allowed size of the larger image
+ dimension.
method: (optional) interpolation method used in resizing. Defaults to
- BILINEAR.
- align_corners: bool. If true, exactly align all 4 corners of the input
- and output. Defaults to False.
- pad_to_max_dimension: Whether to resize the image and pad it with zeros
- so the resulting image is of the spatial size
- [max_dimension, max_dimension]. If masks are included they are padded
- similarly.
+ BILINEAR.
+ align_corners: bool. If true, exactly align all 4 corners of the input and
+ output. Defaults to False.
+ pad_to_max_dimension: Whether to resize the image and pad it with zeros so
+ the resulting image is of the spatial size [max_dimension, max_dimension].
+ If masks are included they are padded similarly.
Returns:
Note that the position of the resized_image_shape changes based on whether
@@ -410,8 +407,8 @@ def resize_to_range(image,
new_image = tf.image.resize(image, new_size[:-1], method=method)
if pad_to_max_dimension:
- new_image = tf.image.pad_to_bounding_box(
- new_image, 0, 0, max_dimension, max_dimension)
+ new_image = tf.image.pad_to_bounding_box(new_image, 0, 0, max_dimension,
+ max_dimension)
result = [new_image]
if masks is not None:
@@ -422,8 +419,8 @@ def resize_to_range(image,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
new_masks = tf.squeeze(new_masks, 3)
if pad_to_max_dimension:
- new_masks = tf.image.pad_to_bounding_box(
- new_masks, 0, 0, max_dimension, max_dimension)
+ new_masks = tf.image.pad_to_bounding_box(new_masks, 0, 0, max_dimension,
+ max_dimension)
result.append(new_masks)
result.append(new_size)
@@ -500,11 +497,10 @@ def scale_boxes_to_pixel_coordinates(image, boxes, keypoints=None):
Args:
image: A 3D float32 tensor of shape [height, width, channels].
boxes: A 2D float32 tensor of shape [num_boxes, 4] containing the bounding
- boxes in normalized coordinates. Each row is of the form
- [ymin, xmin, ymax, xmax].
- keypoints: (optional) rank 3 float32 tensor with shape
- [num_instances, num_keypoints, 2]. The keypoints are in y-x normalized
- coordinates.
+ boxes in normalized coordinates. Each row is of the form [ymin, xmin,
+ ymax, xmax].
+ keypoints: (optional) rank 3 float32 tensor with shape [num_instances,
+ num_keypoints, 2]. The keypoints are in y-x normalized coordinates.
Returns:
image: unchanged input image.
diff --git a/official/vision/detection/utils/object_detection/region_similarity_calculator.py b/official/vision/detection/utils/object_detection/region_similarity_calculator.py
index 0af2ce495ad53c9df0f8d2eb79f7431b02ab430e..9b26b4c65f9c0dde20b1d2d1916b0c3aa3d9e23f 100644
--- a/official/vision/detection/utils/object_detection/region_similarity_calculator.py
+++ b/official/vision/detection/utils/object_detection/region_similarity_calculator.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Region Similarity Calculators for BoxLists.
diff --git a/official/vision/detection/utils/object_detection/shape_utils.py b/official/vision/detection/utils/object_detection/shape_utils.py
index e30b62b7acc15b7f9f98b6c27b1a22efaf2998a8..6bf7c49d0e1d0eb9f10524e7889ba74c461bfc50 100644
--- a/official/vision/detection/utils/object_detection/shape_utils.py
+++ b/official/vision/detection/utils/object_detection/shape_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Utils used to manipulate tensor shapes."""
@@ -42,7 +41,8 @@ def assert_shape_equal(shape_a, shape_b):
all(isinstance(dim, int) for dim in shape_b)):
if shape_a != shape_b:
raise ValueError('Unequal shapes {}, {}'.format(shape_a, shape_b))
- else: return tf.no_op()
+ else:
+ return tf.no_op()
else:
return tf.assert_equal(shape_a, shape_b)
@@ -87,9 +87,7 @@ def pad_or_clip_nd(tensor, output_shape):
if shape is not None else -1 for i, shape in enumerate(output_shape)
]
clipped_tensor = tf.slice(
- tensor,
- begin=tf.zeros(len(clip_size), dtype=tf.int32),
- size=clip_size)
+ tensor, begin=tf.zeros(len(clip_size), dtype=tf.int32), size=clip_size)
# Pad tensor if the shape of clipped tensor is smaller than the expected
# shape.
@@ -99,10 +97,7 @@ def pad_or_clip_nd(tensor, output_shape):
for i, shape in enumerate(output_shape)
]
paddings = tf.stack(
- [
- tf.zeros(len(trailing_paddings), dtype=tf.int32),
- trailing_paddings
- ],
+ [tf.zeros(len(trailing_paddings), dtype=tf.int32), trailing_paddings],
axis=1)
padded_tensor = tf.pad(tensor=clipped_tensor, paddings=paddings)
output_static_shape = [
diff --git a/official/vision/detection/utils/object_detection/target_assigner.py b/official/vision/detection/utils/object_detection/target_assigner.py
index c04448efb052b45da65366b26e7d773b62015773..43aeace3751d4932e69de2c4b4d79372c6e849aa 100644
--- a/official/vision/detection/utils/object_detection/target_assigner.py
+++ b/official/vision/detection/utils/object_detection/target_assigner.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""Base target assigner module.
@@ -31,35 +30,39 @@ Note that TargetAssigners only operate on detections from a single
image at a time, so any logic for applying a TargetAssigner to multiple
images must be handled externally.
"""
+
import tensorflow as tf
from official.vision.detection.utils.object_detection import box_list
from official.vision.detection.utils.object_detection import shape_utils
-
KEYPOINTS_FIELD_NAME = 'keypoints'
class TargetAssigner(object):
"""Target assigner to compute classification and regression targets."""
- def __init__(self, similarity_calc, matcher, box_coder,
- negative_class_weight=1.0, unmatched_cls_target=None):
+ def __init__(self,
+ similarity_calc,
+ matcher,
+ box_coder,
+ negative_class_weight=1.0,
+ unmatched_cls_target=None):
"""Construct Object Detection Target Assigner.
Args:
similarity_calc: a RegionSimilarityCalculator
matcher: Matcher used to match groundtruth to anchors.
- box_coder: BoxCoder used to encode matching groundtruth boxes with
- respect to anchors.
+ box_coder: BoxCoder used to encode matching groundtruth boxes with respect
+ to anchors.
negative_class_weight: classification weight to be associated to negative
anchors (default: 1.0). The weight must be in [0., 1.].
unmatched_cls_target: a float32 tensor with shape [d_1, d_2, ..., d_k]
- which is consistent with the classification target for each
- anchor (and can be empty for scalar targets). This shape must thus be
- compatible with the groundtruth labels that are passed to the "assign"
- function (which have shape [num_gt_boxes, d_1, d_2, ..., d_k]).
- If set to None, unmatched_cls_target is set to be [0] for each anchor.
+ which is consistent with the classification target for each anchor (and
+ can be empty for scalar targets). This shape must thus be compatible
+ with the groundtruth labels that are passed to the "assign" function
+ (which have shape [num_gt_boxes, d_1, d_2, ..., d_k]). If set to None,
+ unmatched_cls_target is set to be [0] for each anchor.
Raises:
ValueError: if similarity_calc is not a RegionSimilarityCalculator or
@@ -78,8 +81,12 @@ class TargetAssigner(object):
def box_coder(self):
return self._box_coder
- def assign(self, anchors, groundtruth_boxes, groundtruth_labels=None,
- groundtruth_weights=None, **params):
+ def assign(self,
+ anchors,
+ groundtruth_boxes,
+ groundtruth_labels=None,
+ groundtruth_weights=None,
+ **params):
"""Assign classification and regression targets to each anchor.
For a given set of anchors and groundtruth detections, match anchors
@@ -93,16 +100,16 @@ class TargetAssigner(object):
Args:
anchors: a BoxList representing N anchors
groundtruth_boxes: a BoxList representing M groundtruth boxes
- groundtruth_labels: a tensor of shape [M, d_1, ... d_k]
- with labels for each of the ground_truth boxes. The subshape
- [d_1, ... d_k] can be empty (corresponding to scalar inputs). When set
- to None, groundtruth_labels assumes a binary problem where all
- ground_truth boxes get a positive label (of 1).
+ groundtruth_labels: a tensor of shape [M, d_1, ... d_k] with labels for
+ each of the ground_truth boxes. The subshape [d_1, ... d_k] can be empty
+ (corresponding to scalar inputs). When set to None, groundtruth_labels
+ assumes a binary problem where all ground_truth boxes get a positive
+ label (of 1).
groundtruth_weights: a float tensor of shape [M] indicating the weight to
assign to all anchors match to a particular groundtruth box. The weights
must be in [0., 1.]. If None, all weights are set to 1.
- **params: Additional keyword arguments for specific implementations of
- the Matcher.
+ **params: Additional keyword arguments for specific implementations of the
+ Matcher.
Returns:
cls_targets: a float32 tensor with shape [num_anchors, d_1, d_2 ... d_k],
@@ -125,16 +132,15 @@ class TargetAssigner(object):
raise ValueError('groundtruth_boxes must be an BoxList')
if groundtruth_labels is None:
- groundtruth_labels = tf.ones(tf.expand_dims(groundtruth_boxes.num_boxes(),
- 0))
+ groundtruth_labels = tf.ones(
+ tf.expand_dims(groundtruth_boxes.num_boxes(), 0))
groundtruth_labels = tf.expand_dims(groundtruth_labels, -1)
unmatched_shape_assert = shape_utils.assert_shape_equal(
shape_utils.combined_static_and_dynamic_shape(groundtruth_labels)[1:],
shape_utils.combined_static_and_dynamic_shape(
self._unmatched_cls_target))
labels_and_box_shapes_assert = shape_utils.assert_shape_equal(
- shape_utils.combined_static_and_dynamic_shape(
- groundtruth_labels)[:1],
+ shape_utils.combined_static_and_dynamic_shape(groundtruth_labels)[:1],
shape_utils.combined_static_and_dynamic_shape(
groundtruth_boxes.get())[:1])
@@ -145,11 +151,10 @@ class TargetAssigner(object):
groundtruth_weights = tf.ones([num_gt_boxes], dtype=tf.float32)
with tf.control_dependencies(
[unmatched_shape_assert, labels_and_box_shapes_assert]):
- match_quality_matrix = self._similarity_calc.compare(groundtruth_boxes,
- anchors)
+ match_quality_matrix = self._similarity_calc(
+ groundtruth_boxes.get(), anchors.get())
match = self._matcher.match(match_quality_matrix, **params)
- reg_targets = self._create_regression_targets(anchors,
- groundtruth_boxes,
+ reg_targets = self._create_regression_targets(anchors, groundtruth_boxes,
match)
cls_targets = self._create_classification_targets(groundtruth_labels,
match)
@@ -210,8 +215,8 @@ class TargetAssigner(object):
match.match_results)
# Zero out the unmatched and ignored regression targets.
- unmatched_ignored_reg_targets = tf.tile(
- self._default_regression_target(), [match_results_shape[0], 1])
+ unmatched_ignored_reg_targets = tf.tile(self._default_regression_target(),
+ [match_results_shape[0], 1])
matched_anchors_mask = match.matched_column_indicator()
# To broadcast matched_anchors_mask to the same shape as
# matched_reg_targets.
@@ -233,7 +238,7 @@ class TargetAssigner(object):
Returns:
default_target: a float32 tensor with shape [1, box_code_dimension]
"""
- return tf.constant([self._box_coder.code_size*[0]], tf.float32)
+ return tf.constant([self._box_coder.code_size * [0]], tf.float32)
def _create_classification_targets(self, groundtruth_labels, match):
"""Create classification targets for each anchor.
@@ -243,11 +248,11 @@ class TargetAssigner(object):
to anything are given the target self._unmatched_cls_target
Args:
- groundtruth_labels: a tensor of shape [num_gt_boxes, d_1, ... d_k]
- with labels for each of the ground_truth boxes. The subshape
- [d_1, ... d_k] can be empty (corresponding to scalar labels).
- match: a matcher.Match object that provides a matching between anchors
- and groundtruth boxes.
+ groundtruth_labels: a tensor of shape [num_gt_boxes, d_1, ... d_k] with
+ labels for each of the ground_truth boxes. The subshape [d_1, ... d_k]
+ can be empty (corresponding to scalar labels).
+ match: a matcher.Match object that provides a matching between anchors and
+ groundtruth boxes.
Returns:
a float32 tensor with shape [num_anchors, d_1, d_2 ... d_k], where the
@@ -267,8 +272,8 @@ class TargetAssigner(object):
negative anchor.
Args:
- match: a matcher.Match object that provides a matching between anchors
- and groundtruth boxes.
+ match: a matcher.Match object that provides a matching between anchors and
+ groundtruth boxes.
groundtruth_weights: a float tensor of shape [M] indicating the weight to
assign to all anchors match to a particular groundtruth box.
@@ -278,9 +283,7 @@ class TargetAssigner(object):
return match.gather_based_on_match(
groundtruth_weights, ignored_value=0., unmatched_value=0.)
- def _create_classification_weights(self,
- match,
- groundtruth_weights):
+ def _create_classification_weights(self, match, groundtruth_weights):
"""Create classification weights for each anchor.
Positive (matched) anchors are associated with a weight of
@@ -291,8 +294,8 @@ class TargetAssigner(object):
the case in object detection).
Args:
- match: a matcher.Match object that provides a matching between anchors
- and groundtruth boxes.
+ match: a matcher.Match object that provides a matching between anchors and
+ groundtruth boxes.
groundtruth_weights: a float tensor of shape [M] indicating the weight to
assign to all anchors match to a particular groundtruth box.
@@ -312,3 +315,209 @@ class TargetAssigner(object):
BoxCoder object.
"""
return self._box_coder
+
+
+class OlnTargetAssigner(TargetAssigner):
+ """Target assigner to compute classification and regression targets."""
+
+ def __init__(self,
+ similarity_calc,
+ matcher,
+ box_coder,
+ negative_class_weight=1.0,
+ unmatched_cls_target=None,
+ center_matcher=None):
+ """Construct Object Detection Target Assigner.
+
+ Args:
+ similarity_calc: a RegionSimilarityCalculator
+ matcher: Matcher used to match groundtruth to anchors.
+ box_coder: BoxCoder used to encode matching groundtruth boxes with respect
+ to anchors.
+ negative_class_weight: classification weight to be associated to negative
+ anchors (default: 1.0). The weight must be in [0., 1.].
+ unmatched_cls_target: a float32 tensor with shape [d_1, d_2, ..., d_k]
+ which is consistent with the classification target for each anchor (and
+ can be empty for scalar targets). This shape must thus be compatible
+ with the groundtruth labels that are passed to the "assign" function
+ (which have shape [num_gt_boxes, d_1, d_2, ..., d_k]). If set to None,
+ unmatched_cls_target is set to be [0] for each anchor.
+ center_matcher: Matcher used to match groundtruth to anchors to sample and
+ assign the regression targets of centerness to each anchor.
+
+ Raises:
+ ValueError: if similarity_calc is not a RegionSimilarityCalculator or
+ if matcher is not a Matcher or if box_coder is not a BoxCoder
+ """
+ super(OlnTargetAssigner, self).__init__(
+ similarity_calc=similarity_calc,
+ matcher=matcher,
+ box_coder=box_coder,
+ negative_class_weight=negative_class_weight,
+ unmatched_cls_target=unmatched_cls_target)
+
+ # centerness-matcher with independent sampling IoU threshold.
+ self._center_matcher = center_matcher
+
+ def assign(self,
+ anchors,
+ groundtruth_boxes,
+ groundtruth_labels=None,
+ groundtruth_weights=None,
+ **params):
+ """Assign classification and regression targets to each anchor.
+
+ For a given set of anchors and groundtruth detections, match anchors
+ to groundtruth_boxes and assign classification and regression targets to
+ each anchor as well as weights based on the resulting match (specifying,
+ e.g., which anchors should not contribute to training loss).
+
+ Anchors that are not matched to anything are given a classification target
+ of self._unmatched_cls_target which can be specified via the constructor.
+
+ Args:
+ anchors: a BoxList representing N anchors
+ groundtruth_boxes: a BoxList representing M groundtruth boxes
+ groundtruth_labels: a tensor of shape [M, d_1, ... d_k] with labels for
+ each of the ground_truth boxes. The subshape [d_1, ... d_k] can be empty
+ (corresponding to scalar inputs). When set to None, groundtruth_labels
+ assumes a binary problem where all ground_truth boxes get a positive
+ label (of 1).
+ groundtruth_weights: a float tensor of shape [M] indicating the weight to
+ assign to all anchors match to a particular groundtruth box. The weights
+ must be in [0., 1.]. If None, all weights are set to 1.
+ **params: Additional keyword arguments for specific implementations of the
+ Matcher.
+
+ Returns:
+ cls_targets: a float32 tensor with shape [num_anchors, d_1, d_2 ... d_k],
+ where the subshape [d_1, ..., d_k] is compatible with groundtruth_labels
+ which has shape [num_gt_boxes, d_1, d_2, ... d_k].
+ cls_weights: a float32 tensor with shape [num_anchors]
+ reg_targets: a float32 tensor with shape [num_anchors, box_code_dimension]
+ reg_weights: a float32 tensor with shape [num_anchors]
+ match: a matcher.Match object encoding the match between anchors and
+ groundtruth boxes, with rows corresponding to groundtruth boxes
+ and columns corresponding to anchors.
+ matched_gt_boxlist: a BoxList object with data of float32 tensor with
+ shape [num_anchors, box_dimension] which encodes the coordinates of the
+ matched groundtruth boxes.
+ matched_anchors_mask: a Bool tensor with shape [num_anchors] which
+ indicates whether an anchor is matched or not.
+ center_matched_gt_boxlist: a BoxList object with data of float32 tensor
+ with shape [num_anchors, box_dimension] which encodes the coordinates of
+ the groundtruth boxes matched for centerness target assignment.
+ center_matched_anchors_mask: a Boolean tensor with shape [num_anchors]
+ which indicates whether an anchor is matched or not for centerness
+ target assignment.
+ matched_ious: a float32 tensor with shape [num_anchors] which encodes the
+ ious between each anchor and the matched groundtruth boxes.
+
+ Raises:
+ ValueError: if anchors or groundtruth_boxes are not of type
+ box_list.BoxList
+ """
+ if not isinstance(anchors, box_list.BoxList):
+ raise ValueError('anchors must be an BoxList')
+ if not isinstance(groundtruth_boxes, box_list.BoxList):
+ raise ValueError('groundtruth_boxes must be an BoxList')
+
+ if groundtruth_labels is None:
+ groundtruth_labels = tf.ones(
+ tf.expand_dims(groundtruth_boxes.num_boxes(), 0))
+ groundtruth_labels = tf.expand_dims(groundtruth_labels, -1)
+ unmatched_shape_assert = shape_utils.assert_shape_equal(
+ shape_utils.combined_static_and_dynamic_shape(groundtruth_labels)[1:],
+ shape_utils.combined_static_and_dynamic_shape(
+ self._unmatched_cls_target))
+ labels_and_box_shapes_assert = shape_utils.assert_shape_equal(
+ shape_utils.combined_static_and_dynamic_shape(groundtruth_labels)[:1],
+ shape_utils.combined_static_and_dynamic_shape(
+ groundtruth_boxes.get())[:1])
+
+ if groundtruth_weights is None:
+ num_gt_boxes = groundtruth_boxes.num_boxes_static()
+ if not num_gt_boxes:
+ num_gt_boxes = groundtruth_boxes.num_boxes()
+ groundtruth_weights = tf.ones([num_gt_boxes], dtype=tf.float32)
+ with tf.control_dependencies(
+ [unmatched_shape_assert, labels_and_box_shapes_assert]):
+ match_quality_matrix = self._similarity_calc(
+ groundtruth_boxes.get(), anchors.get())
+ match = self._matcher.match(match_quality_matrix, **params)
+ reg_targets, matched_gt_boxlist, matched_anchors_mask = (
+ self._create_regression_targets(anchors,
+ groundtruth_boxes,
+ match))
+ cls_targets = self._create_classification_targets(groundtruth_labels,
+ match)
+ reg_weights = self._create_regression_weights(match, groundtruth_weights)
+ cls_weights = self._create_classification_weights(match,
+ groundtruth_weights)
+ # Match for creation of centerness regression targets.
+ if self._center_matcher is not None:
+ center_match = self._center_matcher.match(
+ match_quality_matrix, **params)
+ center_matched_gt_boxes = center_match.gather_based_on_match(
+ groundtruth_boxes.get(),
+ unmatched_value=tf.zeros(4),
+ ignored_value=tf.zeros(4))
+ center_matched_gt_boxlist = box_list.BoxList(center_matched_gt_boxes)
+ center_matched_anchors_mask = center_match.matched_column_indicator()
+
+ num_anchors = anchors.num_boxes_static()
+ if num_anchors is not None:
+ reg_targets = self._reset_target_shape(reg_targets, num_anchors)
+ cls_targets = self._reset_target_shape(cls_targets, num_anchors)
+ reg_weights = self._reset_target_shape(reg_weights, num_anchors)
+ cls_weights = self._reset_target_shape(cls_weights, num_anchors)
+
+ if self._center_matcher is not None:
+ matched_ious = tf.reduce_max(match_quality_matrix, 0)
+ return (cls_targets, cls_weights, reg_targets, reg_weights, match,
+ matched_gt_boxlist, matched_anchors_mask,
+ center_matched_gt_boxlist, center_matched_anchors_mask,
+ matched_ious)
+ else:
+ return (cls_targets, cls_weights, reg_targets, reg_weights, match)
+
+ def _create_regression_targets(self, anchors, groundtruth_boxes, match):
+ """Returns a regression target for each anchor.
+
+ Args:
+ anchors: a BoxList representing N anchors
+ groundtruth_boxes: a BoxList representing M groundtruth_boxes
+ match: a matcher.Match object
+
+ Returns:
+ reg_targets: a float32 tensor with shape [N, box_code_dimension]
+ """
+ matched_gt_boxes = match.gather_based_on_match(
+ groundtruth_boxes.get(),
+ unmatched_value=tf.zeros(4),
+ ignored_value=tf.zeros(4))
+ matched_gt_boxlist = box_list.BoxList(matched_gt_boxes)
+ if groundtruth_boxes.has_field(KEYPOINTS_FIELD_NAME):
+ groundtruth_keypoints = groundtruth_boxes.get_field(KEYPOINTS_FIELD_NAME)
+ matched_keypoints = match.gather_based_on_match(
+ groundtruth_keypoints,
+ unmatched_value=tf.zeros(groundtruth_keypoints.get_shape()[1:]),
+ ignored_value=tf.zeros(groundtruth_keypoints.get_shape()[1:]))
+ matched_gt_boxlist.add_field(KEYPOINTS_FIELD_NAME, matched_keypoints)
+ matched_reg_targets = self._box_coder.encode(matched_gt_boxlist, anchors)
+ match_results_shape = shape_utils.combined_static_and_dynamic_shape(
+ match.match_results)
+
+ # Zero out the unmatched and ignored regression targets.
+ unmatched_ignored_reg_targets = tf.tile(self._default_regression_target(),
+ [match_results_shape[0], 1])
+ matched_anchors_mask = match.matched_column_indicator()
+ # To broadcast matched_anchors_mask to the same shape as
+ # matched_reg_targets.
+ matched_anchors_mask_tiled = tf.tile(
+ tf.expand_dims(matched_anchors_mask, 1),
+ [1, tf.shape(matched_reg_targets)[1]])
+ reg_targets = tf.where(matched_anchors_mask_tiled,
+ matched_reg_targets,
+ unmatched_ignored_reg_targets)
+ return reg_targets, matched_gt_boxlist, matched_anchors_mask
diff --git a/official/vision/detection/utils/object_detection/visualization_utils.py b/official/vision/detection/utils/object_detection/visualization_utils.py
index db4af8089df673cd5c57c4a020b5d7e8f03846c9..d063325b37efe509160bd32186d1b47e304e4f27 100644
--- a/official/vision/detection/utils/object_detection/visualization_utils.py
+++ b/official/vision/detection/utils/object_detection/visualization_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
"""A set of functions that are used for visualization.
@@ -21,9 +20,11 @@ The functions do not return a value, instead they modify the image itself.
"""
import collections
import functools
+
from absl import logging
# Set headless-friendly backend.
-import matplotlib; matplotlib.use('Agg') # pylint: disable=multiple-statements
+import matplotlib
+matplotlib.use('Agg') # pylint: disable=multiple-statements
import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top
import numpy as np
import PIL.Image as Image
@@ -36,7 +37,6 @@ import tensorflow as tf
from official.vision.detection.utils import box_utils
from official.vision.detection.utils.object_detection import shape_utils
-
_TITLE_LEFT_MARGIN = 10
_TITLE_TOP_MARGIN = 10
STANDARD_COLORS = [
@@ -99,9 +99,9 @@ def visualize_images_with_bounding_boxes(images, box_outputs, step,
summary_writer):
"""Records subset of evaluation images with bounding boxes."""
if not isinstance(images, list):
- logging.warning('visualize_images_with_bounding_boxes expects list of '
- 'images but received type: %s and value: %s',
- type(images), images)
+ logging.warning(
+ 'visualize_images_with_bounding_boxes expects list of '
+ 'images but received type: %s and value: %s', type(images), images)
return
image_shape = tf.shape(images[0])
@@ -140,11 +140,11 @@ def draw_bounding_box_on_image_array(image,
xmax: xmax of bounding box.
color: color to draw bounding box. Default is red.
thickness: line thickness. Default value is 4.
- display_str_list: list of strings to display in box
- (each to be shown on its own line).
- use_normalized_coordinates: If True (default), treat coordinates
- ymin, xmin, ymax, xmax as relative to the image. Otherwise treat
- coordinates as absolute.
+ display_str_list: list of strings to display in box (each to be shown on its
+ own line).
+ use_normalized_coordinates: If True (default), treat coordinates ymin, xmin,
+ ymax, xmax as relative to the image. Otherwise treat coordinates as
+ absolute.
"""
image_pil = Image.fromarray(np.uint8(image)).convert('RGB')
draw_bounding_box_on_image(image_pil, ymin, xmin, ymax, xmax, color,
@@ -180,11 +180,11 @@ def draw_bounding_box_on_image(image,
xmax: xmax of bounding box.
color: color to draw bounding box. Default is red.
thickness: line thickness. Default value is 4.
- display_str_list: list of strings to display in box
- (each to be shown on its own line).
- use_normalized_coordinates: If True (default), treat coordinates
- ymin, xmin, ymax, xmax as relative to the image. Otherwise treat
- coordinates as absolute.
+ display_str_list: list of strings to display in box (each to be shown on its
+ own line).
+ use_normalized_coordinates: If True (default), treat coordinates ymin, xmin,
+ ymax, xmax as relative to the image. Otherwise treat coordinates as
+ absolute.
"""
draw = ImageDraw.Draw(image)
im_width, im_height = image.size
@@ -193,8 +193,10 @@ def draw_bounding_box_on_image(image,
ymin * im_height, ymax * im_height)
else:
(left, right, top, bottom) = (xmin, xmax, ymin, ymax)
- draw.line([(left, top), (left, bottom), (right, bottom),
- (right, top), (left, top)], width=thickness, fill=color)
+ draw.line([(left, top), (left, bottom), (right, bottom), (right, top),
+ (left, top)],
+ width=thickness,
+ fill=color)
try:
font = ImageFont.truetype('arial.ttf', 24)
except IOError:
@@ -215,15 +217,13 @@ def draw_bounding_box_on_image(image,
for display_str in display_str_list[::-1]:
text_width, text_height = font.getsize(display_str)
margin = np.ceil(0.05 * text_height)
- draw.rectangle(
- [(left, text_bottom - text_height - 2 * margin), (left + text_width,
- text_bottom)],
- fill=color)
- draw.text(
- (left + margin, text_bottom - text_height - margin),
- display_str,
- fill='black',
- font=font)
+ draw.rectangle([(left, text_bottom - text_height - 2 * margin),
+ (left + text_width, text_bottom)],
+ fill=color)
+ draw.text((left + margin, text_bottom - text_height - margin),
+ display_str,
+ fill='black',
+ font=font)
text_bottom -= text_height - 2 * margin
@@ -236,15 +236,13 @@ def draw_bounding_boxes_on_image_array(image,
Args:
image: a numpy array object.
- boxes: a 2 dimensional numpy array of [N, 4]: (ymin, xmin, ymax, xmax).
- The coordinates are in normalized format between [0, 1].
+ boxes: a 2 dimensional numpy array of [N, 4]: (ymin, xmin, ymax, xmax). The
+ coordinates are in normalized format between [0, 1].
color: color to draw bounding box. Default is red.
thickness: line thickness. Default value is 4.
- display_str_list_list: list of list of strings.
- a list of strings for each bounding box.
- The reason to pass a list of strings for a
- bounding box is that it might contain
- multiple labels.
+ display_str_list_list: list of list of strings. a list of strings for each
+ bounding box. The reason to pass a list of strings for a bounding box is
+ that it might contain multiple labels.
Raises:
ValueError: if boxes is not a [N, 4] array
@@ -264,15 +262,13 @@ def draw_bounding_boxes_on_image(image,
Args:
image: a PIL.Image object.
- boxes: a 2 dimensional numpy array of [N, 4]: (ymin, xmin, ymax, xmax).
- The coordinates are in normalized format between [0, 1].
+ boxes: a 2 dimensional numpy array of [N, 4]: (ymin, xmin, ymax, xmax). The
+ coordinates are in normalized format between [0, 1].
color: color to draw bounding box. Default is red.
thickness: line thickness. Default value is 4.
- display_str_list_list: list of list of strings.
- a list of strings for each bounding box.
- The reason to pass a list of strings for a
- bounding box is that it might contain
- multiple labels.
+ display_str_list_list: list of list of strings. a list of strings for each
+ bounding box. The reason to pass a list of strings for a bounding box is
+ that it might contain multiple labels.
Raises:
ValueError: if boxes is not a [N, 4] array
@@ -319,8 +315,9 @@ def _visualize_boxes_and_keypoints(image, boxes, classes, scores, keypoints,
**kwargs)
-def _visualize_boxes_and_masks_and_keypoints(
- image, boxes, classes, scores, masks, keypoints, category_index, **kwargs):
+def _visualize_boxes_and_masks_and_keypoints(image, boxes, classes, scores,
+ masks, keypoints, category_index,
+ **kwargs):
return visualize_boxes_and_labels_on_image_array(
image,
boxes,
@@ -374,8 +371,8 @@ def draw_bounding_boxes_on_image_tensors(images,
max_boxes_to_draw: Maximum number of boxes to draw on an image. Default 20.
min_score_thresh: Minimum score threshold for visualization. Default 0.2.
use_normalized_coordinates: Whether to assume boxes and kepoints are in
- normalized coordinates (as opposed to absolute coordiantes).
- Default is True.
+ normalized coordinates (as opposed to absolute coordiantes). Default is
+ True.
Returns:
4D image tensor of type uint8, with boxes drawn on top.
@@ -432,17 +429,15 @@ def draw_bounding_boxes_on_image_tensors(images,
_visualize_boxes,
category_index=category_index,
**visualization_keyword_args)
- elems = [
- true_shapes, original_shapes, images, boxes, classes, scores
- ]
+ elems = [true_shapes, original_shapes, images, boxes, classes, scores]
def draw_boxes(image_and_detections):
"""Draws boxes on image."""
true_shape = image_and_detections[0]
original_shape = image_and_detections[1]
if true_image_shape is not None:
- image = shape_utils.pad_or_clip_nd(
- image_and_detections[2], [true_shape[0], true_shape[1], 3])
+ image = shape_utils.pad_or_clip_nd(image_and_detections[2],
+ [true_shape[0], true_shape[1], 3])
if original_image_spatial_shape is not None:
image_and_detections[2] = _resize_original_image(image, original_shape)
@@ -500,7 +495,8 @@ def draw_keypoints_on_image(image,
for keypoint_x, keypoint_y in zip(keypoints_x, keypoints_y):
draw.ellipse([(keypoint_x - radius, keypoint_y - radius),
(keypoint_x + radius, keypoint_y + radius)],
- outline=color, fill=color)
+ outline=color,
+ fill=color)
def draw_mask_on_image_array(image, mask, color='red', alpha=0.4):
@@ -508,8 +504,8 @@ def draw_mask_on_image_array(image, mask, color='red', alpha=0.4):
Args:
image: uint8 numpy array with shape (img_height, img_height, 3)
- mask: a uint8 numpy array of shape (img_height, img_height) with
- values between either 0 or 1.
+ mask: a uint8 numpy array of shape (img_height, img_height) with values
+ between either 0 or 1.
color: color to draw the keypoints with. Default is red.
alpha: transparency value between 0 and 1. (default: 0.4)
@@ -531,7 +527,7 @@ def draw_mask_on_image_array(image, mask, color='red', alpha=0.4):
solid_color = np.expand_dims(
np.ones_like(mask), axis=2) * np.reshape(list(rgb), [1, 1, 3])
pil_solid_color = Image.fromarray(np.uint8(solid_color)).convert('RGBA')
- pil_mask = Image.fromarray(np.uint8(255.0*alpha*mask)).convert('L')
+ pil_mask = Image.fromarray(np.uint8(255.0 * alpha * mask)).convert('L')
pil_image = Image.composite(pil_solid_color, pil_image, pil_mask)
np.copyto(image, np.array(pil_image.convert('RGB')))
@@ -565,21 +561,20 @@ def visualize_boxes_and_labels_on_image_array(
boxes: a numpy array of shape [N, 4]
classes: a numpy array of shape [N]. Note that class indices are 1-based,
and match the keys in the label map.
- scores: a numpy array of shape [N] or None. If scores=None, then
- this function assumes that the boxes to be plotted are groundtruth
- boxes and plot all boxes as black with no classes or scores.
+ scores: a numpy array of shape [N] or None. If scores=None, then this
+ function assumes that the boxes to be plotted are groundtruth boxes and
+ plot all boxes as black with no classes or scores.
category_index: a dict containing category dictionaries (each holding
category index `id` and category name `name`) keyed by category indices.
instance_masks: a numpy array of shape [N, image_height, image_width] with
values ranging between 0 and 1, can be None.
instance_boundaries: a numpy array of shape [N, image_height, image_width]
with values ranging between 0 and 1, can be None.
- keypoints: a numpy array of shape [N, num_keypoints, 2], can
- be None
- use_normalized_coordinates: whether boxes is to be interpreted as
- normalized coordinates or not.
- max_boxes_to_draw: maximum number of boxes to visualize. If None, draw
- all boxes.
+ keypoints: a numpy array of shape [N, num_keypoints, 2], can be None
+ use_normalized_coordinates: whether boxes is to be interpreted as normalized
+ coordinates or not.
+ max_boxes_to_draw: maximum number of boxes to visualize. If None, draw all
+ boxes.
min_score_thresh: minimum score threshold for a box to be visualized
agnostic_mode: boolean (default: False) controlling whether to evaluate in
class-agnostic mode or not. This mode will display scores but ignore
@@ -624,32 +619,25 @@ def visualize_boxes_and_labels_on_image_array(
display_str = str(class_name)
if not skip_scores:
if not display_str:
- display_str = '{}%'.format(int(100*scores[i]))
+ display_str = '{}%'.format(int(100 * scores[i]))
else:
- display_str = '{}: {}%'.format(display_str, int(100*scores[i]))
+ display_str = '{}: {}%'.format(display_str, int(100 * scores[i]))
box_to_display_str_map[box].append(display_str)
if agnostic_mode:
box_to_color_map[box] = 'DarkOrange'
else:
- box_to_color_map[box] = STANDARD_COLORS[
- classes[i] % len(STANDARD_COLORS)]
+ box_to_color_map[box] = STANDARD_COLORS[classes[i] %
+ len(STANDARD_COLORS)]
# Draw all boxes onto image.
for box, color in box_to_color_map.items():
ymin, xmin, ymax, xmax = box
if instance_masks is not None:
draw_mask_on_image_array(
- image,
- box_to_instance_masks_map[box],
- color=color
- )
+ image, box_to_instance_masks_map[box], color=color)
if instance_boundaries is not None:
draw_mask_on_image_array(
- image,
- box_to_instance_boundaries_map[box],
- color='red',
- alpha=1.0
- )
+ image, box_to_instance_boundaries_map[box], color='red', alpha=1.0)
draw_bounding_box_on_image_array(
image,
ymin,
@@ -681,13 +669,15 @@ def add_cdf_image_summary(values, name):
values: a 1-D float32 tensor containing the values.
name: name for the image summary.
"""
+
def cdf_plot(values):
"""Numpy function to plot CDF."""
normalized_values = values / np.sum(values)
sorted_values = np.sort(normalized_values)
cumulative_values = np.cumsum(sorted_values)
- fraction_of_examples = (np.arange(cumulative_values.size, dtype=np.float32)
- / cumulative_values.size)
+ fraction_of_examples = (
+ np.arange(cumulative_values.size, dtype=np.float32) /
+ cumulative_values.size)
fig = plt.figure(frameon=False)
ax = fig.add_subplot('111')
ax.plot(fraction_of_examples, cumulative_values)
@@ -695,8 +685,9 @@ def add_cdf_image_summary(values, name):
ax.set_xlabel('fraction of examples')
fig.canvas.draw()
width, height = fig.get_size_inches() * fig.get_dpi()
- image = np.fromstring(fig.canvas.tostring_rgb(), dtype='uint8').reshape(
- 1, int(height), int(width), 3)
+ image = np.fromstring(
+ fig.canvas.tostring_rgb(),
+ dtype='uint8').reshape(1, int(height), int(width), 3)
return image
cdf_plot = tf.compat.v1.py_func(cdf_plot, [values], tf.uint8)
@@ -725,8 +716,8 @@ def add_hist_image_summary(values, bins, name):
fig.canvas.draw()
width, height = fig.get_size_inches() * fig.get_dpi()
image = np.fromstring(
- fig.canvas.tostring_rgb(), dtype='uint8').reshape(
- 1, int(height), int(width), 3)
+ fig.canvas.tostring_rgb(),
+ dtype='uint8').reshape(1, int(height), int(width), 3)
return image
hist_plot = tf.compat.v1.py_func(hist_plot, [values, bins], tf.uint8)
diff --git a/official/vision/image_classification/README.md b/official/vision/image_classification/README.md
index eb061d5b5f3284255bdb484cfbbb20bb3e157268..78bfe1f27e6543cb999698c9fec9e039d1ce9955 100644
--- a/official/vision/image_classification/README.md
+++ b/official/vision/image_classification/README.md
@@ -43,7 +43,7 @@ builder to 'records' or 'tfds' in the configurations.
Note: These models will **not** work with TPUs on Colab.
You can train image classification models on Cloud TPUs using
-[tf.distribute.experimental.TPUStrategy](https://www.tensorflow.org/api_docs/python/tf/distribute/experimental/TPUStrategy?version=nightly).
+[tf.distribute.TPUStrategy](https://www.tensorflow.org/api_docs/python/tf.distribute.TPUStrategy?version=nightly).
If you are not familiar with Cloud TPUs, it is strongly recommended that you go
through the
[quickstart](https://cloud.google.com/tpu/docs/quickstart) to learn how to
diff --git a/official/vision/image_classification/__init__.py b/official/vision/image_classification/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/image_classification/__init__.py
+++ b/official/vision/image_classification/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/official/vision/image_classification/augment.py b/official/vision/image_classification/augment.py
index b6ef23a229c80bcc1fec92d431996688dc34eaad..d6acd0813875250cbfb49696cd01df541b39893a 100644
--- a/official/vision/image_classification/augment.py
+++ b/official/vision/image_classification/augment.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""AutoAugment and RandAugment policies for enhanced image preprocessing.
AutoAugment Reference: https://arxiv.org/abs/1805.09501
@@ -24,6 +24,7 @@ from __future__ import division
from __future__ import print_function
import math
+
import tensorflow as tf
from typing import Any, Dict, List, Optional, Text, Tuple
@@ -120,10 +121,8 @@ def _convert_translation_to_transform(translations: tf.Tensor) -> tf.Tensor:
)
-def _convert_angles_to_transform(
- angles: tf.Tensor,
- image_width: tf.Tensor,
- image_height: tf.Tensor) -> tf.Tensor:
+def _convert_angles_to_transform(angles: tf.Tensor, image_width: tf.Tensor,
+ image_height: tf.Tensor) -> tf.Tensor:
"""Converts an angle or angles to a projective transform.
Args:
@@ -173,9 +172,7 @@ def transform(image: tf.Tensor, transforms) -> tf.Tensor:
transforms = transforms[None]
image = to_4d(image)
image = image_ops.transform(
- images=image,
- transforms=transforms,
- interpolation='nearest')
+ images=image, transforms=transforms, interpolation='nearest')
return from_4d(image, original_ndims)
@@ -216,9 +213,8 @@ def rotate(image: tf.Tensor, degrees: float) -> tf.Tensor:
image_height = tf.cast(tf.shape(image)[1], tf.float32)
image_width = tf.cast(tf.shape(image)[2], tf.float32)
- transforms = _convert_angles_to_transform(angles=radians,
- image_width=image_width,
- image_height=image_height)
+ transforms = _convert_angles_to_transform(
+ angles=radians, image_width=image_width, image_height=image_height)
# In practice, we should randomize the rotation degrees by flipping
# it negatively half the time, but that's done on 'degrees' outside
# of the function.
@@ -279,11 +275,10 @@ def cutout(image: tf.Tensor, pad_size: int, replace: int = 0) -> tf.Tensor:
Args:
image: An image Tensor of type uint8.
- pad_size: Specifies how big the zero mask that will be generated is that
- is applied to the image. The mask will be of size
- (2*pad_size x 2*pad_size).
- replace: What pixel value to fill in the image in the area that has
- the cutout mask applied to it.
+ pad_size: Specifies how big the zero mask that will be generated is that is
+ applied to the image. The mask will be of size (2*pad_size x 2*pad_size).
+ replace: What pixel value to fill in the image in the area that has the
+ cutout mask applied to it.
Returns:
An image Tensor that is of type uint8.
@@ -293,30 +288,30 @@ def cutout(image: tf.Tensor, pad_size: int, replace: int = 0) -> tf.Tensor:
# Sample the center location in the image where the zero mask will be applied.
cutout_center_height = tf.random.uniform(
- shape=[], minval=0, maxval=image_height,
- dtype=tf.int32)
+ shape=[], minval=0, maxval=image_height, dtype=tf.int32)
cutout_center_width = tf.random.uniform(
- shape=[], minval=0, maxval=image_width,
- dtype=tf.int32)
+ shape=[], minval=0, maxval=image_width, dtype=tf.int32)
lower_pad = tf.maximum(0, cutout_center_height - pad_size)
upper_pad = tf.maximum(0, image_height - cutout_center_height - pad_size)
left_pad = tf.maximum(0, cutout_center_width - pad_size)
right_pad = tf.maximum(0, image_width - cutout_center_width - pad_size)
- cutout_shape = [image_height - (lower_pad + upper_pad),
- image_width - (left_pad + right_pad)]
+ cutout_shape = [
+ image_height - (lower_pad + upper_pad),
+ image_width - (left_pad + right_pad)
+ ]
padding_dims = [[lower_pad, upper_pad], [left_pad, right_pad]]
mask = tf.pad(
tf.zeros(cutout_shape, dtype=image.dtype),
- padding_dims, constant_values=1)
+ padding_dims,
+ constant_values=1)
mask = tf.expand_dims(mask, -1)
mask = tf.tile(mask, [1, 1, 3])
image = tf.where(
tf.equal(mask, 0),
- tf.ones_like(image, dtype=image.dtype) * replace,
- image)
+ tf.ones_like(image, dtype=image.dtype) * replace, image)
return image
@@ -398,8 +393,8 @@ def shear_x(image: tf.Tensor, level: float, replace: int) -> tf.Tensor:
# with a matrix form of:
# [1 level
# 0 1].
- image = transform(image=wrap(image),
- transforms=[1., level, 0., 0., 1., 0., 0., 0.])
+ image = transform(
+ image=wrap(image), transforms=[1., level, 0., 0., 1., 0., 0., 0.])
return unwrap(image, replace)
@@ -409,8 +404,8 @@ def shear_y(image: tf.Tensor, level: float, replace: int) -> tf.Tensor:
# with a matrix form of:
# [1 0
# level 1].
- image = transform(image=wrap(image),
- transforms=[1., 0., 0., level, 1., 0., 0., 0.])
+ image = transform(
+ image=wrap(image), transforms=[1., 0., 0., level, 1., 0., 0., 0.])
return unwrap(image, replace)
@@ -460,9 +455,9 @@ def sharpness(image: tf.Tensor, factor: float) -> tf.Tensor:
# Make image 4D for conv operation.
image = tf.expand_dims(image, 0)
# SMOOTH PIL Kernel.
- kernel = tf.constant(
- [[1, 1, 1], [1, 5, 1], [1, 1, 1]], dtype=tf.float32,
- shape=[3, 3, 1, 1]) / 13.
+ kernel = tf.constant([[1, 1, 1], [1, 5, 1], [1, 1, 1]],
+ dtype=tf.float32,
+ shape=[3, 3, 1, 1]) / 13.
# Tile across channel dimension.
kernel = tf.tile(kernel, [1, 1, 3, 1])
strides = [1, 1, 1, 1]
@@ -484,6 +479,7 @@ def sharpness(image: tf.Tensor, factor: float) -> tf.Tensor:
def equalize(image: tf.Tensor) -> tf.Tensor:
"""Implements Equalize function from PIL using TF ops."""
+
def scale_channel(im, c):
"""Scale the data in the channel to implement equalize."""
im = tf.cast(im[:, :, c], tf.int32)
@@ -507,9 +503,9 @@ def equalize(image: tf.Tensor) -> tf.Tensor:
# If step is zero, return the original image. Otherwise, build
# lut from the full histogram and step and then index from it.
- result = tf.cond(tf.equal(step, 0),
- lambda: im,
- lambda: tf.gather(build_lut(histo, step), im))
+ result = tf.cond(
+ tf.equal(step, 0), lambda: im,
+ lambda: tf.gather(build_lut(histo, step), im))
return tf.cast(result, tf.uint8)
@@ -582,7 +578,7 @@ def _randomly_negate_tensor(tensor):
def _rotate_level_to_arg(level: float):
- level = (level/_MAX_LEVEL) * 30.
+ level = (level / _MAX_LEVEL) * 30.
level = _randomly_negate_tensor(level)
return (level,)
@@ -597,18 +593,18 @@ def _shrink_level_to_arg(level: float):
def _enhance_level_to_arg(level: float):
- return ((level/_MAX_LEVEL) * 1.8 + 0.1,)
+ return ((level / _MAX_LEVEL) * 1.8 + 0.1,)
def _shear_level_to_arg(level: float):
- level = (level/_MAX_LEVEL) * 0.3
+ level = (level / _MAX_LEVEL) * 0.3
# Flip level to negative with 50% chance.
level = _randomly_negate_tensor(level)
return (level,)
def _translate_level_to_arg(level: float, translate_const: float):
- level = (level/_MAX_LEVEL) * float(translate_const)
+ level = (level / _MAX_LEVEL) * float(translate_const)
# Flip level to negative with 50% chance.
level = _randomly_negate_tensor(level)
return (level,)
@@ -618,20 +614,15 @@ def _mult_to_arg(level: float, multiplier: float = 1.):
return (int((level / _MAX_LEVEL) * multiplier),)
-def _apply_func_with_prob(func: Any,
- image: tf.Tensor,
- args: Any,
- prob: float):
+def _apply_func_with_prob(func: Any, image: tf.Tensor, args: Any, prob: float):
"""Apply `func` to image w/ `args` as input with probability `prob`."""
assert isinstance(args, tuple)
# Apply the function with probability `prob`.
should_apply_op = tf.cast(
tf.floor(tf.random.uniform([], dtype=tf.float32) + prob), tf.bool)
- augmented_image = tf.cond(
- should_apply_op,
- lambda: func(image, *args),
- lambda: image)
+ augmented_image = tf.cond(should_apply_op, lambda: func(image, *args),
+ lambda: image)
return augmented_image
@@ -709,11 +700,8 @@ def level_to_arg(cutout_const: float, translate_const: float):
return args
-def _parse_policy_info(name: Text,
- prob: float,
- level: float,
- replace_value: List[int],
- cutout_const: float,
+def _parse_policy_info(name: Text, prob: float, level: float,
+ replace_value: List[int], cutout_const: float,
translate_const: float) -> Tuple[Any, float, Any]:
"""Return the function that corresponds to `name` and update `level` param."""
func = NAME_TO_FUNC[name]
@@ -969,8 +957,9 @@ class RandAugment(ImageAugment):
min_prob, max_prob = 0.2, 0.8
for _ in range(self.num_layers):
- op_to_select = tf.random.uniform(
- [], maxval=len(self.available_ops) + 1, dtype=tf.int32)
+ op_to_select = tf.random.uniform([],
+ maxval=len(self.available_ops) + 1,
+ dtype=tf.int32)
branch_fns = []
for (i, op_name) in enumerate(self.available_ops):
@@ -978,11 +967,8 @@ class RandAugment(ImageAugment):
minval=min_prob,
maxval=max_prob,
dtype=tf.float32)
- func, _, args = _parse_policy_info(op_name,
- prob,
- self.magnitude,
- replace_value,
- self.cutout_const,
+ func, _, args = _parse_policy_info(op_name, prob, self.magnitude,
+ replace_value, self.cutout_const,
self.translate_const)
branch_fns.append((
i,
@@ -991,9 +977,10 @@ class RandAugment(ImageAugment):
image, *selected_args)))
# pylint:enable=g-long-lambda
- image = tf.switch_case(branch_index=op_to_select,
- branch_fns=branch_fns,
- default=lambda: tf.identity(image))
+ image = tf.switch_case(
+ branch_index=op_to_select,
+ branch_fns=branch_fns,
+ default=lambda: tf.identity(image))
image = tf.cast(image, dtype=input_image_type)
return image
diff --git a/official/vision/image_classification/augment_test.py b/official/vision/image_classification/augment_test.py
index 76bdb2b7b9db4fc109f39674c68ae0c1169f3f12..dceb14eeaddb1ce21fd40b4c40b6c463b62f64fb 100644
--- a/official/vision/image_classification/augment_test.py
+++ b/official/vision/image_classification/augment_test.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Tests for autoaugment."""
from __future__ import absolute_import
@@ -49,24 +49,15 @@ class TransformsTest(parameterized.TestCase, tf.test.TestCase):
def test_transform(self, dtype):
image = tf.constant([[1, 2], [3, 4]], dtype=dtype)
- self.assertAllEqual(augment.transform(image, transforms=[1]*8),
- [[4, 4], [4, 4]])
+ self.assertAllEqual(
+ augment.transform(image, transforms=[1] * 8), [[4, 4], [4, 4]])
def test_translate(self, dtype):
image = tf.constant(
- [[1, 0, 1, 0],
- [0, 1, 0, 1],
- [1, 0, 1, 0],
- [0, 1, 0, 1]],
- dtype=dtype)
+ [[1, 0, 1, 0], [0, 1, 0, 1], [1, 0, 1, 0], [0, 1, 0, 1]], dtype=dtype)
translations = [-1, -1]
- translated = augment.translate(image=image,
- translations=translations)
- expected = [
- [1, 0, 1, 1],
- [0, 1, 0, 0],
- [1, 0, 1, 1],
- [1, 0, 1, 1]]
+ translated = augment.translate(image=image, translations=translations)
+ expected = [[1, 0, 1, 1], [0, 1, 0, 0], [1, 0, 1, 1], [1, 0, 1, 1]]
self.assertAllEqual(translated, expected)
def test_translate_shapes(self, dtype):
@@ -85,9 +76,7 @@ class TransformsTest(parameterized.TestCase, tf.test.TestCase):
image = tf.reshape(tf.cast(tf.range(9), dtype), (3, 3))
rotation = 90.
transformed = augment.rotate(image=image, degrees=rotation)
- expected = [[2, 5, 8],
- [1, 4, 7],
- [0, 3, 6]]
+ expected = [[2, 5, 8], [1, 4, 7], [0, 3, 6]]
self.assertAllEqual(transformed, expected)
def test_rotate_shapes(self, dtype):
@@ -129,15 +118,13 @@ class AutoaugmentTest(tf.test.TestCase):
image = tf.ones((224, 224, 3), dtype=tf.uint8)
for op_name in augment.NAME_TO_FUNC:
- func, _, args = augment._parse_policy_info(op_name,
- prob,
- magnitude,
- replace_value,
- cutout_const,
+ func, _, args = augment._parse_policy_info(op_name, prob, magnitude,
+ replace_value, cutout_const,
translate_const)
image = func(image, *args)
self.assertEqual((224, 224, 3), image.shape)
+
if __name__ == '__main__':
tf.test.main()
diff --git a/official/vision/image_classification/callbacks.py b/official/vision/image_classification/callbacks.py
index 985d0c60cc0b866e10ad350986c004e4ea4ac161..033a2dd714f596e5cabbac4b0205099831fdb16c 100644
--- a/official/vision/image_classification/callbacks.py
+++ b/official/vision/image_classification/callbacks.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,7 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Common modules for callbacks."""
from __future__ import absolute_import
from __future__ import division
@@ -21,37 +21,46 @@ from __future__ import print_function
import os
from typing import Any, List, MutableMapping, Text
+
from absl import logging
import tensorflow as tf
+from official.modeling import optimization
from official.utils.misc import keras_utils
-from official.vision.image_classification import optimizer_factory
-
-
-def get_callbacks(model_checkpoint: bool = True,
- include_tensorboard: bool = True,
- time_history: bool = True,
- track_lr: bool = True,
- write_model_weights: bool = True,
- apply_moving_average: bool = False,
- initial_step: int = 0,
- batch_size: int = 0,
- log_steps: int = 0,
- model_dir: str = None) -> List[tf.keras.callbacks.Callback]:
+
+
+def get_callbacks(
+ model_checkpoint: bool = True,
+ include_tensorboard: bool = True,
+ time_history: bool = True,
+ track_lr: bool = True,
+ write_model_weights: bool = True,
+ apply_moving_average: bool = False,
+ initial_step: int = 0,
+ batch_size: int = 0,
+ log_steps: int = 0,
+ model_dir: str = None,
+ backup_and_restore: bool = False) -> List[tf.keras.callbacks.Callback]:
"""Get all callbacks."""
model_dir = model_dir or ''
callbacks = []
if model_checkpoint:
ckpt_full_path = os.path.join(model_dir, 'model.ckpt-{epoch:04d}')
- callbacks.append(tf.keras.callbacks.ModelCheckpoint(
- ckpt_full_path, save_weights_only=True, verbose=1))
+ callbacks.append(
+ tf.keras.callbacks.ModelCheckpoint(
+ ckpt_full_path, save_weights_only=True, verbose=1))
+ if backup_and_restore:
+ backup_dir = os.path.join(model_dir, 'tmp')
+ callbacks.append(
+ tf.keras.callbacks.experimental.BackupAndRestore(backup_dir))
if include_tensorboard:
callbacks.append(
CustomTensorBoard(
log_dir=model_dir,
track_lr=track_lr,
initial_step=initial_step,
- write_images=write_model_weights))
+ write_images=write_model_weights,
+ profile_batch=0))
if time_history:
callbacks.append(
keras_utils.TimeHistory(
@@ -61,13 +70,14 @@ def get_callbacks(model_checkpoint: bool = True,
if apply_moving_average:
# Save moving average model to a different file so that
# we can resume training from a checkpoint
- ckpt_full_path = os.path.join(
- model_dir, 'average', 'model.ckpt-{epoch:04d}')
- callbacks.append(AverageModelCheckpoint(
- update_weights=False,
- filepath=ckpt_full_path,
- save_weights_only=True,
- verbose=1))
+ ckpt_full_path = os.path.join(model_dir, 'average',
+ 'model.ckpt-{epoch:04d}')
+ callbacks.append(
+ AverageModelCheckpoint(
+ update_weights=False,
+ filepath=ckpt_full_path,
+ save_weights_only=True,
+ verbose=1))
callbacks.append(MovingAverageCallback())
return callbacks
@@ -162,7 +172,7 @@ class CustomTensorBoard(tf.keras.callbacks.TensorBoard):
class MovingAverageCallback(tf.keras.callbacks.Callback):
- """A Callback to be used with a `MovingAverage` optimizer.
+ """A Callback to be used with a `ExponentialMovingAverage` optimizer.
Applies moving average weights to the model during validation time to test
and predict on the averaged weights rather than the current model weights.
@@ -175,16 +185,14 @@ class MovingAverageCallback(tf.keras.callbacks.Callback):
**kwargs: Any additional callback arguments.
"""
- def __init__(self,
- overwrite_weights_on_train_end: bool = False,
- **kwargs):
+ def __init__(self, overwrite_weights_on_train_end: bool = False, **kwargs):
super(MovingAverageCallback, self).__init__(**kwargs)
self.overwrite_weights_on_train_end = overwrite_weights_on_train_end
def set_model(self, model: tf.keras.Model):
super(MovingAverageCallback, self).set_model(model)
assert isinstance(self.model.optimizer,
- optimizer_factory.MovingAverage)
+ optimization.ExponentialMovingAverage)
self.model.optimizer.shadow_copy(self.model)
def on_test_begin(self, logs: MutableMapping[Text, Any] = None):
@@ -204,44 +212,35 @@ class AverageModelCheckpoint(tf.keras.callbacks.ModelCheckpoint):
Taken from tfa.callbacks.AverageModelCheckpoint.
Attributes:
- update_weights: If True, assign the moving average weights
- to the model, and save them. If False, keep the old
- non-averaged weights, but the saved model uses the
- average weights.
- See `tf.keras.callbacks.ModelCheckpoint` for the other args.
+ update_weights: If True, assign the moving average weights to the model, and
+ save them. If False, keep the old non-averaged weights, but the saved
+ model uses the average weights. See `tf.keras.callbacks.ModelCheckpoint`
+ for the other args.
"""
- def __init__(
- self,
- update_weights: bool,
- filepath: str,
- monitor: str = 'val_loss',
- verbose: int = 0,
- save_best_only: bool = False,
- save_weights_only: bool = False,
- mode: str = 'auto',
- save_freq: str = 'epoch',
- **kwargs):
+ def __init__(self,
+ update_weights: bool,
+ filepath: str,
+ monitor: str = 'val_loss',
+ verbose: int = 0,
+ save_best_only: bool = False,
+ save_weights_only: bool = False,
+ mode: str = 'auto',
+ save_freq: str = 'epoch',
+ **kwargs):
self.update_weights = update_weights
- super().__init__(
- filepath,
- monitor,
- verbose,
- save_best_only,
- save_weights_only,
- mode,
- save_freq,
- **kwargs)
+ super().__init__(filepath, monitor, verbose, save_best_only,
+ save_weights_only, mode, save_freq, **kwargs)
def set_model(self, model):
- if not isinstance(model.optimizer, optimizer_factory.MovingAverage):
- raise TypeError(
- 'AverageModelCheckpoint is only used when training'
- 'with MovingAverage')
+ if not isinstance(model.optimizer, optimization.ExponentialMovingAverage):
+ raise TypeError('AverageModelCheckpoint is only used when training'
+ 'with MovingAverage')
return super().set_model(model)
def _save_model(self, epoch, logs):
- assert isinstance(self.model.optimizer, optimizer_factory.MovingAverage)
+ assert isinstance(self.model.optimizer,
+ optimization.ExponentialMovingAverage)
if self.update_weights:
self.model.optimizer.assign_average_vars(self.model.variables)
diff --git a/official/vision/image_classification/classifier_trainer.py b/official/vision/image_classification/classifier_trainer.py
index c4b87ad6068d3d1beda0e4f0dec20f363466f7f8..ab6fbaea960e7d894d69e213e95c313d7fe9893c 100644
--- a/official/vision/image_classification/classifier_trainer.py
+++ b/official/vision/image_classification/classifier_trainer.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,7 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Runs an Image Classification model."""
import os
@@ -23,11 +23,10 @@ from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
-
+from official.common import distribute_utils
from official.modeling import hyperparams
from official.modeling import performance
from official.utils import hyperparams_flags
-from official.utils.misc import distribution_utils
from official.utils.misc import keras_utils
from official.vision.image_classification import callbacks as custom_callbacks
from official.vision.image_classification import dataset_factory
@@ -41,7 +40,7 @@ from official.vision.image_classification.resnet import resnet_model
def get_models() -> Mapping[str, tf.keras.Model]:
"""Returns the mapping from model type name to Keras model."""
- return {
+ return {
'efficientnet': efficientnet_model.EfficientNet.from_name,
'resnet': resnet_model.resnet50,
}
@@ -55,7 +54,7 @@ def get_dtype_map() -> Mapping[str, tf.dtypes.DType]:
'float16': tf.float16,
'fp32': tf.float32,
'bf16': tf.bfloat16,
- }
+ }
def _get_metrics(one_hot: bool) -> Mapping[Text, Any]:
@@ -63,22 +62,28 @@ def _get_metrics(one_hot: bool) -> Mapping[Text, Any]:
if one_hot:
return {
# (name, metric_fn)
- 'acc': tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
- 'accuracy': tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
- 'top_1': tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
- 'top_5': tf.keras.metrics.TopKCategoricalAccuracy(
- k=5,
- name='top_5_accuracy'),
+ 'acc':
+ tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
+ 'accuracy':
+ tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
+ 'top_1':
+ tf.keras.metrics.CategoricalAccuracy(name='accuracy'),
+ 'top_5':
+ tf.keras.metrics.TopKCategoricalAccuracy(
+ k=5, name='top_5_accuracy'),
}
else:
return {
# (name, metric_fn)
- 'acc': tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
- 'accuracy': tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
- 'top_1': tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
- 'top_5': tf.keras.metrics.SparseTopKCategoricalAccuracy(
- k=5,
- name='top_5_accuracy'),
+ 'acc':
+ tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
+ 'accuracy':
+ tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
+ 'top_1':
+ tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy'),
+ 'top_5':
+ tf.keras.metrics.SparseTopKCategoricalAccuracy(
+ k=5, name='top_5_accuracy'),
}
@@ -94,8 +99,7 @@ def get_image_size_from_model(
def _get_dataset_builders(params: base_configs.ExperimentConfig,
strategy: tf.distribute.Strategy,
- one_hot: bool
- ) -> Tuple[Any, Any]:
+ one_hot: bool) -> Tuple[Any, Any]:
"""Create and return train and validation dataset builders."""
if one_hot:
logging.warning('label_smoothing > 0, so datasets will be one hot encoded.')
@@ -107,9 +111,7 @@ def _get_dataset_builders(params: base_configs.ExperimentConfig,
image_size = get_image_size_from_model(params)
- dataset_configs = [
- params.train_dataset, params.validation_dataset
- ]
+ dataset_configs = [params.train_dataset, params.validation_dataset]
builders = []
for config in dataset_configs:
@@ -171,8 +173,7 @@ def _get_params_from_flags(flags_obj: flags.FlagValues):
},
}
- overriding_configs = (flags_obj.config_file,
- flags_obj.params_override,
+ overriding_configs = (flags_obj.config_file, flags_obj.params_override,
flags_overrides)
pp = pprint.PrettyPrinter()
@@ -190,8 +191,7 @@ def _get_params_from_flags(flags_obj: flags.FlagValues):
return params
-def resume_from_checkpoint(model: tf.keras.Model,
- model_dir: str,
+def resume_from_checkpoint(model: tf.keras.Model, model_dir: str,
train_steps: int) -> int:
"""Resumes from the latest checkpoint, if possible.
@@ -226,10 +226,8 @@ def resume_from_checkpoint(model: tf.keras.Model,
def initialize(params: base_configs.ExperimentConfig,
dataset_builder: dataset_factory.DatasetBuilder):
"""Initializes backend related initializations."""
- keras_utils.set_session_config(
- enable_xla=params.runtime.enable_xla)
- performance.set_mixed_precision_policy(dataset_builder.dtype,
- get_loss_scale(params))
+ keras_utils.set_session_config(enable_xla=params.runtime.enable_xla)
+ performance.set_mixed_precision_policy(dataset_builder.dtype)
if tf.config.list_physical_devices('GPU'):
data_format = 'channels_first'
else:
@@ -244,7 +242,8 @@ def initialize(params: base_configs.ExperimentConfig,
per_gpu_thread_count=params.runtime.per_gpu_thread_count,
gpu_thread_mode=params.runtime.gpu_thread_mode,
num_gpus=params.runtime.num_gpus,
- datasets_num_private_threads=params.runtime.dataset_num_private_threads) # pylint:disable=line-too-long
+ datasets_num_private_threads=params.runtime
+ .dataset_num_private_threads) # pylint:disable=line-too-long
if params.runtime.batchnorm_spatial_persistent:
os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1'
@@ -253,9 +252,7 @@ def define_classifier_flags():
"""Defines common flags for image classification."""
hyperparams_flags.initialize_common_flags()
flags.DEFINE_string(
- 'data_dir',
- default=None,
- help='The location of the input data.')
+ 'data_dir', default=None, help='The location of the input data.')
flags.DEFINE_string(
'mode',
default=None,
@@ -278,8 +275,7 @@ def define_classifier_flags():
help='The interval of steps between logging of batch level stats.')
-def serialize_config(params: base_configs.ExperimentConfig,
- model_dir: str):
+def serialize_config(params: base_configs.ExperimentConfig, model_dir: str):
"""Serializes and saves the experiment config."""
params_save_path = os.path.join(model_dir, 'params.yaml')
logging.info('Saving experiment configuration to %s', params_save_path)
@@ -293,18 +289,17 @@ def train_and_eval(
"""Runs the train and eval path using compile/fit."""
logging.info('Running train and eval.')
- distribution_utils.configure_cluster(
- params.runtime.worker_hosts,
- params.runtime.task_index)
+ distribute_utils.configure_cluster(params.runtime.worker_hosts,
+ params.runtime.task_index)
# Note: for TPUs, strategy and scope should be created before the dataset
- strategy = strategy_override or distribution_utils.get_distribution_strategy(
+ strategy = strategy_override or distribute_utils.get_distribution_strategy(
distribution_strategy=params.runtime.distribution_strategy,
all_reduce_alg=params.runtime.all_reduce_alg,
num_gpus=params.runtime.num_gpus,
tpu_address=params.runtime.tpu)
- strategy_scope = distribution_utils.get_strategy_scope(strategy)
+ strategy_scope = distribute_utils.get_strategy_scope(strategy)
logging.info('Detected %d devices.',
strategy.num_replicas_in_sync if strategy else 1)
@@ -313,8 +308,9 @@ def train_and_eval(
one_hot = label_smoothing and label_smoothing > 0
builders = _get_dataset_builders(params, strategy, one_hot)
- datasets = [builder.build(strategy)
- if builder else None for builder in builders]
+ datasets = [
+ builder.build(strategy) if builder else None for builder in builders
+ ]
# Unpack datasets and builders based on train/val/test splits
train_builder, validation_builder = builders # pylint: disable=unbalanced-tuple-unpacking
@@ -341,6 +337,10 @@ def train_and_eval(
base_learning_rate=learning_rate,
params=params.model.optimizer.as_dict(),
model=model)
+ optimizer = performance.configure_optimizer(
+ optimizer,
+ use_float16=train_builder.dtype == 'float16',
+ loss_scale=get_loss_scale(params))
metrics_map = _get_metrics(one_hot)
metrics = [metrics_map[metric] for metric in params.train.metrics]
@@ -351,16 +351,16 @@ def train_and_eval(
label_smoothing=params.model.loss.label_smoothing)
else:
loss_obj = tf.keras.losses.SparseCategoricalCrossentropy()
- model.compile(optimizer=optimizer,
- loss=loss_obj,
- metrics=metrics,
- experimental_steps_per_execution=steps_per_loop)
+ model.compile(
+ optimizer=optimizer,
+ loss=loss_obj,
+ metrics=metrics,
+ steps_per_execution=steps_per_loop)
initial_epoch = 0
if params.train.resume_checkpoint:
- initial_epoch = resume_from_checkpoint(model=model,
- model_dir=params.model_dir,
- train_steps=train_steps)
+ initial_epoch = resume_from_checkpoint(
+ model=model, model_dir=params.model_dir, train_steps=train_steps)
callbacks = custom_callbacks.get_callbacks(
model_checkpoint=params.train.callbacks.enable_checkpoint_and_export,
@@ -371,7 +371,8 @@ def train_and_eval(
initial_step=initial_epoch * train_steps,
batch_size=train_builder.global_batch_size,
log_steps=params.train.time_history.log_steps,
- model_dir=params.model_dir)
+ model_dir=params.model_dir,
+ backup_and_restore=params.train.callbacks.enable_backup_and_restore)
serialize_config(params=params, model_dir=params.model_dir)
@@ -399,9 +400,7 @@ def train_and_eval(
validation_dataset, steps=validation_steps, verbose=2)
# TODO(dankondratyuk): eval and save final test accuracy
- stats = common.build_stats(history,
- validation_output,
- callbacks)
+ stats = common.build_stats(history, validation_output, callbacks)
return stats
diff --git a/official/vision/image_classification/classifier_trainer_test.py b/official/vision/image_classification/classifier_trainer_test.py
index 244425feef76bf89d4de939cb8a1914a6f0f47c6..06227c154427db3057269f9e9250a179a52264c9 100644
--- a/official/vision/image_classification/classifier_trainer_test.py
+++ b/official/vision/image_classification/classifier_trainer_test.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,14 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Unit tests for the classifier trainer models."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
-import copy
import functools
import json
@@ -29,6 +28,7 @@ import sys
from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional, Tuple
from absl import flags
+from absl.testing import flagsaver
from absl.testing import parameterized
import tensorflow as tf
@@ -36,9 +36,7 @@ from tensorflow.python.distribute import combinations
from tensorflow.python.distribute import strategy_combinations
from official.utils.flags import core as flags_core
from official.vision.image_classification import classifier_trainer
-from official.vision.image_classification import dataset_factory
-from official.vision.image_classification import test_utils
-from official.vision.image_classification.configs import base_configs
+
classifier_trainer.define_classifier_flags()
@@ -48,7 +46,7 @@ def distribution_strategy_combinations() -> Iterable[Tuple[Any, ...]]:
return combinations.combine(
distribution=[
strategy_combinations.default_strategy,
- strategy_combinations.tpu_strategy,
+ strategy_combinations.cloud_tpu_strategy,
strategy_combinations.one_device_strategy_gpu,
strategy_combinations.mirrored_strategy_with_two_gpus,
],
@@ -56,7 +54,6 @@ def distribution_strategy_combinations() -> Iterable[Tuple[Any, ...]]:
'efficientnet',
'resnet',
],
- mode='eager',
dataset=[
'imagenet',
],
@@ -99,33 +96,7 @@ def basic_params_override(dtype: str = 'float32') -> MutableMapping[str, Any]:
}
-def get_trivial_model(num_classes: int) -> tf.keras.Model:
- """Creates and compiles trivial model for ImageNet dataset."""
- model = test_utils.trivial_model(num_classes=num_classes)
- lr = 0.01
- optimizer = tf.keras.optimizers.SGD(learning_rate=lr)
- loss_obj = tf.keras.losses.SparseCategoricalCrossentropy()
- model.compile(optimizer=optimizer,
- loss=loss_obj,
- run_eagerly=True)
- return model
-
-
-def get_trivial_data() -> tf.data.Dataset:
- """Gets trivial data in the ImageNet size."""
- def generate_data(_) -> tf.data.Dataset:
- image = tf.zeros(shape=(224, 224, 3), dtype=tf.float32)
- label = tf.zeros([1], dtype=tf.int32)
- return image, label
-
- dataset = tf.data.Dataset.range(1)
- dataset = dataset.repeat()
- dataset = dataset.map(generate_data,
- num_parallel_calls=tf.data.experimental.AUTOTUNE)
- dataset = dataset.prefetch(buffer_size=1).batch(1)
- return dataset
-
-
+@flagsaver.flagsaver
def run_end_to_end(main: Callable[[Any], None],
extra_flags: Optional[Iterable[str]] = None,
model_dir: Optional[str] = None):
@@ -154,7 +125,7 @@ class ClassifierTest(tf.test.TestCase, parameterized.TestCase):
# Some parameters are not defined as flags (e.g. cannot run
# classifier_train.py --batch_size=...) by design, so use
# "--params_override=..." instead
- model_dir = self.get_temp_dir()
+ model_dir = self.create_tempdir().full_path
base_flags = [
'--data_dir=not_used',
'--model_type=' + model,
@@ -165,11 +136,10 @@ class ClassifierTest(tf.test.TestCase, parameterized.TestCase):
'--mode=train_and_eval',
]
- run = functools.partial(classifier_trainer.run,
- strategy_override=distribution)
- run_end_to_end(main=run,
- extra_flags=train_and_eval_flags,
- model_dir=model_dir)
+ run = functools.partial(
+ classifier_trainer.run, strategy_override=distribution)
+ run_end_to_end(
+ main=run, extra_flags=train_and_eval_flags, model_dir=model_dir)
@combinations.generate(
combinations.combine(
@@ -180,7 +150,6 @@ class ClassifierTest(tf.test.TestCase, parameterized.TestCase):
'efficientnet',
'resnet',
],
- mode='eager',
dataset='imagenet',
dtype='float16',
))
@@ -189,7 +158,7 @@ class ClassifierTest(tf.test.TestCase, parameterized.TestCase):
# Some parameters are not defined as flags (e.g. cannot run
# classifier_train.py --batch_size=...) by design, so use
# "--params_override=..." instead
- model_dir = self.get_temp_dir()
+ model_dir = self.create_tempdir().full_path
base_flags = [
'--data_dir=not_used',
'--model_type=' + model,
@@ -209,35 +178,31 @@ class ClassifierTest(tf.test.TestCase, parameterized.TestCase):
get_params_override(export_params)
]
- run = functools.partial(classifier_trainer.run,
- strategy_override=distribution)
- run_end_to_end(main=run,
- extra_flags=train_and_eval_flags,
- model_dir=model_dir)
- run_end_to_end(main=run,
- extra_flags=export_flags,
- model_dir=model_dir)
+ run = functools.partial(
+ classifier_trainer.run, strategy_override=distribution)
+ run_end_to_end(
+ main=run, extra_flags=train_and_eval_flags, model_dir=model_dir)
+ run_end_to_end(main=run, extra_flags=export_flags, model_dir=model_dir)
self.assertTrue(os.path.exists(export_path))
@combinations.generate(
combinations.combine(
- distribution=[
- strategy_combinations.tpu_strategy,
- ],
- model=[
- 'efficientnet',
- 'resnet',
- ],
- mode='eager',
- dataset='imagenet',
- dtype='bfloat16',
- ))
+ distribution=[
+ strategy_combinations.cloud_tpu_strategy,
+ ],
+ model=[
+ 'efficientnet',
+ 'resnet',
+ ],
+ dataset='imagenet',
+ dtype='bfloat16',
+ ))
def test_tpu_train(self, distribution, model, dataset, dtype):
"""Test train_and_eval and export for Keras classifier models."""
# Some parameters are not defined as flags (e.g. cannot run
# classifier_train.py --batch_size=...) by design, so use
# "--params_override=..." instead
- model_dir = self.get_temp_dir()
+ model_dir = self.create_tempdir().full_path
base_flags = [
'--data_dir=not_used',
'--model_type=' + model,
@@ -248,16 +213,15 @@ class ClassifierTest(tf.test.TestCase, parameterized.TestCase):
'--mode=train_and_eval',
]
- run = functools.partial(classifier_trainer.run,
- strategy_override=distribution)
- run_end_to_end(main=run,
- extra_flags=train_and_eval_flags,
- model_dir=model_dir)
+ run = functools.partial(
+ classifier_trainer.run, strategy_override=distribution)
+ run_end_to_end(
+ main=run, extra_flags=train_and_eval_flags, model_dir=model_dir)
@combinations.generate(distribution_strategy_combinations())
def test_end_to_end_invalid_mode(self, distribution, model, dataset):
"""Test the Keras EfficientNet model with `strategy`."""
- model_dir = self.get_temp_dir()
+ model_dir = self.create_tempdir().full_path
extra_flags = [
'--data_dir=not_used',
'--mode=invalid_mode',
@@ -266,122 +230,11 @@ class ClassifierTest(tf.test.TestCase, parameterized.TestCase):
get_params_override(basic_params_override()),
]
- run = functools.partial(classifier_trainer.run,
- strategy_override=distribution)
+ run = functools.partial(
+ classifier_trainer.run, strategy_override=distribution)
with self.assertRaises(ValueError):
run_end_to_end(main=run, extra_flags=extra_flags, model_dir=model_dir)
-class UtilTests(parameterized.TestCase, tf.test.TestCase):
- """Tests for individual utility functions within classifier_trainer.py."""
-
- @parameterized.named_parameters(
- ('efficientnet-b0', 'efficientnet', 'efficientnet-b0', 224),
- ('efficientnet-b1', 'efficientnet', 'efficientnet-b1', 240),
- ('efficientnet-b2', 'efficientnet', 'efficientnet-b2', 260),
- ('efficientnet-b3', 'efficientnet', 'efficientnet-b3', 300),
- ('efficientnet-b4', 'efficientnet', 'efficientnet-b4', 380),
- ('efficientnet-b5', 'efficientnet', 'efficientnet-b5', 456),
- ('efficientnet-b6', 'efficientnet', 'efficientnet-b6', 528),
- ('efficientnet-b7', 'efficientnet', 'efficientnet-b7', 600),
- ('resnet', 'resnet', '', None),
- )
- def test_get_model_size(self, model, model_name, expected):
- config = base_configs.ExperimentConfig(
- model_name=model,
- model=base_configs.ModelConfig(
- model_params={
- 'model_name': model_name,
- },
- )
- )
- size = classifier_trainer.get_image_size_from_model(config)
- self.assertEqual(size, expected)
-
- @parameterized.named_parameters(
- ('dynamic', 'dynamic', None, 'dynamic'),
- ('scalar', 128., None, 128.),
- ('float32', None, 'float32', 1),
- ('float16', None, 'float16', 128),
- )
- def test_get_loss_scale(self, loss_scale, dtype, expected):
- config = base_configs.ExperimentConfig(
- runtime=base_configs.RuntimeConfig(
- loss_scale=loss_scale),
- train_dataset=dataset_factory.DatasetConfig(dtype=dtype))
- ls = classifier_trainer.get_loss_scale(config, fp16_default=128)
- self.assertEqual(ls, expected)
-
- @parameterized.named_parameters(
- ('float16', 'float16'),
- ('bfloat16', 'bfloat16')
- )
- def test_initialize(self, dtype):
- config = base_configs.ExperimentConfig(
- runtime=base_configs.RuntimeConfig(
- run_eagerly=False,
- enable_xla=False,
- per_gpu_thread_count=1,
- gpu_thread_mode='gpu_private',
- num_gpus=1,
- dataset_num_private_threads=1,
- ),
- train_dataset=dataset_factory.DatasetConfig(dtype=dtype),
- model=base_configs.ModelConfig(),
- )
-
- class EmptyClass:
- pass
- fake_ds_builder = EmptyClass()
- fake_ds_builder.dtype = dtype
- fake_ds_builder.config = EmptyClass()
- classifier_trainer.initialize(config, fake_ds_builder)
-
- def test_resume_from_checkpoint(self):
- """Tests functionality for resuming from checkpoint."""
- # Set the keras policy
- policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')
- tf.keras.mixed_precision.experimental.set_policy(policy)
-
- # Get the model, datasets, and compile it.
- model = get_trivial_model(10)
-
- # Create the checkpoint
- model_dir = self.get_temp_dir()
- train_epochs = 1
- train_steps = 10
- ds = get_trivial_data()
- callbacks = [
- tf.keras.callbacks.ModelCheckpoint(
- os.path.join(model_dir, 'model.ckpt-{epoch:04d}'),
- save_weights_only=True)
- ]
- model.fit(
- ds,
- callbacks=callbacks,
- epochs=train_epochs,
- steps_per_epoch=train_steps)
-
- # Test load from checkpoint
- clean_model = get_trivial_model(10)
- weights_before_load = copy.deepcopy(clean_model.get_weights())
- initial_epoch = classifier_trainer.resume_from_checkpoint(
- model=clean_model,
- model_dir=model_dir,
- train_steps=train_steps)
- self.assertEqual(initial_epoch, 1)
- self.assertNotAllClose(weights_before_load, clean_model.get_weights())
-
- tf.io.gfile.rmtree(model_dir)
-
- def test_serialize_config(self):
- """Tests functionality for serializing data."""
- config = base_configs.ExperimentConfig()
- model_dir = self.get_temp_dir()
- classifier_trainer.serialize_config(params=config, model_dir=model_dir)
- saved_params_path = os.path.join(model_dir, 'params.yaml')
- self.assertTrue(os.path.exists(saved_params_path))
- tf.io.gfile.rmtree(model_dir)
-
if __name__ == '__main__':
tf.test.main()
diff --git a/official/vision/image_classification/classifier_trainer_util_test.py b/official/vision/image_classification/classifier_trainer_util_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..d3624c286fdc716e4a09df56fbb8157fa35602aa
--- /dev/null
+++ b/official/vision/image_classification/classifier_trainer_util_test.py
@@ -0,0 +1,166 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Unit tests for the classifier trainer models."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+import os
+
+from absl.testing import parameterized
+import tensorflow as tf
+
+from official.vision.image_classification import classifier_trainer
+from official.vision.image_classification import dataset_factory
+from official.vision.image_classification import test_utils
+from official.vision.image_classification.configs import base_configs
+
+
+def get_trivial_model(num_classes: int) -> tf.keras.Model:
+ """Creates and compiles trivial model for ImageNet dataset."""
+ model = test_utils.trivial_model(num_classes=num_classes)
+ lr = 0.01
+ optimizer = tf.keras.optimizers.SGD(learning_rate=lr)
+ loss_obj = tf.keras.losses.SparseCategoricalCrossentropy()
+ model.compile(optimizer=optimizer, loss=loss_obj, run_eagerly=True)
+ return model
+
+
+def get_trivial_data() -> tf.data.Dataset:
+ """Gets trivial data in the ImageNet size."""
+
+ def generate_data(_) -> tf.data.Dataset:
+ image = tf.zeros(shape=(224, 224, 3), dtype=tf.float32)
+ label = tf.zeros([1], dtype=tf.int32)
+ return image, label
+
+ dataset = tf.data.Dataset.range(1)
+ dataset = dataset.repeat()
+ dataset = dataset.map(
+ generate_data, num_parallel_calls=tf.data.experimental.AUTOTUNE)
+ dataset = dataset.prefetch(buffer_size=1).batch(1)
+ return dataset
+
+
+class UtilTests(parameterized.TestCase, tf.test.TestCase):
+ """Tests for individual utility functions within classifier_trainer.py."""
+
+ @parameterized.named_parameters(
+ ('efficientnet-b0', 'efficientnet', 'efficientnet-b0', 224),
+ ('efficientnet-b1', 'efficientnet', 'efficientnet-b1', 240),
+ ('efficientnet-b2', 'efficientnet', 'efficientnet-b2', 260),
+ ('efficientnet-b3', 'efficientnet', 'efficientnet-b3', 300),
+ ('efficientnet-b4', 'efficientnet', 'efficientnet-b4', 380),
+ ('efficientnet-b5', 'efficientnet', 'efficientnet-b5', 456),
+ ('efficientnet-b6', 'efficientnet', 'efficientnet-b6', 528),
+ ('efficientnet-b7', 'efficientnet', 'efficientnet-b7', 600),
+ ('resnet', 'resnet', '', None),
+ )
+ def test_get_model_size(self, model, model_name, expected):
+ config = base_configs.ExperimentConfig(
+ model_name=model,
+ model=base_configs.ModelConfig(
+ model_params={
+ 'model_name': model_name,
+ },))
+ size = classifier_trainer.get_image_size_from_model(config)
+ self.assertEqual(size, expected)
+
+ @parameterized.named_parameters(
+ ('dynamic', 'dynamic', None, 'dynamic'),
+ ('scalar', 128., None, 128.),
+ ('float32', None, 'float32', 1),
+ ('float16', None, 'float16', 128),
+ )
+ def test_get_loss_scale(self, loss_scale, dtype, expected):
+ config = base_configs.ExperimentConfig(
+ runtime=base_configs.RuntimeConfig(loss_scale=loss_scale),
+ train_dataset=dataset_factory.DatasetConfig(dtype=dtype))
+ ls = classifier_trainer.get_loss_scale(config, fp16_default=128)
+ self.assertEqual(ls, expected)
+
+ @parameterized.named_parameters(('float16', 'float16'),
+ ('bfloat16', 'bfloat16'))
+ def test_initialize(self, dtype):
+ config = base_configs.ExperimentConfig(
+ runtime=base_configs.RuntimeConfig(
+ run_eagerly=False,
+ enable_xla=False,
+ per_gpu_thread_count=1,
+ gpu_thread_mode='gpu_private',
+ num_gpus=1,
+ dataset_num_private_threads=1,
+ ),
+ train_dataset=dataset_factory.DatasetConfig(dtype=dtype),
+ model=base_configs.ModelConfig(),
+ )
+
+ class EmptyClass:
+ pass
+
+ fake_ds_builder = EmptyClass()
+ fake_ds_builder.dtype = dtype
+ fake_ds_builder.config = EmptyClass()
+ classifier_trainer.initialize(config, fake_ds_builder)
+
+ def test_resume_from_checkpoint(self):
+ """Tests functionality for resuming from checkpoint."""
+ # Set the keras policy
+ tf.keras.mixed_precision.set_global_policy('mixed_bfloat16')
+
+ # Get the model, datasets, and compile it.
+ model = get_trivial_model(10)
+
+ # Create the checkpoint
+ model_dir = self.create_tempdir().full_path
+ train_epochs = 1
+ train_steps = 10
+ ds = get_trivial_data()
+ callbacks = [
+ tf.keras.callbacks.ModelCheckpoint(
+ os.path.join(model_dir, 'model.ckpt-{epoch:04d}'),
+ save_weights_only=True)
+ ]
+ model.fit(
+ ds,
+ callbacks=callbacks,
+ epochs=train_epochs,
+ steps_per_epoch=train_steps)
+
+ # Test load from checkpoint
+ clean_model = get_trivial_model(10)
+ weights_before_load = copy.deepcopy(clean_model.get_weights())
+ initial_epoch = classifier_trainer.resume_from_checkpoint(
+ model=clean_model, model_dir=model_dir, train_steps=train_steps)
+ self.assertEqual(initial_epoch, 1)
+ self.assertNotAllClose(weights_before_load, clean_model.get_weights())
+
+ tf.io.gfile.rmtree(model_dir)
+
+ def test_serialize_config(self):
+ """Tests functionality for serializing data."""
+ config = base_configs.ExperimentConfig()
+ model_dir = self.create_tempdir().full_path
+ classifier_trainer.serialize_config(params=config, model_dir=model_dir)
+ saved_params_path = os.path.join(model_dir, 'params.yaml')
+ self.assertTrue(os.path.exists(saved_params_path))
+ tf.io.gfile.rmtree(model_dir)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/image_classification/configs/__init__.py b/official/vision/image_classification/configs/__init__.py
index 931c2ef11db4a949e6c2e95bca44e36bac1241e9..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/image_classification/configs/__init__.py
+++ b/official/vision/image_classification/configs/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,4 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
diff --git a/official/vision/image_classification/configs/base_configs.py b/official/vision/image_classification/configs/base_configs.py
index 11fcb5305660ec71153ebfc12631f455a3464115..6ae085260e13ebb1623867c88525c99f8b5d2698 100644
--- a/official/vision/image_classification/configs/base_configs.py
+++ b/official/vision/image_classification/configs/base_configs.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,22 +11,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""Definitions for high level configuration groups.."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
+# Lint as: python3
+"""Definitions for high level configuration groups.."""
from typing import Any, List, Mapping, Optional
import dataclasses
+from official.core import config_definitions
from official.modeling import hyperparams
-from official.modeling.hyperparams import config_definitions
+from official.modeling.hyperparams import config_definitions as legacy_cfg
-CallbacksConfig = config_definitions.CallbacksConfig
-TensorboardConfig = config_definitions.TensorboardConfig
+CallbacksConfig = legacy_cfg.CallbacksConfig
+TensorboardConfig = legacy_cfg.TensorboardConfig
RuntimeConfig = config_definitions.RuntimeConfig
@@ -79,7 +75,7 @@ class TrainConfig(hyperparams.Config):
callbacks: An instance of CallbacksConfig.
metrics: An instance of MetricsConfig.
tensorboard: An instance of TensorboardConfig.
- set_epoch_loop: Whether or not to set `experimental_steps_per_execution` to
+ set_epoch_loop: Whether or not to set `steps_per_execution` to
equal the number of training steps in `model.compile`. This reduces the
number of callbacks run per epoch which significantly improves end-to-end
TPU training time.
diff --git a/official/vision/image_classification/configs/configs.py b/official/vision/image_classification/configs/configs.py
index 8a79a1cd9b563a554614b9d4f2f0b93acf016791..413de5f542aed3f25e2f84d194a0e60c2c862da6 100644
--- a/official/vision/image_classification/configs/configs.py
+++ b/official/vision/image_classification/configs/configs.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,7 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Configuration utils for image classification experiments."""
from __future__ import absolute_import
from __future__ import division
@@ -37,7 +37,6 @@ class EfficientNetImageNetConfig(base_configs.ExperimentConfig):
train: A `TrainConfig` instance.
evaluation: An `EvalConfig` instance.
model: A `ModelConfig` instance.
-
"""
export: base_configs.ExportConfig = base_configs.ExportConfig()
runtime: base_configs.RuntimeConfig = base_configs.RuntimeConfig()
@@ -49,16 +48,15 @@ class EfficientNetImageNetConfig(base_configs.ExperimentConfig):
resume_checkpoint=True,
epochs=500,
steps=None,
- callbacks=base_configs.CallbacksConfig(enable_checkpoint_and_export=True,
- enable_tensorboard=True),
+ callbacks=base_configs.CallbacksConfig(
+ enable_checkpoint_and_export=True, enable_tensorboard=True),
metrics=['accuracy', 'top_5'],
time_history=base_configs.TimeHistoryConfig(log_steps=100),
- tensorboard=base_configs.TensorboardConfig(track_lr=True,
- write_model_weights=False),
+ tensorboard=base_configs.TensorboardConfig(
+ track_lr=True, write_model_weights=False),
set_epoch_loop=False)
evaluation: base_configs.EvalConfig = base_configs.EvalConfig(
- epochs_between_evals=1,
- steps=None)
+ epochs_between_evals=1, steps=None)
model: base_configs.ModelConfig = \
efficientnet_config.EfficientNetModelConfig()
@@ -82,16 +80,15 @@ class ResNetImagenetConfig(base_configs.ExperimentConfig):
resume_checkpoint=True,
epochs=90,
steps=None,
- callbacks=base_configs.CallbacksConfig(enable_checkpoint_and_export=True,
- enable_tensorboard=True),
+ callbacks=base_configs.CallbacksConfig(
+ enable_checkpoint_and_export=True, enable_tensorboard=True),
metrics=['accuracy', 'top_5'],
time_history=base_configs.TimeHistoryConfig(log_steps=100),
- tensorboard=base_configs.TensorboardConfig(track_lr=True,
- write_model_weights=False),
+ tensorboard=base_configs.TensorboardConfig(
+ track_lr=True, write_model_weights=False),
set_epoch_loop=False)
evaluation: base_configs.EvalConfig = base_configs.EvalConfig(
- epochs_between_evals=1,
- steps=None)
+ epochs_between_evals=1, steps=None)
model: base_configs.ModelConfig = resnet_config.ResNetModelConfig()
@@ -109,10 +106,8 @@ def get_config(model: str, dataset: str) -> base_configs.ExperimentConfig:
if dataset not in dataset_model_config_map:
raise KeyError('Invalid dataset received. Received: {}. Supported '
'datasets include: {}'.format(
- dataset,
- ', '.join(dataset_model_config_map.keys())))
+ dataset, ', '.join(dataset_model_config_map.keys())))
raise KeyError('Invalid model received. Received: {}. Supported models for'
'{} include: {}'.format(
- model,
- dataset,
+ model, dataset,
', '.join(dataset_model_config_map[dataset].keys())))
diff --git a/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml b/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml
index 56844b81db70fbd5e8291a4c1c2eb60e3c488088..2037d6b5d1c39b9ff898eaf49ec7a68e3987356b 100644
--- a/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml
+++ b/official/vision/image_classification/configs/examples/resnet/imagenet/gpu.yaml
@@ -40,8 +40,6 @@ model:
momentum: 0.9
decay: 0.9
epsilon: 0.001
- learning_rate:
- name: 'piecewise_constant_with_warmup'
loss:
label_smoothing: 0.1
train:
diff --git a/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml b/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml
index ae975c16251ac0a23877bf8f6804cdea6b2baadf..0a3030333bb42ce59e67cfbe12a12be877ab19d0 100644
--- a/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml
+++ b/official/vision/image_classification/configs/examples/resnet/imagenet/tpu.yaml
@@ -43,8 +43,6 @@ model:
epsilon: 0.001
moving_average_decay: 0.
lookahead: False
- learning_rate:
- name: 'piecewise_constant_with_warmup'
loss:
label_smoothing: 0.1
train:
diff --git a/official/vision/image_classification/dataset_factory.py b/official/vision/image_classification/dataset_factory.py
index e9dad1268a7bed86f622f80ca28f4d485a0fab31..463de95c77ec1e4eb7b6187181a2fe54121bb6a1 100644
--- a/official/vision/image_classification/dataset_factory.py
+++ b/official/vision/image_classification/dataset_factory.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,7 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Dataset utilities for vision tasks using TFDS and tf.data.Dataset."""
from __future__ import absolute_import
from __future__ import division
@@ -21,6 +21,7 @@ from __future__ import print_function
import os
from typing import Any, List, Optional, Tuple, Mapping, Union
+
from absl import logging
from dataclasses import dataclass
import tensorflow as tf
@@ -30,7 +31,6 @@ from official.modeling.hyperparams import base_config
from official.vision.image_classification import augment
from official.vision.image_classification import preprocessing
-
AUGMENTERS = {
'autoaugment': augment.AutoAugment,
'randaugment': augment.RandAugment,
@@ -42,8 +42,8 @@ class AugmentConfig(base_config.Config):
"""Configuration for image augmenters.
Attributes:
- name: The name of the image augmentation to use. Possible options are
- None (default), 'autoaugment', or 'randaugment'.
+ name: The name of the image augmentation to use. Possible options are None
+ (default), 'autoaugment', or 'randaugment'.
params: Any paramaters used to initialize the augmenter.
"""
name: Optional[str] = None
@@ -68,17 +68,17 @@ class DatasetConfig(base_config.Config):
'tfds' (load using TFDS), 'records' (load from TFRecords), or 'synthetic'
(generate dummy synthetic data without reading from files).
split: The split of the dataset. Usually 'train', 'validation', or 'test'.
- image_size: The size of the image in the dataset. This assumes that
- `width` == `height`. Set to 'infer' to infer the image size from TFDS
- info. This requires `name` to be a registered dataset in TFDS.
- num_classes: The number of classes given by the dataset. Set to 'infer'
- to infer the image size from TFDS info. This requires `name` to be a
+ image_size: The size of the image in the dataset. This assumes that `width`
+ == `height`. Set to 'infer' to infer the image size from TFDS info. This
+ requires `name` to be a registered dataset in TFDS.
+ num_classes: The number of classes given by the dataset. Set to 'infer' to
+ infer the image size from TFDS info. This requires `name` to be a
registered dataset in TFDS.
- num_channels: The number of channels given by the dataset. Set to 'infer'
- to infer the image size from TFDS info. This requires `name` to be a
+ num_channels: The number of channels given by the dataset. Set to 'infer' to
+ infer the image size from TFDS info. This requires `name` to be a
registered dataset in TFDS.
- num_examples: The number of examples given by the dataset. Set to 'infer'
- to infer the image size from TFDS info. This requires `name` to be a
+ num_examples: The number of examples given by the dataset. Set to 'infer' to
+ infer the image size from TFDS info. This requires `name` to be a
registered dataset in TFDS.
batch_size: The base batch size for the dataset.
use_per_replica_batch_size: Whether to scale the batch size based on
@@ -143,6 +143,9 @@ class ImageNetConfig(DatasetConfig):
# Note: for large datasets like ImageNet, using records is faster than tfds
builder: str = 'records'
image_size: int = 224
+ num_channels: int = 3
+ num_examples: int = 1281167
+ num_classes: int = 1000
batch_size: int = 128
@@ -267,8 +270,14 @@ class DatasetBuilder:
@property
def info(self) -> tfds.core.DatasetInfo:
"""The TFDS dataset info, if available."""
- if self.builder_info is None:
- self.builder_info = tfds.builder(self.config.name).info
+ try:
+ if self.builder_info is None:
+ self.builder_info = tfds.builder(self.config.name).info
+ except ConnectionError as e:
+ logging.error('Failed to use TFDS to load info. Please set dataset info '
+ '(image_size, num_channels, num_examples, num_classes) in '
+ 'the dataset config.')
+ raise e
return self.builder_info
def build(self, strategy: tf.distribute.Strategy = None) -> tf.data.Dataset:
@@ -284,19 +293,19 @@ class DatasetBuilder:
"""
if strategy:
if strategy.num_replicas_in_sync != self.config.num_devices:
- logging.warn('Passed a strategy with %d devices, but expected'
- '%d devices.',
- strategy.num_replicas_in_sync,
- self.config.num_devices)
- dataset = strategy.experimental_distribute_datasets_from_function(
- self._build)
+ logging.warn(
+ 'Passed a strategy with %d devices, but expected'
+ '%d devices.', strategy.num_replicas_in_sync,
+ self.config.num_devices)
+ dataset = strategy.distribute_datasets_from_function(self._build)
else:
dataset = self._build()
return dataset
- def _build(self, input_context: tf.distribute.InputContext = None
- ) -> tf.data.Dataset:
+ def _build(
+ self,
+ input_context: tf.distribute.InputContext = None) -> tf.data.Dataset:
"""Construct a dataset end-to-end and return it.
Args:
@@ -328,8 +337,7 @@ class DatasetBuilder:
logging.info('Using TFDS to load data.')
- builder = tfds.builder(self.config.name,
- data_dir=self.config.data_dir)
+ builder = tfds.builder(self.config.name, data_dir=self.config.data_dir)
if self.config.download:
builder.download_and_prepare()
@@ -380,8 +388,8 @@ class DatasetBuilder:
dataset = tf.data.Dataset.range(1)
dataset = dataset.repeat()
- dataset = dataset.map(generate_data,
- num_parallel_calls=tf.data.experimental.AUTOTUNE)
+ dataset = dataset.map(
+ generate_data, num_parallel_calls=tf.data.experimental.AUTOTUNE)
return dataset
def pipeline(self, dataset: tf.data.Dataset) -> tf.data.Dataset:
@@ -393,14 +401,14 @@ class DatasetBuilder:
Returns:
A TensorFlow dataset outputting batched images and labels.
"""
- if (self.config.builder != 'tfds' and self.input_context
- and self.input_context.num_input_pipelines > 1):
+ if (self.config.builder != 'tfds' and self.input_context and
+ self.input_context.num_input_pipelines > 1):
dataset = dataset.shard(self.input_context.num_input_pipelines,
self.input_context.input_pipeline_id)
- logging.info('Sharding the dataset: input_pipeline_id=%d '
- 'num_input_pipelines=%d',
- self.input_context.num_input_pipelines,
- self.input_context.input_pipeline_id)
+ logging.info(
+ 'Sharding the dataset: input_pipeline_id=%d '
+ 'num_input_pipelines=%d', self.input_context.num_input_pipelines,
+ self.input_context.input_pipeline_id)
if self.is_training and self.config.builder == 'records':
# Shuffle the input files.
@@ -429,8 +437,8 @@ class DatasetBuilder:
preprocess = self.parse_record
else:
preprocess = self.preprocess
- dataset = dataset.map(preprocess,
- num_parallel_calls=tf.data.experimental.AUTOTUNE)
+ dataset = dataset.map(
+ preprocess, num_parallel_calls=tf.data.experimental.AUTOTUNE)
if self.input_context and self.config.num_devices > 1:
if not self.config.use_per_replica_batch_size:
@@ -444,11 +452,11 @@ class DatasetBuilder:
# The batch size of the dataset will be multiplied by the number of
# replicas automatically when strategy.distribute_datasets_from_function
# is called, so we use local batch size here.
- dataset = dataset.batch(self.local_batch_size,
- drop_remainder=self.is_training)
+ dataset = dataset.batch(
+ self.local_batch_size, drop_remainder=self.is_training)
else:
- dataset = dataset.batch(self.global_batch_size,
- drop_remainder=self.is_training)
+ dataset = dataset.batch(
+ self.global_batch_size, drop_remainder=self.is_training)
# Prefetch overlaps in-feed with training
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
@@ -470,24 +478,15 @@ class DatasetBuilder:
def parse_record(self, record: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]:
"""Parse an ImageNet record from a serialized string Tensor."""
keys_to_features = {
- 'image/encoded':
- tf.io.FixedLenFeature((), tf.string, ''),
- 'image/format':
- tf.io.FixedLenFeature((), tf.string, 'jpeg'),
- 'image/class/label':
- tf.io.FixedLenFeature([], tf.int64, -1),
- 'image/class/text':
- tf.io.FixedLenFeature([], tf.string, ''),
- 'image/object/bbox/xmin':
- tf.io.VarLenFeature(dtype=tf.float32),
- 'image/object/bbox/ymin':
- tf.io.VarLenFeature(dtype=tf.float32),
- 'image/object/bbox/xmax':
- tf.io.VarLenFeature(dtype=tf.float32),
- 'image/object/bbox/ymax':
- tf.io.VarLenFeature(dtype=tf.float32),
- 'image/object/class/label':
- tf.io.VarLenFeature(dtype=tf.int64),
+ 'image/encoded': tf.io.FixedLenFeature((), tf.string, ''),
+ 'image/format': tf.io.FixedLenFeature((), tf.string, 'jpeg'),
+ 'image/class/label': tf.io.FixedLenFeature([], tf.int64, -1),
+ 'image/class/text': tf.io.FixedLenFeature([], tf.string, ''),
+ 'image/object/bbox/xmin': tf.io.VarLenFeature(dtype=tf.float32),
+ 'image/object/bbox/ymin': tf.io.VarLenFeature(dtype=tf.float32),
+ 'image/object/bbox/xmax': tf.io.VarLenFeature(dtype=tf.float32),
+ 'image/object/bbox/ymax': tf.io.VarLenFeature(dtype=tf.float32),
+ 'image/object/class/label': tf.io.VarLenFeature(dtype=tf.int64),
}
parsed = tf.io.parse_single_example(record, keys_to_features)
@@ -502,8 +501,8 @@ class DatasetBuilder:
return image, label
- def preprocess(self, image: tf.Tensor, label: tf.Tensor
- ) -> Tuple[tf.Tensor, tf.Tensor]:
+ def preprocess(self, image: tf.Tensor,
+ label: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]:
"""Apply image preprocessing and augmentation to the image and label."""
if self.is_training:
image = preprocessing.preprocess_for_train(
diff --git a/official/vision/image_classification/efficientnet/__init__.py b/official/vision/image_classification/efficientnet/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/image_classification/efficientnet/__init__.py
+++ b/official/vision/image_classification/efficientnet/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/official/vision/image_classification/efficientnet/common_modules.py b/official/vision/image_classification/efficientnet/common_modules.py
index 9c9c2097d2398ec78cae5e1265478f804860f944..e3657bd862b4dca1cb678e14de8e0bd7568eceee 100644
--- a/official/vision/image_classification/efficientnet/common_modules.py
+++ b/official/vision/image_classification/efficientnet/common_modules.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Common modeling utilities."""
from __future__ import absolute_import
from __future__ import division
@@ -79,7 +79,7 @@ def get_batch_norm(batch_norm_type: Text) -> tf.keras.layers.BatchNormalization:
Args:
batch_norm_type: The type of batch normalization layer implementation. `tpu`
- will use `TpuBatchNormalization`.
+ will use `TpuBatchNormalization`.
Returns:
An instance of `tf.keras.layers.BatchNormalization`.
@@ -95,8 +95,10 @@ def count_params(model, trainable_only=True):
if not trainable_only:
return model.count_params()
else:
- return int(np.sum([tf.keras.backend.count_params(p)
- for p in model.trainable_weights]))
+ return int(
+ np.sum([
+ tf.keras.backend.count_params(p) for p in model.trainable_weights
+ ]))
def load_weights(model: tf.keras.Model,
@@ -107,8 +109,8 @@ def load_weights(model: tf.keras.Model,
Args:
model: the model to load weights into
model_weights_path: the path of the model weights
- weights_format: the model weights format. One of 'saved_model', 'h5',
- or 'checkpoint'.
+ weights_format: the model weights format. One of 'saved_model', 'h5', or
+ 'checkpoint'.
"""
if weights_format == 'saved_model':
loaded_model = tf.keras.models.load_model(model_weights_path)
diff --git a/official/vision/image_classification/efficientnet/efficientnet_config.py b/official/vision/image_classification/efficientnet/efficientnet_config.py
index a758cc63c944463ebf184eaeae26cebd5935031a..47cfd740221d3581db585e90bc6df0711c289019 100644
--- a/official/vision/image_classification/efficientnet/efficientnet_config.py
+++ b/official/vision/image_classification/efficientnet/efficientnet_config.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,7 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Configuration definitions for EfficientNet losses, learning rates, and optimizers."""
from __future__ import absolute_import
from __future__ import division
diff --git a/official/vision/image_classification/efficientnet/efficientnet_model.py b/official/vision/image_classification/efficientnet/efficientnet_model.py
index ab81fc25d1200557c99f77424d34c74cf8774d84..e5f2c2c69fdea82288f286971395b6f9aec3f500 100644
--- a/official/vision/image_classification/efficientnet/efficientnet_model.py
+++ b/official/vision/image_classification/efficientnet/efficientnet_model.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,7 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Contains definitions for EfficientNet model.
[1] Mingxing Tan, Quoc V. Le
@@ -64,11 +64,11 @@ class ModelConfig(base_config.Config):
# (input_filters, output_filters, kernel_size, num_repeat,
# expand_ratio, strides, se_ratio)
# pylint: disable=bad-whitespace
- BlockConfig.from_args(32, 16, 3, 1, 1, (1, 1), 0.25),
- BlockConfig.from_args(16, 24, 3, 2, 6, (2, 2), 0.25),
- BlockConfig.from_args(24, 40, 5, 2, 6, (2, 2), 0.25),
- BlockConfig.from_args(40, 80, 3, 3, 6, (2, 2), 0.25),
- BlockConfig.from_args(80, 112, 5, 3, 6, (1, 1), 0.25),
+ BlockConfig.from_args(32, 16, 3, 1, 1, (1, 1), 0.25),
+ BlockConfig.from_args(16, 24, 3, 2, 6, (2, 2), 0.25),
+ BlockConfig.from_args(24, 40, 5, 2, 6, (2, 2), 0.25),
+ BlockConfig.from_args(40, 80, 3, 3, 6, (2, 2), 0.25),
+ BlockConfig.from_args(80, 112, 5, 3, 6, (1, 1), 0.25),
BlockConfig.from_args(112, 192, 5, 4, 6, (2, 2), 0.25),
BlockConfig.from_args(192, 320, 3, 1, 6, (1, 1), 0.25),
# pylint: enable=bad-whitespace
@@ -128,8 +128,7 @@ DENSE_KERNEL_INITIALIZER = {
}
-def round_filters(filters: int,
- config: ModelConfig) -> int:
+def round_filters(filters: int, config: ModelConfig) -> int:
"""Round number of filters based on width coefficient."""
width_coefficient = config.width_coefficient
min_depth = config.min_depth
@@ -189,21 +188,24 @@ def conv2d_block(inputs: tf.Tensor,
init_kwargs.update({'depthwise_initializer': CONV_KERNEL_INITIALIZER})
else:
conv2d = tf.keras.layers.Conv2D
- init_kwargs.update({'filters': conv_filters,
- 'kernel_initializer': CONV_KERNEL_INITIALIZER})
+ init_kwargs.update({
+ 'filters': conv_filters,
+ 'kernel_initializer': CONV_KERNEL_INITIALIZER
+ })
x = conv2d(**init_kwargs)(inputs)
if use_batch_norm:
bn_axis = 1 if data_format == 'channels_first' else -1
- x = batch_norm(axis=bn_axis,
- momentum=bn_momentum,
- epsilon=bn_epsilon,
- name=name + '_bn')(x)
+ x = batch_norm(
+ axis=bn_axis,
+ momentum=bn_momentum,
+ epsilon=bn_epsilon,
+ name=name + '_bn')(
+ x)
if activation is not None:
- x = tf.keras.layers.Activation(activation,
- name=name + '_activation')(x)
+ x = tf.keras.layers.Activation(activation, name=name + '_activation')(x)
return x
@@ -235,42 +237,43 @@ def mb_conv_block(inputs: tf.Tensor,
if block.fused_conv:
# If we use fused mbconv, skip expansion and use regular conv.
- x = conv2d_block(x,
- filters,
- config,
- kernel_size=block.kernel_size,
- strides=block.strides,
- activation=activation,
- name=prefix + 'fused')
+ x = conv2d_block(
+ x,
+ filters,
+ config,
+ kernel_size=block.kernel_size,
+ strides=block.strides,
+ activation=activation,
+ name=prefix + 'fused')
else:
if block.expand_ratio != 1:
# Expansion phase
kernel_size = (1, 1) if use_depthwise else (3, 3)
- x = conv2d_block(x,
- filters,
- config,
- kernel_size=kernel_size,
- activation=activation,
- name=prefix + 'expand')
+ x = conv2d_block(
+ x,
+ filters,
+ config,
+ kernel_size=kernel_size,
+ activation=activation,
+ name=prefix + 'expand')
# Depthwise Convolution
if use_depthwise:
- x = conv2d_block(x,
- conv_filters=None,
- config=config,
- kernel_size=block.kernel_size,
- strides=block.strides,
- activation=activation,
- depthwise=True,
- name=prefix + 'depthwise')
+ x = conv2d_block(
+ x,
+ conv_filters=None,
+ config=config,
+ kernel_size=block.kernel_size,
+ strides=block.strides,
+ activation=activation,
+ depthwise=True,
+ name=prefix + 'depthwise')
# Squeeze and Excitation phase
if use_se:
assert block.se_ratio is not None
assert 0 < block.se_ratio <= 1
- num_reduced_filters = max(1, int(
- block.input_filters * block.se_ratio
- ))
+ num_reduced_filters = max(1, int(block.input_filters * block.se_ratio))
if data_format == 'channels_first':
se_shape = (filters, 1, 1)
@@ -280,53 +283,51 @@ def mb_conv_block(inputs: tf.Tensor,
se = tf.keras.layers.GlobalAveragePooling2D(name=prefix + 'se_squeeze')(x)
se = tf.keras.layers.Reshape(se_shape, name=prefix + 'se_reshape')(se)
- se = conv2d_block(se,
- num_reduced_filters,
- config,
- use_bias=True,
- use_batch_norm=False,
- activation=activation,
- name=prefix + 'se_reduce')
- se = conv2d_block(se,
- filters,
- config,
- use_bias=True,
- use_batch_norm=False,
- activation='sigmoid',
- name=prefix + 'se_expand')
+ se = conv2d_block(
+ se,
+ num_reduced_filters,
+ config,
+ use_bias=True,
+ use_batch_norm=False,
+ activation=activation,
+ name=prefix + 'se_reduce')
+ se = conv2d_block(
+ se,
+ filters,
+ config,
+ use_bias=True,
+ use_batch_norm=False,
+ activation='sigmoid',
+ name=prefix + 'se_expand')
x = tf.keras.layers.multiply([x, se], name=prefix + 'se_excite')
# Output phase
- x = conv2d_block(x,
- block.output_filters,
- config,
- activation=None,
- name=prefix + 'project')
+ x = conv2d_block(
+ x, block.output_filters, config, activation=None, name=prefix + 'project')
# Add identity so that quantization-aware training can insert quantization
# ops correctly.
- x = tf.keras.layers.Activation(tf_utils.get_activation('identity'),
- name=prefix + 'id')(x)
+ x = tf.keras.layers.Activation(
+ tf_utils.get_activation('identity'), name=prefix + 'id')(
+ x)
- if (block.id_skip
- and all(s == 1 for s in block.strides)
- and block.input_filters == block.output_filters):
+ if (block.id_skip and all(s == 1 for s in block.strides) and
+ block.input_filters == block.output_filters):
if drop_connect_rate and drop_connect_rate > 0:
# Apply dropconnect
# The only difference between dropout and dropconnect in TF is scaling by
# drop_connect_rate during training. See:
# https://github.com/keras-team/keras/pull/9898#issuecomment-380577612
- x = tf.keras.layers.Dropout(drop_connect_rate,
- noise_shape=(None, 1, 1, 1),
- name=prefix + 'drop')(x)
+ x = tf.keras.layers.Dropout(
+ drop_connect_rate, noise_shape=(None, 1, 1, 1), name=prefix + 'drop')(
+ x)
x = tf.keras.layers.add([x, inputs], name=prefix + 'add')
return x
-def efficientnet(image_input: tf.keras.layers.Input,
- config: ModelConfig):
+def efficientnet(image_input: tf.keras.layers.Input, config: ModelConfig):
"""Creates an EfficientNet graph given the model parameters.
This function is wrapped by the `EfficientNet` class to make a tf.keras.Model.
@@ -357,19 +358,18 @@ def efficientnet(image_input: tf.keras.layers.Input,
# Happens on GPU/TPU if available.
x = tf.keras.layers.Permute((3, 1, 2))(x)
if rescale_input:
- x = preprocessing.normalize_images(x,
- num_channels=input_channels,
- dtype=dtype,
- data_format=data_format)
+ x = preprocessing.normalize_images(
+ x, num_channels=input_channels, dtype=dtype, data_format=data_format)
# Build stem
- x = conv2d_block(x,
- round_filters(stem_base_filters, config),
- config,
- kernel_size=[3, 3],
- strides=[2, 2],
- activation=activation,
- name='stem')
+ x = conv2d_block(
+ x,
+ round_filters(stem_base_filters, config),
+ config,
+ kernel_size=[3, 3],
+ strides=[2, 2],
+ activation=activation,
+ name='stem')
# Build blocks
num_blocks_total = sum(
@@ -391,10 +391,7 @@ def efficientnet(image_input: tf.keras.layers.Input,
x = mb_conv_block(x, block, config, block_prefix)
block_num += 1
if block.num_repeat > 1:
- block = block.replace(
- input_filters=block.output_filters,
- strides=[1, 1]
- )
+ block = block.replace(input_filters=block.output_filters, strides=[1, 1])
for block_idx in range(block.num_repeat - 1):
drop_rate = drop_connect_rate * float(block_num) / num_blocks_total
@@ -404,11 +401,12 @@ def efficientnet(image_input: tf.keras.layers.Input,
block_num += 1
# Build top
- x = conv2d_block(x,
- round_filters(top_base_filters, config),
- config,
- activation=activation,
- name='top')
+ x = conv2d_block(
+ x,
+ round_filters(top_base_filters, config),
+ config,
+ activation=activation,
+ name='top')
# Build classifier
x = tf.keras.layers.GlobalAveragePooling2D(name='top_pool')(x)
@@ -419,7 +417,8 @@ def efficientnet(image_input: tf.keras.layers.Input,
kernel_initializer=DENSE_KERNEL_INITIALIZER,
kernel_regularizer=tf.keras.regularizers.l2(weight_decay),
bias_regularizer=tf.keras.regularizers.l2(weight_decay),
- name='logits')(x)
+ name='logits')(
+ x)
x = tf.keras.layers.Activation('softmax', name='probs')(x)
return x
@@ -439,8 +438,7 @@ class EfficientNet(tf.keras.Model):
Args:
config: (optional) the main model parameters to create the model
- overrides: (optional) a dict containing keys that can override
- config
+ overrides: (optional) a dict containing keys that can override config
"""
overrides = overrides or {}
config = config or ModelConfig()
@@ -457,9 +455,7 @@ class EfficientNet(tf.keras.Model):
# Cast to float32 in case we have a different model dtype
output = tf.cast(output, tf.float32)
- logging.info('Building model %s with params %s',
- model_name,
- self.config)
+ logging.info('Building model %s with params %s', model_name, self.config)
super(EfficientNet, self).__init__(
inputs=image_input, outputs=output, name=model_name)
@@ -477,8 +473,8 @@ class EfficientNet(tf.keras.Model):
Args:
model_name: the predefined model name
model_weights_path: the path to the weights (h5 file or saved model dir)
- weights_format: the model weights format. One of 'saved_model', 'h5',
- or 'checkpoint'.
+ weights_format: the model weights format. One of 'saved_model', 'h5', or
+ 'checkpoint'.
overrides: (optional) a dict containing keys that can override config
Returns:
@@ -498,8 +494,7 @@ class EfficientNet(tf.keras.Model):
model = cls(config=config, overrides=overrides)
if model_weights_path:
- common_modules.load_weights(model,
- model_weights_path,
- weights_format=weights_format)
+ common_modules.load_weights(
+ model, model_weights_path, weights_format=weights_format)
return model
diff --git a/official/vision/image_classification/efficientnet/tfhub_export.py b/official/vision/image_classification/efficientnet/tfhub_export.py
index 3be8608a5cfc25442f5f936b4052f90b89c6cfce..691e568fa0a32abaa536deff6ee8a13621c1ee7e 100644
--- a/official/vision/image_classification/efficientnet/tfhub_export.py
+++ b/official/vision/image_classification/efficientnet/tfhub_export.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""A script to export TF-Hub SavedModel."""
from __future__ import absolute_import
@@ -30,10 +30,8 @@ from official.vision.image_classification.efficientnet import efficientnet_model
FLAGS = flags.FLAGS
-flags.DEFINE_string("model_name", None,
- "EfficientNet model name.")
-flags.DEFINE_string("model_path", None,
- "File path to TF model checkpoint.")
+flags.DEFINE_string("model_name", None, "EfficientNet model name.")
+flags.DEFINE_string("model_path", None, "File path to TF model checkpoint.")
flags.DEFINE_string("export_path", None,
"TF-Hub SavedModel destination path to export.")
@@ -65,5 +63,6 @@ def main(argv):
export_tfhub(FLAGS.model_path, FLAGS.export_path, FLAGS.model_name)
+
if __name__ == "__main__":
app.run(main)
diff --git a/official/vision/image_classification/learning_rate.py b/official/vision/image_classification/learning_rate.py
index 1c78b04bc6297a08a8bc7823dccc00f464e05ad4..72f7e95187521eeebefa1e698ca5382f10642e88 100644
--- a/official/vision/image_classification/learning_rate.py
+++ b/official/vision/image_classification/learning_rate.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,13 +11,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Learning rate utilities for vision tasks."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
-from typing import Any, List, Mapping
+from typing import Any, Mapping, Optional
import numpy as np
import tensorflow as tf
@@ -29,32 +29,39 @@ BASE_LEARNING_RATE = 0.1
class WarmupDecaySchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
"""A wrapper for LearningRateSchedule that includes warmup steps."""
- def __init__(
- self,
- lr_schedule: tf.keras.optimizers.schedules.LearningRateSchedule,
- warmup_steps: int):
+ def __init__(self,
+ lr_schedule: tf.keras.optimizers.schedules.LearningRateSchedule,
+ warmup_steps: int,
+ warmup_lr: Optional[float] = None):
"""Add warmup decay to a learning rate schedule.
Args:
lr_schedule: base learning rate scheduler
warmup_steps: number of warmup steps
-
+ warmup_lr: an optional field for the final warmup learning rate. This
+ should be provided if the base `lr_schedule` does not contain this
+ field.
"""
super(WarmupDecaySchedule, self).__init__()
self._lr_schedule = lr_schedule
self._warmup_steps = warmup_steps
+ self._warmup_lr = warmup_lr
def __call__(self, step: int):
lr = self._lr_schedule(step)
if self._warmup_steps:
- initial_learning_rate = tf.convert_to_tensor(
- self._lr_schedule.initial_learning_rate, name="initial_learning_rate")
+ if self._warmup_lr is not None:
+ initial_learning_rate = tf.convert_to_tensor(
+ self._warmup_lr, name="initial_learning_rate")
+ else:
+ initial_learning_rate = tf.convert_to_tensor(
+ self._lr_schedule.initial_learning_rate,
+ name="initial_learning_rate")
dtype = initial_learning_rate.dtype
global_step_recomp = tf.cast(step, dtype)
warmup_steps = tf.cast(self._warmup_steps, dtype)
warmup_lr = initial_learning_rate * global_step_recomp / warmup_steps
- lr = tf.cond(global_step_recomp < warmup_steps,
- lambda: warmup_lr,
+ lr = tf.cond(global_step_recomp < warmup_steps, lambda: warmup_lr,
lambda: lr)
return lr
@@ -62,65 +69,11 @@ class WarmupDecaySchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
config = self._lr_schedule.get_config()
config.update({
"warmup_steps": self._warmup_steps,
+ "warmup_lr": self._warmup_lr,
})
return config
-# TODO(b/149030439) - refactor this with
-# tf.keras.optimizers.schedules.PiecewiseConstantDecay + WarmupDecaySchedule.
-class PiecewiseConstantDecayWithWarmup(
- tf.keras.optimizers.schedules.LearningRateSchedule):
- """Piecewise constant decay with warmup schedule."""
-
- def __init__(self,
- batch_size: int,
- epoch_size: int,
- warmup_epochs: int,
- boundaries: List[int],
- multipliers: List[float]):
- """Piecewise constant decay with warmup.
-
- Args:
- batch_size: The training batch size used in the experiment.
- epoch_size: The size of an epoch, or the number of examples in an epoch.
- warmup_epochs: The number of warmup epochs to apply.
- boundaries: The list of floats with strictly increasing entries.
- multipliers: The list of multipliers/learning rates to use for the
- piecewise portion. The length must be 1 less than that of boundaries.
-
- """
- super(PiecewiseConstantDecayWithWarmup, self).__init__()
- if len(boundaries) != len(multipliers) - 1:
- raise ValueError("The length of boundaries must be 1 less than the "
- "length of multipliers")
-
- base_lr_batch_size = 256
- steps_per_epoch = epoch_size // batch_size
-
- self._rescaled_lr = BASE_LEARNING_RATE * batch_size / base_lr_batch_size
- self._step_boundaries = [float(steps_per_epoch) * x for x in boundaries]
- self._lr_values = [self._rescaled_lr * m for m in multipliers]
- self._warmup_steps = warmup_epochs * steps_per_epoch
-
- def __call__(self, step: int):
- """Compute learning rate at given step."""
- def warmup_lr():
- return self._rescaled_lr * (
- step / tf.cast(self._warmup_steps, tf.float32))
- def piecewise_lr():
- return tf.compat.v1.train.piecewise_constant(
- tf.cast(step, tf.float32), self._step_boundaries, self._lr_values)
- return tf.cond(step < self._warmup_steps, warmup_lr, piecewise_lr)
-
- def get_config(self) -> Mapping[str, Any]:
- return {
- "rescaled_lr": self._rescaled_lr,
- "step_boundaries": self._step_boundaries,
- "lr_values": self._lr_values,
- "warmup_steps": self._warmup_steps,
- }
-
-
class CosineDecayWithWarmup(tf.keras.optimizers.schedules.LearningRateSchedule):
"""Class to generate learning rate tensor."""
diff --git a/official/vision/image_classification/learning_rate_test.py b/official/vision/image_classification/learning_rate_test.py
index 272d2935fd7f1e6a7f1810e9247c4ef505021fde..6c33ed24b8e46b8ecb58005a1f528e62a66f0005 100644
--- a/official/vision/image_classification/learning_rate_test.py
+++ b/official/vision/image_classification/learning_rate_test.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Tests for learning_rate."""
from __future__ import absolute_import
@@ -37,52 +37,13 @@ class LearningRateTests(tf.test.TestCase):
decay_steps=decay_steps,
decay_rate=decay_rate)
lr = learning_rate.WarmupDecaySchedule(
- lr_schedule=base_lr,
- warmup_steps=warmup_steps)
+ lr_schedule=base_lr, warmup_steps=warmup_steps)
for step in range(warmup_steps - 1):
config = lr.get_config()
self.assertEqual(config['warmup_steps'], warmup_steps)
- self.assertAllClose(self.evaluate(lr(step)),
- step / warmup_steps * initial_lr)
-
- def test_piecewise_constant_decay_with_warmup(self):
- """Basic computational test for piecewise constant decay with warmup."""
- boundaries = [1, 2, 3]
- warmup_epochs = boundaries[0]
- learning_rate_multipliers = [1.0, 0.1, 0.001]
- expected_keys = [
- 'rescaled_lr', 'step_boundaries', 'lr_values', 'warmup_steps',
- ]
-
- expected_lrs = [0.0, 0.1, 0.1]
-
- lr = learning_rate.PiecewiseConstantDecayWithWarmup(
- batch_size=256,
- epoch_size=256,
- warmup_epochs=warmup_epochs,
- boundaries=boundaries[1:],
- multipliers=learning_rate_multipliers)
-
- step = 0
-
- config = lr.get_config()
- self.assertAllInSet(list(config.keys()), expected_keys)
-
- for boundary, expected_lr in zip(boundaries, expected_lrs):
- for _ in range(step, boundary):
- self.assertAllClose(self.evaluate(lr(step)), expected_lr)
- step += 1
-
- def test_piecewise_constant_decay_invalid_boundaries(self):
- with self.assertRaisesRegex(ValueError,
- 'The length of boundaries must be 1 less '):
- learning_rate.PiecewiseConstantDecayWithWarmup(
- batch_size=256,
- epoch_size=256,
- warmup_epochs=1,
- boundaries=[1, 2],
- multipliers=[1, 2])
+ self.assertAllClose(
+ self.evaluate(lr(step)), step / warmup_steps * initial_lr)
def test_cosine_decay_with_warmup(self):
"""Basic computational test for cosine decay with warmup."""
diff --git a/official/vision/image_classification/mnist_main.py b/official/vision/image_classification/mnist_main.py
index 1470c02d05b431e95de3c5807b68678a96d2b520..3eba80b06a9215cb5dc4d3b13facb2f2a4f3058c 100644
--- a/official/vision/image_classification/mnist_main.py
+++ b/official/vision/image_classification/mnist_main.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Runs a simple model on the MNIST dataset."""
from __future__ import absolute_import
from __future__ import division
@@ -19,14 +19,14 @@ from __future__ import print_function
import os
+# Import libraries
from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
import tensorflow_datasets as tfds
-
+from official.common import distribute_utils
from official.utils.flags import core as flags_core
-from official.utils.misc import distribution_utils
from official.utils.misc import model_helpers
from official.vision.image_classification.resnet import common
@@ -81,12 +81,15 @@ def run(flags_obj, datasets_override=None, strategy_override=None):
Returns:
Dictionary of training and eval stats.
"""
- strategy = strategy_override or distribution_utils.get_distribution_strategy(
+ # Start TF profiler server.
+ tf.profiler.experimental.server.start(flags_obj.profiler_port)
+
+ strategy = strategy_override or distribute_utils.get_distribution_strategy(
distribution_strategy=flags_obj.distribution_strategy,
num_gpus=flags_obj.num_gpus,
tpu_address=flags_obj.tpu)
- strategy_scope = distribution_utils.get_strategy_scope(strategy)
+ strategy_scope = distribute_utils.get_strategy_scope(strategy)
mnist = tfds.builder('mnist', data_dir=flags_obj.data_dir)
if flags_obj.download:
@@ -154,8 +157,10 @@ def define_mnist_flags():
distribution_strategy=True)
flags_core.define_device()
flags_core.define_distribution()
- flags.DEFINE_bool('download', False,
+ flags.DEFINE_bool('download', True,
'Whether to download data to `--data_dir`.')
+ flags.DEFINE_integer('profiler_port', 9012,
+ 'Port to start profiler server on.')
FLAGS.set_default('batch_size', 1024)
diff --git a/official/vision/image_classification/mnist_test.py b/official/vision/image_classification/mnist_test.py
index c05efcfe5d68fbbb3c181c19b59444db1abe5702..c94396a444294b37259ba849bd8ea2f6f76997d0 100644
--- a/official/vision/image_classification/mnist_test.py
+++ b/official/vision/image_classification/mnist_test.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Test the Keras MNIST model on GPU."""
from __future__ import absolute_import
@@ -29,15 +29,16 @@ from official.utils.testing import integration
from official.vision.image_classification import mnist_main
+mnist_main.define_mnist_flags()
+
+
def eager_strategy_combinations():
return combinations.combine(
distribution=[
strategy_combinations.default_strategy,
- strategy_combinations.tpu_strategy,
+ strategy_combinations.cloud_tpu_strategy,
strategy_combinations.one_device_strategy_gpu,
- ],
- mode="eager",
- )
+ ],)
class KerasMnistTest(tf.test.TestCase, parameterized.TestCase):
@@ -47,7 +48,6 @@ class KerasMnistTest(tf.test.TestCase, parameterized.TestCase):
@classmethod
def setUpClass(cls): # pylint: disable=invalid-name
super(KerasMnistTest, cls).setUpClass()
- mnist_main.define_mnist_flags()
def tearDown(self):
super(KerasMnistTest, self).tearDown()
@@ -58,7 +58,8 @@ class KerasMnistTest(tf.test.TestCase, parameterized.TestCase):
"""Test Keras MNIST model with `strategy`."""
extra_flags = [
- "-train_epochs", "1",
+ "-train_epochs",
+ "1",
# Let TFDS find the metadata folder automatically
"--data_dir="
]
@@ -72,14 +73,15 @@ class KerasMnistTest(tf.test.TestCase, parameterized.TestCase):
tf.data.Dataset.from_tensor_slices(dummy_data),
)
- run = functools.partial(mnist_main.run,
- datasets_override=datasets,
- strategy_override=distribution)
+ run = functools.partial(
+ mnist_main.run,
+ datasets_override=datasets,
+ strategy_override=distribution)
integration.run_synthetic(
main=run,
synth=False,
- tmp_root=self.get_temp_dir(),
+ tmp_root=self.create_tempdir().full_path,
extra_flags=extra_flags)
diff --git a/official/vision/image_classification/optimizer_factory.py b/official/vision/image_classification/optimizer_factory.py
index 29b19e22daf2605ba430506c8b2d6545b1cc0074..e3eaba944b5c22fb2543972e33eca0a1256f062d 100644
--- a/official/vision/image_classification/optimizer_factory.py
+++ b/official/vision/image_classification/optimizer_factory.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,243 +11,26 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Optimizer factory for vision tasks."""
from __future__ import absolute_import
from __future__ import division
# from __future__ import google_type_annotations
from __future__ import print_function
-from typing import Any, Dict, Text, List
+from typing import Any, Dict, Text
from absl import logging
import tensorflow as tf
import tensorflow_addons as tfa
+from official.modeling import optimization
from official.vision.image_classification import learning_rate
from official.vision.image_classification.configs import base_configs
# pylint: disable=protected-access
-class MovingAverage(tf.keras.optimizers.Optimizer):
- """Optimizer that computes a moving average of the variables.
-
- Empirically it has been found that using the moving average of the trained
- parameters of a deep network is better than using its trained parameters
- directly. This optimizer allows you to compute this moving average and swap
- the variables at save time so that any code outside of the training loop
- will use by default the average values instead of the original ones.
-
- Example of usage for training:
- ```python
- opt = tf.keras.optimizers.SGD(learning_rate)
- opt = MovingAverage(opt)
-
- opt.shadow_copy(model)
- ```
-
- At test time, swap the shadow variables to evaluate on the averaged weights:
- ```python
- opt.swap_weights()
- # Test eval the model here
- opt.swap_weights()
- ```
- """
-
- def __init__(self,
- optimizer: tf.keras.optimizers.Optimizer,
- average_decay: float = 0.99,
- start_step: int = 0,
- dynamic_decay: bool = True,
- name: Text = 'moving_average',
- **kwargs):
- """Construct a new MovingAverage optimizer.
-
- Args:
- optimizer: `tf.keras.optimizers.Optimizer` that will be
- used to compute and apply gradients.
- average_decay: float. Decay to use to maintain the moving averages
- of trained variables.
- start_step: int. What step to start the moving average.
- dynamic_decay: bool. Whether to change the decay based on the number
- of optimizer updates. Decay will start at 0.1 and gradually increase
- up to `average_decay` after each optimizer update. This behavior is
- similar to `tf.train.ExponentialMovingAverage` in TF 1.x.
- name: Optional name for the operations created when applying
- gradients. Defaults to "moving_average".
- **kwargs: keyword arguments. Allowed to be {`clipnorm`,
- `clipvalue`, `lr`, `decay`}.
- """
- super(MovingAverage, self).__init__(name, **kwargs)
- self._optimizer = optimizer
- self._average_decay = average_decay
- self._start_step = tf.constant(start_step, tf.float32)
- self._dynamic_decay = dynamic_decay
-
- def shadow_copy(self, model: tf.keras.Model):
- """Creates shadow variables for the given model weights."""
- for var in model.weights:
- self.add_slot(var, 'average', initializer='zeros')
- self._average_weights = [
- self.get_slot(var, 'average') for var in model.weights
- ]
- self._model_weights = model.weights
-
- @property
- def has_shadow_copy(self):
- """Whether this optimizer has created shadow variables."""
- return self._model_weights is not None
-
- def _create_slots(self, var_list):
- self._optimizer._create_slots(var_list=var_list) # pylint: disable=protected-access
-
- def apply_gradients(self, grads_and_vars, name: Text = None):
- result = self._optimizer.apply_gradients(grads_and_vars, name)
- self.update_average(self._optimizer.iterations)
- return result
-
- @tf.function
- def update_average(self, step: tf.Tensor):
- step = tf.cast(step, tf.float32)
- if step < self._start_step:
- decay = tf.constant(0., tf.float32)
- elif self._dynamic_decay:
- decay = step - self._start_step
- decay = tf.minimum(self._average_decay, (1. + decay) / (10. + decay))
- else:
- decay = self._average_decay
-
- def _apply_moving(v_moving, v_normal):
- diff = v_moving - v_normal
- v_moving.assign_sub(tf.cast(1. - decay, v_moving.dtype) * diff)
- return v_moving
-
- def _update(strategy, v_moving_and_v_normal):
- for v_moving, v_normal in v_moving_and_v_normal:
- strategy.extended.update(v_moving, _apply_moving, args=(v_normal,))
-
- ctx = tf.distribute.get_replica_context()
- return ctx.merge_call(_update, args=(zip(self._average_weights,
- self._model_weights),))
-
- def swap_weights(self):
- """Swap the average and moving weights.
-
- This is a convenience method to allow one to evaluate the averaged weights
- at test time. Loads the weights stored in `self._average` into the model,
- keeping a copy of the original model weights. Swapping twice will return
- the original weights.
- """
- if tf.distribute.in_cross_replica_context():
- strategy = tf.distribute.get_strategy()
- strategy.run(self._swap_weights, args=())
- else:
- raise ValueError('Swapping weights must occur under a '
- 'tf.distribute.Strategy')
-
- @tf.function
- def _swap_weights(self):
- def fn_0(a, b):
- a.assign_add(b)
- return a
- def fn_1(b, a):
- b.assign(a - b)
- return b
- def fn_2(a, b):
- a.assign_sub(b)
- return a
-
- def swap(strategy, a_and_b):
- """Swap `a` and `b` and mirror to all devices."""
- for a, b in a_and_b:
- strategy.extended.update(a, fn_0, args=(b,)) # a = a + b
- strategy.extended.update(b, fn_1, args=(a,)) # b = a - b
- strategy.extended.update(a, fn_2, args=(b,)) # a = a - b
-
- ctx = tf.distribute.get_replica_context()
- return ctx.merge_call(
- swap, args=(zip(self._average_weights, self._model_weights),))
-
- def assign_average_vars(self, var_list: List[tf.Variable]):
- """Assign variables in var_list with their respective averages.
-
- Args:
- var_list: List of model variables to be assigned to their average.
- Returns:
- assign_op: The op corresponding to the assignment operation of
- variables to their average.
- """
- assign_op = tf.group([
- var.assign(self.get_slot(var, 'average')) for var in var_list
- if var.trainable
- ])
- return assign_op
-
- def _create_hypers(self):
- self._optimizer._create_hypers() # pylint: disable=protected-access
-
- def _prepare(self, var_list):
- return self._optimizer._prepare(var_list=var_list) # pylint: disable=protected-access
-
- @property
- def iterations(self):
- return self._optimizer.iterations
-
- @iterations.setter
- def iterations(self, variable):
- self._optimizer.iterations = variable
-
- @property
- def weights(self):
- # return self._weights + self._optimizer.weights
- return self._optimizer.weights
-
- @property
- def lr(self):
- return self._optimizer._get_hyper('learning_rate')
-
- @lr.setter
- def lr(self, lr):
- self._optimizer._set_hyper('learning_rate', lr)
-
- @property
- def learning_rate(self):
- return self._optimizer._get_hyper('learning_rate')
-
- @learning_rate.setter
- def learning_rate(self, learning_rate): # pylint: disable=redefined-outer-name
- self._optimizer._set_hyper('learning_rate', learning_rate)
-
- def _resource_apply_dense(self, grad, var):
- return self._optimizer._resource_apply_dense(grad, var)
-
- def _resource_apply_sparse(self, grad, var, indices):
- return self._optimizer._resource_apply_sparse(grad, var, indices)
-
- def _resource_apply_sparse_duplicate_indices(self, grad, var, indices):
- return self._optimizer._resource_apply_sparse_duplicate_indices(
- grad, var, indices)
-
- def get_config(self):
- config = {
- 'optimizer': tf.keras.optimizers.serialize(self._optimizer),
- 'average_decay': self._average_decay,
- 'start_step': self._start_step,
- 'dynamic_decay': self._dynamic_decay,
- }
- base_config = super(MovingAverage, self).get_config()
- return dict(list(base_config.items()) + list(config.items()))
-
- @classmethod
- def from_config(cls, config, custom_objects=None):
- optimizer = tf.keras.optimizers.deserialize(
- config.pop('optimizer'),
- custom_objects=custom_objects,
- )
- return cls(optimizer, **config)
-
-
def build_optimizer(
optimizer_name: Text,
base_learning_rate: tf.keras.optimizers.schedules.LearningRateSchedule,
@@ -256,15 +39,15 @@ def build_optimizer(
"""Build the optimizer based on name.
Args:
- optimizer_name: String representation of the optimizer name. Examples:
- sgd, momentum, rmsprop.
+ optimizer_name: String representation of the optimizer name. Examples: sgd,
+ momentum, rmsprop.
base_learning_rate: `tf.keras.optimizers.schedules.LearningRateSchedule`
base learning rate.
- params: String -> Any dictionary representing the optimizer params.
- This should contain optimizer specific parameters such as
- `base_learning_rate`, `decay`, etc.
+ params: String -> Any dictionary representing the optimizer params. This
+ should contain optimizer specific parameters such as `base_learning_rate`,
+ `decay`, etc.
model: The `tf.keras.Model`. This is used for the shadow copy if using
- `MovingAverage`.
+ `ExponentialMovingAverage`.
Returns:
A tf.keras.Optimizer.
@@ -279,43 +62,47 @@ def build_optimizer(
if optimizer_name == 'sgd':
logging.info('Using SGD optimizer')
nesterov = params.get('nesterov', False)
- optimizer = tf.keras.optimizers.SGD(learning_rate=base_learning_rate,
- nesterov=nesterov)
+ optimizer = tf.keras.optimizers.SGD(
+ learning_rate=base_learning_rate, nesterov=nesterov)
elif optimizer_name == 'momentum':
logging.info('Using momentum optimizer')
nesterov = params.get('nesterov', False)
- optimizer = tf.keras.optimizers.SGD(learning_rate=base_learning_rate,
- momentum=params['momentum'],
- nesterov=nesterov)
+ optimizer = tf.keras.optimizers.SGD(
+ learning_rate=base_learning_rate,
+ momentum=params['momentum'],
+ nesterov=nesterov)
elif optimizer_name == 'rmsprop':
logging.info('Using RMSProp')
rho = params.get('decay', None) or params.get('rho', 0.9)
momentum = params.get('momentum', 0.9)
epsilon = params.get('epsilon', 1e-07)
- optimizer = tf.keras.optimizers.RMSprop(learning_rate=base_learning_rate,
- rho=rho,
- momentum=momentum,
- epsilon=epsilon)
+ optimizer = tf.keras.optimizers.RMSprop(
+ learning_rate=base_learning_rate,
+ rho=rho,
+ momentum=momentum,
+ epsilon=epsilon)
elif optimizer_name == 'adam':
logging.info('Using Adam')
beta_1 = params.get('beta_1', 0.9)
beta_2 = params.get('beta_2', 0.999)
epsilon = params.get('epsilon', 1e-07)
- optimizer = tf.keras.optimizers.Adam(learning_rate=base_learning_rate,
- beta_1=beta_1,
- beta_2=beta_2,
- epsilon=epsilon)
+ optimizer = tf.keras.optimizers.Adam(
+ learning_rate=base_learning_rate,
+ beta_1=beta_1,
+ beta_2=beta_2,
+ epsilon=epsilon)
elif optimizer_name == 'adamw':
logging.info('Using AdamW')
weight_decay = params.get('weight_decay', 0.01)
beta_1 = params.get('beta_1', 0.9)
beta_2 = params.get('beta_2', 0.999)
epsilon = params.get('epsilon', 1e-07)
- optimizer = tfa.optimizers.AdamW(weight_decay=weight_decay,
- learning_rate=base_learning_rate,
- beta_1=beta_1,
- beta_2=beta_2,
- epsilon=epsilon)
+ optimizer = tfa.optimizers.AdamW(
+ weight_decay=weight_decay,
+ learning_rate=base_learning_rate,
+ beta_1=beta_1,
+ beta_2=beta_2,
+ epsilon=epsilon)
else:
raise ValueError('Unknown optimizer %s' % optimizer_name)
@@ -327,11 +114,11 @@ def build_optimizer(
moving_average_decay = params.get('moving_average_decay', 0.)
if moving_average_decay is not None and moving_average_decay > 0.:
if model is None:
- raise ValueError('`model` must be provided if using `MovingAverage`.')
+ raise ValueError(
+ '`model` must be provided if using `ExponentialMovingAverage`.')
logging.info('Including moving average decay.')
- optimizer = MovingAverage(
- optimizer=optimizer,
- average_decay=moving_average_decay)
+ optimizer = optimization.ExponentialMovingAverage(
+ optimizer=optimizer, average_decay=moving_average_decay)
optimizer.shadow_copy(model)
return optimizer
@@ -358,41 +145,38 @@ def build_learning_rate(params: base_configs.LearningRateConfig,
if lr_multiplier and lr_multiplier > 0:
# Scale the learning rate based on the batch size and a multiplier
base_lr *= lr_multiplier * batch_size
- logging.info('Scaling the learning rate based on the batch size '
- 'multiplier. New base_lr: %f', base_lr)
+ logging.info(
+ 'Scaling the learning rate based on the batch size '
+ 'multiplier. New base_lr: %f', base_lr)
if decay_type == 'exponential':
- logging.info('Using exponential learning rate with: '
- 'initial_learning_rate: %f, decay_steps: %d, '
- 'decay_rate: %f', base_lr, decay_steps, decay_rate)
+ logging.info(
+ 'Using exponential learning rate with: '
+ 'initial_learning_rate: %f, decay_steps: %d, '
+ 'decay_rate: %f', base_lr, decay_steps, decay_rate)
lr = tf.keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate=base_lr,
decay_steps=decay_steps,
decay_rate=decay_rate,
staircase=params.staircase)
- elif decay_type == 'piecewise_constant_with_warmup':
- logging.info('Using Piecewise constant decay with warmup. '
- 'Parameters: batch_size: %d, epoch_size: %d, '
- 'warmup_epochs: %d, boundaries: %s, multipliers: %s',
- batch_size, params.examples_per_epoch,
- params.warmup_epochs, params.boundaries,
- params.multipliers)
- lr = learning_rate.PiecewiseConstantDecayWithWarmup(
- batch_size=batch_size,
- epoch_size=params.examples_per_epoch,
- warmup_epochs=params.warmup_epochs,
- boundaries=params.boundaries,
- multipliers=params.multipliers)
+ elif decay_type == 'stepwise':
+ steps_per_epoch = params.examples_per_epoch // batch_size
+ boundaries = [boundary * steps_per_epoch for boundary in params.boundaries]
+ multipliers = [batch_size * multiplier for multiplier in params.multipliers]
+ logging.info(
+ 'Using stepwise learning rate. Parameters: '
+ 'boundaries: %s, values: %s', boundaries, multipliers)
+ lr = tf.keras.optimizers.schedules.PiecewiseConstantDecay(
+ boundaries=boundaries, values=multipliers)
elif decay_type == 'cosine_with_warmup':
lr = learning_rate.CosineDecayWithWarmup(
batch_size=batch_size,
total_steps=train_epochs * train_steps,
warmup_steps=warmup_steps)
if warmup_steps > 0:
- if decay_type not in [
- 'piecewise_constant_with_warmup', 'cosine_with_warmup'
- ]:
+ if decay_type not in ['cosine_with_warmup']:
logging.info('Applying %d warmup steps to the learning rate',
warmup_steps)
- lr = learning_rate.WarmupDecaySchedule(lr, warmup_steps)
+ lr = learning_rate.WarmupDecaySchedule(
+ lr, warmup_steps, warmup_lr=base_lr)
return lr
diff --git a/official/vision/image_classification/optimizer_factory_test.py b/official/vision/image_classification/optimizer_factory_test.py
index a620728482f66febe402c20e2f01717f6a1393e5..a98d23d9bff0f7d339b1f5d5d648b18c04c4e08c 100644
--- a/official/vision/image_classification/optimizer_factory_test.py
+++ b/official/vision/image_classification/optimizer_factory_test.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Tests for optimizer_factory."""
from __future__ import absolute_import
@@ -35,10 +35,8 @@ class OptimizerFactoryTest(tf.test.TestCase, parameterized.TestCase):
return model
@parameterized.named_parameters(
- ('sgd', 'sgd', 0., False),
- ('momentum', 'momentum', 0., False),
- ('rmsprop', 'rmsprop', 0., False),
- ('adam', 'adam', 0., False),
+ ('sgd', 'sgd', 0., False), ('momentum', 'momentum', 0., False),
+ ('rmsprop', 'rmsprop', 0., False), ('adam', 'adam', 0., False),
('adamw', 'adamw', 0., False),
('momentum_lookahead', 'momentum', 0., True),
('sgd_ema', 'sgd', 0.999, False),
@@ -84,17 +82,13 @@ class OptimizerFactoryTest(tf.test.TestCase, parameterized.TestCase):
train_steps = 1
lr = optimizer_factory.build_learning_rate(
- params=params,
- batch_size=batch_size,
- train_steps=train_steps)
+ params=params, batch_size=batch_size, train_steps=train_steps)
self.assertTrue(
issubclass(
type(lr), tf.keras.optimizers.schedules.LearningRateSchedule))
- @parameterized.named_parameters(
- ('exponential', 'exponential'),
- ('piecewise_constant_with_warmup', 'piecewise_constant_with_warmup'),
- ('cosine_with_warmup', 'cosine_with_warmup'))
+ @parameterized.named_parameters(('exponential', 'exponential'),
+ ('cosine_with_warmup', 'cosine_with_warmup'))
def test_learning_rate_with_decay_and_warmup(self, lr_decay_type):
"""Basic smoke test for syntax."""
params = base_configs.LearningRateConfig(
diff --git a/official/vision/image_classification/preprocessing.py b/official/vision/image_classification/preprocessing.py
index 3f2019189d4e5f9c269a67276531b4344ede7e32..dece1fbc1199a137e5ee86ce7312b39cdeac27c9 100644
--- a/official/vision/image_classification/preprocessing.py
+++ b/official/vision/image_classification/preprocessing.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Preprocessing functions for images."""
from __future__ import absolute_import
@@ -379,12 +379,12 @@ def preprocess_for_train(image_bytes: tf.Tensor,
"""
images = decode_crop_and_flip(image_bytes=image_bytes)
images = resize_image(images, height=image_size, width=image_size)
+ if augmenter is not None:
+ images = augmenter.distort(images)
if mean_subtract:
images = mean_image_subtraction(image_bytes=images, means=MEAN_RGB)
if standardize:
images = standardize_image(image_bytes=images, stddev=STDDEV_RGB)
- if augmenter is not None:
- images = augmenter.distort(images)
if dtype is not None:
images = tf.image.convert_image_dtype(images, dtype)
diff --git a/official/vision/image_classification/resnet/__init__.py b/official/vision/image_classification/resnet/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e419af524b5f349fe04abfa820c3cb51b777d422 100644
--- a/official/vision/image_classification/resnet/__init__.py
+++ b/official/vision/image_classification/resnet/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/official/vision/image_classification/resnet/common.py b/official/vision/image_classification/resnet/common.py
index a9a64aa4064978863332a8024f4e46d64b9baaef..a034ba7dd0be5b2b2536727137497c84519001a5 100644
--- a/official/vision/image_classification/resnet/common.py
+++ b/official/vision/image_classification/resnet/common.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Common util functions and classes used by both keras cifar and imagenet."""
from __future__ import absolute_import
from __future__ import division
@@ -22,7 +22,6 @@ import os
from absl import flags
import tensorflow as tf
-from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2
import tensorflow_model_optimization as tfmot
from official.utils.flags import core as flags_core
from official.utils.misc import keras_utils
@@ -30,7 +29,7 @@ from official.utils.misc import keras_utils
FLAGS = flags.FLAGS
BASE_LEARNING_RATE = 0.1 # This matches Jing's version.
TRAIN_TOP_1 = 'training_accuracy_top_1'
-LR_SCHEDULE = [ # (multiplier, epoch to start) tuples
+LR_SCHEDULE = [ # (multiplier, epoch to start) tuples
(1.0, 5), (0.1, 30), (0.01, 60), (0.001, 80)
]
@@ -39,8 +38,14 @@ class PiecewiseConstantDecayWithWarmup(
tf.keras.optimizers.schedules.LearningRateSchedule):
"""Piecewise constant decay with warmup schedule."""
- def __init__(self, batch_size, epoch_size, warmup_epochs, boundaries,
- multipliers, compute_lr_on_cpu=True, name=None):
+ def __init__(self,
+ batch_size,
+ epoch_size,
+ warmup_epochs,
+ boundaries,
+ multipliers,
+ compute_lr_on_cpu=True,
+ name=None):
super(PiecewiseConstantDecayWithWarmup, self).__init__()
if len(boundaries) != len(multipliers) - 1:
raise ValueError('The length of boundaries must be 1 less than the '
@@ -77,14 +82,16 @@ class PiecewiseConstantDecayWithWarmup(
def _get_learning_rate(self, step):
"""Compute learning rate at given step."""
with tf.name_scope('PiecewiseConstantDecayWithWarmup'):
+
def warmup_lr(step):
return self.rescaled_lr * (
tf.cast(step, tf.float32) / tf.cast(self.warmup_steps, tf.float32))
+
def piecewise_lr(step):
- return tf.compat.v1.train.piecewise_constant(
- step, self.step_boundaries, self.lr_values)
- return tf.cond(step < self.warmup_steps,
- lambda: warmup_lr(step),
+ return tf.compat.v1.train.piecewise_constant(step, self.step_boundaries,
+ self.lr_values)
+
+ return tf.cond(step < self.warmup_steps, lambda: warmup_lr(step),
lambda: piecewise_lr(step))
def get_config(self):
@@ -101,13 +108,12 @@ class PiecewiseConstantDecayWithWarmup(
def get_optimizer(learning_rate=0.1):
"""Returns optimizer to use."""
# The learning_rate is overwritten at the beginning of each step by callback.
- return gradient_descent_v2.SGD(learning_rate=learning_rate, momentum=0.9)
+ return tf.keras.optimizers.SGD(learning_rate=learning_rate, momentum=0.9)
-def get_callbacks(
- pruning_method=None,
- enable_checkpoint_and_export=False,
- model_dir=None):
+def get_callbacks(pruning_method=None,
+ enable_checkpoint_and_export=False,
+ model_dir=None):
"""Returns common callbacks."""
time_callback = keras_utils.TimeHistory(
FLAGS.batch_size,
@@ -117,23 +123,23 @@ def get_callbacks(
if FLAGS.enable_tensorboard:
tensorboard_callback = tf.keras.callbacks.TensorBoard(
- log_dir=FLAGS.model_dir,
- profile_batch=FLAGS.profile_steps)
+ log_dir=FLAGS.model_dir, profile_batch=FLAGS.profile_steps)
callbacks.append(tensorboard_callback)
is_pruning_enabled = pruning_method is not None
if is_pruning_enabled:
callbacks.append(tfmot.sparsity.keras.UpdatePruningStep())
if model_dir is not None:
- callbacks.append(tfmot.sparsity.keras.PruningSummaries(
- log_dir=model_dir, profile_batch=0))
+ callbacks.append(
+ tfmot.sparsity.keras.PruningSummaries(
+ log_dir=model_dir, profile_batch=0))
if enable_checkpoint_and_export:
if model_dir is not None:
ckpt_full_path = os.path.join(model_dir, 'model.ckpt-{epoch:04d}')
callbacks.append(
- tf.keras.callbacks.ModelCheckpoint(ckpt_full_path,
- save_weights_only=True))
+ tf.keras.callbacks.ModelCheckpoint(
+ ckpt_full_path, save_weights_only=True))
return callbacks
@@ -182,28 +188,30 @@ def build_stats(history, eval_output, callbacks):
return stats
-def define_keras_flags(
- dynamic_loss_scale=True,
- model=False,
- optimizer=False,
- pretrained_filepath=False):
+def define_keras_flags(model=False,
+ optimizer=False,
+ pretrained_filepath=False):
"""Define flags for Keras models."""
- flags_core.define_base(clean=True, num_gpu=True, run_eagerly=True,
- train_epochs=True, epochs_between_evals=True,
- distribution_strategy=True)
- flags_core.define_performance(num_parallel_calls=False,
- synthetic_data=True,
- dtype=True,
- all_reduce_alg=True,
- num_packs=True,
- tf_gpu_thread_mode=True,
- datasets_num_private_threads=True,
- dynamic_loss_scale=dynamic_loss_scale,
- loss_scale=True,
- fp16_implementation=True,
- tf_data_experimental_slack=True,
- enable_xla=True,
- training_dataset_cache=True)
+ flags_core.define_base(
+ clean=True,
+ num_gpu=True,
+ run_eagerly=True,
+ train_epochs=True,
+ epochs_between_evals=True,
+ distribution_strategy=True)
+ flags_core.define_performance(
+ num_parallel_calls=False,
+ synthetic_data=True,
+ dtype=True,
+ all_reduce_alg=True,
+ num_packs=True,
+ tf_gpu_thread_mode=True,
+ datasets_num_private_threads=True,
+ loss_scale=True,
+ fp16_implementation=True,
+ tf_data_experimental_slack=True,
+ enable_xla=True,
+ training_dataset_cache=True)
flags_core.define_image()
flags_core.define_benchmark()
flags_core.define_distribution()
@@ -214,23 +222,33 @@ def define_keras_flags(
# TODO(b/135607288): Remove this flag once we understand the root cause of
# slowdown when setting the learning phase in Keras backend.
flags.DEFINE_boolean(
- name='set_learning_phase_to_train', default=True,
+ name='set_learning_phase_to_train',
+ default=True,
help='If skip eval, also set Keras learning phase to 1 (training).')
flags.DEFINE_boolean(
- name='explicit_gpu_placement', default=False,
+ name='explicit_gpu_placement',
+ default=False,
help='If not using distribution strategy, explicitly set device scope '
'for the Keras training loop.')
- flags.DEFINE_boolean(name='use_trivial_model', default=False,
- help='Whether to use a trivial Keras model.')
- flags.DEFINE_boolean(name='report_accuracy_metrics', default=True,
- help='Report metrics during training and evaluation.')
- flags.DEFINE_boolean(name='use_tensor_lr', default=True,
- help='Use learning rate tensor instead of a callback.')
flags.DEFINE_boolean(
- name='enable_tensorboard', default=False,
+ name='use_trivial_model',
+ default=False,
+ help='Whether to use a trivial Keras model.')
+ flags.DEFINE_boolean(
+ name='report_accuracy_metrics',
+ default=True,
+ help='Report metrics during training and evaluation.')
+ flags.DEFINE_boolean(
+ name='use_tensor_lr',
+ default=True,
+ help='Use learning rate tensor instead of a callback.')
+ flags.DEFINE_boolean(
+ name='enable_tensorboard',
+ default=False,
help='Whether to enable Tensorboard callback.')
flags.DEFINE_string(
- name='profile_steps', default=None,
+ name='profile_steps',
+ default=None,
help='Save profiling data to model dir at given range of global steps. The '
'value must be a comma separated pair of positive integers, specifying '
'the first and last step to profile. For example, "--profile_steps=2,4" '
@@ -238,24 +256,27 @@ def define_keras_flags(
'Note that profiler has a non-trivial performance overhead, and the '
'output file can be gigantic if profiling many steps.')
flags.DEFINE_integer(
- name='train_steps', default=None,
+ name='train_steps',
+ default=None,
help='The number of steps to run for training. If it is larger than '
'# batches per epoch, then use # batches per epoch. This flag will be '
'ignored if train_epochs is set to be larger than 1. ')
flags.DEFINE_boolean(
- name='batchnorm_spatial_persistent', default=True,
+ name='batchnorm_spatial_persistent',
+ default=True,
help='Enable the spacial persistent mode for CuDNN batch norm kernel.')
flags.DEFINE_boolean(
- name='enable_get_next_as_optional', default=False,
+ name='enable_get_next_as_optional',
+ default=False,
help='Enable get_next_as_optional behavior in DistributedIterator.')
flags.DEFINE_boolean(
- name='enable_checkpoint_and_export', default=False,
+ name='enable_checkpoint_and_export',
+ default=False,
help='Whether to enable a checkpoint callback and export the savedmodel.')
- flags.DEFINE_string(
- name='tpu', default='', help='TPU address to connect to.')
+ flags.DEFINE_string(name='tpu', default='', help='TPU address to connect to.')
flags.DEFINE_integer(
name='steps_per_loop',
- default=500,
+ default=None,
help='Number of steps per training loop. Only training step happens '
'inside the loop. Callbacks will not be called inside. Will be capped at '
'steps per epoch.')
@@ -270,20 +291,20 @@ def define_keras_flags(
flags.DEFINE_string('model', 'resnet50_v1.5',
'Name of model preset. (mobilenet, resnet50_v1.5)')
if optimizer:
- flags.DEFINE_string('optimizer', 'resnet50_default',
- 'Name of optimizer preset. '
- '(mobilenet_default, resnet50_default)')
+ flags.DEFINE_string(
+ 'optimizer', 'resnet50_default', 'Name of optimizer preset. '
+ '(mobilenet_default, resnet50_default)')
# TODO(kimjaehong): Replace as general hyper-params not only for mobilenet.
- flags.DEFINE_float('initial_learning_rate_per_sample', 0.00007,
- 'Initial value of learning rate per sample for '
- 'mobilenet_default.')
+ flags.DEFINE_float(
+ 'initial_learning_rate_per_sample', 0.00007,
+ 'Initial value of learning rate per sample for '
+ 'mobilenet_default.')
flags.DEFINE_float('lr_decay_factor', 0.94,
'Learning rate decay factor for mobilenet_default.')
flags.DEFINE_float('num_epochs_per_decay', 2.5,
'Number of epochs per decay for mobilenet_default.')
if pretrained_filepath:
- flags.DEFINE_string('pretrained_filepath', '',
- 'Pretrained file path.')
+ flags.DEFINE_string('pretrained_filepath', '', 'Pretrained file path.')
def get_synth_data(height, width, num_channels, num_classes, dtype):
@@ -317,23 +338,31 @@ def get_synth_data(height, width, num_channels, num_classes, dtype):
def define_pruning_flags():
"""Define flags for pruning methods."""
- flags.DEFINE_string('pruning_method', None,
- 'Pruning method.'
- 'None (no pruning) or polynomial_decay.')
+ flags.DEFINE_string(
+ 'pruning_method', None, 'Pruning method.'
+ 'None (no pruning) or polynomial_decay.')
flags.DEFINE_float('pruning_initial_sparsity', 0.0,
'Initial sparsity for pruning.')
flags.DEFINE_float('pruning_final_sparsity', 0.5,
'Final sparsity for pruning.')
- flags.DEFINE_integer('pruning_begin_step', 0,
- 'Begin step for pruning.')
- flags.DEFINE_integer('pruning_end_step', 100000,
- 'End step for pruning.')
- flags.DEFINE_integer('pruning_frequency', 100,
- 'Frequency for pruning.')
+ flags.DEFINE_integer('pruning_begin_step', 0, 'Begin step for pruning.')
+ flags.DEFINE_integer('pruning_end_step', 100000, 'End step for pruning.')
+ flags.DEFINE_integer('pruning_frequency', 100, 'Frequency for pruning.')
+
+def define_clustering_flags():
+ """Define flags for clustering methods."""
+ flags.DEFINE_string('clustering_method', None,
+ 'None (no clustering) or selective_clustering '
+ '(cluster last three Conv2D layers of the model).')
-def get_synth_input_fn(height, width, num_channels, num_classes,
- dtype=tf.float32, drop_remainder=True):
+
+def get_synth_input_fn(height,
+ width,
+ num_channels,
+ num_classes,
+ dtype=tf.float32,
+ drop_remainder=True):
"""Returns an input function that returns a dataset with random data.
This input_fn returns a data set that iterates over a set of random data and
@@ -355,14 +384,16 @@ def get_synth_input_fn(height, width, num_channels, num_classes,
An input_fn that can be used in place of a real one to return a dataset
that can be used for iteration.
"""
+
# pylint: disable=unused-argument
def input_fn(is_training, data_dir, batch_size, *args, **kwargs):
"""Returns dataset filled with random data."""
- inputs, labels = get_synth_data(height=height,
- width=width,
- num_channels=num_channels,
- num_classes=num_classes,
- dtype=dtype)
+ inputs, labels = get_synth_data(
+ height=height,
+ width=width,
+ num_channels=num_channels,
+ num_classes=num_classes,
+ dtype=dtype)
# Cast to float32 for Keras model.
labels = tf.cast(labels, dtype=tf.float32)
data = tf.data.Dataset.from_tensors((inputs, labels)).repeat()
diff --git a/official/vision/image_classification/resnet/imagenet_preprocessing.py b/official/vision/image_classification/resnet/imagenet_preprocessing.py
index f1490c22d8d769f32a6f6a1c6d29455519e8743a..86ba3ed98084987ea5d63edf8fd5f515d58fba93 100644
--- a/official/vision/image_classification/resnet/imagenet_preprocessing.py
+++ b/official/vision/image_classification/resnet/imagenet_preprocessing.py
@@ -1,4 +1,4 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Provides utilities to preprocess images.
Training images are sampled using the provided bounding boxes, and subsequently
@@ -36,6 +36,7 @@ from __future__ import division
from __future__ import print_function
import os
+
from absl import logging
import tensorflow as tf
@@ -78,17 +79,17 @@ def process_record_dataset(dataset,
is_training: A boolean denoting whether the input is for training.
batch_size: The number of samples per batch.
shuffle_buffer: The buffer size to use when shuffling records. A larger
- value results in better randomness, but smaller values reduce startup
- time and use less memory.
+ value results in better randomness, but smaller values reduce startup time
+ and use less memory.
parse_record_fn: A function that takes a raw record and returns the
corresponding (image, label) pair.
dtype: Data type to use for images/features.
- datasets_num_private_threads: Number of threads for a private
- threadpool created for all datasets computation.
+ datasets_num_private_threads: Number of threads for a private threadpool
+ created for all datasets computation.
drop_remainder: A boolean indicates whether to drop the remainder of the
batches. If True, the batch dimension will be static.
- tf_data_experimental_slack: Whether to enable tf.data's
- `experimental_slack` option.
+ tf_data_experimental_slack: Whether to enable tf.data's `experimental_slack`
+ option.
Returns:
Dataset of (image, label) pairs ready for iteration.
@@ -99,8 +100,8 @@ def process_record_dataset(dataset,
options.experimental_threading.private_threadpool_size = (
datasets_num_private_threads)
dataset = dataset.with_options(options)
- logging.info(
- 'datasets_num_private_threads: %s', datasets_num_private_threads)
+ logging.info('datasets_num_private_threads: %s',
+ datasets_num_private_threads)
if is_training:
# Shuffles records before repeating to respect epoch boundaries.
@@ -134,11 +135,13 @@ def get_filenames(is_training, data_dir):
if is_training:
return [
os.path.join(data_dir, 'train-%05d-of-01024' % i)
- for i in range(_NUM_TRAIN_FILES)]
+ for i in range(_NUM_TRAIN_FILES)
+ ]
else:
return [
os.path.join(data_dir, 'validation-%05d-of-00128' % i)
- for i in range(128)]
+ for i in range(128)
+ ]
def parse_example_proto(example_serialized):
@@ -165,8 +168,8 @@ def parse_example_proto(example_serialized):
image/encoded:
Args:
- example_serialized: scalar Tensor tf.string containing a serialized
- Example protocol buffer.
+ example_serialized: scalar Tensor tf.string containing a serialized Example
+ protocol buffer.
Returns:
image_buffer: Tensor tf.string containing the contents of a JPEG file.
@@ -177,22 +180,24 @@ def parse_example_proto(example_serialized):
"""
# Dense features in Example proto.
feature_map = {
- 'image/encoded': tf.io.FixedLenFeature([], dtype=tf.string,
- default_value=''),
- 'image/class/label': tf.io.FixedLenFeature([], dtype=tf.int64,
- default_value=-1),
- 'image/class/text': tf.io.FixedLenFeature([], dtype=tf.string,
- default_value=''),
+ 'image/encoded':
+ tf.io.FixedLenFeature([], dtype=tf.string, default_value=''),
+ 'image/class/label':
+ tf.io.FixedLenFeature([], dtype=tf.int64, default_value=-1),
+ 'image/class/text':
+ tf.io.FixedLenFeature([], dtype=tf.string, default_value=''),
}
sparse_float32 = tf.io.VarLenFeature(dtype=tf.float32)
# Sparse features in Example proto.
- feature_map.update(
- {k: sparse_float32 for k in [
+ feature_map.update({
+ k: sparse_float32 for k in [
'image/object/bbox/xmin', 'image/object/bbox/ymin',
- 'image/object/bbox/xmax', 'image/object/bbox/ymax']})
+ 'image/object/bbox/xmax', 'image/object/bbox/ymax'
+ ]
+ })
- features = tf.io.parse_single_example(serialized=example_serialized,
- features=feature_map)
+ features = tf.io.parse_single_example(
+ serialized=example_serialized, features=feature_map)
label = tf.cast(features['image/class/label'], dtype=tf.int32)
xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0)
@@ -218,8 +223,8 @@ def parse_record(raw_record, is_training, dtype):
through preprocessing steps (cropping, flipping, and so on).
Args:
- raw_record: scalar Tensor tf.string containing a serialized
- Example protocol buffer.
+ raw_record: scalar Tensor tf.string containing a serialized Example protocol
+ buffer.
is_training: A boolean denoting whether the input is for training.
dtype: data type to use for images/features.
@@ -240,8 +245,9 @@ def parse_record(raw_record, is_training, dtype):
# Subtract one so that labels are in [0, 1000), and cast to float32 for
# Keras model.
- label = tf.cast(tf.cast(tf.reshape(label, shape=[1]), dtype=tf.int32) - 1,
- dtype=tf.float32)
+ label = tf.cast(
+ tf.cast(tf.reshape(label, shape=[1]), dtype=tf.int32) - 1,
+ dtype=tf.float32)
return image, label
@@ -262,12 +268,14 @@ def get_parse_record_fn(use_keras_image_data_format=False):
Returns:
Function to use for parsing the records.
"""
+
def parse_record_fn(raw_record, is_training, dtype):
image, label = parse_record(raw_record, is_training, dtype)
if use_keras_image_data_format:
if tf.keras.backend.image_data_format() == 'channels_first':
image = tf.transpose(image, perm=[2, 0, 1])
return image, label
+
return parse_record_fn
@@ -295,11 +303,11 @@ def input_fn(is_training,
`tf.distribute.Strategy`.
drop_remainder: A boolean indicates whether to drop the remainder of the
batches. If True, the batch dimension will be static.
- tf_data_experimental_slack: Whether to enable tf.data's
- `experimental_slack` option.
+ tf_data_experimental_slack: Whether to enable tf.data's `experimental_slack`
+ option.
training_dataset_cache: Whether to cache the training dataset on workers.
- Typically used to improve training performance when training data is in
- remote storage and can fit into worker memory.
+ Typically used to improve training performance when training data is in
+ remote storage and can fit into worker memory.
filenames: Optional field for providing the file names of the TFRecords.
Returns:
@@ -357,8 +365,8 @@ def _decode_crop_and_flip(image_buffer, bbox, num_channels):
Args:
image_buffer: scalar string Tensor representing the raw JPEG image buffer.
bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
- where each coordinate is [0, 1) and the coordinates are arranged as
- [ymin, xmin, ymax, xmax].
+ where each coordinate is [0, 1) and the coordinates are arranged as [ymin,
+ xmin, ymax, xmax].
num_channels: Integer depth of the image buffer for decoding.
Returns:
@@ -414,8 +422,8 @@ def _central_crop(image, crop_height, crop_width):
crop_top = amount_to_be_cropped_h // 2
amount_to_be_cropped_w = (width - crop_width)
crop_left = amount_to_be_cropped_w // 2
- return tf.slice(
- image, [crop_top, crop_left, 0], [crop_height, crop_width, -1])
+ return tf.slice(image, [crop_top, crop_left, 0],
+ [crop_height, crop_width, -1])
def _mean_image_subtraction(image, means, num_channels):
@@ -463,8 +471,8 @@ def _smallest_size_at_least(height, width, resize_min):
Args:
height: an int32 scalar tensor indicating the current height.
width: an int32 scalar tensor indicating the current width.
- resize_min: A python integer or scalar `Tensor` indicating the size of
- the smallest side after resize.
+ resize_min: A python integer or scalar `Tensor` indicating the size of the
+ smallest side after resize.
Returns:
new_height: an int32 scalar tensor indicating the new height.
@@ -490,8 +498,8 @@ def _aspect_preserving_resize(image, resize_min):
Args:
image: A 3-D image `Tensor`.
- resize_min: A python integer or scalar `Tensor` indicating the size of
- the smallest side after resize.
+ resize_min: A python integer or scalar `Tensor` indicating the size of the
+ smallest side after resize.
Returns:
resized_image: A 3-D tensor containing the resized image.
@@ -520,12 +528,17 @@ def _resize_image(image, height, width):
dimensions have the shape [height, width].
"""
return tf.compat.v1.image.resize(
- image, [height, width], method=tf.image.ResizeMethod.BILINEAR,
+ image, [height, width],
+ method=tf.image.ResizeMethod.BILINEAR,
align_corners=False)
-def preprocess_image(image_buffer, bbox, output_height, output_width,
- num_channels, is_training=False):
+def preprocess_image(image_buffer,
+ bbox,
+ output_height,
+ output_width,
+ num_channels,
+ is_training=False):
"""Preprocesses the given image.
Preprocessing includes decoding, cropping, and resizing for both training
@@ -535,8 +548,8 @@ def preprocess_image(image_buffer, bbox, output_height, output_width,
Args:
image_buffer: scalar string Tensor representing the raw JPEG image buffer.
bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
- where each coordinate is [0, 1) and the coordinates are arranged as
- [ymin, xmin, ymax, xmax].
+ where each coordinate is [0, 1) and the coordinates are arranged as [ymin,
+ xmin, ymax, xmax].
output_height: The height of the image after preprocessing.
output_width: The width of the image after preprocessing.
num_channels: Integer depth of the image buffer for decoding.
diff --git a/official/vision/image_classification/resnet/resnet_config.py b/official/vision/image_classification/resnet/resnet_config.py
index a746257f02b85eddfc72192b9474638b92378644..e39db3955f9fe9c312ea307c8ac3196d45447cf3 100644
--- a/official/vision/image_classification/resnet/resnet_config.py
+++ b/official/vision/image_classification/resnet/resnet_config.py
@@ -1,5 +1,4 @@
-# Lint as: python3
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,28 +11,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+# Lint as: python3
"""Configuration definitions for ResNet losses, learning rates, and optimizers."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
-from typing import Any, Mapping
-
import dataclasses
from official.modeling.hyperparams import base_config
from official.vision.image_classification.configs import base_configs
-_RESNET_LR_SCHEDULE = [ # (multiplier, epoch to start) tuples
- (1.0, 5), (0.1, 30), (0.01, 60), (0.001, 80)
-]
-_RESNET_LR_BOUNDARIES = list(p[1] for p in _RESNET_LR_SCHEDULE[1:])
-_RESNET_LR_MULTIPLIERS = list(p[0] for p in _RESNET_LR_SCHEDULE)
-_RESNET_LR_WARMUP_EPOCHS = _RESNET_LR_SCHEDULE[0][1]
-
-
@dataclasses.dataclass
class ResNetModelConfig(base_configs.ModelConfig):
"""Configuration for the ResNet model."""
@@ -56,8 +46,10 @@ class ResNetModelConfig(base_configs.ModelConfig):
moving_average_decay=None)
learning_rate: base_configs.LearningRateConfig = (
base_configs.LearningRateConfig(
- name='piecewise_constant_with_warmup',
+ name='stepwise',
+ initial_lr=0.1,
examples_per_epoch=1281167,
- warmup_epochs=_RESNET_LR_WARMUP_EPOCHS,
- boundaries=_RESNET_LR_BOUNDARIES,
- multipliers=_RESNET_LR_MULTIPLIERS))
+ boundaries=[30, 60, 80],
+ warmup_epochs=5,
+ scale_by_batch_size=1. / 256.,
+ multipliers=[0.1 / 256, 0.01 / 256, 0.001 / 256, 0.0001 / 256]))
diff --git a/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py b/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py
index c128dc0b99535d806634b42b99a2e56211c567ca..a66461df17a3fe5fc0d75969e99920310a694e71 100644
--- a/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py
+++ b/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,23 +11,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""Runs a ResNet model on the ImageNet dataset using custom training loops."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Runs a ResNet model on the ImageNet dataset using custom training loops."""
import math
+import os
+
+# Import libraries
from absl import app
from absl import flags
from absl import logging
+import orbit
import tensorflow as tf
-
+from official.common import distribute_utils
from official.modeling import performance
-from official.staging.training import controller
from official.utils.flags import core as flags_core
-from official.utils.misc import distribution_utils
from official.utils.misc import keras_utils
from official.utils.misc import model_helpers
from official.vision.image_classification.resnet import common
@@ -87,15 +85,6 @@ def get_num_train_iterations(flags_obj):
return train_steps, train_epochs, eval_steps
-def _steps_to_run(steps_in_current_epoch, steps_per_epoch, steps_per_loop):
- """Calculates steps to run on device."""
- if steps_per_loop <= 0:
- raise ValueError('steps_per_loop should be positive integer.')
- if steps_per_loop == 1:
- return steps_per_loop
- return min(steps_per_loop, steps_per_epoch - steps_in_current_epoch)
-
-
def run(flags_obj):
"""Run ResNet ImageNet training and eval loop using custom training loops.
@@ -108,8 +97,7 @@ def run(flags_obj):
Returns:
Dictionary of training and eval stats.
"""
- keras_utils.set_session_config(
- enable_xla=flags_obj.enable_xla)
+ keras_utils.set_session_config()
performance.set_mixed_precision_policy(flags_core.get_tf_dtype(flags_obj))
if tf.config.list_physical_devices('GPU'):
@@ -121,14 +109,13 @@ def run(flags_obj):
datasets_num_private_threads=flags_obj.datasets_num_private_threads)
common.set_cudnn_batchnorm_mode()
- # TODO(anj-s): Set data_format without using Keras.
data_format = flags_obj.data_format
if data_format is None:
data_format = ('channels_first' if tf.config.list_physical_devices('GPU')
else 'channels_last')
tf.keras.backend.set_image_data_format(data_format)
- strategy = distribution_utils.get_distribution_strategy(
+ strategy = distribute_utils.get_distribution_strategy(
distribution_strategy=flags_obj.distribution_strategy,
num_gpus=flags_obj.num_gpus,
all_reduce_alg=flags_obj.all_reduce_alg,
@@ -137,7 +124,14 @@ def run(flags_obj):
per_epoch_steps, train_epochs, eval_steps = get_num_train_iterations(
flags_obj)
- steps_per_loop = min(flags_obj.steps_per_loop, per_epoch_steps)
+ if flags_obj.steps_per_loop is None:
+ steps_per_loop = per_epoch_steps
+ elif flags_obj.steps_per_loop > per_epoch_steps:
+ steps_per_loop = per_epoch_steps
+ logging.warn('Setting steps_per_loop to %d to respect epoch boundary.',
+ steps_per_loop)
+ else:
+ steps_per_loop = flags_obj.steps_per_loop
logging.info(
'Training %d epochs, each epoch has %d steps, '
@@ -148,14 +142,14 @@ def run(flags_obj):
flags_obj.batch_size,
flags_obj.log_steps,
logdir=flags_obj.model_dir if flags_obj.enable_tensorboard else None)
- with distribution_utils.get_strategy_scope(strategy):
+ with distribute_utils.get_strategy_scope(strategy):
runnable = resnet_runnable.ResnetRunnable(flags_obj, time_callback,
per_epoch_steps)
eval_interval = flags_obj.epochs_between_evals * per_epoch_steps
checkpoint_interval = (
- per_epoch_steps if flags_obj.enable_checkpoint_and_export else None)
- summary_interval = per_epoch_steps if flags_obj.enable_tensorboard else None
+ steps_per_loop * 5 if flags_obj.enable_checkpoint_and_export else None)
+ summary_interval = steps_per_loop if flags_obj.enable_tensorboard else None
checkpoint_manager = tf.train.CheckpointManager(
runnable.checkpoint,
@@ -164,20 +158,25 @@ def run(flags_obj):
step_counter=runnable.global_step,
checkpoint_interval=checkpoint_interval)
- resnet_controller = controller.Controller(
- strategy,
- runnable.train,
- runnable.evaluate if not flags_obj.skip_eval else None,
+ resnet_controller = orbit.Controller(
+ strategy=strategy,
+ trainer=runnable,
+ evaluator=runnable if not flags_obj.skip_eval else None,
global_step=runnable.global_step,
steps_per_loop=steps_per_loop,
- train_steps=per_epoch_steps * train_epochs,
checkpoint_manager=checkpoint_manager,
summary_interval=summary_interval,
- eval_steps=eval_steps,
- eval_interval=eval_interval)
+ summary_dir=flags_obj.model_dir,
+ eval_summary_dir=os.path.join(flags_obj.model_dir, 'eval'))
time_callback.on_train_begin()
- resnet_controller.train(evaluate=not flags_obj.skip_eval)
+ if not flags_obj.skip_eval:
+ resnet_controller.train_and_evaluate(
+ train_steps=per_epoch_steps * train_epochs,
+ eval_steps=eval_steps,
+ eval_interval=eval_interval)
+ else:
+ resnet_controller.train(steps=per_epoch_steps * train_epochs)
time_callback.on_train_end()
stats = build_stats(runnable, time_callback)
diff --git a/official/vision/image_classification/resnet/resnet_model.py b/official/vision/image_classification/resnet/resnet_model.py
index 10f1233356ece188cce51ec254f0064739cd6f41..17d124bb20a649885623277f5e32c5de05703819 100644
--- a/official/vision/image_classification/resnet/resnet_model.py
+++ b/official/vision/image_classification/resnet/resnet_model.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""ResNet50 model for Keras.
Adapted from tf.keras.applications.resnet50.ResNet50().
@@ -28,18 +28,14 @@ from __future__ import division
from __future__ import print_function
import tensorflow as tf
-
-from tensorflow.python.keras import backend
-from tensorflow.python.keras import initializers
-from tensorflow.python.keras import models
-from tensorflow.python.keras import regularizers
from official.vision.image_classification.resnet import imagenet_preprocessing
layers = tf.keras.layers
def _gen_l2_regularizer(use_l2_regularizer=True, l2_weight_decay=1e-4):
- return regularizers.l2(l2_weight_decay) if use_l2_regularizer else None
+ return tf.keras.regularizers.L2(
+ l2_weight_decay) if use_l2_regularizer else None
def identity_block(input_tensor,
@@ -66,7 +62,7 @@ def identity_block(input_tensor,
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
- if backend.image_data_format() == 'channels_last':
+ if tf.keras.backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
@@ -154,7 +150,7 @@ def conv_block(input_tensor,
Output tensor for the block.
"""
filters1, filters2, filters3 = filters
- if backend.image_data_format() == 'channels_last':
+ if tf.keras.backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
@@ -253,7 +249,7 @@ def resnet50(num_classes,
# Hub image modules expect inputs in the range [0, 1]. This rescales these
# inputs to the range expected by the trained model.
x = layers.Lambda(
- lambda x: x * 255.0 - backend.constant(
+ lambda x: x * 255.0 - tf.keras.backend.constant( # pylint: disable=g-long-lambda
imagenet_preprocessing.CHANNEL_MEANS,
shape=[1, 1, 3],
dtype=x.dtype),
@@ -262,7 +258,7 @@ def resnet50(num_classes,
else:
x = img_input
- if backend.image_data_format() == 'channels_first':
+ if tf.keras.backend.image_data_format() == 'channels_first':
x = layers.Permute((3, 1, 2))(x)
bn_axis = 1
else: # channels_last
@@ -315,7 +311,8 @@ def resnet50(num_classes,
x = layers.GlobalAveragePooling2D()(x)
x = layers.Dense(
num_classes,
- kernel_initializer=initializers.RandomNormal(stddev=0.01),
+ kernel_initializer=tf.compat.v1.keras.initializers.random_normal(
+ stddev=0.01),
kernel_regularizer=_gen_l2_regularizer(use_l2_regularizer),
bias_regularizer=_gen_l2_regularizer(use_l2_regularizer),
name='fc1000')(
@@ -326,4 +323,4 @@ def resnet50(num_classes,
x = layers.Activation('softmax', dtype='float32')(x)
# Create model.
- return models.Model(img_input, x, name='resnet50')
+ return tf.keras.Model(img_input, x, name='resnet50')
diff --git a/official/vision/image_classification/resnet/resnet_runnable.py b/official/vision/image_classification/resnet/resnet_runnable.py
index 473b18daf7aaf02bfb1dc86110b3ae0fd2704359..6fa40b98cb76377a9b11e3f4a35b38094eca5cc5 100644
--- a/official/vision/image_classification/resnet/resnet_runnable.py
+++ b/official/vision/image_classification/resnet/resnet_runnable.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,36 +11,24 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""Runs a ResNet model on the ImageNet dataset using custom training loops."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Runs a ResNet model on the ImageNet dataset using custom training loops."""
+import orbit
import tensorflow as tf
from official.modeling import performance
from official.staging.training import grad_utils
-from official.staging.training import standard_runnable
-from official.staging.training import utils
from official.utils.flags import core as flags_core
from official.vision.image_classification.resnet import common
from official.vision.image_classification.resnet import imagenet_preprocessing
from official.vision.image_classification.resnet import resnet_model
-class ResnetRunnable(standard_runnable.StandardTrainable,
- standard_runnable.StandardEvaluable):
+class ResnetRunnable(orbit.StandardTrainer, orbit.StandardEvaluator):
"""Implements the training and evaluation APIs for Resnet model."""
def __init__(self, flags_obj, time_callback, epoch_steps):
- standard_runnable.StandardTrainable.__init__(self,
- flags_obj.use_tf_while_loop,
- flags_obj.use_tf_function)
- standard_runnable.StandardEvaluable.__init__(self,
- flags_obj.use_tf_function)
-
self.strategy = tf.distribute.get_strategy()
self.flags_obj = flags_obj
self.dtype = flags_core.get_tf_dtype(flags_obj)
@@ -54,7 +42,7 @@ class ResnetRunnable(standard_runnable.StandardTrainable,
self.strategy.num_replicas_in_sync))
# As auto rebatching is not supported in
- # `experimental_distribute_datasets_from_function()` API, which is
+ # `distribute_datasets_from_function()` API, which is
# required when cloning dataset to multiple workers in eager mode,
# we use per-replica batch size.
self.batch_size = int(batch_size / self.strategy.num_replicas_in_sync)
@@ -107,11 +95,8 @@ class ResnetRunnable(standard_runnable.StandardTrainable,
# Handling epochs.
self.epoch_steps = epoch_steps
- self.epoch_helper = utils.EpochHelper(epoch_steps, self.global_step)
-
- def build_train_dataset(self):
- """See base class."""
- return utils.make_distributed_dataset(
+ self.epoch_helper = orbit.utils.EpochHelper(epoch_steps, self.global_step)
+ train_dataset = orbit.utils.make_distributed_dataset(
self.strategy,
self.input_fn,
is_training=True,
@@ -122,17 +107,26 @@ class ResnetRunnable(standard_runnable.StandardTrainable,
.datasets_num_private_threads,
dtype=self.dtype,
drop_remainder=True)
-
- def build_eval_dataset(self):
- """See base class."""
- return utils.make_distributed_dataset(
- self.strategy,
- self.input_fn,
- is_training=False,
- data_dir=self.flags_obj.data_dir,
- batch_size=self.batch_size,
- parse_record_fn=imagenet_preprocessing.parse_record,
- dtype=self.dtype)
+ orbit.StandardTrainer.__init__(
+ self,
+ train_dataset,
+ options=orbit.StandardTrainerOptions(
+ use_tf_while_loop=flags_obj.use_tf_while_loop,
+ use_tf_function=flags_obj.use_tf_function))
+ if not flags_obj.skip_eval:
+ eval_dataset = orbit.utils.make_distributed_dataset(
+ self.strategy,
+ self.input_fn,
+ is_training=False,
+ data_dir=self.flags_obj.data_dir,
+ batch_size=self.batch_size,
+ parse_record_fn=imagenet_preprocessing.parse_record,
+ dtype=self.dtype)
+ orbit.StandardEvaluator.__init__(
+ self,
+ eval_dataset,
+ options=orbit.StandardEvaluatorOptions(
+ use_tf_function=flags_obj.use_tf_function))
def train_loop_begin(self):
"""See base class."""
@@ -173,7 +167,8 @@ class ResnetRunnable(standard_runnable.StandardTrainable,
tape, self.optimizer, loss, self.model.trainable_variables)
self.train_loss.update_state(loss)
self.train_accuracy.update_state(labels, logits)
-
+ if self.flags_obj.enable_xla:
+ step_fn = tf.function(step_fn, jit_compile=True)
self.strategy.run(step_fn, args=(next(iterator),))
def train_loop_end(self):
diff --git a/official/vision/image_classification/resnet/tfhub_export.py b/official/vision/image_classification/resnet/tfhub_export.py
index ff1f124a1d67c93b9deee453a23cf71133bb6434..3f79a791304a8092bd5af808693a156d027a6ed1 100644
--- a/official/vision/image_classification/resnet/tfhub_export.py
+++ b/official/vision/image_classification/resnet/tfhub_export.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""A script to export TF-Hub SavedModel."""
from __future__ import absolute_import
@@ -21,6 +21,7 @@ from __future__ import print_function
import os
+# Import libraries
from absl import app
from absl import flags
diff --git a/official/vision/image_classification/test_utils.py b/official/vision/image_classification/test_utils.py
index a6dc91dc775ce25950a8918450548c19992eb2c4..8d7180c9d4e10c3241c4d6dd31d2cd013439df7a 100644
--- a/official/vision/image_classification/test_utils.py
+++ b/official/vision/image_classification/test_utils.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,28 +11,27 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Test utilities for image classification tasks."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
-from tensorflow.python.keras import backend
-from tensorflow.python.keras import layers
-from tensorflow.python.keras import models
+import tensorflow as tf
def trivial_model(num_classes):
"""Trivial model for ImageNet dataset."""
input_shape = (224, 224, 3)
- img_input = layers.Input(shape=input_shape)
+ img_input = tf.keras.layers.Input(shape=input_shape)
- x = layers.Lambda(lambda x: backend.reshape(x, [-1, 224 * 224 * 3]),
- name='reshape')(img_input)
- x = layers.Dense(1, name='fc1')(x)
- x = layers.Dense(num_classes, name='fc1000')(x)
- x = layers.Activation('softmax', dtype='float32')(x)
+ x = tf.keras.layers.Lambda(
+ lambda x: tf.keras.backend.reshape(x, [-1, 224 * 224 * 3]),
+ name='reshape')(img_input)
+ x = tf.keras.layers.Dense(1, name='fc1')(x)
+ x = tf.keras.layers.Dense(num_classes, name='fc1000')(x)
+ x = tf.keras.layers.Activation('softmax', dtype='float32')(x)
- return models.Model(img_input, x, name='trivial')
+ return tf.keras.models.Model(img_input, x, name='trivial')
diff --git a/official/vision/keras_cv/LICENSE b/official/vision/keras_cv/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..0b1ba4429805c3f80ea21528070e0791c484021b
--- /dev/null
+++ b/official/vision/keras_cv/LICENSE
@@ -0,0 +1,203 @@
+Copyright 2020 The TensorFlow Authors. All rights reserved.
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2015, The TensorFlow Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
\ No newline at end of file
diff --git a/official/vision/keras_cv/README.md b/official/vision/keras_cv/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..1132d521cd78605417b4c5a301d61d790a0410e5
--- /dev/null
+++ b/official/vision/keras_cv/README.md
@@ -0,0 +1,13 @@
+# keras-cv
+
+## Losses
+
+* [FocalLoss](losses/focal_loss.py) implements Focal loss as described in
+ ["Focal Loss for Dense Object Detection"](https://arxiv.org/abs/1708.02002).
+
+
+## Ops
+
+Ops are used in data pipeline for pre-compute labels, weights.
+
+* [IOUSimilarity](ops/iou_similarity.py) implements Intersection-Over-Union.
diff --git a/official/vision/keras_cv/__init__.py b/official/vision/keras_cv/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d448259a474eaee1b54d332df0e658221f21587
--- /dev/null
+++ b/official/vision/keras_cv/__init__.py
@@ -0,0 +1,20 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Keras-CV package definition."""
+# pylint: disable=wildcard-import
+from official.vision.keras_cv import layers
+from official.vision.keras_cv import losses
+from official.vision.keras_cv import metrics
+from official.vision.keras_cv import ops
diff --git a/official/vision/keras_cv/contributing.md b/official/vision/keras_cv/contributing.md
new file mode 100644
index 0000000000000000000000000000000000000000..d9efe9b0691c8fdb580513d52d6a35271c096bbb
--- /dev/null
+++ b/official/vision/keras_cv/contributing.md
@@ -0,0 +1,21 @@
+## Contributing to KerasCV
+
+Patches to KerasCV are welcome!
+
+The source-of-truth repository lives under
+[TF Model Garden Vision](https://github.com/tensorflow/models/official/vision/keras_cv),
+and is mirrored as a read-only repository under
+[keras-team/keras-cv](https://github.com/keras-team/keras-cv).
+Contributions should be made as PRs to the TF Model Garden repository.
+This is to ensure the codebase is rigorously tested with state-of-art models
+on different accelerators.
+In the long run, we will move development to the current repository `keras-team/keras-cv`.
+
+## :heavy_check_mark: Contributor checklist
+
+1. Ensure you have signed the [Contributor License Agreement](https://cla.developers.google.com/about/google-individual?csw=1).
+ * All code contributors are required to sign a Contributor License Agreement.
+ * Please read this [troubleshooting guide](Contributor-License-Agreements#troubleshooting-clas)
+ if you encounter an issue.
+2. Please review the [contribution guidelines](https://github.com/tensorflow/models/wiki/How-to-contribute).
+3. Check if your changes are consistent with the [TensorFlow coding style](https://www.tensorflow.org/community/contribute/code_style).
diff --git a/official/vision/keras_cv/layers/__init__.py b/official/vision/keras_cv/layers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..db606e5ada05387059ccbdcd5774e3c76484b0ec
--- /dev/null
+++ b/official/vision/keras_cv/layers/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Keras-CV layers package definition."""
+from official.vision.keras_cv.layers.deeplab import SpatialPyramidPooling
diff --git a/official/vision/keras_cv/layers/deeplab.py b/official/vision/keras_cv/layers/deeplab.py
new file mode 100644
index 0000000000000000000000000000000000000000..40b820c20862aa56cb3194d5d110120ac62dfaf7
--- /dev/null
+++ b/official/vision/keras_cv/layers/deeplab.py
@@ -0,0 +1,193 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Layers for DeepLabV3."""
+
+import tensorflow as tf
+
+
+@tf.keras.utils.register_keras_serializable(package='keras_cv')
+class SpatialPyramidPooling(tf.keras.layers.Layer):
+ """Implements the Atrous Spatial Pyramid Pooling.
+
+ Reference:
+ [Rethinking Atrous Convolution for Semantic Image Segmentation](
+ https://arxiv.org/pdf/1706.05587.pdf)
+ """
+
+ def __init__(
+ self,
+ output_channels,
+ dilation_rates,
+ pool_kernel_size=None,
+ use_sync_bn=False,
+ batchnorm_momentum=0.99,
+ batchnorm_epsilon=0.001,
+ activation='relu',
+ dropout=0.5,
+ kernel_initializer='glorot_uniform',
+ kernel_regularizer=None,
+ interpolation='bilinear',
+ **kwargs):
+ """Initializes `SpatialPyramidPooling`.
+
+ Args:
+ output_channels: Number of channels produced by SpatialPyramidPooling.
+ dilation_rates: A list of integers for parallel dilated conv.
+ pool_kernel_size: A list of integers or None. If None, global average
+ pooling is applied, otherwise an average pooling of pool_kernel_size
+ is applied.
+ use_sync_bn: A bool, whether or not to use sync batch normalization.
+ batchnorm_momentum: A float for the momentum in BatchNorm. Defaults to
+ 0.99.
+ batchnorm_epsilon: A float for the epsilon value in BatchNorm. Defaults to
+ 0.001.
+ activation: A `str` for type of activation to be used. Defaults to 'relu'.
+ dropout: A float for the dropout rate before output. Defaults to 0.5.
+ kernel_initializer: Kernel initializer for conv layers. Defaults to
+ `glorot_uniform`.
+ kernel_regularizer: Kernel regularizer for conv layers. Defaults to None.
+ interpolation: The interpolation method for upsampling. Defaults to
+ `bilinear`.
+ **kwargs: Other keyword arguments for the layer.
+ """
+ super(SpatialPyramidPooling, self).__init__(**kwargs)
+
+ self.output_channels = output_channels
+ self.dilation_rates = dilation_rates
+ self.use_sync_bn = use_sync_bn
+ self.batchnorm_momentum = batchnorm_momentum
+ self.batchnorm_epsilon = batchnorm_epsilon
+ self.activation = activation
+ self.dropout = dropout
+ self.kernel_initializer = tf.keras.initializers.get(kernel_initializer)
+ self.kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer)
+ self.interpolation = interpolation
+ self.input_spec = tf.keras.layers.InputSpec(ndim=4)
+ self.pool_kernel_size = pool_kernel_size
+
+ def build(self, input_shape):
+ height = input_shape[1]
+ width = input_shape[2]
+ channels = input_shape[3]
+
+ self.aspp_layers = []
+
+ if self.use_sync_bn:
+ bn_op = tf.keras.layers.experimental.SyncBatchNormalization
+ else:
+ bn_op = tf.keras.layers.BatchNormalization
+
+ if tf.keras.backend.image_data_format() == 'channels_last':
+ bn_axis = -1
+ else:
+ bn_axis = 1
+
+ conv_sequential = tf.keras.Sequential([
+ tf.keras.layers.Conv2D(
+ filters=self.output_channels, kernel_size=(1, 1),
+ kernel_initializer=self.kernel_initializer,
+ kernel_regularizer=self.kernel_regularizer,
+ use_bias=False),
+ bn_op(
+ axis=bn_axis,
+ momentum=self.batchnorm_momentum,
+ epsilon=self.batchnorm_epsilon),
+ tf.keras.layers.Activation(self.activation)
+ ])
+ self.aspp_layers.append(conv_sequential)
+
+ for dilation_rate in self.dilation_rates:
+ conv_sequential = tf.keras.Sequential([
+ tf.keras.layers.Conv2D(
+ filters=self.output_channels, kernel_size=(3, 3),
+ padding='same', kernel_regularizer=self.kernel_regularizer,
+ kernel_initializer=self.kernel_initializer,
+ dilation_rate=dilation_rate, use_bias=False),
+ bn_op(axis=bn_axis, momentum=self.batchnorm_momentum,
+ epsilon=self.batchnorm_epsilon),
+ tf.keras.layers.Activation(self.activation)])
+ self.aspp_layers.append(conv_sequential)
+
+ if self.pool_kernel_size is None:
+ pool_sequential = tf.keras.Sequential([
+ tf.keras.layers.GlobalAveragePooling2D(),
+ tf.keras.layers.Reshape((1, 1, channels))])
+ else:
+ pool_sequential = tf.keras.Sequential([
+ tf.keras.layers.AveragePooling2D(self.pool_kernel_size)])
+
+ pool_sequential.add(
+ tf.keras.Sequential([
+ tf.keras.layers.Conv2D(
+ filters=self.output_channels,
+ kernel_size=(1, 1),
+ kernel_initializer=self.kernel_initializer,
+ kernel_regularizer=self.kernel_regularizer,
+ use_bias=False),
+ bn_op(
+ axis=bn_axis,
+ momentum=self.batchnorm_momentum,
+ epsilon=self.batchnorm_epsilon),
+ tf.keras.layers.Activation(self.activation),
+ tf.keras.layers.experimental.preprocessing.Resizing(
+ height,
+ width,
+ interpolation=self.interpolation,
+ dtype=tf.float32)
+ ]))
+
+ self.aspp_layers.append(pool_sequential)
+
+ self.projection = tf.keras.Sequential([
+ tf.keras.layers.Conv2D(
+ filters=self.output_channels, kernel_size=(1, 1),
+ kernel_initializer=self.kernel_initializer,
+ kernel_regularizer=self.kernel_regularizer,
+ use_bias=False),
+ bn_op(
+ axis=bn_axis,
+ momentum=self.batchnorm_momentum,
+ epsilon=self.batchnorm_epsilon),
+ tf.keras.layers.Activation(self.activation),
+ tf.keras.layers.Dropout(rate=self.dropout)])
+
+ def call(self, inputs, training=None):
+ if training is None:
+ training = tf.keras.backend.learning_phase()
+ result = []
+ for layer in self.aspp_layers:
+ result.append(tf.cast(layer(inputs, training=training), inputs.dtype))
+ result = tf.concat(result, axis=-1)
+ result = self.projection(result, training=training)
+ return result
+
+ def get_config(self):
+ config = {
+ 'output_channels': self.output_channels,
+ 'dilation_rates': self.dilation_rates,
+ 'pool_kernel_size': self.pool_kernel_size,
+ 'use_sync_bn': self.use_sync_bn,
+ 'batchnorm_momentum': self.batchnorm_momentum,
+ 'batchnorm_epsilon': self.batchnorm_epsilon,
+ 'activation': self.activation,
+ 'dropout': self.dropout,
+ 'kernel_initializer': tf.keras.initializers.serialize(
+ self.kernel_initializer),
+ 'kernel_regularizer': tf.keras.regularizers.serialize(
+ self.kernel_regularizer),
+ 'interpolation': self.interpolation,
+ }
+ base_config = super(SpatialPyramidPooling, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
diff --git a/official/vision/keras_cv/layers/deeplab_test.py b/official/vision/keras_cv/layers/deeplab_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..858382ebf1b52ecbadb5bbf1605631f39f4b6f0e
--- /dev/null
+++ b/official/vision/keras_cv/layers/deeplab_test.py
@@ -0,0 +1,53 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for ASPP."""
+
+import tensorflow as tf
+
+from tensorflow.python.keras import keras_parameterized
+from official.vision.keras_cv.layers import deeplab
+
+
+@keras_parameterized.run_all_keras_modes
+class DeeplabTest(keras_parameterized.TestCase):
+
+ @keras_parameterized.parameterized.parameters(
+ (None,),
+ ([32, 32],),
+ )
+ def test_aspp(self, pool_kernel_size):
+ inputs = tf.keras.Input(shape=(64, 64, 128), dtype=tf.float32)
+ layer = deeplab.SpatialPyramidPooling(output_channels=256,
+ dilation_rates=[6, 12, 18],
+ pool_kernel_size=None)
+ output = layer(inputs)
+ self.assertAllEqual([None, 64, 64, 256], output.shape)
+
+ def test_aspp_invalid_shape(self):
+ inputs = tf.keras.Input(shape=(64, 64), dtype=tf.float32)
+ layer = deeplab.SpatialPyramidPooling(output_channels=256,
+ dilation_rates=[6, 12, 18])
+ with self.assertRaises(ValueError):
+ _ = layer(inputs)
+
+ def test_config_with_custom_name(self):
+ layer = deeplab.SpatialPyramidPooling(256, [5], name='aspp')
+ config = layer.get_config()
+ layer_1 = deeplab.SpatialPyramidPooling.from_config(config)
+ self.assertEqual(layer_1.name, layer.name)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/keras_cv/losses/__init__.py b/official/vision/keras_cv/losses/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd95f0a0249a696516a9cdcc2271d922fbf27d50
--- /dev/null
+++ b/official/vision/keras_cv/losses/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Keras-CV layers package definition."""
+from official.vision.keras_cv.losses.focal_loss import FocalLoss
+from official.vision.keras_cv.losses.loss_utils import multi_level_flatten
diff --git a/official/vision/keras_cv/losses/focal_loss.py b/official/vision/keras_cv/losses/focal_loss.py
new file mode 100644
index 0000000000000000000000000000000000000000..7241d9ae261e6644bb973e85587a3f6de535f603
--- /dev/null
+++ b/official/vision/keras_cv/losses/focal_loss.py
@@ -0,0 +1,84 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Losses used for detection models."""
+
+import tensorflow as tf
+
+
+class FocalLoss(tf.keras.losses.Loss):
+ """Implements a Focal loss for classification problems.
+
+ Reference:
+ [Focal Loss for Dense Object Detection](https://arxiv.org/abs/1708.02002).
+ """
+
+ def __init__(self,
+ alpha,
+ gamma,
+ reduction=tf.keras.losses.Reduction.AUTO,
+ name=None):
+ """Initializes `FocalLoss`.
+
+ Args:
+ alpha: The `alpha` weight factor for binary class imbalance.
+ gamma: The `gamma` focusing parameter to re-weight loss.
+ reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to
+ loss. Default value is `AUTO`. `AUTO` indicates that the reduction
+ option will be determined by the usage context. For almost all cases
+ this defaults to `SUM_OVER_BATCH_SIZE`. When used with
+ `tf.distribute.Strategy`, outside of built-in training loops such as
+ `tf.keras` `compile` and `fit`, using `AUTO` or `SUM_OVER_BATCH_SIZE`
+ will raise an error. Please see this custom training [tutorial](
+ https://www.tensorflow.org/tutorials/distribute/custom_training) for
+ more details.
+ name: Optional name for the op. Defaults to 'retinanet_class_loss'.
+ """
+ self._alpha = alpha
+ self._gamma = gamma
+ super(FocalLoss, self).__init__(reduction=reduction, name=name)
+
+ def call(self, y_true, y_pred):
+ """Invokes the `FocalLoss`.
+
+ Args:
+ y_true: A tensor of size [batch, num_anchors, num_classes]
+ y_pred: A tensor of size [batch, num_anchors, num_classes]
+
+ Returns:
+ Summed loss float `Tensor`.
+ """
+ with tf.name_scope('focal_loss'):
+ y_true = tf.cast(y_true, dtype=tf.float32)
+ y_pred = tf.cast(y_pred, dtype=tf.float32)
+ positive_label_mask = tf.equal(y_true, 1.0)
+ cross_entropy = (
+ tf.nn.sigmoid_cross_entropy_with_logits(labels=y_true, logits=y_pred))
+ probs = tf.sigmoid(y_pred)
+ probs_gt = tf.where(positive_label_mask, probs, 1.0 - probs)
+ # With small gamma, the implementation could produce NaN during back prop.
+ modulator = tf.pow(1.0 - probs_gt, self._gamma)
+ loss = modulator * cross_entropy
+ weighted_loss = tf.where(positive_label_mask, self._alpha * loss,
+ (1.0 - self._alpha) * loss)
+
+ return weighted_loss
+
+ def get_config(self):
+ config = {
+ 'alpha': self._alpha,
+ 'gamma': self._gamma,
+ }
+ base_config = super(FocalLoss, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
diff --git a/official/vision/keras_cv/losses/loss_utils.py b/official/vision/keras_cv/losses/loss_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..70bc1ce5cad1d26de41a41b4d58750fb6c9c2928
--- /dev/null
+++ b/official/vision/keras_cv/losses/loss_utils.py
@@ -0,0 +1,42 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Losses utilities for detection models."""
+
+import tensorflow as tf
+
+
+def multi_level_flatten(multi_level_inputs, last_dim=None):
+ """Flattens a multi-level input.
+
+ Args:
+ multi_level_inputs: Ordered Dict with level to [batch, d1, ..., dm].
+ last_dim: Whether the output should be [batch_size, None], or [batch_size,
+ None, last_dim]. Defaults to `None`.
+
+ Returns:
+ Concatenated output [batch_size, None], or [batch_size, None, dm]
+ """
+ flattened_inputs = []
+ batch_size = None
+ for level in multi_level_inputs.keys():
+ single_input = multi_level_inputs[level]
+ if batch_size is None:
+ batch_size = single_input.shape[0] or tf.shape(single_input)[0]
+ if last_dim is not None:
+ flattened_input = tf.reshape(single_input, [batch_size, -1, last_dim])
+ else:
+ flattened_input = tf.reshape(single_input, [batch_size, -1])
+ flattened_inputs.append(flattened_input)
+ return tf.concat(flattened_inputs, axis=1)
diff --git a/official/vision/keras_cv/metrics/__init__.py b/official/vision/keras_cv/metrics/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..507349b4ab4b35bd91b7bb77b396db96ad16eacf
--- /dev/null
+++ b/official/vision/keras_cv/metrics/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Keras-CV metrics package definition."""
+from official.vision.keras_cv.metrics.iou import PerClassIoU
diff --git a/official/vision/keras_cv/metrics/iou.py b/official/vision/keras_cv/metrics/iou.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1d94e7ea446cb292a5ea7e3722a5ab1df696138
--- /dev/null
+++ b/official/vision/keras_cv/metrics/iou.py
@@ -0,0 +1,129 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""IOU Metrics used for semantic segmentation models."""
+
+import numpy as np
+import tensorflow as tf
+
+
+class PerClassIoU(tf.keras.metrics.Metric):
+ """Computes the per-class Intersection-Over-Union metric.
+
+ Mean Intersection-Over-Union is a common evaluation metric for semantic image
+ segmentation, which first computes the IOU for each semantic class.
+ IOU is defined as follows:
+ IOU = true_positive / (true_positive + false_positive + false_negative).
+ The predictions are accumulated in a confusion matrix, weighted by
+ `sample_weight` and the metric is then calculated from it.
+
+ If `sample_weight` is `None`, weights default to 1.
+ Use `sample_weight` of 0 to mask values.
+
+ Example:
+
+ >>> # cm = [[1, 1],
+ >>> # [1, 1]]
+ >>> # sum_row = [2, 2], sum_col = [2, 2], true_positives = [1, 1]
+ >>> # iou = true_positives / (sum_row + sum_col - true_positives))
+ >>> # result = [(1 / (2 + 2 - 1), 1 / (2 + 2 - 1)] = 0.33
+ >>> m = tf.keras.metrics.MeanIoU(num_classes=2)
+ >>> m.update_state([0, 0, 1, 1], [0, 1, 0, 1])
+ >>> m.result().numpy()
+ [0.33333334, 0.33333334]
+
+ """
+
+ def __init__(self, num_classes, name=None, dtype=None):
+ """Initializes `PerClassIoU`.
+
+ Args:
+ num_classes: The possible number of labels the prediction task can have.
+ This value must be provided, since a confusion matrix of dimension =
+ [num_classes, num_classes] will be allocated.
+ name: (Optional) string name of the metric instance.
+ dtype: (Optional) data type of the metric result.
+
+ """
+
+ super(PerClassIoU, self).__init__(name=name, dtype=dtype)
+ self.num_classes = num_classes
+
+ # Variable to accumulate the predictions in the confusion matrix.
+ self.total_cm = self.add_weight(
+ 'total_confusion_matrix',
+ shape=(num_classes, num_classes),
+ initializer=tf.compat.v1.zeros_initializer)
+
+ def update_state(self, y_true, y_pred, sample_weight=None):
+ """Accumulates the confusion matrix statistics.
+
+ Args:
+ y_true: The ground truth values.
+ y_pred: The predicted values.
+ sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+ `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+ be broadcastable to `y_true`.
+
+ Returns:
+ IOU per class.
+ """
+
+ y_true = tf.cast(y_true, self._dtype)
+ y_pred = tf.cast(y_pred, self._dtype)
+
+ # Flatten the input if its rank > 1.
+ if y_pred.shape.ndims > 1:
+ y_pred = tf.reshape(y_pred, [-1])
+
+ if y_true.shape.ndims > 1:
+ y_true = tf.reshape(y_true, [-1])
+
+ if sample_weight is not None:
+ sample_weight = tf.cast(sample_weight, self._dtype)
+ if sample_weight.shape.ndims > 1:
+ sample_weight = tf.reshape(sample_weight, [-1])
+
+ # Accumulate the prediction to current confusion matrix.
+ current_cm = tf.math.confusion_matrix(
+ y_true,
+ y_pred,
+ self.num_classes,
+ weights=sample_weight,
+ dtype=self._dtype)
+ return self.total_cm.assign_add(current_cm)
+
+ def result(self):
+ """Compute the mean intersection-over-union via the confusion matrix."""
+ sum_over_row = tf.cast(
+ tf.reduce_sum(self.total_cm, axis=0), dtype=self._dtype)
+ sum_over_col = tf.cast(
+ tf.reduce_sum(self.total_cm, axis=1), dtype=self._dtype)
+ true_positives = tf.cast(
+ tf.linalg.tensor_diag_part(self.total_cm), dtype=self._dtype)
+
+ # sum_over_row + sum_over_col =
+ # 2 * true_positives + false_positives + false_negatives.
+ denominator = sum_over_row + sum_over_col - true_positives
+
+ return tf.math.divide_no_nan(true_positives, denominator)
+
+ def reset_states(self):
+ tf.keras.backend.set_value(
+ self.total_cm, np.zeros((self.num_classes, self.num_classes)))
+
+ def get_config(self):
+ config = {'num_classes': self.num_classes}
+ base_config = super(PerClassIoU, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
diff --git a/official/vision/keras_cv/metrics/iou_test.py b/official/vision/keras_cv/metrics/iou_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..20c2aa39970cdd453636e56984d70497460e00a5
--- /dev/null
+++ b/official/vision/keras_cv/metrics/iou_test.py
@@ -0,0 +1,99 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for Keras metrics functions."""
+
+import tensorflow as tf
+
+from official.vision.keras_cv.metrics import iou
+
+
+class MeanIoUTest(tf.test.TestCase):
+
+ def test_config(self):
+ m_obj = iou.PerClassIoU(num_classes=2, name='per_class_iou')
+ self.assertEqual(m_obj.name, 'per_class_iou')
+ self.assertEqual(m_obj.num_classes, 2)
+
+ m_obj2 = iou.PerClassIoU.from_config(m_obj.get_config())
+ self.assertEqual(m_obj2.name, 'per_class_iou')
+ self.assertEqual(m_obj2.num_classes, 2)
+
+ def test_unweighted(self):
+ y_pred = [0, 1, 0, 1]
+ y_true = [0, 0, 1, 1]
+
+ m_obj = iou.PerClassIoU(num_classes=2)
+
+ result = m_obj(y_true, y_pred)
+
+ # cm = [[1, 1],
+ # [1, 1]]
+ # sum_row = [2, 2], sum_col = [2, 2], true_positives = [1, 1]
+ # iou = true_positives / (sum_row + sum_col - true_positives))
+ expected_result = [1 / (2 + 2 - 1), 1 / (2 + 2 - 1)]
+ self.assertAllClose(expected_result, result, atol=1e-3)
+
+ def test_weighted(self):
+ y_pred = tf.constant([0, 1, 0, 1], dtype=tf.float32)
+ y_true = tf.constant([0, 0, 1, 1])
+ sample_weight = tf.constant([0.2, 0.3, 0.4, 0.1])
+
+ m_obj = iou.PerClassIoU(num_classes=2)
+
+ result = m_obj(y_true, y_pred, sample_weight=sample_weight)
+
+ # cm = [[0.2, 0.3],
+ # [0.4, 0.1]]
+ # sum_row = [0.6, 0.4], sum_col = [0.5, 0.5], true_positives = [0.2, 0.1]
+ # iou = true_positives / (sum_row + sum_col - true_positives))
+ expected_result = [0.2 / (0.6 + 0.5 - 0.2), 0.1 / (0.4 + 0.5 - 0.1)]
+ self.assertAllClose(expected_result, result, atol=1e-3)
+
+ def test_multi_dim_input(self):
+ y_pred = tf.constant([[0, 1], [0, 1]], dtype=tf.float32)
+ y_true = tf.constant([[0, 0], [1, 1]])
+ sample_weight = tf.constant([[0.2, 0.3], [0.4, 0.1]])
+
+ m_obj = iou.PerClassIoU(num_classes=2)
+
+ result = m_obj(y_true, y_pred, sample_weight=sample_weight)
+
+ # cm = [[0.2, 0.3],
+ # [0.4, 0.1]]
+ # sum_row = [0.6, 0.4], sum_col = [0.5, 0.5], true_positives = [0.2, 0.1]
+ # iou = true_positives / (sum_row + sum_col - true_positives))
+ expected_result = [0.2 / (0.6 + 0.5 - 0.2), 0.1 / (0.4 + 0.5 - 0.1)]
+ self.assertAllClose(expected_result, result, atol=1e-3)
+
+ def test_zero_valid_entries(self):
+ m_obj = iou.PerClassIoU(num_classes=2)
+ self.assertAllClose(m_obj.result(), [0, 0], atol=1e-3)
+
+ def test_zero_and_non_zero_entries(self):
+ y_pred = tf.constant([1], dtype=tf.float32)
+ y_true = tf.constant([1])
+
+ m_obj = iou.PerClassIoU(num_classes=2)
+ result = m_obj(y_true, y_pred)
+
+ # cm = [[0, 0],
+ # [0, 1]]
+ # sum_row = [0, 1], sum_col = [0, 1], true_positives = [0, 1]
+ # iou = true_positives / (sum_row + sum_col - true_positives))
+ expected_result = [0, 1 / (1 + 1 - 1)]
+ self.assertAllClose(expected_result, result, atol=1e-3)
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/keras_cv/ops/__init__.py b/official/vision/keras_cv/ops/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2fc6dcd8a5f4ecb8ff565fe43a86b772ae16dc72
--- /dev/null
+++ b/official/vision/keras_cv/ops/__init__.py
@@ -0,0 +1,19 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Keras-CV layers package definition."""
+from official.vision.keras_cv.ops.anchor_generator import AnchorGenerator
+from official.vision.keras_cv.ops.box_matcher import BoxMatcher
+from official.vision.keras_cv.ops.iou_similarity import IouSimilarity
+from official.vision.keras_cv.ops.target_gather import TargetGather
diff --git a/official/vision/keras_cv/ops/anchor_generator.py b/official/vision/keras_cv/ops/anchor_generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..c15b178da09913cd080bf54ccd2ee261f17751d6
--- /dev/null
+++ b/official/vision/keras_cv/ops/anchor_generator.py
@@ -0,0 +1,182 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Multi scale anchor generator definition."""
+
+import tensorflow as tf
+
+
+# (TODO/tanzheny): consider having customized anchor offset.
+class _SingleAnchorGenerator:
+ """Utility to generate anchors for a single feature map.
+
+ Example:
+ ```python
+ anchor_gen = _SingleAnchorGenerator(32, [.5, 1., 2.], stride=16)
+ anchors = anchor_gen([512, 512, 3])
+ ```
+ """
+
+ def __init__(self,
+ anchor_size,
+ scales,
+ aspect_ratios,
+ stride,
+ clip_boxes=False):
+ """Constructs single scale anchor.
+
+ Args:
+ anchor_size: A single int represents the base anchor size. The anchor
+ height will be `anchor_size / sqrt(aspect_ratio)`, anchor width will be
+ `anchor_size * sqrt(aspect_ratio)`.
+ scales: A list/tuple, or a list/tuple of a list/tuple of positive
+ floats representing the actual anchor size to the base `anchor_size`.
+ aspect_ratios: a list/tuple of positive floats representing the ratio of
+ anchor width to anchor height.
+ stride: A single int represents the anchor stride size between center of
+ each anchor.
+ clip_boxes: Boolean to represent whether the anchor coordinates should be
+ clipped to the image size. Defaults to `True`.
+ Input shape: the size of the image, `[H, W, C]`
+ Output shape: the size of anchors, `[(H / stride) * (W / stride), 4]`
+ """
+ self.anchor_size = anchor_size
+ self.scales = scales
+ self.aspect_ratios = aspect_ratios
+ self.stride = stride
+ self.clip_boxes = clip_boxes
+
+ def __call__(self, image_size):
+ image_height = tf.cast(image_size[0], tf.float32)
+ image_width = tf.cast(image_size[1], tf.float32)
+
+ k = len(self.scales) * len(self.aspect_ratios)
+ aspect_ratios_sqrt = tf.cast(tf.sqrt(self.aspect_ratios), dtype=tf.float32)
+ anchor_size = tf.cast(self.anchor_size, tf.float32)
+
+ # [K]
+ anchor_heights = []
+ anchor_widths = []
+ for scale in self.scales:
+ anchor_size_t = anchor_size * scale
+ anchor_height = anchor_size_t / aspect_ratios_sqrt
+ anchor_width = anchor_size_t * aspect_ratios_sqrt
+ anchor_heights.append(anchor_height)
+ anchor_widths.append(anchor_width)
+ anchor_heights = tf.concat(anchor_heights, axis=0)
+ anchor_widths = tf.concat(anchor_widths, axis=0)
+ half_anchor_heights = tf.reshape(0.5 * anchor_heights, [1, 1, k])
+ half_anchor_widths = tf.reshape(0.5 * anchor_widths, [1, 1, k])
+
+ stride = tf.cast(self.stride, tf.float32)
+ # [W]
+ cx = tf.range(0.5 * stride, image_width, stride)
+ # [H]
+ cy = tf.range(0.5 * stride, image_height, stride)
+ # [H, W]
+ cx_grid, cy_grid = tf.meshgrid(cx, cy)
+ # [H, W, 1]
+ cx_grid = tf.expand_dims(cx_grid, axis=-1)
+ cy_grid = tf.expand_dims(cy_grid, axis=-1)
+
+ # [H, W, K, 1]
+ y_min = tf.expand_dims(cy_grid - half_anchor_heights, axis=-1)
+ y_max = tf.expand_dims(cy_grid + half_anchor_heights, axis=-1)
+ x_min = tf.expand_dims(cx_grid - half_anchor_widths, axis=-1)
+ x_max = tf.expand_dims(cx_grid + half_anchor_widths, axis=-1)
+
+ if self.clip_boxes:
+ y_min = tf.maximum(tf.minimum(y_min, image_height), 0.)
+ y_max = tf.maximum(tf.minimum(y_max, image_height), 0.)
+ x_min = tf.maximum(tf.minimum(x_min, image_width), 0.)
+ x_max = tf.maximum(tf.minimum(x_max, image_width), 0.)
+
+ # [H, W, K, 4]
+ result = tf.concat([y_min, x_min, y_max, x_max], axis=-1)
+ shape = result.shape.as_list()
+ # [H, W, K * 4]
+ return tf.reshape(result, [shape[0], shape[1], shape[2] * shape[3]])
+
+
+class AnchorGenerator():
+ """Utility to generate anchors for a multiple feature maps.
+
+ Example:
+ ```python
+ anchor_gen = AnchorGenerator([32, 64], [.5, 1., 2.],
+ strides=[16, 32])
+ anchors = anchor_gen([512, 512, 3])
+ ```
+
+ """
+
+ def __init__(self,
+ anchor_sizes,
+ scales,
+ aspect_ratios,
+ strides,
+ clip_boxes=False):
+ """Constructs multiscale anchors.
+
+ Args:
+ anchor_sizes: A list of int represents the anchor size for each scale. The
+ anchor height will be `anchor_size / sqrt(aspect_ratio)`, anchor width
+ will be `anchor_size * sqrt(aspect_ratio)` for each scale.
+ scales: A list/tuple, or a list/tuple of a list/tuple of positive
+ floats representing the actual anchor size to the base `anchor_size`.
+ aspect_ratios: A list/tuple, or a list/tuple of a list/tuple of positive
+ floats representing the ratio of anchor width to anchor height.
+ strides: A list/tuple of ints represent the anchor stride size between
+ center of anchors at each scale.
+ clip_boxes: Boolean to represents whether the anchor coordinates should be
+ clipped to the image size. Defaults to `False`.
+ Input shape: the size of the image, `[H, W, C]`
+ Output shape: the size of anchors concat on each level, `[(H /
+ strides) * (W / strides), K * 4]`
+ """
+ # aspect_ratio is a single list that is the same across all levels.
+ aspect_ratios = maybe_map_structure_for_anchor(aspect_ratios, anchor_sizes)
+ scales = maybe_map_structure_for_anchor(scales, anchor_sizes)
+ if isinstance(anchor_sizes, dict):
+ self.anchor_generators = {}
+ for k in anchor_sizes.keys():
+ self.anchor_generators[k] = _SingleAnchorGenerator(
+ anchor_sizes[k], scales[k], aspect_ratios[k], strides[k],
+ clip_boxes)
+ elif isinstance(anchor_sizes, (list, tuple)):
+ self.anchor_generators = []
+ for anchor_size, scale_list, ar_list, stride in zip(
+ anchor_sizes, scales, aspect_ratios, strides):
+ self.anchor_generators.append(
+ _SingleAnchorGenerator(anchor_size, scale_list, ar_list, stride,
+ clip_boxes))
+
+ def __call__(self, image_size):
+ anchor_generators = tf.nest.flatten(self.anchor_generators)
+ results = [anchor_gen(image_size) for anchor_gen in anchor_generators]
+ return tf.nest.pack_sequence_as(self.anchor_generators, results)
+
+
+def maybe_map_structure_for_anchor(params, anchor_sizes):
+ """broadcast the params to match anchor_sizes."""
+ if all(isinstance(param, (int, float)) for param in params):
+ if isinstance(anchor_sizes, (tuple, list)):
+ return [params] * len(anchor_sizes)
+ elif isinstance(anchor_sizes, dict):
+ return tf.nest.map_structure(lambda _: params, anchor_sizes)
+ else:
+ raise ValueError("the structure of `anchor_sizes` must be a tuple, "
+ "list, or dict, given {}".format(anchor_sizes))
+ else:
+ return params
diff --git a/official/vision/keras_cv/ops/anchor_generator_test.py b/official/vision/keras_cv/ops/anchor_generator_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..5de6cb905c2fb01b1110638bc02d25b16f2d4e1e
--- /dev/null
+++ b/official/vision/keras_cv/ops/anchor_generator_test.py
@@ -0,0 +1,137 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for anchor_generator.py."""
+
+from absl.testing import parameterized
+import tensorflow as tf
+from official.vision.keras_cv.ops import anchor_generator
+
+
+class AnchorGeneratorTest(parameterized.TestCase, tf.test.TestCase):
+
+ @parameterized.parameters(
+ # Single scale anchor.
+ (5, [1.0], [[[-16., -16., 48., 48.], [-16., 16., 48., 80.]],
+ [[16., -16., 80., 48.], [16., 16., 80., 80.]]]),
+ # # Multi aspect ratio anchor.
+ (6, [1.0, 4.0, 0.25],
+ [[[-32., -32., 96., 96., 0., -96., 64., 160., -96., 0., 160., 64.]]]),
+ )
+ def testAnchorGeneration(self, level, aspect_ratios, expected_boxes):
+ image_size = [64, 64]
+ anchor_size = 2**(level + 1)
+ stride = 2**level
+ anchor_gen = anchor_generator._SingleAnchorGenerator(
+ anchor_size=anchor_size,
+ scales=[1.],
+ aspect_ratios=aspect_ratios,
+ stride=stride,
+ clip_boxes=False)
+ anchors = anchor_gen(image_size).numpy()
+ self.assertAllClose(expected_boxes, anchors)
+
+ @parameterized.parameters(
+ # Single scale anchor.
+ (5, [1.0], [[[0., 0., 48., 48.], [0., 16., 48., 64.]],
+ [[16., 0., 64., 48.], [16., 16., 64., 64.]]]),
+ # # Multi aspect ratio anchor.
+ (6, [1.0, 4.0, 0.25
+ ], [[[0., 0., 64., 64., 0., 0., 64., 64., 0., 0., 64., 64.]]]),
+ )
+ def testAnchorGenerationClipped(self, level, aspect_ratios, expected_boxes):
+ image_size = [64, 64]
+ anchor_size = 2**(level + 1)
+ stride = 2**level
+ anchor_gen = anchor_generator._SingleAnchorGenerator(
+ anchor_size=anchor_size,
+ scales=[1.],
+ aspect_ratios=aspect_ratios,
+ stride=stride,
+ clip_boxes=True)
+ anchors = anchor_gen(image_size).numpy()
+ self.assertAllClose(expected_boxes, anchors)
+
+
+class MultiScaleAnchorGeneratorTest(parameterized.TestCase, tf.test.TestCase):
+
+ @parameterized.parameters(
+ # Multi scale anchor.
+ (5, 6, [[1.0], [1.0]], [[-16, -16, 48, 48], [-16, 16, 48, 80],
+ [16, -16, 80, 48], [16, 16, 80, 80],
+ [-32, -32, 96, 96]]),)
+ def testAnchorGeneration(self, min_level, max_level, aspect_ratios,
+ expected_boxes):
+ image_size = [64, 64]
+ levels = range(min_level, max_level + 1)
+ anchor_sizes = [2**(level + 1) for level in levels]
+ strides = [2**level for level in levels]
+ anchor_gen = anchor_generator.AnchorGenerator(
+ anchor_sizes=anchor_sizes,
+ scales=[1.],
+ aspect_ratios=aspect_ratios,
+ strides=strides)
+ anchors = anchor_gen(image_size)
+ anchors = [tf.reshape(anchor, [-1, 4]) for anchor in anchors]
+ anchors = tf.concat(anchors, axis=0).numpy()
+ self.assertAllClose(expected_boxes, anchors)
+
+ @parameterized.parameters(
+ # Multi scale anchor.
+ (5, 6, [[1.0], [1.0]], [[-16, -16, 48, 48], [-16, 16, 48, 80],
+ [16, -16, 80, 48], [16, 16, 80, 80],
+ [-32, -32, 96, 96]]),)
+ def testAnchorGenerationClipped(self, min_level, max_level, aspect_ratios,
+ expected_boxes):
+ image_size = [64, 64]
+ levels = range(min_level, max_level + 1)
+ anchor_sizes = [2**(level + 1) for level in levels]
+ strides = [2**level for level in levels]
+ anchor_gen = anchor_generator.AnchorGenerator(
+ anchor_sizes=anchor_sizes,
+ scales=[1.],
+ aspect_ratios=aspect_ratios,
+ strides=strides,
+ clip_boxes=False)
+ anchors = anchor_gen(image_size)
+ anchors = [tf.reshape(anchor, [-1, 4]) for anchor in anchors]
+ anchors = tf.concat(anchors, axis=0).numpy()
+ self.assertAllClose(expected_boxes, anchors)
+
+ @parameterized.parameters(
+ # Multi scale anchor.
+ (5, 6, [1.0], {
+ '5': [[[-16., -16., 48., 48.], [-16., 16., 48., 80.]],
+ [[16., -16., 80., 48.], [16., 16., 80., 80.]]],
+ '6': [[[-32, -32, 96, 96]]]
+ }),)
+ def testAnchorGenerationDict(self, min_level, max_level, aspect_ratios,
+ expected_boxes):
+ image_size = [64, 64]
+ levels = range(min_level, max_level + 1)
+ anchor_sizes = dict((str(level), 2**(level + 1)) for level in levels)
+ strides = dict((str(level), 2**level) for level in levels)
+ anchor_gen = anchor_generator.AnchorGenerator(
+ anchor_sizes=anchor_sizes,
+ scales=[1.],
+ aspect_ratios=aspect_ratios,
+ strides=strides,
+ clip_boxes=False)
+ anchors = anchor_gen(image_size)
+ for k in expected_boxes.keys():
+ self.assertAllClose(expected_boxes[k], anchors[k].numpy())
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/keras_cv/ops/box_matcher.py b/official/vision/keras_cv/ops/box_matcher.py
new file mode 100644
index 0000000000000000000000000000000000000000..d788577d2f9701146252b52bd6ac0b738937143b
--- /dev/null
+++ b/official/vision/keras_cv/ops/box_matcher.py
@@ -0,0 +1,191 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Box matcher implementation."""
+
+
+import tensorflow as tf
+
+
+class BoxMatcher:
+ """Matcher based on highest value.
+
+ This class computes matches from a similarity matrix. Each column is matched
+ to a single row.
+
+ To support object detection target assignment this class enables setting both
+ positive_threshold (upper threshold) and negative_threshold (lower thresholds)
+ defining three categories of similarity which define whether examples are
+ positive, negative, or ignored, for example:
+ (1) thresholds=[negative_threshold, positive_threshold], and
+ indicators=[negative_value, ignore_value, positive_value]: The similarity
+ metrics below negative_threshold will be assigned with negative_value,
+ the metrics between negative_threshold and positive_threshold will be
+ assigned ignore_value, and the metrics above positive_threshold will be
+ assigned positive_value.
+ (2) thresholds=[negative_threshold, positive_threshold], and
+ indicators=[ignore_value, negative_value, positive_value]: The similarity
+ metric below negative_threshold will be assigned with ignore_value,
+ the metrics between negative_threshold and positive_threshold will be
+ assigned negative_value, and the metrics above positive_threshold will be
+ assigned positive_value.
+ """
+
+ def __init__(self, thresholds, indicators, force_match_for_each_col=False):
+ """Construct BoxMatcher.
+
+ Args:
+ thresholds: A list of thresholds to classify boxes into
+ different buckets. The list needs to be sorted, and will be prepended
+ with -Inf and appended with +Inf.
+ indicators: A list of values to assign for each bucket. len(`indicators`)
+ must equal to len(`thresholds`) + 1.
+ force_match_for_each_col: If True, ensures that each column is matched to
+ at least one row (which is not guaranteed otherwise if the
+ positive_threshold is high). Defaults to False. If True, all force
+ matched row will be assigned to `indicators[-1]`.
+
+ Raises:
+ ValueError: If `threshold` not sorted,
+ or len(indicators) != len(threshold) + 1
+ """
+ if not all([lo <= hi for (lo, hi) in zip(thresholds[:-1], thresholds[1:])]):
+ raise ValueError('`threshold` must be sorted, got {}'.format(thresholds))
+ self.indicators = indicators
+ if len(indicators) != len(thresholds) + 1:
+ raise ValueError('len(`indicators`) must be len(`thresholds`) + 1, got '
+ 'indicators {}, thresholds {}'.format(
+ indicators, thresholds))
+ thresholds = thresholds[:]
+ thresholds.insert(0, -float('inf'))
+ thresholds.append(float('inf'))
+ self.thresholds = thresholds
+ self._force_match_for_each_col = force_match_for_each_col
+
+ def __call__(self, similarity_matrix):
+ """Tries to match each column of the similarity matrix to a row.
+
+ Args:
+ similarity_matrix: A float tensor of shape [N, M] representing any
+ similarity metric.
+
+ Returns:
+ A integer tensor of shape [N] with corresponding match indices for each
+ of M columns, for positive match, the match result will be the
+ corresponding row index, for negative match, the match will be
+ `negative_value`, for ignored match, the match result will be
+ `ignore_value`.
+ """
+ squeeze_result = False
+ if len(similarity_matrix.shape) == 2:
+ squeeze_result = True
+ similarity_matrix = tf.expand_dims(similarity_matrix, axis=0)
+
+ static_shape = similarity_matrix.shape.as_list()
+ num_rows = static_shape[1] or tf.shape(similarity_matrix)[1]
+ batch_size = static_shape[0] or tf.shape(similarity_matrix)[0]
+
+ def _match_when_rows_are_empty():
+ """Performs matching when the rows of similarity matrix are empty.
+
+ When the rows are empty, all detections are false positives. So we return
+ a tensor of -1's to indicate that the columns do not match to any rows.
+
+ Returns:
+ matches: int32 tensor indicating the row each column matches to.
+ """
+ with tf.name_scope('empty_gt_boxes'):
+ matches = tf.zeros([batch_size, num_rows], dtype=tf.int32)
+ match_labels = -tf.ones([batch_size, num_rows], dtype=tf.int32)
+ return matches, match_labels
+
+ def _match_when_rows_are_non_empty():
+ """Performs matching when the rows of similarity matrix are non empty.
+
+ Returns:
+ matches: int32 tensor indicating the row each column matches to.
+ """
+ # Matches for each column
+ with tf.name_scope('non_empty_gt_boxes'):
+ matches = tf.argmax(similarity_matrix, axis=-1, output_type=tf.int32)
+
+ # Get logical indices of ignored and unmatched columns as tf.int64
+ matched_vals = tf.reduce_max(similarity_matrix, axis=-1)
+ matched_indicators = tf.zeros([batch_size, num_rows], tf.int32)
+
+ match_dtype = matched_vals.dtype
+ for (ind, low, high) in zip(self.indicators, self.thresholds[:-1],
+ self.thresholds[1:]):
+ low_threshold = tf.cast(low, match_dtype)
+ high_threshold = tf.cast(high, match_dtype)
+ mask = tf.logical_and(
+ tf.greater_equal(matched_vals, low_threshold),
+ tf.less(matched_vals, high_threshold))
+ matched_indicators = self._set_values_using_indicator(
+ matched_indicators, mask, ind)
+
+ if self._force_match_for_each_col:
+ # [batch_size, M], for each col (groundtruth_box), find the best
+ # matching row (anchor).
+ force_match_column_ids = tf.argmax(
+ input=similarity_matrix, axis=1, output_type=tf.int32)
+ # [batch_size, M, N]
+ force_match_column_indicators = tf.one_hot(
+ force_match_column_ids, depth=num_rows)
+ # [batch_size, N], for each row (anchor), find the largest column
+ # index for groundtruth box
+ force_match_row_ids = tf.argmax(
+ input=force_match_column_indicators, axis=1, output_type=tf.int32)
+ # [batch_size, N]
+ force_match_column_mask = tf.cast(
+ tf.reduce_max(force_match_column_indicators, axis=1),
+ tf.bool)
+ # [batch_size, N]
+ final_matches = tf.where(force_match_column_mask, force_match_row_ids,
+ matches)
+ final_matched_indicators = tf.where(
+ force_match_column_mask, self.indicators[-1] *
+ tf.ones([batch_size, num_rows], dtype=tf.int32),
+ matched_indicators)
+ return final_matches, final_matched_indicators
+ else:
+ return matches, matched_indicators
+
+ num_gt_boxes = similarity_matrix.shape.as_list()[-1] or tf.shape(
+ similarity_matrix)[-1]
+ result_match, result_matched_indicators = tf.cond(
+ pred=tf.greater(num_gt_boxes, 0),
+ true_fn=_match_when_rows_are_non_empty,
+ false_fn=_match_when_rows_are_empty)
+
+ if squeeze_result:
+ result_match = tf.squeeze(result_match, axis=0)
+ result_matched_indicators = tf.squeeze(result_matched_indicators, axis=0)
+
+ return result_match, result_matched_indicators
+
+ def _set_values_using_indicator(self, x, indicator, val):
+ """Set the indicated fields of x to val.
+
+ Args:
+ x: tensor.
+ indicator: boolean with same shape as x.
+ val: scalar with value to set.
+
+ Returns:
+ modified tensor.
+ """
+ indicator = tf.cast(indicator, x.dtype)
+ return tf.add(tf.multiply(x, 1 - indicator), val * indicator)
diff --git a/official/vision/keras_cv/ops/box_matcher_test.py b/official/vision/keras_cv/ops/box_matcher_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..baf4fe1136051b6bc175f9380177bab5b0e2c7ca
--- /dev/null
+++ b/official/vision/keras_cv/ops/box_matcher_test.py
@@ -0,0 +1,78 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for box_matcher.py."""
+
+import tensorflow as tf
+
+from official.vision.keras_cv.ops import box_matcher
+
+
+class BoxMatcherTest(tf.test.TestCase):
+
+ def test_box_matcher_unbatched(self):
+ sim_matrix = tf.constant(
+ [[0.04, 0, 0, 0],
+ [0, 0, 1., 0]],
+ dtype=tf.float32)
+
+ fg_threshold = 0.5
+ bg_thresh_hi = 0.2
+ bg_thresh_lo = 0.0
+
+ matcher = box_matcher.BoxMatcher(
+ thresholds=[bg_thresh_lo, bg_thresh_hi, fg_threshold],
+ indicators=[-3, -2, -1, 1])
+ match_indices, match_indicators = matcher(sim_matrix)
+ positive_matches = tf.greater_equal(match_indicators, 0)
+ negative_matches = tf.equal(match_indicators, -2)
+
+ self.assertAllEqual(
+ positive_matches.numpy(), [False, True])
+ self.assertAllEqual(
+ negative_matches.numpy(), [True, False])
+ self.assertAllEqual(
+ match_indices.numpy(), [0, 2])
+ self.assertAllEqual(
+ match_indicators.numpy(), [-2, 1])
+
+ def test_box_matcher_batched(self):
+ sim_matrix = tf.constant(
+ [[[0.04, 0, 0, 0],
+ [0, 0, 1., 0]]],
+ dtype=tf.float32)
+
+ fg_threshold = 0.5
+ bg_thresh_hi = 0.2
+ bg_thresh_lo = 0.0
+
+ matcher = box_matcher.BoxMatcher(
+ thresholds=[bg_thresh_lo, bg_thresh_hi, fg_threshold],
+ indicators=[-3, -2, -1, 1])
+ match_indices, match_indicators = matcher(sim_matrix)
+ positive_matches = tf.greater_equal(match_indicators, 0)
+ negative_matches = tf.equal(match_indicators, -2)
+
+ self.assertAllEqual(
+ positive_matches.numpy(), [[False, True]])
+ self.assertAllEqual(
+ negative_matches.numpy(), [[True, False]])
+ self.assertAllEqual(
+ match_indices.numpy(), [[0, 2]])
+ self.assertAllEqual(
+ match_indicators.numpy(), [[-2, 1]])
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/keras_cv/ops/iou_similarity.py b/official/vision/keras_cv/ops/iou_similarity.py
new file mode 100644
index 0000000000000000000000000000000000000000..beb69da73d40a781995dde50e80a8d5fae21ae16
--- /dev/null
+++ b/official/vision/keras_cv/ops/iou_similarity.py
@@ -0,0 +1,164 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Region Similarity Calculators."""
+
+import tensorflow as tf
+
+
+def area(box):
+ """Computes area of boxes.
+
+ B: batch_size
+ N: number of boxes
+
+ Args:
+ box: a float Tensor with [N, 4], or [B, N, 4].
+
+ Returns:
+ a float Tensor with [N], or [B, N]
+ """
+ with tf.name_scope('Area'):
+ y_min, x_min, y_max, x_max = tf.split(
+ value=box, num_or_size_splits=4, axis=-1)
+ return tf.squeeze((y_max - y_min) * (x_max - x_min), axis=-1)
+
+
+def intersection(gt_boxes, boxes):
+ """Compute pairwise intersection areas between boxes.
+
+ B: batch_size
+ N: number of groundtruth boxes.
+ M: number of anchor boxes.
+
+ Args:
+ gt_boxes: a float Tensor with [N, 4], or [B, N, 4]
+ boxes: a float Tensor with [M, 4], or [B, M, 4]
+
+ Returns:
+ a float Tensor with shape [N, M] or [B, N, M] representing pairwise
+ intersections.
+ """
+ with tf.name_scope('Intersection'):
+ y_min1, x_min1, y_max1, x_max1 = tf.split(
+ value=gt_boxes, num_or_size_splits=4, axis=-1)
+ y_min2, x_min2, y_max2, x_max2 = tf.split(
+ value=boxes, num_or_size_splits=4, axis=-1)
+
+ boxes_rank = len(boxes.shape)
+ perm = [1, 0] if boxes_rank == 2 else [0, 2, 1]
+ # [N, M] or [B, N, M]
+ y_min_max = tf.minimum(y_max1, tf.transpose(y_max2, perm))
+ y_max_min = tf.maximum(y_min1, tf.transpose(y_min2, perm))
+ x_min_max = tf.minimum(x_max1, tf.transpose(x_max2, perm))
+ x_max_min = tf.maximum(x_min1, tf.transpose(x_min2, perm))
+
+ intersect_heights = y_min_max - y_max_min
+ intersect_widths = x_min_max - x_max_min
+ zeros_t = tf.cast(0, intersect_heights.dtype)
+ intersect_heights = tf.maximum(zeros_t, intersect_heights)
+ intersect_widths = tf.maximum(zeros_t, intersect_widths)
+ return intersect_heights * intersect_widths
+
+
+def iou(gt_boxes, boxes):
+ """Computes pairwise intersection-over-union between box collections.
+
+ Args:
+ gt_boxes: a float Tensor with [N, 4].
+ boxes: a float Tensor with [M, 4].
+
+ Returns:
+ a Tensor with shape [N, M] representing pairwise iou scores.
+ """
+ with tf.name_scope('IOU'):
+ intersections = intersection(gt_boxes, boxes)
+ gt_boxes_areas = area(gt_boxes)
+ boxes_areas = area(boxes)
+ boxes_rank = len(boxes_areas.shape)
+ boxes_axis = 1 if (boxes_rank == 2) else 0
+ gt_boxes_areas = tf.expand_dims(gt_boxes_areas, -1)
+ boxes_areas = tf.expand_dims(boxes_areas, boxes_axis)
+ unions = gt_boxes_areas + boxes_areas
+ unions = unions - intersections
+ return tf.where(
+ tf.equal(intersections, 0.0), tf.zeros_like(intersections),
+ tf.truediv(intersections, unions))
+
+
+class IouSimilarity:
+ """Class to compute similarity based on Intersection over Union (IOU) metric.
+
+ """
+
+ def __init__(self, mask_val=-1):
+ self.mask_val = mask_val
+
+ def __call__(self, boxes_1, boxes_2, boxes_1_masks=None, boxes_2_masks=None):
+ """Compute pairwise IOU similarity between ground truth boxes and anchors.
+
+ B: batch_size
+ N: Number of groundtruth boxes.
+ M: Number of anchor boxes.
+
+ Args:
+ boxes_1: a float Tensor with M or B * M boxes.
+ boxes_2: a float Tensor with N or B * N boxes, the rank must be less than
+ or equal to rank of `boxes_1`.
+ boxes_1_masks: a boolean Tensor with M or B * M boxes. Optional.
+ boxes_2_masks: a boolean Tensor with N or B * N boxes. Optional.
+
+ Returns:
+ A Tensor with shape [M, N] or [B, M, N] representing pairwise
+ iou scores, anchor per row and groundtruth_box per colulmn.
+
+ Input shape:
+ boxes_1: [N, 4], or [B, N, 4]
+ boxes_2: [M, 4], or [B, M, 4]
+ boxes_1_masks: [N, 1], or [B, N, 1]
+ boxes_2_masks: [M, 1], or [B, M, 1]
+
+ Output shape:
+ [M, N], or [B, M, N]
+ """
+ boxes_1_rank = len(boxes_1.shape)
+ boxes_2_rank = len(boxes_2.shape)
+ if boxes_1_rank < 2 or boxes_1_rank > 3:
+ raise ValueError(
+ '`groudtruth_boxes` must be rank 2 or 3, got {}'.format(boxes_1_rank))
+ if boxes_2_rank < 2 or boxes_2_rank > 3:
+ raise ValueError(
+ '`anchors` must be rank 2 or 3, got {}'.format(boxes_2_rank))
+ if boxes_1_rank < boxes_2_rank:
+ raise ValueError('`groundtruth_boxes` is unbatched while `anchors` is '
+ 'batched is not a valid use case, got groundtruth_box '
+ 'rank {}, and anchors rank {}'.format(
+ boxes_1_rank, boxes_2_rank))
+
+ result = iou(boxes_1, boxes_2)
+ if boxes_1_masks is None and boxes_2_masks is None:
+ return result
+ background_mask = None
+ mask_val_t = tf.cast(self.mask_val, result.dtype) * tf.ones_like(result)
+ perm = [1, 0] if boxes_2_rank == 2 else [0, 2, 1]
+ if boxes_1_masks is not None and boxes_2_masks is not None:
+ background_mask = tf.logical_or(boxes_1_masks,
+ tf.transpose(boxes_2_masks, perm))
+ elif boxes_1_masks is not None:
+ background_mask = boxes_1_masks
+ else:
+ background_mask = tf.logical_or(
+ tf.zeros(tf.shape(boxes_2)[:-1], dtype=tf.bool),
+ tf.transpose(boxes_2_masks, perm))
+ return tf.where(background_mask, mask_val_t, result)
diff --git a/official/vision/keras_cv/ops/iou_similarity_test.py b/official/vision/keras_cv/ops/iou_similarity_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..137f75646a286e302924f3d1db6538fb82a5baba
--- /dev/null
+++ b/official/vision/keras_cv/ops/iou_similarity_test.py
@@ -0,0 +1,76 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for iou_similarity.py."""
+
+import tensorflow as tf
+
+from official.vision.keras_cv.ops import iou_similarity
+
+
+class BoxMatcherTest(tf.test.TestCase):
+
+ def test_similarity_unbatched(self):
+ boxes = tf.constant(
+ [
+ [0, 0, 1, 1],
+ [5, 0, 10, 5],
+ ],
+ dtype=tf.float32)
+
+ gt_boxes = tf.constant(
+ [
+ [0, 0, 5, 5],
+ [0, 5, 5, 10],
+ [5, 0, 10, 5],
+ [5, 5, 10, 10],
+ ],
+ dtype=tf.float32)
+
+ sim_calc = iou_similarity.IouSimilarity()
+ sim_matrix = sim_calc(boxes, gt_boxes)
+
+ self.assertAllClose(
+ sim_matrix.numpy(),
+ [[0.04, 0, 0, 0],
+ [0, 0, 1., 0]])
+
+ def test_similarity_batched(self):
+ boxes = tf.constant(
+ [[
+ [0, 0, 1, 1],
+ [5, 0, 10, 5],
+ ]],
+ dtype=tf.float32)
+
+ gt_boxes = tf.constant(
+ [[
+ [0, 0, 5, 5],
+ [0, 5, 5, 10],
+ [5, 0, 10, 5],
+ [5, 5, 10, 10],
+ ]],
+ dtype=tf.float32)
+
+ sim_calc = iou_similarity.IouSimilarity()
+ sim_matrix = sim_calc(boxes, gt_boxes)
+
+ self.assertAllClose(
+ sim_matrix.numpy(),
+ [[[0.04, 0, 0, 0],
+ [0, 0, 1., 0]]])
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/keras_cv/ops/target_gather.py b/official/vision/keras_cv/ops/target_gather.py
new file mode 100644
index 0000000000000000000000000000000000000000..e9cbbe4c62d1cd53e621bd4970a19f623a35e357
--- /dev/null
+++ b/official/vision/keras_cv/ops/target_gather.py
@@ -0,0 +1,103 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Definition of target gather, which gathers targets from indices."""
+
+import tensorflow as tf
+
+
+class TargetGather:
+ """Targer gather for dense object detector."""
+
+ def __call__(self, labels, match_indices, mask=None, mask_val=0.0):
+ """Labels anchors with ground truth inputs.
+
+ B: batch_size
+ N: number of groundtruth boxes.
+
+ Args:
+ labels: An integer tensor with shape [N, dims] or [B, N, ...] representing
+ groundtruth labels.
+ match_indices: An integer tensor with shape [M] or [B, M] representing
+ match label index.
+ mask: An boolean tensor with shape [M, dims] or [B, M,...] representing
+ match labels.
+ mask_val: An integer to fill in for mask.
+
+ Returns:
+ target: An integer Tensor with shape [M] or [B, M]
+ Raises:
+ ValueError: If `labels` is higher than rank 3.
+ """
+ if len(labels.shape) <= 2:
+ return self._gather_unbatched(labels, match_indices, mask, mask_val)
+ elif len(labels.shape) == 3:
+ return self._gather_batched(labels, match_indices, mask, mask_val)
+ else:
+ raise ValueError("`TargetGather` does not support `labels` with rank "
+ "larger than 3, got {}".format(len(labels.shape)))
+
+ def _gather_unbatched(self, labels, match_indices, mask, mask_val):
+ """Gather based on unbatched labels and boxes."""
+ num_gt_boxes = tf.shape(labels)[0]
+
+ def _assign_when_rows_empty():
+ if len(labels.shape) > 1:
+ mask_shape = [match_indices.shape[0], labels.shape[-1]]
+ else:
+ mask_shape = [match_indices.shape[0]]
+ return tf.cast(mask_val, labels.dtype) * tf.ones(
+ mask_shape, dtype=labels.dtype)
+
+ def _assign_when_rows_not_empty():
+ targets = tf.gather(labels, match_indices)
+ if mask is None:
+ return targets
+ else:
+ masked_targets = tf.cast(mask_val, labels.dtype) * tf.ones_like(
+ mask, dtype=labels.dtype)
+ return tf.where(mask, masked_targets, targets)
+
+ return tf.cond(tf.greater(num_gt_boxes, 0),
+ _assign_when_rows_not_empty,
+ _assign_when_rows_empty)
+
+ def _gather_batched(self, labels, match_indices, mask, mask_val):
+ """Gather based on batched labels."""
+ batch_size = labels.shape[0]
+ if batch_size == 1:
+ if mask is not None:
+ result = self._gather_unbatched(
+ tf.squeeze(labels, axis=0), tf.squeeze(match_indices, axis=0),
+ tf.squeeze(mask, axis=0), mask_val)
+ else:
+ result = self._gather_unbatched(
+ tf.squeeze(labels, axis=0), tf.squeeze(match_indices, axis=0),
+ None, mask_val)
+ return tf.expand_dims(result, axis=0)
+ else:
+ indices_shape = tf.shape(match_indices)
+ indices_dtype = match_indices.dtype
+ batch_indices = (tf.expand_dims(
+ tf.range(indices_shape[0], dtype=indices_dtype), axis=-1) *
+ tf.ones([1, indices_shape[-1]], dtype=indices_dtype))
+ gather_nd_indices = tf.stack(
+ [batch_indices, match_indices], axis=-1)
+ targets = tf.gather_nd(labels, gather_nd_indices)
+ if mask is None:
+ return targets
+ else:
+ masked_targets = tf.cast(mask_val, labels.dtype) * tf.ones_like(
+ mask, dtype=labels.dtype)
+ return tf.where(mask, masked_targets, targets)
diff --git a/official/vision/keras_cv/ops/target_gather_test.py b/official/vision/keras_cv/ops/target_gather_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..cf5693b12b1092ecf88db9cca5bb6c81a6d5e508
--- /dev/null
+++ b/official/vision/keras_cv/ops/target_gather_test.py
@@ -0,0 +1,77 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for target_gather.py."""
+
+import tensorflow as tf
+
+from official.vision.keras_cv.ops import target_gather
+
+
+class TargetGatherTest(tf.test.TestCase):
+
+ def test_target_gather_batched(self):
+ gt_boxes = tf.constant(
+ [[
+ [0, 0, 5, 5],
+ [0, 5, 5, 10],
+ [5, 0, 10, 5],
+ [5, 5, 10, 10],
+ ]],
+ dtype=tf.float32)
+ gt_classes = tf.constant([[[2], [10], [3], [-1]]], dtype=tf.int32)
+
+ labeler = target_gather.TargetGather()
+
+ match_indices = tf.constant([[0, 2]], dtype=tf.int32)
+ match_indicators = tf.constant([[-2, 1]])
+ mask = tf.less_equal(match_indicators, 0)
+ cls_mask = tf.expand_dims(mask, -1)
+ matched_gt_classes = labeler(gt_classes, match_indices, cls_mask)
+ box_mask = tf.tile(cls_mask, [1, 1, 4])
+ matched_gt_boxes = labeler(gt_boxes, match_indices, box_mask)
+
+ self.assertAllEqual(
+ matched_gt_classes.numpy(), [[[0], [3]]])
+ self.assertAllClose(
+ matched_gt_boxes.numpy(), [[[0, 0, 0, 0], [5, 0, 10, 5]]])
+
+ def test_target_gather_unbatched(self):
+ gt_boxes = tf.constant(
+ [
+ [0, 0, 5, 5],
+ [0, 5, 5, 10],
+ [5, 0, 10, 5],
+ [5, 5, 10, 10],
+ ],
+ dtype=tf.float32)
+ gt_classes = tf.constant([[2], [10], [3], [-1]], dtype=tf.int32)
+
+ labeler = target_gather.TargetGather()
+
+ match_indices = tf.constant([0, 2], dtype=tf.int32)
+ match_indicators = tf.constant([-2, 1])
+ mask = tf.less_equal(match_indicators, 0)
+ cls_mask = tf.expand_dims(mask, -1)
+ matched_gt_classes = labeler(gt_classes, match_indices, cls_mask)
+ box_mask = tf.tile(cls_mask, [1, 4])
+ matched_gt_boxes = labeler(gt_boxes, match_indices, box_mask)
+
+ self.assertAllEqual(
+ matched_gt_classes.numpy(), [[0], [3]])
+ self.assertAllClose(
+ matched_gt_boxes.numpy(), [[0, 0, 0, 0], [5, 0, 10, 5]])
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/official/vision/keras_cv/requirements.txt b/official/vision/keras_cv/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6bad10388ecb1eefd890a797d833976a5e631541
--- /dev/null
+++ b/official/vision/keras_cv/requirements.txt
@@ -0,0 +1,2 @@
+numpy
+scipy
diff --git a/official/vision/keras_cv/setup.py b/official/vision/keras_cv/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..030b22dc4064a092a72ebebfb93821be61e41f2a
--- /dev/null
+++ b/official/vision/keras_cv/setup.py
@@ -0,0 +1,70 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Setup script."""
+
+import os
+
+from setuptools import find_packages
+from setuptools import setup
+
+version = '0.0.1'
+
+
+def _get_requirements():
+ """Parses requirements.txt file."""
+ install_requires_tmp = []
+ dependency_links_tmp = []
+ with open(
+ os.path.join(os.path.dirname(__file__), './requirements.txt'), 'r') as f:
+ for line in f:
+ package_name = line.strip()
+ # Skip empty line or comments starting with "#".
+ if not package_name or package_name[0] == '#':
+ continue
+ if package_name.startswith('-e '):
+ dependency_links_tmp.append(package_name[3:].strip())
+ else:
+ install_requires_tmp.append(package_name)
+ return install_requires_tmp, dependency_links_tmp
+
+install_requires, dependency_links = _get_requirements()
+
+install_requires.append('tf-nightly')
+install_requires.append('tensorflow-datasets')
+
+setup(
+ name='keras-cv',
+ version=version,
+ description='Keras Computer Vision Library',
+ url='https://github.com/keras-team/keras-cv',
+ author='The Keras authors',
+ author_email='keras-team@google.com',
+ license='Apache License 2.0',
+ install_requires=install_requires,
+ classifiers=[
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 3.6',
+ 'Operating System :: Unix',
+ 'Operating System :: Microsoft :: Windows',
+ 'Operating System :: MacOS',
+ 'Intended Audience :: Science/Research',
+ 'Topic :: Scientific/Engineering',
+ 'Topic :: Software Development'
+ ],
+ packages=find_packages(exclude=('tests',)),
+ exclude_package_data={'': ['*_test.py',],},
+ dependency_links=dependency_links,
+ python_requires='>=3.6',
+)
diff --git a/orbit/README.md b/orbit/README.md
index 539c924d3062933a0070d7cc1f6b9ab45504cf40..9412860036e7540578b494631c12977ad364396e 100644
--- a/orbit/README.md
+++ b/orbit/README.md
@@ -1,9 +1,11 @@
-
-
# Orbit
-Orbit is a customized training loop library built on top of Tensorflow 2. It
-provides a flexible lightweight library that users can easily use or fork when
-writing [customized training loop code](https://www.tensorflow.org/tutorials/distribute/custom_training)
-in TF2. It intergates with `tf.distribute` seamlessly and supports running on
-different device types (CPU, GPU, and TPU).
+Orbit is a flexible, lightweight library designed to make it easy to write
+[custom training loops][custom_training] in TensorFlow 2. Orbit handles common
+model training tasks such as saving checkpoints, running model evaluations, and
+setting up summary writing, while giving users full control over implementing
+the inner training loop. It integrates with `tf.distribute` seamlessly and
+supports running on different device types (CPU, GPU, and TPU). The core code is
+intended to be easy to read and fork.
+
+[custom_training]: https://www.tensorflow.org/tutorials/distribute/custom_training
diff --git a/orbit/__init__.py b/orbit/__init__.py
index 6e7bbe53cd6328f5d5bc9387619c567b4a88b98c..a97bb719d7aaa385fe12a670b83b83707412738d 100644
--- a/orbit/__init__.py
+++ b/orbit/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The Orbit Authors. All Rights Reserved.
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,9 +11,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
+"""Defines exported symbols for the `orbit` package."""
from orbit import utils
+
from orbit.controller import Controller
-from orbit.runner import *
-from orbit.standard_runner import *
+
+from orbit.runner import AbstractEvaluator
+from orbit.runner import AbstractTrainer
+
+from orbit.standard_runner import StandardEvaluator
+from orbit.standard_runner import StandardEvaluatorOptions
+from orbit.standard_runner import StandardTrainer
+from orbit.standard_runner import StandardTrainerOptions
diff --git a/orbit/controller.py b/orbit/controller.py
index 6e2840f0470464193f30102d7e4cce9828306463..5242a7a7e42cace0c4ccde22ef7582fff98e9374 100644
--- a/orbit/controller.py
+++ b/orbit/controller.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The Orbit Authors. All Rights Reserved.
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,105 +11,152 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""A light weight utilities to train TF2 models."""
-from __future__ import absolute_import
-from __future__ import division
-# from __future__ import google_type_annotations
-from __future__ import print_function
+"""Provides a `Controller` class for managing the outer training loop."""
+import pprint
import time
-from typing import Callable, Optional, Text, Union
+
+from typing import Callable, Optional, Union
from absl import logging
+
from orbit import runner
from orbit import utils
import tensorflow as tf
-def _log_info(message: Text):
+def _log(message: str):
"""Logs `message` to the `info` log, and also prints to stdout."""
logging.info(message)
print(message)
-def _validate_interval(interval: Optional[int], steps_per_loop: Optional[int],
- interval_name: str):
- if interval and steps_per_loop and (interval % steps_per_loop != 0):
- raise ValueError("The {} interval ({}) must be a multiple "
- "of the steps_per_loop ({})".format(
- interval_name, interval, steps_per_loop))
+logging.ABSLLogger.register_frame_to_skip(__file__, _log.__name__)
+
+
+def _format_output(output, indent=4):
+ """Formats `output`, either on one line, or indented across multiple lines."""
+ formatted = pprint.pformat(output)
+ lines = formatted.splitlines()
+ if len(lines) == 1:
+ return formatted
+ lines = [" " * indent + line for line in lines]
+ return "\n" + "\n".join(lines)
+
+class Controller:
+ """Class that controls the outer loop of model training and evaluation.
-class Controller(object):
- """Class that facilitates training and evaluation of models."""
+ Orbit divides training and evaluation into "inner" and "outer" loops. Inner
+ loops are implemented by users in the form of `AbstractTrainer` and
+ `AbstractEvaluator` subclasses, and define how to run a given number of
+ training or evaluation steps. The outer loop is provided by this `Controller`,
+ and interleaves calls to the user provided inner loops with additional actions
+ such as saving checkpoints, running evaluations, and writing summaries
+ (depending on the arguments passed to `Controller.__init__` and the method
+ being called).
+
+ There are four top-level "outer loops" provided:
+
+ - `train`, which trains until a specified number of global steps is reached;
+ - `evaluate`, for one-off model evaluation;
+ - `train_and_evaluate`, for interleaved training and evaluation;
+ - `evaluate_continuously`, for monitoring a given directory and running
+ evaluations on new model checkpoints.
+
+ While this class attempts to provide out-of-the-box solutions for common
+ training and evaluation use cases, the internal details and method
+ implementations are also intended to be simple enough to make subclassing or
+ other custom outer loop implementations easy to achieve.
+ """
def __init__(
self,
- strategy: Optional[tf.distribute.Strategy] = None,
+ *, # Makes all args keyword only.
+ global_step: tf.Variable,
trainer: Optional[runner.AbstractTrainer] = None,
evaluator: Optional[runner.AbstractEvaluator] = None,
- global_step: Optional[tf.Variable] = None,
+ strategy: Optional[tf.distribute.Strategy] = None,
# Train related
steps_per_loop: Optional[int] = None,
checkpoint_manager: Optional[tf.train.CheckpointManager] = None,
# Summary related
summary_interval: Optional[int] = None,
- summary_dir: Optional[Text] = None,
+ summary_dir: Optional[str] = None,
# Evaluation related
- eval_summary_dir: Optional[Text] = None):
- """Constructs a `Controller` instance.
+ eval_summary_dir: Optional[str] = None):
+ """Initializes a `Controller` instance.
+
+ Note that if `checkpoint_manager` is provided and there are checkpoints in
+ the associated model directory, the model will be restored from the most
+ recent checkpoint during this `__init__` method.
Args:
- strategy: An instance of `tf.distribute.Strategy`.
- trainer: An instance of `orbit.AbstractTrainer`, which represents model
- training details.
- evaluator: An instance of `orbit.AbstractEvaluator`, which represents
- model evaluation details.
- global_step: An integer `tf.Variable` indicating the global training step
- number. Usually this can be obtained from `iterations` property of the
- model's optimizer (e.g. `self.optimizer.iterations`), or users can
- create their own global step variable as well. If the users create their
- own global step variable, it is recommended to create the `tf.Variable`
- inside strategy scope, and with
- `aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA`.
- steps_per_loop: The number of steps to run in each "inner loop" of
- training (passed to the `num_steps` parameter of `trainer.train`).
- checkpoint_manager: An instance of `tf.train.CheckpointManager`.
+ global_step: An integer `tf.Variable` storing the global training step
+ number. Usually this can be obtained from the `iterations` property of
+ the model's optimizer (e.g. `trainer.optimizer.iterations`). In cases
+ where multiple optimizers are used, or if one model "step" corresponds
+ to more than one update to model parameters, users can create and
+ increment their own global step variable as well. In this case it is
+ recommended to create the `tf.Variable` inside the distribution strategy
+ scope, with `aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA` (see
+ also `orbit.utils.create_global_step()`).
+ trainer: An instance of `orbit.AbstractTrainer`, which implements the
+ inner training loop.
+ evaluator: An instance of `orbit.AbstractEvaluator`, which implements
+ evaluation.
+ strategy: An instance of `tf.distribute.Strategy`. If not provided, the
+ strategy will be initialized from the current in-scope strategy using
+ `tf.distribute.get_strategy()`.
+ steps_per_loop: The number of steps to run in each inner loop of training
+ (passed as the `num_steps` parameter of `trainer.train`).
+ checkpoint_manager: An instance of `tf.train.CheckpointManager`. If
+ provided and there are checkpoints in the associated model directory,
+ the model will be restored from the most recent checkpoint inside this
+ `__init__` method. If not provided, the `Controller` will not
+ automatically save to or restore from checkpoints.
summary_interval: Step interval for training summaries. Note that this
- argument only applies to the summaries inside `trainer.train` function.
- Summaries outside like "steps_per_second" and outputs from
- `trainer.train` function will always be enabled. If set, the value
- should be divisible by steps_per_loop.
- summary_dir: The directory to restore and write checkpoints and summaries.
- If None, it will be set to `checkpoint_manager.directory`.
- eval_summary_dir: The directory to write eval summaries. If None, it will
- be set to `summary_dir`.
+ argument only applies to `tf.summary` calls inside the `trainer.train`
+ function. Summaries written by the `Controller` (specifically
+ "steps_per_second" and output from the `trainer.train` method) will
+ always be enabled unless the `summary_dir` parameter is `None`. If set,
+ the value must be divisible by `steps_per_loop`.
+ summary_dir: The directory to write summaries to. To use the same
+ directory as for checkpointing, pass `checkpoint_manager.directory`. If
+ `None`, no training summaries will be written.
+ eval_summary_dir: The directory to write eval summaries to. If `None`, it
+ will be set to `summary_dir`. If both `summary_dir` and
+ `eval_summary_dir` are `None`, no eval summaries will be written.
Raises:
- ValueError: If both `trainer` and `evaluator` are None.
+ ValueError: If both `trainer` and `evaluator` are `None`.
ValueError: If `steps_per_loop` is not a positive integer.
- ValueError: If `summary_interval` is not a positive integer or it cannot
- be divisible by `steps_per_loop`.
+ ValueError: If `summary_interval` is not a positive integer or is not
+ divisible by `steps_per_loop`.
"""
if trainer is None and evaluator is None:
- raise ValueError("`trainer` and `evaluator` should not both be None")
-
+ raise ValueError("`trainer` and `evaluator` should not both be `None`.")
if trainer is not None:
if steps_per_loop is None:
- raise ValueError("`steps_per_loop` is required when `trainer` is "
- "provided.")
-
- if not isinstance(steps_per_loop, int) or steps_per_loop < 1:
- raise ValueError("`steps_per_loop` should be a positive integer")
+ raise ValueError(
+ "`steps_per_loop` is required when `trainer` is provided.")
+ elif not isinstance(steps_per_loop, int) or steps_per_loop < 1:
+ raise ValueError(
+ f"`steps_per_loop` ({steps_per_loop}) must be a positive integer.")
if summary_interval is not None:
if summary_interval <= 0:
- raise ValueError("`summary_interval` should be larger than 0")
- _validate_interval(
- summary_interval, steps_per_loop, interval_name="summary")
+ raise ValueError(
+ f"`summary_interval` ({summary_interval}) must be larger than 0.")
+ elif summary_interval % steps_per_loop != 0:
+ raise ValueError(
+ f"`summary interval` ({summary_interval}) must be a multiple "
+ f"of `steps_per_loop` ({steps_per_loop}).")
+
+ if not isinstance(global_step, tf.Variable):
+ raise ValueError("`global_step` must be a `tf.Variable`.")
self.trainer = trainer
self.evaluator = evaluator
@@ -119,9 +166,6 @@ class Controller(object):
self.global_step = global_step
self.checkpoint_manager = checkpoint_manager
- if summary_dir is None and checkpoint_manager:
- summary_dir = checkpoint_manager.directory
-
if self.trainer is not None:
self.step_timer = None
self.steps_per_loop = steps_per_loop
@@ -129,7 +173,6 @@ class Controller(object):
self.summary_manager = utils.SummaryManager(
summary_dir, tf.summary.scalar, global_step=self.global_step)
- eval_summary_writer = None
if self.evaluator is not None:
eval_summary_dir = eval_summary_dir or summary_dir
if eval_summary_dir == summary_dir and self.trainer is not None:
@@ -140,171 +183,141 @@ class Controller(object):
self.eval_summary_manager = utils.SummaryManager(
eval_summary_dir, tf.summary.scalar, global_step=self.global_step)
- if self.global_step is not None:
- tf.summary.experimental.set_step(self.global_step)
+ tf.summary.experimental.set_step(self.global_step)
# Restores the model if needed.
- # TODO(momernick): We probably only want to do this on certain occasions?
if self.checkpoint_manager is not None:
- checkpoint_interval = self.checkpoint_manager.checkpoint_interval
- _validate_interval(
- checkpoint_interval, steps_per_loop, interval_name="checkpoint")
-
- model_restored = self.restore_checkpoint()
- if not model_restored and checkpoint_interval:
- # If the model is not restored from a checkpoint, save an initial
- # checkpoint.
- self.save_checkpoint()
+ restored_path = self.restore_checkpoint()
+ if restored_path:
+ _log(f"restored from checkpoint: {restored_path}")
def train(self, steps: int, checkpoint_at_completion: bool = True):
- """Runs training.
+ """Runs training until the specified global step count has been reached.
- This method calls the `train` method on the Trainable object until the
- global step count is equal to `steps`. It will optionally save checkpoints,
- if a CheckpointManager was passed to the Controller instance's `__init__`.
+ This method makes calls to `self.trainer.train()` until the global step
+ count is equal to `steps`. It will additionally save checkpoints (if a
+ `CheckpointManager` was passed to `Controller.__init__`) and summarize
+ training output (if `summary_dir` is set).
Args:
steps: The global step count to train up to.
checkpoint_at_completion: Whether to save a checkpoint when this method
- returns. Defaults to True (write the checkpoint). This is always
- triggered, regardless of the checkpointing interval.
+ returns (regardless of the checkpointing interval). Defaults to `True`.
"""
- if self.trainer is None:
- raise ValueError("`self.trainer` is required when calling `train` "
- "method.")
- if self.global_step is None:
- raise ValueError("`self.global_step` is required when calling `train` "
- "method.")
+ self._require("trainer", for_method="train")
# TODO(momernick): Support steps=None or -1 (training to exhaustion).
- current_step = self.global_step.numpy() # This is an expensive access.
+ current_step = self.global_step.numpy() # Cache, since this is expensive.
+ _log(f"train | step: {current_step: 6d} | training until step {steps}...")
while current_step < steps:
- logging.info("Train at step %s of %s", current_step, steps)
# Calculates steps to run for the next train loop.
num_steps = min(steps - current_step, self.steps_per_loop)
self._train_n_steps(num_steps)
self._maybe_save_checkpoint()
- current_step = self.global_step.numpy() # This is an expensive access.
+ current_step = self.global_step.numpy()
if checkpoint_at_completion:
- self.save_checkpoint()
+ self._maybe_save_checkpoint(check_interval=False)
- def evaluate(self, steps: int = None):
- """Runs evaluation.
+ def evaluate(self, steps: int = -1) -> Optional[runner.Output]:
+ """Runs evaluation for the given number of steps.
- This method calls the `evaluate` method on the Evaluator object for `steps`
- steps, then writes the returned summaries (if any).
+ This method calls `self.evaluator.evaluate(steps)`, then writes the returned
+ summaries (if any).
Args:
- steps: The number of steps to evaluate for.
+ steps: The number of evaluation steps to run. The value `-1` is reserved
+ as a special sentinel to indicate a "complete" evaluation that runs
+ until the underlying dataset is exhausted. Support for this is dependent
+ on the specific `evaluator` being used.
- Raises:
- ValueError: If no checkpoint found in `self.checkpoint_manager.directory`.
- ValueError: If `evaluator` is not provided.
+ Returns:
+ The evaluation results as a dictionary mapping names to NumPy values.
+ Raises:
+ ValueError: If `evaluator` was not provided to `Controller.__init__`.
+ ValueError: If no checkpoint is present in `checkpoint_manager.directory`.
+ ValueError: If `steps` is not a positive value or -1.
"""
- if self.evaluator is None:
- raise ValueError("`evaluator` must be provided to call `evaluate()` "
- "method.")
+ self._require("evaluator", for_method="evaluate")
- steps = steps or -1
- current_step = self.global_step.numpy()
if steps > 0:
- logging.info("Running %s steps of evaluation at train step: %s", steps,
- current_step)
- steps = tf.convert_to_tensor(steps, dtype=tf.int32)
+ steps_msg = f"running {steps} steps of evaluation..."
+ elif steps == -1:
+ steps_msg = "running complete evaluation..."
else:
- logging.info("Evaluating at train step: %s", current_step)
+ raise ValueError(f"`steps` ({steps}) should be > 0, or == -1.")
- with self.eval_summary_manager.summary_writer.as_default():
- eval_outputs = self.evaluator.evaluate(steps)
+ current_step = self.global_step.numpy()
+ _log(f" eval | step: {current_step: 6d} | {steps_msg}")
- if eval_outputs:
- eval_outputs = tf.nest.map_structure(utils.get_value, eval_outputs)
+ start = time.time()
+ with self.eval_summary_manager.summary_writer().as_default():
+ steps_tensor = tf.convert_to_tensor(steps, dtype=tf.int32)
+ eval_output = self.evaluator.evaluate(steps_tensor)
+ eval_output = tf.nest.map_structure(utils.get_value, eval_output or {})
+ elapsed = time.time() - start
- info = "step: {} evaluation metric: {}".format(
- current_step, eval_outputs)
- _log_info(info)
+ _log(f" eval | step: {current_step: 6d} | "
+ f"eval time: {elapsed: 6.1f} sec | "
+ f"output: {_format_output(eval_output)}")
- self.eval_summary_manager.write_summaries(eval_outputs)
+ self.eval_summary_manager.write_summaries(eval_output)
self.eval_summary_manager.flush()
- def restore_checkpoint(self, checkpoint_path: Text = None):
- """Restore or initialize the model.
-
- Args:
- checkpoint_path: An optional string indicates the checkpoint path to
- restore. If None, will restore from `self.checkpoint_manager`.
-
- Returns:
- The path to the restored checkpoint if a restore happened, or None
- if no restore occurred.
- """
- with self.strategy.scope():
- # Checkpoint restoring should be inside scope. b/139450638
- if checkpoint_path is not None:
- self.checkpoint_manager.checkpoint.restore(checkpoint_path)
- return checkpoint_path
- return self.checkpoint_manager.restore_or_initialize()
-
- def save_checkpoint(self):
- """Checkpoint the model.
-
- This method will write a checkpoint containing the current state of the
- model.
-
- Raises:
- ValueError: if no CheckpointManager was provided to this Controller's
- init args.
- """
- self._maybe_save_checkpoint(force_trigger=True)
+ return eval_output
def train_and_evaluate(self,
- train_steps: int = None,
- eval_steps: int = None,
- eval_interval: int = None):
- """Train and evaluate in an interleaved manner.
+ train_steps: int,
+ eval_steps: int = -1,
+ eval_interval: Optional[int] = None) -> None:
+ """Runs interleaved training and evaluation.
- This method will train the model until the global step count equals
- `train_steps`, running an evaluation for `eval_steps` every `eval_interval`
- training steps. In addition, this method will run a final evaluation at the
- end of the training sequence.
+ This method interleaves calls to `self.train()` and `self.evaluate()`,
+ training the model until the global step count equals `train_steps`, and
+ running an evaluation for `eval_steps` every `eval_interval` training steps.
+ In addition, this method will run a final evaluation at the end of the
+ training sequence.
Args:
train_steps: The global step count to train up to.
- eval_steps: The number of steps to run during an evaluation. If None,
- this method will evaluate over the entire evaluation dataset.
- eval_interval: The number of training steps to run between evalutions.
- Must be a multiple of the controller's `steps_per_loop` init arg. If
- None, evaluation will only be performed after training is complete.
+ eval_steps: The number of steps to run during an evaluation. If -1, this
+ method will evaluate over the entire evaluation dataset.
+ eval_interval: The number of training steps to run between evaluations. If
+ set, training will always stop every `eval_interval` steps, even if this
+ results in a shorter inner loop than specified by `steps_per_loop`
+ setting. If None, evaluation will only be performed after training is
+ complete.
Raises:
ValueError: If eval_interval is not a multiple of self.steps_per_loop.
"""
- _validate_interval(eval_interval, self.steps_per_loop, interval_name="eval")
+ self._require("trainer", for_method="train_and_evaluate")
+ self._require("evaluator", for_method="train_and_evaluate")
- current_step = self.global_step.numpy() # This is an expensive access.
+ current_step = self.global_step.numpy() # Cache, since this is expensive.
eval_interval = eval_interval or (train_steps - current_step)
while current_step < train_steps:
interval = min(train_steps - current_step, eval_interval)
num_steps = current_step + interval
self.train(steps=num_steps, checkpoint_at_completion=False)
self.evaluate(steps=eval_steps)
- current_step = self.global_step.numpy() # This is an expensive access.
- self.save_checkpoint()
+ current_step = self.global_step.numpy()
+ self._maybe_save_checkpoint(check_interval=False)
def evaluate_continuously(self,
- steps: int = None,
+ steps: int = -1,
timeout: Optional[Union[int, float]] = None,
timeout_fn: Optional[Callable[[], bool]] = None):
- """Monitor a directory and evaluate on checkpoints in it.
+ """Continuously monitors a directory and evaluates new checkpoints in it.
This method continuously monitors a directory as specified by this
Controller's CheckpointManager init arg and runs evaluation on the
checkpoints found there.
Args:
- steps: The number of steps to run when evaluating.
+ steps: The number of steps to run when evaluating. If -1, this method will
+ evaluate over the entire evaluation dataset.
timeout: The maximum number of seconds to wait between checkpoints. See
tf.train.checkpoints_iterator documentation.
timeout_fn: Optional callable to call after a timeout. If the function
@@ -314,8 +327,10 @@ class Controller(object):
Raises:
ValueError: If no checkpoint found in `self.checkpoint_manager.directory`.
ValueError: If `evaluator` was not provided as a controller init arg.
-
"""
+ self._require("evaluator", for_method="evaluate_continuously")
+ self._require("checkpoint_manager", for_method="evaluate_continuously")
+
for checkpoint_path in tf.train.checkpoints_iterator(
self.checkpoint_manager.directory,
timeout=timeout,
@@ -323,63 +338,110 @@ class Controller(object):
self.restore_checkpoint(checkpoint_path)
self.evaluate(steps)
+ def restore_checkpoint(self, checkpoint_path: str = None):
+ """Restores the model from a checkpoint.
+
+ Args:
+ checkpoint_path: An optional string specifying the checkpoint path to
+ restore from. If `None`, will restore from the most recent checkpoint
+ (or initialize the model using a custom `init_fn` if no checkpoints can
+ be found) using `self.checkpoint_manager.restore_or_initialize()`.
+
+ Returns:
+ The path to the restored checkpoint if a restore happened, or `None` if no
+ restore occurred.
+ """
+ self._require("checkpoint_manager", for_method="restore_checkpoint")
+
+ with self.strategy.scope():
+ # Checkpoint restoring should be inside scope (b/139450638).
+ if checkpoint_path is not None:
+ _log(f"restoring model from {checkpoint_path}...")
+ self.checkpoint_manager.checkpoint.restore(checkpoint_path)
+ else:
+ _log("restoring or initializing model...")
+ checkpoint_path = self.checkpoint_manager.restore_or_initialize()
+
+ if checkpoint_path is not None:
+ _log(f"restored model from {checkpoint_path}.")
+ else:
+ _log("initialized model.")
+
+ return checkpoint_path
+
+ def save_checkpoint(self):
+ """Saves the model to a checkpoint.
+
+ This method will save a checkpoint containing the current state of the
+ model.
+
+ Raises:
+ ValueError: If no `checkpoint_manager` was provided to
+ `Controller.__init__`.
+ """
+ self._require("checkpoint_manager", for_method="save_checkpoint")
+ self._maybe_save_checkpoint(check_interval=False)
+
def _train_n_steps(self, num_steps: int):
- """Run training for `num_steps`.
+ """Runs training for `num_steps` steps.
- It will also write training outputs to summaries if there is any.
+ Also prints/logs updates about training progress, and summarizes training
+ output (if output is returned from `self.trainer.train()`, and if
+ `self.summary_dir` is set).
Args:
- num_steps: An integer indicates how many steps to run for this training
- loop.
+ num_steps: An integer specifying how many steps of training to run.
Raises:
- RuntimeError: If `global_step` is not updated correctly in
- `trainer.train`.
+ RuntimeError: If `global_step` is not properly incremented by `num_steps`
+ after calling `self.trainer.train(num_steps)`.
"""
if not self.step_timer:
self.step_timer = StepTimer(self.global_step)
-
- # Calculates steps to run for the next train loop.
current_step = self.global_step.numpy()
- logging.info("Entering training loop at step %s of %s", current_step,
- num_steps)
- current_step += num_steps
- num_steps = tf.convert_to_tensor(num_steps, dtype=tf.int32)
- with self.summary_manager.summary_writer.as_default():
- # Create a lambda that returns true when summaries should be written.
+ with self.summary_manager.summary_writer().as_default():
should_record = False # Allows static optimization in no-summary cases.
if self.summary_interval:
+ # Create a predicate to determine when summaries should be written.
should_record = lambda: (self.global_step % self.summary_interval == 0)
with tf.summary.record_if(should_record):
- train_outputs = self.trainer.train(num_steps)
-
- # Updates and verifies the current step after a training loop finishes.
- if current_step != self.global_step.numpy():
- raise RuntimeError("`trainer.train` function is not updating "
- "`global_step` correctly, expected: %s, actual: %s" %
- (current_step, self.global_step.numpy()))
-
- # Print information like metrics and steps_per_second after a training
- # loop.
- if train_outputs:
- train_outputs = tf.nest.map_structure(utils.get_value, train_outputs)
-
- train_outputs = train_outputs or {}
+ num_steps_tensor = tf.convert_to_tensor(num_steps, dtype=tf.int32)
+ train_output = self.trainer.train(num_steps_tensor)
+ train_output = tf.nest.map_structure(utils.get_value, train_output or {})
+
+ # Verify that global_step was updated properly, then update current_step.
+ expected_step = current_step + num_steps
+ if self.global_step.numpy() != expected_step:
+ message = (
+ f"`trainer.train({num_steps})` did not update `global_step` by "
+ f"{num_steps}. Old value was {current_step}, expected updated value "
+ f"to be {expected_step}, but it was {self.global_step.numpy()}.")
+ logging.warning(message)
+ return
+
+ current_step = expected_step
steps_per_second = self.step_timer.steps_per_second()
- info = "step: {} steps_per_second: {:.2f} {}".format(
- current_step, steps_per_second, train_outputs)
- _log_info(info)
+ _log(f"train | step: {current_step: 6d} | "
+ f"steps/sec: {steps_per_second: 6.1f} | "
+ f"output: {_format_output(train_output)}")
- train_outputs["steps_per_second"] = steps_per_second
- self.summary_manager.write_summaries(train_outputs)
+ train_output["steps_per_second"] = steps_per_second
+ self.summary_manager.write_summaries(train_output)
+ self.summary_manager.flush()
- def _maybe_save_checkpoint(self, force_trigger: bool = False):
- """Save checkpoints if necessary.
+ def _maybe_save_checkpoint(self, check_interval: bool = True):
+ """Conditionally saves a checkpoint.
+
+ A checkpoint is saved if a `CheckpointManager` is available, and if the
+ required number of steps has elapsed since the last checkpoint was saved
+ (although this condition can be disabled by setting `check_interval=False`).
Args:
- force_trigger: A boolean indicates whether to force saving checkpoints
- regardless of the checkpoint interval.
+ check_interval: Whether to check if the checkpoint interval has fully
+ elapsed. If `False`, a checkpoint is saved regardless of the elapsed
+ steps since the most recent checkpoint, unless no `checkpoint_manager`
+ was provided to `Controller.__init__`.
Returns:
A boolean indicating whether a checkpoint was saved.
@@ -387,14 +449,21 @@ class Controller(object):
if self.checkpoint_manager and self.checkpoint_manager.checkpoint_interval:
ckpt_path = self.checkpoint_manager.save(
checkpoint_number=self.global_step.numpy(),
- check_interval=not force_trigger)
+ check_interval=check_interval)
if ckpt_path is not None:
- logging.info("Saved checkpoints in %s", ckpt_path)
+ _log(f"saved checkpoint to {ckpt_path}.")
return True
return False
+ def _require(self, attribute, for_method):
+ """Utility method to raise an error if the given `attribute` is not set."""
+ if getattr(self, attribute, None) is None:
+ raise ValueError(
+ f"`{attribute}` is not set. Pass `{attribute}` to "
+ f"`Controller.__init__` before calling `{for_method}()`.")
+
-class StepTimer(object):
+class StepTimer:
"""Utility class for measuring steps/second."""
def __init__(self, step):
diff --git a/orbit/controller_test.py b/orbit/controller_test.py
index 74c7d4194d4b6d0e0866b299200c7174eefad407..b4620b83bd7164e820424edb97ac670a3bf141ed 100644
--- a/orbit/controller_test.py
+++ b/orbit/controller_test.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The Orbit Authors. All Rights Reserved.
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,18 +11,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""Tests for orbit.controller."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Tests for orbit.controller."""
import os
+
from absl import logging
from absl.testing import parameterized
+
import numpy as np
+
from orbit import controller
+from orbit import runner
from orbit import standard_runner
import tensorflow as tf
@@ -36,19 +36,15 @@ def create_model():
def summaries_with_matching_keyword(keyword, summary_dir):
- """Yields summary protos matching given keyword from event file."""
+ """Returns summary protos matching given keyword from event file."""
+ matches = []
event_paths = tf.io.gfile.glob(os.path.join(summary_dir, "events*"))
for event in tf.compat.v1.train.summary_iterator(event_paths[-1]):
if event.summary is not None:
for value in event.summary.value:
if keyword in value.tag:
- logging.info(event)
- yield event.summary
-
-
-def check_eventfile_for_keyword(keyword, summary_dir):
- """Checks event files for the keyword."""
- return any(summaries_with_matching_keyword(keyword, summary_dir))
+ matches.append(event.summary)
+ return matches
def dataset_fn(ctx):
@@ -73,12 +69,8 @@ class TestRunner(standard_runner.StandardTrainer,
self.train_loss = tf.keras.metrics.Mean("train_loss", dtype=tf.float32)
self.eval_loss = tf.keras.metrics.Mean("eval_loss", dtype=tf.float32)
self.return_numpy = return_numpy
- train_dataset = (
- self.strategy.experimental_distribute_datasets_from_function(dataset_fn)
- )
- eval_dataset = (
- self.strategy.experimental_distribute_datasets_from_function(dataset_fn)
- )
+ train_dataset = self.strategy.distribute_datasets_from_function(dataset_fn)
+ eval_dataset = self.strategy.distribute_datasets_from_function(dataset_fn)
standard_runner.StandardTrainer.__init__(self, train_dataset)
standard_runner.StandardEvaluator.__init__(self, eval_dataset)
@@ -103,8 +95,7 @@ class TestRunner(standard_runner.StandardTrainer,
}
def build_eval_dataset(self):
- return self.strategy.experimental_distribute_datasets_from_function(
- dataset_fn)
+ return self.strategy.distribute_datasets_from_function(dataset_fn)
def eval_begin(self):
self.eval_loss.reset_states()
@@ -133,8 +124,7 @@ class TestEvaluator(standard_runner.StandardEvaluator):
def __init__(self):
self.strategy = tf.distribute.get_strategy()
self.model = create_model()
- eval_dataset = self.strategy.experimental_distribute_datasets_from_function(
- dataset_fn)
+ eval_dataset = self.strategy.distribute_datasets_from_function(dataset_fn)
standard_runner.StandardEvaluator.__init__(self, eval_dataset)
def eval_reduce(self, state, output):
@@ -165,6 +155,61 @@ class TestEvaluator(standard_runner.StandardEvaluator):
}
+class TestEvaluatorNoOutput(runner.AbstractEvaluator):
+
+ def evaluate(self, num_steps):
+ pass
+
+
+class TestEvaluatorWithNestedSummary(standard_runner.StandardEvaluator):
+ """Implements the training and evaluation APIs for the test model."""
+
+ def __init__(self):
+ self.strategy = tf.distribute.get_strategy()
+ self.model = create_model()
+ dataset = self.strategy.distribute_datasets_from_function(dataset_fn)
+ dataset2 = self.strategy.distribute_datasets_from_function(dataset_fn)
+ self.loss = tf.keras.metrics.Mean("loss", dtype=tf.float32)
+ self.accuracy = tf.keras.metrics.CategoricalAccuracy(
+ "accuracy", dtype=tf.float32)
+ self.loss2 = tf.keras.metrics.Mean("loss", dtype=tf.float32)
+ self.accuracy2 = tf.keras.metrics.CategoricalAccuracy(
+ "accuracy", dtype=tf.float32)
+ standard_runner.StandardEvaluator.__init__(
+ self, eval_dataset={
+ "dataset": dataset,
+ "dataset2": dataset2
+ })
+
+ def eval_step(self, iterator):
+
+ def _replicated_step(loss, accuracy, inputs):
+ """Replicated evaluation step."""
+ inputs, targets = inputs
+ outputs = self.model(inputs)
+ loss.update_state(tf.keras.losses.MSE(targets, outputs))
+ accuracy.update_state(targets, outputs)
+
+ self.strategy.run(
+ lambda inputs: _replicated_step(self.loss, self.accuracy, inputs),
+ args=(next(iterator["dataset"]),))
+ self.strategy.run(
+ lambda inputs: _replicated_step(self.loss2, self.accuracy2, inputs),
+ args=(next(iterator["dataset2"]),))
+
+ def eval_end(self):
+ return {
+ "dataset": {
+ "loss": self.loss.result(),
+ "accuracy": self.accuracy.result()
+ },
+ "dataset2": {
+ "loss": self.loss2.result(),
+ "accuracy": self.accuracy2.result()
+ },
+ }
+
+
class TestTrainerWithSummaries(standard_runner.StandardTrainer):
"""A Trainer model with summaries for testing purposes."""
@@ -174,15 +219,15 @@ class TestTrainerWithSummaries(standard_runner.StandardTrainer):
self.optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.1)
self.global_step = self.optimizer.iterations
self.train_loss = tf.keras.metrics.Mean("train_loss", dtype=tf.float32)
- train_dataset = (
- self.strategy.experimental_distribute_datasets_from_function(dataset_fn)
- )
+ train_dataset = self.strategy.distribute_datasets_from_function(dataset_fn)
standard_runner.StandardTrainer.__init__(
- self, train_dataset, use_tpu_summary_optimization=True)
+ self,
+ train_dataset,
+ options=standard_runner.StandardTrainerOptions(
+ use_tpu_summary_optimization=True))
def build_train_dataset(self):
- return self.strategy.experimental_distribute_datasets_from_function(
- dataset_fn)
+ return self.strategy.distribute_datasets_from_function(dataset_fn)
def train_step(self, iterator):
@@ -203,7 +248,7 @@ class TestTrainerWithSummaries(standard_runner.StandardTrainer):
class ControllerTest(tf.test.TestCase, parameterized.TestCase):
def setUp(self):
- super(ControllerTest, self).setUp()
+ super().setUp()
self.model_dir = self.get_temp_dir()
def test_no_checkpoint(self):
@@ -222,13 +267,13 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
# Loss and accuracy values should be written into summaries.
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/train")))
- self.assertTrue(
- check_eventfile_for_keyword(
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
"loss", os.path.join(self.model_dir, "summaries/train")))
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/eval")))
- self.assertTrue(
- check_eventfile_for_keyword(
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
"eval_loss", os.path.join(self.model_dir, "summaries/eval")))
# No checkpoint, so global step starts from 0.
test_runner.global_step.assign(0)
@@ -248,6 +293,76 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
train_steps=10, eval_steps=2, eval_interval=6)
self.assertEqual(test_runner.global_step, 10)
+ def test_has_checkpoint_no_summaries(self):
+ test_runner = TestRunner()
+ # Has checkpoint, but no summary directories.
+ checkpoint = tf.train.Checkpoint(model=test_runner.model)
+ checkpoint_manager = tf.train.CheckpointManager(
+ checkpoint,
+ self.model_dir,
+ max_to_keep=None,
+ step_counter=test_runner.global_step)
+ test_controller = controller.Controller(
+ trainer=test_runner,
+ evaluator=test_runner,
+ global_step=test_runner.global_step,
+ checkpoint_manager=checkpoint_manager,
+ steps_per_loop=2)
+ test_controller.train_and_evaluate(
+ train_steps=10, eval_steps=2, eval_interval=6)
+ self.assertEqual(test_runner.global_step, 10)
+
+ # No summaries are saved.
+ self.assertEmpty(tf.io.gfile.glob(
+ os.path.join(checkpoint_manager.directory, "events.*")))
+
+ def test_has_checkpoint_eval_summary_only(self):
+ test_runner = TestRunner()
+ # Has checkpoint, but no summary directories.
+ checkpoint = tf.train.Checkpoint(model=test_runner.model)
+ checkpoint_manager = tf.train.CheckpointManager(
+ checkpoint,
+ self.model_dir,
+ max_to_keep=None,
+ step_counter=test_runner.global_step)
+ test_controller = controller.Controller(
+ trainer=test_runner,
+ evaluator=test_runner,
+ global_step=test_runner.global_step,
+ checkpoint_manager=checkpoint_manager,
+ eval_summary_dir=os.path.join(self.model_dir, "summaries/eval"),
+ steps_per_loop=2)
+ test_controller.train_and_evaluate(
+ train_steps=10, eval_steps=2, eval_interval=6)
+ self.assertEqual(test_runner.global_step, 10)
+
+ # Training summaries are not saved.
+ self.assertEmpty(tf.io.gfile.glob(
+ os.path.join(checkpoint_manager.directory, "events.*")))
+ # Evaluation summaries are saved.
+ self.assertNotEmpty(tf.io.gfile.glob(
+ os.path.join(self.model_dir, "summaries/eval/events.*")))
+
+ def test_restore_from_most_recent_checkpoint(self):
+ test_runner = TestRunner()
+ checkpoint = tf.train.Checkpoint(model=test_runner.model)
+ checkpoint_manager = tf.train.CheckpointManager(
+ checkpoint,
+ self.model_dir,
+ max_to_keep=None,
+ step_counter=test_runner.global_step,
+ checkpoint_interval=5)
+ test_controller = controller.Controller(
+ trainer=test_runner,
+ global_step=test_runner.global_step,
+ checkpoint_manager=checkpoint_manager,
+ eval_summary_dir=os.path.join(self.model_dir, "summaries/eval"),
+ steps_per_loop=5)
+ test_controller.train(20)
+ self.assertLen(checkpoint_manager.checkpoints, 4)
+ restored_path = test_controller.restore_checkpoint()
+ self.assertEqual(restored_path, checkpoint_manager.checkpoints[-1])
+
@parameterized.named_parameters(("return_numpy", True),
("return_tensor", False))
def test_train_and_evaluate(self, return_numpy):
@@ -278,13 +393,13 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
# Loss and accuracy values should be written into summaries.
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/train")))
- self.assertTrue(
- check_eventfile_for_keyword(
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
"loss", os.path.join(self.model_dir, "summaries/train")))
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/eval")))
- self.assertTrue(
- check_eventfile_for_keyword(
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
"eval_loss", os.path.join(self.model_dir, "summaries/eval")))
def test_train_only(self):
@@ -314,8 +429,8 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
# Only train summaries are written.
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/train")))
- self.assertTrue(
- check_eventfile_for_keyword(
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
"loss", os.path.join(self.model_dir, "summaries/train")))
self.assertFalse(
tf.io.gfile.exists(os.path.join(self.model_dir, "summaries/eval")))
@@ -336,16 +451,17 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
checkpoint_manager=checkpoint_manager,
summary_dir=os.path.join(self.model_dir, "summaries/train"),
eval_summary_dir=os.path.join(self.model_dir, "summaries/eval"))
- test_controller.evaluate(steps=2)
+ eval_results = test_controller.evaluate(steps=2)
# Only eval summaries are written
self.assertFalse(
tf.io.gfile.exists(os.path.join(self.model_dir, "summaries/train")))
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/eval")))
- self.assertTrue(
- check_eventfile_for_keyword(
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
"eval_loss", os.path.join(self.model_dir, "summaries/eval")))
+ self.assertIn("eval_loss", eval_results)
# Tests continuous eval with timeout and timeout_fn.
done_file = os.path.join(self.model_dir, "summaries/eval/Done")
@@ -426,8 +542,8 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
# Only train summaries are written.
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/train")))
- self.assertTrue(
- check_eventfile_for_keyword(
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
"loss", os.path.join(self.model_dir, "summaries/train")))
self.assertFalse(
tf.io.gfile.exists(os.path.join(self.model_dir, "summaries/eval")))
@@ -456,12 +572,12 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
# Loss and accuracy values should be written into summaries.
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries")))
- self.assertTrue(
- check_eventfile_for_keyword("loss",
- os.path.join(self.model_dir, "summaries")))
- self.assertTrue(
- check_eventfile_for_keyword("eval_loss",
- os.path.join(self.model_dir, "summaries")))
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
+ "loss", os.path.join(self.model_dir, "summaries")))
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
+ "eval_loss", os.path.join(self.model_dir, "summaries")))
def test_early_stop_on_eval_loss(self):
test_runner = TestRunner()
@@ -504,7 +620,7 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
self.assertLess(test_runner.global_step, 10)
- def test_evaluate_with_loss_outputs(self):
+ def test_evaluate_with_loss_output(self):
test_evaluator = TestEvaluator()
checkpoint = tf.train.Checkpoint(model=test_evaluator.model)
@@ -521,10 +637,17 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
# Only eval summaries are written
self.assertNotEmpty(
tf.io.gfile.listdir(os.path.join(self.model_dir, "summaries/eval")))
- self.assertTrue(
- check_eventfile_for_keyword(
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
"eval_loss", os.path.join(self.model_dir, "summaries/eval")))
+ def test_evaluate_with_no_output(self):
+ test_controller = controller.Controller(
+ evaluator=TestEvaluatorNoOutput(),
+ global_step=tf.Variable(0, dtype=tf.int64),
+ eval_summary_dir=os.path.join(self.model_dir, "summaries/eval"))
+ self.assertEqual(test_controller.evaluate(steps=5), {})
+
def test_train_and_evaluate_reset_datasets(self):
test_runner = TestRunner()
@@ -538,16 +661,68 @@ class ControllerTest(tf.test.TestCase, parameterized.TestCase):
train_steps=10, eval_steps=2, eval_interval=6)
train_dataset = (
- test_runner.strategy.experimental_distribute_datasets_from_function(
- dataset_fn))
+ test_runner.strategy.distribute_datasets_from_function(dataset_fn))
eval_dataset = (
- test_runner.strategy.experimental_distribute_datasets_from_function(
- dataset_fn))
+ test_runner.strategy.distribute_datasets_from_function(dataset_fn))
test_runner.train_dataset = train_dataset
test_runner.eval_dataset = eval_dataset
test_controller.train_and_evaluate(
train_steps=10, eval_steps=2, eval_interval=6)
+ def test_eval_and_checkpoint_interval(self):
+ test_runner = TestRunner()
+
+ checkpoint = tf.train.Checkpoint(
+ model=test_runner.model, optimizer=test_runner.optimizer)
+ checkpoint_manager = tf.train.CheckpointManager(
+ checkpoint,
+ self.model_dir,
+ max_to_keep=None,
+ step_counter=test_runner.global_step,
+ checkpoint_interval=5)
+ test_controller = controller.Controller(
+ trainer=test_runner,
+ evaluator=test_runner,
+ global_step=test_runner.global_step,
+ steps_per_loop=10,
+ checkpoint_manager=checkpoint_manager,
+ summary_dir=self.model_dir)
+ test_controller.train_and_evaluate(
+ train_steps=10, eval_steps=2, eval_interval=5)
+
+ # Expect 3 checkpoints to be saved at step: 5, 10.
+ self.assertLen(
+ tf.io.gfile.glob(os.path.join(self.model_dir, "ckpt-*.data*")), 2)
+ # Expect evaluation is performed 2 times at step: 5, 10.
+ self.assertLen(
+ summaries_with_matching_keyword("eval_loss", self.model_dir), 2)
+
+ def test_evaluate_with_nested_summaries(self):
+ test_evaluator = TestEvaluatorWithNestedSummary()
+ test_controller = controller.Controller(
+ evaluator=test_evaluator,
+ global_step=tf.Variable(0, dtype=tf.int64),
+ eval_summary_dir=self.model_dir)
+ test_controller.evaluate(steps=5)
+
+ self.assertNotEmpty(
+ tf.io.gfile.listdir(os.path.join(self.model_dir, "dataset")))
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
+ "loss", os.path.join(self.model_dir, "dataset")))
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
+ "accuracy", os.path.join(self.model_dir, "dataset")))
+
+ self.assertNotEmpty(
+ tf.io.gfile.listdir(os.path.join(self.model_dir, "dataset2")))
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
+ "loss", os.path.join(self.model_dir, "dataset2")))
+ self.assertNotEmpty(
+ summaries_with_matching_keyword(
+ "accuracy", os.path.join(self.model_dir, "dataset2")))
+
if __name__ == "__main__":
tf.test.main()
diff --git a/orbit/examples/__init__.py b/orbit/examples/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4d9cc3a1b148e5c8c153f2f2357d0475e7a43b6
--- /dev/null
+++ b/orbit/examples/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/orbit/examples/single_task/__init__.py b/orbit/examples/single_task/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4d9cc3a1b148e5c8c153f2f2357d0475e7a43b6
--- /dev/null
+++ b/orbit/examples/single_task/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/orbit/examples/single_task/single_task_evaluator.py b/orbit/examples/single_task/single_task_evaluator.py
new file mode 100644
index 0000000000000000000000000000000000000000..0dcbae063a6282cbf76b8247bdcea43ee9c26c42
--- /dev/null
+++ b/orbit/examples/single_task/single_task_evaluator.py
@@ -0,0 +1,86 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An evaluator object that can evaluate models with a single output."""
+import orbit
+import tensorflow as tf
+
+
+class SingleTaskEvaluator(orbit.StandardEvaluator):
+ """Evaluates a single-output model on a given dataset.
+
+ This evaluator will handle running a model with one output on a single
+ dataset, and will apply the output of that model to one or more
+ `tf.keras.metrics.Metric` objects.
+ """
+
+ def __init__(self,
+ eval_dataset,
+ label_key,
+ model,
+ metrics,
+ evaluator_options=None):
+ """Initializes a `SingleTaskEvaluator` instance.
+
+ If the `SingleTaskEvaluator` should run its model under a distribution
+ strategy, it should be created within that strategy's scope.
+
+ Arguments:
+ eval_dataset: A `tf.data.Dataset` or `DistributedDataset` that contains a
+ string-keyed dict of `Tensor`s.
+ label_key: The key corresponding to the label value in feature
+ dictionaries dequeued from `eval_dataset`. This key will be removed from
+ the dictionary before it is passed to the model.
+ model: A `tf.Module` or Keras `Model` object to evaluate.
+ metrics: A single `tf.keras.metrics.Metric` object, or a list of
+ `tf.keras.metrics.Metric` objects.
+ evaluator_options: An optional `orbit.StandardEvaluatorOptions` object.
+ """
+
+ self.label_key = label_key
+ self.model = model
+ self.metrics = metrics if isinstance(metrics, list) else [metrics]
+
+ # Capture the strategy from the containing scope.
+ self.strategy = tf.distribute.get_strategy()
+
+ super(SingleTaskEvaluator, self).__init__(
+ eval_dataset=eval_dataset, options=evaluator_options)
+
+ def eval_begin(self):
+ """Actions to take once before every eval loop."""
+ for metric in self.metrics:
+ metric.reset_states()
+
+ def eval_step(self, iterator):
+ """One eval step. Called multiple times per eval loop by the superclass."""
+
+ def step_fn(inputs):
+ # Extract the target value and delete it from the input dict, so that
+ # the model never sees it.
+ target = inputs.pop(self.label_key)
+ output = self.model(inputs)
+ for metric in self.metrics:
+ metric.update_state(target, output)
+
+ # This is needed to handle distributed computation.
+ self.strategy.run(step_fn, args=(next(iterator),))
+
+ def eval_end(self):
+ """Actions to take once after an eval loop."""
+ with self.strategy.scope():
+ # Export the metrics.
+ metrics = {metric.name: metric.result() for metric in self.metrics}
+
+ return metrics
diff --git a/orbit/examples/single_task/single_task_evaluator_test.py b/orbit/examples/single_task/single_task_evaluator_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..c074da0fb9de8e054451ac1a07f729c26ff08613
--- /dev/null
+++ b/orbit/examples/single_task/single_task_evaluator_test.py
@@ -0,0 +1,65 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the single_task_evaluator."""
+import orbit
+from orbit.examples.single_task import single_task_evaluator
+from orbit.examples.single_task import single_task_trainer
+
+import tensorflow as tf
+import tensorflow_datasets as tfds
+
+
+class SingleTaskEvaluatorTest(tf.test.TestCase):
+
+ def test_single_task_evaluation(self):
+
+ iris = tfds.load('iris')
+ train_ds = iris['train'].batch(32)
+
+ model = tf.keras.Sequential([
+ tf.keras.Input(shape=(4,), name='features'),
+ tf.keras.layers.Dense(10, activation=tf.nn.relu),
+ tf.keras.layers.Dense(10, activation=tf.nn.relu),
+ tf.keras.layers.Dense(3)
+ ])
+
+ trainer = single_task_trainer.SingleTaskTrainer(
+ train_ds,
+ label_key='label',
+ model=model,
+ loss_fn=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+ optimizer=tf.keras.optimizers.SGD(learning_rate=0.01))
+
+ evaluator = single_task_evaluator.SingleTaskEvaluator(
+ train_ds,
+ label_key='label',
+ model=model,
+ metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
+
+ controller = orbit.Controller(
+ trainer=trainer,
+ evaluator=evaluator,
+ steps_per_loop=100,
+ global_step=trainer.optimizer.iterations)
+
+ controller.train(train_ds.cardinality().numpy())
+ controller.evaluate()
+ accuracy = evaluator.metrics[0].result().numpy()
+
+ self.assertGreater(0.925, accuracy)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/orbit/examples/single_task/single_task_trainer.py b/orbit/examples/single_task/single_task_trainer.py
new file mode 100644
index 0000000000000000000000000000000000000000..f9b29185a760ea391581ee08be94ff1f9df79932
--- /dev/null
+++ b/orbit/examples/single_task/single_task_trainer.py
@@ -0,0 +1,140 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A trainer object that can train models with a single output."""
+
+import orbit
+import tensorflow as tf
+
+
+class SingleTaskTrainer(orbit.StandardTrainer):
+ """Trains a single-output model on a given dataset.
+
+ This trainer will handle running a model with one output on a single
+ dataset. It will apply the provided loss function to the model's output
+ to calculate gradients and will apply them via the provided optimizer. It will
+ also supply the output of that model to one or more `tf.keras.metrics.Metric`
+ objects.
+ """
+
+ def __init__(self,
+ train_dataset,
+ label_key,
+ model,
+ loss_fn,
+ optimizer,
+ metrics=None,
+ trainer_options=None):
+ """Initializes a `SingleTaskTrainer` instance.
+
+ If the `SingleTaskTrainer` should run its model under a distribution
+ strategy, it should be created within that strategy's scope.
+
+ This trainer will also calculate metrics during training. The loss metric
+ is calculated by default, but other metrics can be passed to the `metrics`
+ arg.
+
+ Arguments:
+ train_dataset: A `tf.data.Dataset` or `DistributedDataset` that contains a
+ string-keyed dict of `Tensor`s.
+ label_key: The key corresponding to the label value in feature
+ dictionaries dequeued from `train_dataset`. This key will be removed
+ from the dictionary before it is passed to the model.
+ model: A `tf.Module` or Keras `Model` object to evaluate. It must accept a
+ `training` kwarg.
+ loss_fn: A per-element loss function of the form (target, output). The
+ output of this loss function will be reduced via `tf.reduce_mean` to
+ create the final loss. We recommend using the functions in the
+ `tf.keras.losses` package or `tf.keras.losses.Loss` objects with
+ `reduction=tf.keras.losses.reduction.NONE`.
+ optimizer: A `tf.keras.optimizers.Optimizer` instance.
+ metrics: A single `tf.keras.metrics.Metric` object, or a list of
+ `tf.keras.metrics.Metric` objects.
+ trainer_options: An optional `orbit.utils.StandardTrainerOptions` object.
+ """
+ self.label_key = label_key
+ self.model = model
+ self.loss_fn = loss_fn
+ self.optimizer = optimizer
+
+ # Capture the strategy from the containing scope.
+ self.strategy = tf.distribute.get_strategy()
+
+ # We always want to report training loss.
+ self.train_loss = tf.keras.metrics.Mean('training_loss', dtype=tf.float32)
+
+ # We need self.metrics to be an iterable later, so we handle that here.
+ if metrics is None:
+ self.metrics = []
+ elif isinstance(metrics, list):
+ self.metrics = metrics
+ else:
+ self.metrics = [metrics]
+
+ super(SingleTaskTrainer, self).__init__(
+ train_dataset=train_dataset, options=trainer_options)
+
+ def train_loop_begin(self):
+ """Actions to take once, at the beginning of each train loop."""
+ self.train_loss.reset_states()
+ for metric in self.metrics:
+ metric.reset_states()
+
+ def train_step(self, iterator):
+ """A train step. Called multiple times per train loop by the superclass."""
+
+ def train_fn(inputs):
+ with tf.GradientTape() as tape:
+ # Extract the target value and delete it from the input dict, so that
+ # the model never sees it.
+ target = inputs.pop(self.label_key)
+
+ # Get the outputs of the model.
+ output = self.model(inputs, training=True)
+
+ # Get the average per-batch loss and scale it down by the number of
+ # replicas. This ensures that we don't end up multiplying our loss by
+ # the number of workers - gradients are summed, not averaged, across
+ # replicas during the apply_gradients call.
+ # Note, the reduction of loss is explicitly handled and scaled by
+ # num_replicas_in_sync. Recommend to use a plain loss function.
+ # If you're using tf.keras.losses.Loss object, you may need to set
+ # reduction argument explicitly.
+ loss = tf.reduce_mean(self.loss_fn(target, output))
+ scaled_loss = loss / self.strategy.num_replicas_in_sync
+
+ # Get the gradients by applying the loss to the model's trainable
+ # variables.
+ gradients = tape.gradient(scaled_loss, self.model.trainable_variables)
+
+ # Apply the gradients via the optimizer.
+ self.optimizer.apply_gradients(
+ list(zip(gradients, self.model.trainable_variables)))
+
+ # Update metrics.
+ self.train_loss.update_state(loss)
+ for metric in self.metrics:
+ metric.update_state(target, output)
+
+ # This is needed to handle distributed computation.
+ self.strategy.run(train_fn, args=(next(iterator),))
+
+ def train_loop_end(self):
+ """Actions to take once after a training loop."""
+ with self.strategy.scope():
+ # Export the metrics.
+ metrics = {metric.name: metric.result() for metric in self.metrics}
+ metrics[self.train_loss.name] = self.train_loss.result()
+
+ return metrics
diff --git a/orbit/examples/single_task/single_task_trainer_test.py b/orbit/examples/single_task/single_task_trainer_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..cba34f7b05485d891c9b1c6da4e02522df1f772a
--- /dev/null
+++ b/orbit/examples/single_task/single_task_trainer_test.py
@@ -0,0 +1,60 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the single_task_trainer."""
+import orbit
+from orbit.examples.single_task import single_task_trainer
+
+import tensorflow as tf
+import tensorflow_datasets as tfds
+
+
+class SingleTaskTrainerTest(tf.test.TestCase):
+
+ def test_single_task_training(self):
+ iris = tfds.load('iris')
+ train_ds = iris['train'].batch(32).repeat()
+
+ model = tf.keras.Sequential([
+ tf.keras.Input(shape=(4,), name='features'),
+ tf.keras.layers.Dense(10, activation=tf.nn.relu),
+ tf.keras.layers.Dense(10, activation=tf.nn.relu),
+ tf.keras.layers.Dense(3),
+ tf.keras.layers.Softmax(),
+ ])
+
+ trainer = single_task_trainer.SingleTaskTrainer(
+ train_ds,
+ label_key='label',
+ model=model,
+ loss_fn=tf.keras.losses.sparse_categorical_crossentropy,
+ optimizer=tf.keras.optimizers.SGD(learning_rate=0.01))
+
+ controller = orbit.Controller(
+ trainer=trainer,
+ steps_per_loop=100,
+ global_step=trainer.optimizer.iterations)
+
+ controller.train(1)
+ start_loss = trainer.train_loss.result().numpy()
+ controller.train(500)
+ end_loss = trainer.train_loss.result().numpy()
+
+ # Assert that the model has trained 'significantly' - that the loss
+ # has dropped by over 50%.
+ self.assertLess(end_loss, start_loss / 2)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/orbit/runner.py b/orbit/runner.py
index 36931e464c87d98ce1fd3163877e38d2ed569912..b0377c5218cac2f60ceacdc66721bac5b149c68b 100644
--- a/orbit/runner.py
+++ b/orbit/runner.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The Orbit Authors. All Rights Reserved.
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,69 +11,73 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""An abstraction that users can easily handle their custom training loops."""
-from __future__ import absolute_import
-from __future__ import division
-# from __future__ import google_type_annotations
-from __future__ import print_function
+"""Provides AbstractTrainer/Evaluator base classes, defining train/eval APIs."""
import abc
-from typing import Dict, Optional, Text
-import six
+
+from typing import Dict, Optional, Union
+
+import numpy as np
import tensorflow as tf
-@six.add_metaclass(abc.ABCMeta)
-class AbstractTrainer(tf.Module):
- """An abstract class defining the APIs required for training."""
+Output = Dict[str, Union[tf.Tensor, float, np.number, np.ndarray, 'Output']] # pytype: disable=not-supported-yet
+
+
+class AbstractTrainer(tf.Module, metaclass=abc.ABCMeta):
+ """An abstract class defining the API required for training."""
@abc.abstractmethod
- def train(self,
- num_steps: Optional[tf.Tensor]) -> Optional[Dict[Text, tf.Tensor]]:
- """Implements model training with multiple steps.
-
- In training, it is common to break the total training steps into several
- training loops, so users can do checkpointing, write summaries and run some
- python callbacks. This is necessary for getting good performance in TPU
- training, as the overhead for launching a multi worker tf.function may be
- large in Eager mode. It is usually encouraged to create a host training loop
- (e.g. using a `tf.range` wrapping `strategy.run` inside a
- `tf.function`) in the TPU case. For the cases that don't require host
- training loop to acheive peak performance, users can just implement a simple
- python loop to drive each step.
+ def train(self, num_steps: tf.Tensor) -> Optional[Output]:
+ """Implements `num_steps` steps of training.
+
+ This method will be called by the `Controller` to perform the "inner loop"
+ of training. This inner loop amortizes the cost of bookkeeping associated
+ with checkpointing, evaluation, and writing summaries. Additionally, the
+ inner loop can be implemented (if desired) using TensorFlow's looping
+ constructs (e.g. a `for` loop over a `tf.range` inside a `tf.function`),
+ which can be necessary for getting optimal performance when running on TPU.
+ For cases that don't require peak performance, a simple Python loop can be
+ used instead for simplicity.
Args:
- num_steps: A guideline for how many training steps to run. Note that it is
- up to the model what constitutes a "step" (this may involve more than
- one update to model parameters, e.g. if training a GAN).
+ num_steps: The number of training steps to run. Note that it is up to the
+ model what constitutes a "step", which may involve more than one update
+ to model parameters (e.g., if training a GAN).
Returns:
- The function may return a dictionary of `Tensors` or numpy arrays, which
- will be written to logs and as TensorBoard summaries.
+ Either `None`, or a dictionary mapping names to `Tensor`s or NumPy values.
+ If a dictionary is returned, it will be written to logs and as TensorBoard
+ summaries. The dictionary may also be nested, which will generate a
+ hierarchy of summary directories.
"""
pass
-@six.add_metaclass(abc.ABCMeta)
-class AbstractEvaluator(tf.Module):
- """An abstract class defining the APIs required for evaluation."""
+class AbstractEvaluator(tf.Module, metaclass=abc.ABCMeta):
+ """An abstract class defining the API required for evaluation."""
@abc.abstractmethod
- def evaluate(
- self, num_steps: Optional[tf.Tensor]) -> Optional[Dict[Text, tf.Tensor]]:
- """Implements model evaluation.
+ def evaluate(self, num_steps: tf.Tensor) -> Optional[Output]:
+ """Implements `num_steps` steps of evaluation.
+
+ This method will by called the `Controller` to perform an evaluation. The
+ `num_steps` parameter specifies the number of steps of evaluation to run,
+ which is specified by the user when calling one of the `Controller`'s
+ evaluation methods. A special sentinel value of `-1` is reserved to indicate
+ evaluation should run until the underlying data source is exhausted.
Args:
- num_steps: A guideline for how many evaluation steps to run. Note that it
- is up to the model what constitutes a "step". Generally, it may be
- desirable to support both a limited number of eval steps and iterating
- over a full dataset (however many steps are required) when `num_steps`
- is `None`.
+ num_steps: The number of evaluation steps to run. Note that it is up to
+ the model what constitutes a "step". Evaluations may also want to
+ support "complete" evaluations when `num_steps == -1`, running until a
+ given data source is exhausted.
Returns:
- The function may return a dictionary of `Tensors` or numpy arrays, which
- will be written to logs and as TensorBoard summaries.
+ Either `None`, or a dictionary mapping names to `Tensor`s or NumPy values.
+ If a dictionary is returned, it will be written to logs and as TensorBoard
+ summaries. The dictionary may also be nested, which will generate a
+ hierarchy of summary directories.
"""
pass
diff --git a/orbit/standard_runner.py b/orbit/standard_runner.py
index c0162e9e5ab0954b7d96d33917f47c0a39a93037..ac03707a0f7e0becd87147b1669dca196b1bff72 100644
--- a/orbit/standard_runner.py
+++ b/orbit/standard_runner.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The Orbit Authors. All Rights Reserved.
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,91 +11,146 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""An abstraction that users can easily handle their custom training loops."""
-from __future__ import absolute_import
-from __future__ import division
-# from __future__ import google_type_annotations
-from __future__ import print_function
+"""AbstractTrainer/Evaluator subclasses with added functionality.
+
+The classes in this module provide some additional structure to the bare
+`AbstractTrainer`/`AbstractEvaluator` APIs.
+
+Both `StandardTrainer` and `StandardEvaluator` split the train/eval loops into
+"begin", "step", and "end" methods, and provide an implementation of the loop
+itself that makes calls to the relevant step method.
+
+`StandardTrainer` supports running the loop using the TF while loop construct
+for added performance (particularly on TPUs). It additionally provides some
+functionality to make writing summaries from inside a model more performant when
+running on TPUs.
+
+These classes are intended to work well in common settings, however there may
+be use cases these classes don't support (for instance, `StandardEvaluator` in
+particular doesn't support running full evaluations over multiple different eval
+datasets). Users are encouraged to simply fall back to custom `AbstractTrainer`
+and `AbstractEvaluator` subclasses in these cases.
+"""
import abc
-from typing import Any, Dict, Optional, Text
+
+from typing import Any, Optional
+
+import dataclasses
+
from orbit import runner
-from orbit import utils
-import six
+from orbit.utils import loop_fns
+
import tensorflow as tf
-@six.add_metaclass(abc.ABCMeta)
-class StandardTrainer(runner.AbstractTrainer):
- """Implements the standard functionality of AbstractTrainer APIs."""
+@dataclasses.dataclass(frozen=True)
+class StandardTrainerOptions:
+ """Advanced options for `orbit.StandardTrainer`.
- def __init__(self,
- train_dataset,
- use_tf_while_loop=True,
- use_tf_function=True,
- use_tpu_summary_optimization=False):
- """Construct a `StandardTrainer` object.
+ Attributes:
+ use_tf_function: A boolean indicating whether to apply `tf.function` to the
+ training loop. This will only affect the body of the loop (involving
+ `train_step`); `train_loop_begin` and `train_loop_end` will always be run
+ in eager mode.
+ use_tf_while_loop: A boolean indicating whether to run the training loop
+ using a `tf.while_loop`. If `True`, `use_tf_function` must also be `True`.
+ use_tpu_summary_optimization: A boolean indicating whether to enable a
+ performance optimization for summaries in TPUs. Writing summaries
+ conditionally with outside compilation on TPUs can be extremely slow. If
+ `True`, this optimization creates two `tf.function`s with two XLA programs
+ (one with summary calls, and one without). The program with summaries runs
+ only for one step when summaries should be recorded.
+ """
+ use_tf_function: bool = True
+ use_tf_while_loop: bool = True
+ use_tpu_summary_optimization: bool = False
+
+
+class StandardTrainer(runner.AbstractTrainer, metaclass=abc.ABCMeta):
+ """Implements standard functionality on top of the AbstractTrainer API.
+
+ This class structures the training "inner loop" roughly as follows:
+
+ train_loop_begin()
+ for _ in range(num_steps):
+ train_step(train_iterator)
+ return train_loop_end()
+
+ Calls to `train_loop_begin` and `train_loop_end` are always done in eager
+ mode, while the loop/`train_step` may be implemented using `tf.while` and/or
+ `tf.function`, as determined by the `options` passed to `__init__`.
+ """
+
+ def __init__(self, train_dataset, options: StandardTrainerOptions = None):
+ """Initializes the `StandardTrainer` instance.
Args:
- train_dataset: A tf.nest-compatible structure of tf.data.Dataset or
- DistributedDataset.
- use_tf_while_loop: A boolean indicates whether to wrap the train step with
- a `tf.while_loop`.
- use_tf_function: A boolean indicates whether a `tf.function` will be used.
- If False, training will run on pure eager mode.
- use_tpu_summary_optimization: A boolean indicates whether to enable the
- performance optimization for summaries in TPUs. In TPUs, writing
- summaries with outside compilation inside train step is slow. If True,
- it creates two `tf.function` with two XLA programs: one with summaries
- and one without, and run the program with summaries (slow one) only if
- necessary.
+ train_dataset: A `tf.nest`-compatible structure of `tf.data.Dataset` or
+ `DistributedDataset`.
+ options: An `orbit.StandardTrainerOptions` instance.
"""
- if use_tf_while_loop and not use_tf_function:
+ options = options or StandardTrainerOptions()
+ if options.use_tf_while_loop and not options.use_tf_function:
raise ValueError("`use_tf_while_loop=True` and `use_tf_function=False` "
"is not supported")
- if use_tpu_summary_optimization and not use_tf_while_loop:
+ if options.use_tpu_summary_optimization and not options.use_tf_while_loop:
raise ValueError("`use_tpu_summary_optimization=True` and "
"`use_tf_while_loop=False` is not supported")
- self._use_tf_while_loop = use_tf_while_loop
- self._use_tf_function = use_tf_function
+
+ self._train_options = options
self._train_dataset = train_dataset
self._train_iter = None
self._train_loop_fn = None
- self._use_tpu_summary_optimization = use_tpu_summary_optimization
- def train(self,
- num_steps: Optional[tf.Tensor]) -> Optional[Dict[Text, tf.Tensor]]:
- """See base class."""
+ def create_train_loop_fn(self):
+ """Creates a training loop from the current step function and options.
+
+ Returns:
+ The train loop function, i.e. wrapper of multiple train steps.
+ """
+ train_step_fn = self.train_step
+ if self._train_options.use_tf_while_loop:
+ loop_fn = loop_fns.create_tf_while_loop_fn(train_step_fn)
+ if self._train_options.use_tpu_summary_optimization:
+ loop_fn = loop_fns.LoopFnWithSummaries(loop_fn)
+ else:
+ loop_fn = tf.function(loop_fn)
+ else:
+ if self._train_options.use_tf_function:
+ train_step_fn = tf.function(train_step_fn)
+ loop_fn = loop_fns.create_loop_fn(train_step_fn)
+ return loop_fn
+
+ def train(self, num_steps: tf.Tensor) -> Optional[runner.Output]:
+ """Implements `num_steps` steps of training.
+
+ Args:
+ num_steps: The number of training steps to run. This corresponds directly
+ to the number of calls made to `train_step`.
+
+ Returns:
+ The output of `train_loop_end`.
+ """
self.train_loop_begin()
+ if self._train_loop_fn is None:
+ self._train_loop_fn = self.create_train_loop_fn()
+
if self._train_iter is None:
self._train_iter = tf.nest.map_structure(iter, self.train_dataset)
- if self._train_loop_fn is None:
- train_fn = self.train_step
- if self._use_tf_while_loop:
- self._train_loop_fn = utils.create_tf_while_loop_fn(train_fn)
- if self._use_tpu_summary_optimization:
- self._train_loop_fn = utils.train_function_with_summaries(
- self._train_loop_fn)
- else:
- self._train_loop_fn = tf.function(self._train_loop_fn)
- else:
- if self._use_tf_function:
- train_fn = tf.function(train_fn)
- self._train_loop_fn = utils.create_loop_fn(train_fn)
-
self._train_loop_fn(self._train_iter, num_steps)
return self.train_loop_end()
def train_loop_begin(self):
"""Called once at the beginning of the training loop.
- This method is called before dataset iterators creation.
- This is a good place to reset metrics that accumulate values over multiple
- steps of training.
+ This method is always called in eager mode, and is a good place to reset
+ metrics that accumulate values over multiple steps of training.
+
+ Note that this method is called before dataset iterator creation.
"""
pass
@@ -103,79 +158,187 @@ class StandardTrainer(runner.AbstractTrainer):
def train_step(self, iterator):
"""Implements one step of training.
- What a "step" consists of is up to the implementer. If using distribution
- strategies, the call to this method should take place in the "cross-replica
+ What a "step" consists of is up to the implementer. When using distribution
+ strategies, the call to this method takes place in the "cross-replica
context" for generality, to allow e.g. multiple iterator dequeues and calls
to `strategy.run`.
+ Note that if `use_tf_function=True`, all the code inside `train_step` should
+ be compatible with `tf.function` tracing (and in particular, any state
+ modifications involving `self` should be avoided). In some cases, non-
+ `tf.function` compatible code can be moved to `train_loop_begin` or
+ `train_loop_end`, which always execute eagerly.
+
Args:
- iterator: A tf.nest-compatible structure of tf.data Iterator or
- DistributedIterator.
+ iterator: A `tf.nest`-compatible structure of `tf.data.Iterator` or
+ `DistributedIterator`. The structure of this input matches the structure
+ of `train_dataset` as passed to `__init__`.
"""
pass
- def train_loop_end(self) -> Optional[Dict[Text, tf.Tensor]]:
- """Called at the end of the training loop.
+ def train_loop_end(self) -> Optional[runner.Output]:
+ """Called once at the end of the training loop.
- This is a good place to get metric results. The value returned from this
- function will be returned as-is from the train() method.
+ This method is always called in eager mode, and is a good place to get
+ metric results. The value returned from this function will be returned as-is
+ from the `train` method implementation provided by `StandardTrainer`.
Returns:
The function may return a dictionary of `Tensors`, which will be
- written to logs and as TensorBoard summaries.
+ written to logs and as TensorBoard summaries. It can also be a
+ nested dictionary, yielding a hierarchy of summary directories.
"""
pass
@property
def train_dataset(self):
- """Returns the train_dataset instance."""
+ """The current training dataset."""
return self._train_dataset
@train_dataset.setter
def train_dataset(self, train_dataset):
- """Set a new train dataset and replace with the existing one.
+ """Sets a new training dataset, replacing the current one.
- Any unfinished work in the previous dataset will be discarded.
+ Any unprocessed examples in the current dataset are discarded.
Args:
- train_dataset: A tf.nest-compatible structure of tf.data.Dataset or
- DistributedDataset.
+ train_dataset: A `tf.nest`-compatible structure of `tf.data.Dataset` or
+ `DistributedDataset`.
"""
self._train_dataset = train_dataset
self._train_iter = None
-@six.add_metaclass(abc.ABCMeta)
-class StandardEvaluator(runner.AbstractEvaluator):
- """Implements the standard functionality of AbstractEvaluator APIs."""
-
- def __init__(self, eval_dataset, use_tf_function=True):
- """Construct a `StandardEvaluator` object.
+@dataclasses.dataclass(frozen=True)
+class StandardEvaluatorOptions:
+ """Advanced options for the `orbit.StandardEvaluator`.
+
+ Attributes:
+ use_tf_function: A boolean indicating whether to apply `tf.function` to the
+ evaluation loop. This will only affect the body of the loop (involving
+ `eval_step`); `eval_loop_begin` and `eval_loop_end` will always be run
+ in eager mode.
+ use_tf_while_loop: A boolean indicating whether to run the evaluation loop
+ using a `tf.while_loop`. If `True`, `use_tf_function` must also be `True`.
+ recreate_iterator_for_each_eval: A boolean indicating whether to recreate a
+ new iterator for the evaluation dataset before each round of evaluation,
+ which implies each round of evaluation starts from the beginning of
+ the evaluation dataset. For example, the evaluation dataset is
+ `[1, 2, 3, 4]`, batch size is 1 and evaluation steps is 2. If `True`, the
+ data to be evaluated is [1, 2] every time. If `False`, the iterator
+ state is maintained between calls to `StandardEvaluator.evaluate()`.
+ """
+ use_tf_function: bool = True
+ use_tf_while_loop: bool = False
+ recreate_iterator_for_each_eval: bool = True
+
+
+class StandardEvaluator(runner.AbstractEvaluator, metaclass=abc.ABCMeta):
+ """Implements the standard functionality of AbstractEvaluator APIs.
+
+ This class structures evaluation roughly as follows:
+
+ state = eval_begin()
+ for _ in range(num_steps):
+ step_outputs = eval_step(eval_iterator)
+ state = eval_reduce(state, step_outputs)
+ return eval_end(state)
+
+ Calls to `eval_begin` and `eval_end` are always done in eager
+ mode, while `eval_step` may be compiled with `tf.function` as determined by
+ the `options` passed to `__init__`. `eval_reduce` is in eager mode if
+ `use_tf_while_loop=False` in `StandardEvaluatorOptions`, but in graph mode if
+ `use_tf_while_loop=True`.
+
+ This class does not support completely evaluating multiple different datasets
+ (i.e., where every example of each dataset should be processed, as opposed to
+ running for a fixed number of evaluation steps). A custom `AbstractEvaluator`
+ is recommended in this case.
+ """
+
+ def __init__(self, eval_dataset, options: StandardEvaluatorOptions = None):
+ """Initializes the `StandardEvaluator` instance.
Args:
- eval_dataset: A tf.nest-compatible structure of tf.data.Dataset or
- DistributedDataset.
- use_tf_function: A boolean indicates whether a `tf.function` will be used.
- If False, evaluation will run on pure eager mode.
+ eval_dataset: A `tf.nest`-compatible structure of `tf.data.Dataset` or
+ `DistributedDataset`.
+ options: An `orbit.StandardEvaluatorOptions` instance.
"""
- self._eval_use_tf_function = use_tf_function
+ options = options or StandardEvaluatorOptions()
+ if options.use_tf_while_loop and not options.use_tf_function:
+ raise ValueError("`use_tf_while_loop=True` and `use_tf_function=False` "
+ "is not supported")
+
+ self._eval_options = options
self._eval_dataset = eval_dataset
+ self._eval_iter = None
self._eval_loop_fn = None
- def evaluate(
- self, num_steps: Optional[tf.Tensor]) -> Optional[Dict[Text, tf.Tensor]]:
- """See base class."""
+ def create_eval_loop_fn(self, has_state: bool):
+ """Creates an eval loop from the current step function and options.
+
+ Args:
+ has_state: If the step function has state, state will be kept in the loop.
+
+ Returns:
+ The eval loop function, i.e. wrapper of multiple eval steps.
+ """
+ eval_step_fn = self.eval_step
+ if self._eval_options.use_tf_while_loop:
+ # TODO(b/176126742): tf.while_loop doesn't support `None` as a loop input
+ # even when it is not used inside the loop. To workaround this limitation,
+ # we have to build two tf.functions for it.
+ if has_state:
+ loop_fn = loop_fns.create_tf_while_loop_fn_with_state(eval_step_fn)
+ else:
+ loop_fn = loop_fns.create_tf_while_loop_fn(eval_step_fn)
+ loop_fn = tf.function(loop_fn)
+ else:
+ if self._eval_options.use_tf_function:
+ eval_step_fn = tf.function(eval_step_fn)
+ loop_fn = loop_fns.create_loop_fn(eval_step_fn)
+ return loop_fn
+
+ def evaluate(self, num_steps: tf.Tensor) -> Optional[runner.Output]:
+ """Implements `num_steps` steps of evaluation.
+
+ Args:
+ num_steps: The number of evaluation steps to run. When this is -1,
+ evaluation proceeds until a call to `eval_step` raises a `StopIteration`
+ or `tf.errors.OutOfRangeError`.
+
+ Returns:
+ The output of `self.eval_end()`.
+
+ Raises:
+ ValueError: If `options.use_tf_while_loop` is `True` and `num_steps` is
+ unspecified.
+ """
+ if self._eval_options.use_tf_while_loop and num_steps == -1:
+ raise ValueError("Looping until exhausted is not supported if "
+ "`options.use_tf_while_loop` is `True`")
+
outputs = self.eval_begin() # pylint: disable=assignment-from-no-return
- eval_iter = tf.nest.map_structure(iter, self._eval_dataset)
+ has_state = outputs is not None
if self._eval_loop_fn is None:
- eval_fn = self.eval_step
- if self._eval_use_tf_function:
- eval_fn = tf.function(eval_fn)
- self._eval_loop_fn = utils.create_loop_fn(eval_fn)
+ self._eval_loop_fn = self.create_eval_loop_fn(has_state)
+
+ # If `recreate_iterator_for_each_eval` is `True`, `self._eval_iter` is
+ # always None.
+ if self._eval_iter is None:
+ eval_iter = tf.nest.map_structure(iter, self.eval_dataset)
+ if not self._eval_options.recreate_iterator_for_each_eval:
+ self._eval_iter = eval_iter
+ else:
+ eval_iter = self._eval_iter
+
+ if self._eval_options.use_tf_while_loop and not has_state:
+ self._eval_loop_fn(eval_iter, num_steps)
+ else:
+ outputs = self._eval_loop_fn(
+ eval_iter, num_steps, state=outputs, reduce_fn=self.eval_reduce)
- outputs = self._eval_loop_fn(
- eval_iter, num_steps, state=outputs, reduce_fn=self.eval_reduce)
if outputs is None:
return self.eval_end()
else:
@@ -184,12 +347,13 @@ class StandardEvaluator(runner.AbstractEvaluator):
def eval_begin(self) -> Any:
"""Called once at the beginning of the evaluation.
- This method is called before dataset iterators creation.
- This is a good place to reset metrics that accumulate values over the entire
- evaluation.
+ This method is always called in eager mode, and is a good place to reset
+ metrics that accumulate values over the course of evaluation.
+
+ Note that this method is called before dataset iterator creation.
Returns:
- An output which is passed as `state` argument into `eval_reduce` function.
+ An value to pass as the `state` argument to `eval_reduce`.
"""
pass
@@ -197,14 +361,20 @@ class StandardEvaluator(runner.AbstractEvaluator):
def eval_step(self, iterator) -> Any:
"""Implements one step of evaluation.
- What a "step" consists of is up to the implementer. If using distribution
- strategies, the call to this method should take place in the "cross-replica
+ What a "step" consists of is up to the implementer. When using distribution
+ strategies, the call to this method takes place in the "cross-replica
context" for generality, to allow e.g. multiple iterator dequeues and calls
to `strategy.run`.
+ Note that if `use_tf_function=True`, all the code inside `eval_step` should
+ be compatible with `tf.function` tracing (and in particular, any state
+ modifications involving `self` should be avoided). In some cases, non-
+ `tf.function` compatible code can be moved to `eval_loop_begin`,
+ `eval_reduce`, or `eval_loop_end`, which always execute eagerly.
+
Args:
- iterator: A tf.nest-compatible structure of tf.data Iterator or
- DistributedIterator.
+ iterator: A `tf.nest`-compatible structure of `tf.data.Iterator` or
+ `DistributedIterator`.
Returns:
An output which is passed as `step_outputs` argument into `eval_reduce`
@@ -212,50 +382,62 @@ class StandardEvaluator(runner.AbstractEvaluator):
"""
pass
- def eval_end(self, *args) -> Optional[Dict[Text, tf.Tensor]]:
+ def eval_end(self, *args) -> Optional[runner.Output]:
"""Called at the end of the evaluation.
- This is a good place to get metric results. The value returned from this
- function will be returned as-is from the evaluate() method.
+ Called once at the end of evaluation.
+
+ This method is always called in eager mode, and is a good place to get
+ metric results. The value returned from this function will be returned as-is
+ from the `evaluate` method implementation provided by `StandardEvaluator`.
Args:
- *args: the outputs from `eval_reduce` for the last eval step.
+ *args: The outputs from `eval_reduce` for the last eval step, if they are
+ non-`None` (if they are `None`, nothing is passed).
Returns:
The function may return a dictionary of `Tensors`, which will be
- written to logs and as TensorBoard summaries.
+ written to logs and as TensorBoard summaries. It can also be a
+ nested dictionary, yielding a hierarchy of summary directories.
"""
pass
- def eval_reduce(self, state=None, step_outputs=None) -> Any:
- """A function to do the reduction on the evaluation outputs per step.
+ def eval_reduce(self,
+ state: Any = None,
+ step_outputs: Optional[runner.Output] = None) -> Any:
+ """A function to perform per-step reduction on the evaluation outputs.
- This is useful for passing states throughout evaluation. E.g. it can be used
- to maintain the output losses from all the evaluation steps, and compute the
- mean loss in `eval_end` function.
+ This is useful for passing state throughout evaluation, especially in cases
+ where maintaining or accumulating state is hard to accomplish using
+ `tf.metrics.Metric` or other `tf.Variable`-based approaches. For instance,
+ it can be used to easily accumulate all per-example losses from the full
+ evaluation for subsequent processing in `eval_end()`.
Args:
- state: A maintained state throughout the evaluation.
+ state: A state being mainted throughout the evaluation.
step_outputs: Outputs from the current evaluation step.
Returns:
- An output which is passed as `state` argument into `eval_reduce` function
- for the next step. After evaluation is finished, the output from last step
- will be passed into `eval_end` function.
+ An output which is passed as the `state` argument to this function for the
+ next step. After evaluation is finished, the output from last step will be
+ passed to `eval_end`.
"""
pass
@property
def eval_dataset(self):
- """Returns the train_datase instance."""
+ """The current evaluation dataset."""
return self._eval_dataset
@eval_dataset.setter
def eval_dataset(self, eval_dataset):
- """Set a new eval dataset and replace with the existing one.
+ """Sets a new eval dataset, replacing the current one.
+
+ Any unprocessed examples in the current dataset are discarded.
Args:
- eval_dataset: A tf.nest-compatible structure of tf.data.Dataset or
- DistributedDataset.
+ eval_dataset: A `tf.nest`-compatible structure of `tf.data.Dataset` or
+ `DistributedDataset`.
"""
self._eval_dataset = eval_dataset
+ self._eval_iter = None
diff --git a/orbit/standard_runner_test.py b/orbit/standard_runner_test.py
index 5854a4f963d046f667f273d4fc618dc3a315fc52..ef1335e79c1a85143e9c0cdc35a6271288590f0d 100644
--- a/orbit/standard_runner_test.py
+++ b/orbit/standard_runner_test.py
@@ -1,4 +1,4 @@
-# Copyright 2020 The Orbit Authors. All Rights Reserved.
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,11 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
+
"""Tests for orbit.standard_runner."""
-# pylint: disable=g-bad-import-order
+
+from absl.testing import parameterized
from orbit import standard_runner
+from orbit import utils
import tensorflow as tf
@@ -33,63 +35,118 @@ def dataset_fn(input_context=None):
return dataset
-class TestRunner(standard_runner.StandardTrainer,
- standard_runner.StandardEvaluator):
- """Implements the training and evaluation APIs for tests."""
+class TestTrainer(standard_runner.StandardTrainer):
+ """A StandardTrainer subclass for tests."""
- def __init__(self):
+ def __init__(self, options=None):
self.strategy = tf.distribute.get_strategy()
- self.global_step = tf.Variable(
- 0,
- trainable=False,
- dtype=tf.int64,
- name='global_step',
- aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA)
- standard_runner.StandardTrainer.__init__(self, train_dataset=None)
- standard_runner.StandardEvaluator.__init__(self, eval_dataset=None)
+ self.global_step = utils.create_global_step()
+ dataset = self.strategy.distribute_datasets_from_function(dataset_fn)
+ super().__init__(train_dataset=dataset, options=options)
def train_loop_begin(self):
- self.train_dataset = (
- self.strategy.experimental_distribute_datasets_from_function(dataset_fn)
- )
+ self.global_step.assign(0)
def train_step(self, iterator):
- def _replicated_step(_):
+ def replica_step(_):
self.global_step.assign_add(1)
- self.strategy.run(_replicated_step, args=(next(iterator),))
+ self.strategy.run(replica_step, args=(next(iterator),))
def train_loop_end(self):
return self.global_step.numpy()
+
+class TestEvaluator(standard_runner.StandardEvaluator):
+ """A StandardEvaluator subclass for tests."""
+
+ def __init__(self, options=None):
+ self.strategy = tf.distribute.get_strategy()
+ self.global_step = utils.create_global_step()
+ dataset = self.strategy.distribute_datasets_from_function(dataset_fn)
+ super().__init__(eval_dataset=dataset, options=options)
+
def eval_begin(self):
- self.eval_dataset = self.strategy.experimental_distribute_datasets_from_function(
- dataset_fn)
+ self.global_step.assign(0)
def eval_step(self, iterator):
- def _replicated_step(_):
+ def replica_step(_):
self.global_step.assign_add(1)
- self.strategy.run(_replicated_step, args=(next(iterator),))
+ self.strategy.run(replica_step, args=(next(iterator),))
def eval_end(self):
return self.global_step.numpy()
-class StandardRunnerTest(tf.test.TestCase):
+class TestEvaluatorWithOutputsAggregation(standard_runner.StandardEvaluator):
+ """A StandardEvaluator subclass for tests."""
+
+ def __init__(self, options=None):
+ self.strategy = tf.distribute.get_strategy()
+ dataset = self.strategy.distribute_datasets_from_function(
+ lambda _: tf.data.Dataset.range(10))
+ super().__init__(eval_dataset=dataset, options=options)
+
+ def eval_begin(self):
+ return tf.constant((0.0,))
+
+ def eval_reduce(self, state, step_outputs):
+ state = tf.concat([state, step_outputs], 0)
+ return state
+
+ def eval_step(self, iterator):
+
+ def replica_step(x):
+ x = tf.cast(x, tf.float32)
+ return tf.reduce_sum(x)
+
+ return self.strategy.experimental_local_results(
+ self.strategy.run(replica_step, args=(next(iterator),)))
+
+ def eval_end(self, outputs):
+ return tf.reduce_sum(outputs)
+
+
+class StandardRunnerTest(parameterized.TestCase):
+
+ def test_default_trainer(self):
+ trainer = TestTrainer()
+ self.assertEqual(trainer.train(tf.constant(10)), 10)
+
+ def test_trainer_with_tpu_summary_optimization(self):
+ options = standard_runner.StandardTrainerOptions(
+ use_tpu_summary_optimization=True)
+ trainer = TestTrainer(options)
+ self.assertEqual(trainer.train(tf.constant(10)), 10)
+
+ @parameterized.named_parameters(("use_tf_while_loop", True), ("", False))
+ def test_default_evaluator(self, use_tf_while_loop):
+ options = standard_runner.StandardEvaluatorOptions(
+ use_tf_while_loop=use_tf_while_loop)
+ evaluator = TestEvaluator(options)
+ self.assertEqual(evaluator.evaluate(tf.constant(10)), 10)
- def test_train(self):
- test_runner = TestRunner()
- self.assertEqual(
- test_runner.train(tf.convert_to_tensor(10, dtype=tf.int32)), 10)
+ @parameterized.named_parameters(("use_tf_while_loop", True), ("", False))
+ def test_evaluator_with_outputs_aggregation(self, use_tf_while_loop):
+ options = standard_runner.StandardEvaluatorOptions(
+ use_tf_while_loop=use_tf_while_loop)
+ evaluator = TestEvaluatorWithOutputsAggregation(options)
+ self.assertEqual(evaluator.evaluate(tf.constant(10)), 45)
- def test_eval(self):
- test_runner = TestRunner()
- self.assertEqual(
- test_runner.evaluate(tf.convert_to_tensor(10, dtype=tf.int32)), 10)
+ @parameterized.named_parameters(
+ ("recreate_iterator_for_each_eval", True, 10, 10),
+ ("not_recreate_iterator_for_each_eval", False, 10, 35))
+ def test_evaluator_with_repeat_dataset(self, recreate_iterator_for_each_eval,
+ sum_for_1st_time, sum_for_2nd_time):
+ options = standard_runner.StandardEvaluatorOptions(
+ recreate_iterator_for_each_eval=recreate_iterator_for_each_eval)
+ evaluator = TestEvaluatorWithOutputsAggregation(options)
+ self.assertEqual(evaluator.evaluate(tf.constant(5)), sum_for_1st_time)
+ self.assertEqual(evaluator.evaluate(tf.constant(5)), sum_for_2nd_time)
-if __name__ == '__main__':
+if __name__ == "__main__":
tf.test.main()
diff --git a/orbit/utils.py b/orbit/utils.py
deleted file mode 100644
index e63de82a943d49fc4ca78231a4d45c4b7380739d..0000000000000000000000000000000000000000
--- a/orbit/utils.py
+++ /dev/null
@@ -1,394 +0,0 @@
-# Copyright 2020 The Orbit Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Some layered modules/functions to help users writing custom training loop."""
-
-from __future__ import absolute_import
-from __future__ import division
-# from __future__ import google_type_annotations
-from __future__ import print_function
-
-import abc
-import contextlib
-import functools
-import inspect
-
-import numpy as np
-import six
-import tensorflow as tf
-
-
-def create_loop_fn(step_fn):
- """Creates a multiple steps function driven by the python while loop.
-
- Args:
- step_fn: A function which takes `iterator` as input.
-
- Returns:
- A callable defined as the `loop_fn` defination below.
- """
-
- def loop_fn(iterator, num_steps, state=None, reduce_fn=None):
- """A loop function with multiple steps.
-
- Args:
- iterator: A nested structure of tf.data `Iterator` or
- `DistributedIterator`.
- num_steps: The number of steps in the loop. If `num_steps==-1`, will
- iterate until exausting the iterator.
- state: An optional initial state before running the loop.
- reduce_fn: a callable defined as `def reduce_fn(state, value)`, where
- `value` is the outputs from `step_fn`.
-
- Returns:
- The updated state.
- """
- try:
- step = 0
- # To make sure the OutOfRangeError exception can be handled well with
- # async remote eager, we need to wrap the loop body in a `async_scope`.
- with tf.experimental.async_scope():
- while (num_steps == -1 or step < num_steps):
- outputs = step_fn(iterator)
- if reduce_fn is not None:
- state = reduce_fn(state, outputs)
- step += 1
- return state
- except (StopIteration, tf.errors.OutOfRangeError):
- tf.experimental.async_clear_error()
- return state
-
- return loop_fn
-
-
-def create_tf_while_loop_fn(step_fn):
- """Create a multiple steps function driven by tf.while_loop on the host.
-
- Args:
- step_fn: A function which takes `iterator` as input.
-
- Returns:
- A callable defined as the `loop_fn` defination below.
- """
-
- def loop_fn(iterator, num_steps):
- """A loop function with multiple steps.
-
- Args:
- iterator: A nested structure of tf.data `Iterator` or
- `DistributedIterator`.
- num_steps: The number of steps in the loop. Must be a tf.Tensor.
- """
- if not isinstance(num_steps, tf.Tensor):
- raise ValueError("`num_steps` should be an `tf.Tensor`. Python object "
- "may cause retracing.")
-
- for _ in tf.range(num_steps):
- step_fn(iterator)
-
- return loop_fn
-
-
-def make_distributed_dataset(strategy, dataset_or_fn, *args, **kwargs):
- """A helper function to create distributed dataset.
-
- Args:
- strategy: An instance of `tf.distribute.Strategy`.
- dataset_or_fn: A instance of `tf.data.Dataset` or a function which takes an
- `tf.distribute.InputContext` as input and returns a `tf.data.Dataset`. If
- it is a function, it could optionally have an argument named
- `input_context` which is `tf.distribute.InputContext` argument type.
- *args: The list of arguments to be passed to dataset_or_fn.
- **kwargs: Any keyword arguments to be passed.
-
- Returns:
- A distributed Dataset.
- """
- if strategy is None:
- strategy = tf.distribute.get_strategy()
-
- if isinstance(dataset_or_fn, tf.data.Dataset):
- return strategy.experimental_distribute_dataset(dataset_or_fn)
-
- if not callable(dataset_or_fn):
- raise ValueError("`dataset_or_fn` should be either callable or an instance "
- "of `tf.data.Dataset`")
-
- def dataset_fn(ctx):
- """Wrapped dataset function for creating distributed dataset.."""
-
- # If `dataset_or_fn` is a function and has `input_context` as argument
- # names, pass `ctx` as the value of `input_context` when calling
- # `dataset_or_fn`. Otherwise `ctx` will not be used when calling
- # `dataset_or_fn`.
- if six.PY3:
- argspec = inspect.getfullargspec(dataset_or_fn)
- else:
- argspec = inspect.getargspec(dataset_or_fn) # pylint: disable=deprecated-method
- args_names = argspec.args
-
- if "input_context" in args_names:
- kwargs["input_context"] = ctx
- ds = dataset_or_fn(*args, **kwargs)
- return ds
-
- return strategy.experimental_distribute_datasets_from_function(dataset_fn)
-
-
-class SummaryManager(object):
- """A class manages writing summaries."""
-
- def __init__(self, summary_dir, summary_fn, global_step=None):
- """Construct a summary manager object.
-
- Args:
- summary_dir: the directory to write summaries.
- summary_fn: A callable defined as `def summary_fn(name, tensor,
- step=None)`, which describes the summary operation.
- global_step: A `tf.Variable` instance for the global step.
- """
- self._enabled = (summary_dir is not None)
- self._summary_dir = summary_dir
- self._summary_fn = summary_fn
- self._summary_writer = None
-
- if global_step is None:
- self._global_step = tf.summary.experimental.get_step()
- else:
- self._global_step = global_step
-
- @property
- def summary_writer(self):
- """Returns the underlying summary writer."""
- if self._summary_writer is not None:
- return self._summary_writer
- if self._enabled:
- self._summary_writer = tf.summary.create_file_writer(self._summary_dir)
- else:
- self._summary_writer = tf.summary.create_noop_writer()
- return self._summary_writer
-
- def flush(self):
- """Flush the underlying summary writer."""
- if self._enabled:
- tf.summary.flush(self.summary_writer)
-
- def write_summaries(self, items):
- """Write a bulk of summaries.
-
- Args:
- items: a dictionary of `Tensors` for writing summaries.
- """
- # TODO(rxsang): Support writing summaries with nested structure, so users
- # can split the summaries into different directories for nicer visualization
- # in Tensorboard, like train and eval metrics.
- if not self._enabled:
- return
-
- with self.summary_writer.as_default():
- for name, tensor in items.items():
- self._summary_fn(name, tensor, step=self._global_step)
-
-
-@six.add_metaclass(abc.ABCMeta)
-class Trigger(object):
- """An abstract class representing a "trigger" for some event."""
-
- @abc.abstractmethod
- def __call__(self, value: float, force_trigger=False):
- """Maybe trigger the event based on the given value.
-
- Args:
- value: the value for triggering.
- force_trigger: Whether the trigger is forced triggered.
-
- Returns:
- `True` if the trigger is triggered on the given `value`, and
- `False` otherwise.
- """
-
- @abc.abstractmethod
- def reset(self):
- """Reset states in the trigger."""
-
-
-class IntervalTrigger(Trigger):
- """Triggers on every fixed interval."""
-
- def __init__(self, interval, start=0):
- """Constructs the IntervalTrigger.
-
- Args:
- interval: The triggering interval.
- start: An initial value for the trigger.
- """
- self._interval = interval
- self._last_trigger_value = start
-
- def __call__(self, value, force_trigger=False):
- """Maybe trigger the event based on the given value.
-
- Args:
- value: the value for triggering.
- force_trigger: If True, the trigger will be forced triggered unless the
- last trigger value is equal to `value`.
-
- Returns:
- `True` if the trigger is triggered on the given `value`, and
- `False` otherwise.
- """
- if force_trigger and value != self._last_trigger_value:
- self._last_trigger_value = value
- return True
-
- if self._interval and self._interval > 0:
- if value >= self._last_trigger_value + self._interval:
- self._last_trigger_value = value
- return True
- return False
-
- def reset(self):
- """See base class."""
- self._last_trigger_value = 0
-
-
-class EpochHelper(object):
- """A Helper class to handle epochs in Customized Training Loop."""
-
- def __init__(self, epoch_steps, global_step):
- """Constructs the EpochHelper.
-
- Args:
- epoch_steps: An integer indicates how many steps in an epoch.
- global_step: A `tf.Variable` instance indicates the current global step.
- """
- self._epoch_steps = epoch_steps
- self._global_step = global_step
- self._current_epoch = None
- self._epoch_start_step = None
- self._in_epoch = False
-
- def epoch_begin(self):
- """Returns whether a new epoch should begin."""
- if self._in_epoch:
- return False
- current_step = self._global_step.numpy()
- self._epoch_start_step = current_step
- self._current_epoch = current_step // self._epoch_steps
- self._in_epoch = True
- return True
-
- def epoch_end(self):
- """Returns whether the current epoch should end."""
- if not self._in_epoch:
- raise ValueError("`epoch_end` can only be called inside an epoch")
- current_step = self._global_step.numpy()
- epoch = current_step // self._epoch_steps
-
- if epoch > self._current_epoch:
- self._in_epoch = False
- return True
- return False
-
- @property
- def batch_index(self):
- """Index of the next batch within the current epoch."""
- return self._global_step.numpy() - self._epoch_start_step
-
- @property
- def current_epoch(self):
- return self._current_epoch
-
-
-@contextlib.contextmanager
-def _soft_device_placement():
- """Context manager for soft device placement, allowing summaries on CPU."""
- original_setting = tf.config.get_soft_device_placement()
- try:
- tf.config.set_soft_device_placement(True)
- yield
- finally:
- tf.config.set_soft_device_placement(original_setting)
-
-
-def train_function_with_summaries(*args, **kwargs):
- """Utility function to support TPU summaries via multiple `tf.function`s.
-
- This permits interleaving summaries inside TPU-compatible code, but without
- any performance impact on steps that do not write summaries.
-
- Usage is as a decorator, similar to `tf.function`, and any `tf.function`
- arguments will be passed through if supplied:
-
- @trainer.train_function_with_summaries
- def train(self, num_steps):
- ...
-
- The decorated function is assumed to be a loop method accepting a `num_steps`
- parameter, as for instance would be called within the `Controller`'s outer
- train loop. The implementation here assumes that `summary_frequency` is
- divisible by `steps_per_loop`. The decorated method should accept two
- arguments, `self` and `num_steps`.
-
- Two `tf.function` versions of `train_fn` are created: one inside a summary
- writer scope with soft device placement enabled (used on steps that require
- summary writing), and one with no summary writer present and soft device
- placement disabled (used on all other steps).
-
- Args:
- *args: Arguments to pass through to `tf.function`.
- **kwargs: Keyword arguments to pass through to `tf.function`.
-
- Returns:
- If the first argument is a callable, returns the decorated callable.
- Otherwise, returns a decorator.
- """
-
- def decorator(train_fn):
- # TODO(dhr): Validate the signature of train_fn?
-
- train_fn_with_summaries = tf.function(train_fn, *args, **kwargs)
- train_fn_without_summaries = tf.function(train_fn, *args, **kwargs)
-
- @functools.wraps(train_fn)
- def wrapper(self, num_steps):
- if tf.summary.should_record_summaries():
- with _soft_device_placement():
- output = train_fn_with_summaries(self, tf.constant(1))
- num_steps -= 1
- if num_steps >= 1:
- with tf.summary.record_if(False):
- output = train_fn_without_summaries(self, num_steps)
- return output
-
- return wrapper
-
- if args and callable(args[0]):
- train_fn, args = args[0], args[1:]
- return decorator(train_fn)
- return decorator
-
-
-def get_value(x) -> np.ndarray:
- """Returns the value of a variable/tensor.
-
- Args:
- x: input variable.
-
- Returns:
- A Numpy array.
- """
- if not tf.is_tensor(x):
- return x
- return x.numpy()
diff --git a/orbit/utils/__init__.py b/orbit/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3eeb67c4a284d238e260e9587c4cfda1aab13a9a
--- /dev/null
+++ b/orbit/utils/__init__.py
@@ -0,0 +1,29 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Defines exported symbols for the `orbit.utils` package."""
+
+from orbit.utils.common import create_global_step
+from orbit.utils.common import get_value
+from orbit.utils.common import make_distributed_dataset
+
+from orbit.utils.epoch_helper import EpochHelper
+
+from orbit.utils.loop_fns import create_loop_fn
+from orbit.utils.loop_fns import create_tf_while_loop_fn
+from orbit.utils.loop_fns import LoopFnWithSummaries
+
+from orbit.utils.summary_manager import SummaryManager
+
+from orbit.utils.tpu_summaries import OptionalSummariesFunction
diff --git a/orbit/utils/common.py b/orbit/utils/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..63ee020afe2e0ae7c923d2211cf1750b141f54ef
--- /dev/null
+++ b/orbit/utils/common.py
@@ -0,0 +1,100 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Some layered modules/functions to help users writing custom training loop."""
+
+import inspect
+
+import tensorflow as tf
+
+
+def create_global_step() -> tf.Variable:
+ """Creates a `tf.Variable` suitable for use as a global step counter.
+
+ Creating and managing a global step variable may be necessary for
+ `AbstractTrainer` subclasses that perform multiple parameter updates per
+ `Controller` "step", or use different optimizers on different steps.
+
+ In these cases, an `optimizer.iterations` property generally can't be used
+ directly, since it would correspond to parameter updates instead of iterations
+ in the `Controller`'s training loop. Such use cases should simply call
+ `step.assign_add(1)` at the end of each step.
+
+ Returns:
+ A non-trainable scalar `tf.Variable` of dtype `tf.int64`, with only the
+ first replica's value retained when synchronizing across replicas in
+ a distributed setting.
+ """
+ return tf.Variable(
+ 0,
+ dtype=tf.int64,
+ name="global_step",
+ trainable=False,
+ aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA)
+
+
+def make_distributed_dataset(strategy, dataset_or_fn, *args, **kwargs):
+ """A utility function to help create a `tf.distribute.DistributedDataset`.
+
+ Args:
+ strategy: An instance of `tf.distribute.Strategy`.
+ dataset_or_fn: A instance of `tf.data.Dataset`, or a "dataset function"
+ returning a `tf.data.Dataset`. If it is a function, it may optionally have
+ an argument named `input_context` which will be passed a
+ `tf.distribute.InputContext` instance.
+ *args: Any positional arguments to pass through to `dataset_or_fn`.
+ **kwargs: Any keyword arguments to pass through to `dataset_or_fn`.
+
+ Returns:
+ A distributed Dataset.
+ """
+ if strategy is None:
+ strategy = tf.distribute.get_strategy()
+
+ if isinstance(dataset_or_fn, tf.data.Dataset):
+ return strategy.experimental_distribute_dataset(dataset_or_fn)
+
+ if not callable(dataset_or_fn):
+ raise ValueError("`dataset_or_fn` should be either callable or an instance "
+ "of `tf.data.Dataset`.")
+
+ def dataset_fn(input_context):
+ """Wraps `dataset_or_fn` for strategy.distribute_datasets_from_function."""
+
+ # If `dataset_or_fn` is a function and has an argument named
+ # `input_context`, pass through the given `input_context`. Otherwise
+ # `input_context` will be ignored.
+ argspec = inspect.getfullargspec(dataset_or_fn)
+ arg_names = argspec.args
+
+ if "input_context" in arg_names:
+ kwargs["input_context"] = input_context
+ return dataset_or_fn(*args, **kwargs)
+
+ return strategy.distribute_datasets_from_function(dataset_fn)
+
+
+def get_value(x):
+ """Returns input values, converting any TensorFlow values to NumPy values.
+
+ Args:
+ x: The input. May be a `tf.Tensor` or `tf.Variable`.
+
+ Returns:
+ If the input is a TensorFlow `Tensor`, returns the `Tensor`'s equivalent
+ NumPy value. Otherwise, just returns the input.
+ """
+ if not tf.is_tensor(x):
+ return x
+ return x.numpy()
diff --git a/orbit/utils/common_test.py b/orbit/utils/common_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a68e7c66b20b0814d2618de190128a0dcaa0387
--- /dev/null
+++ b/orbit/utils/common_test.py
@@ -0,0 +1,34 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for orbit.utils.common."""
+
+from orbit.utils import common
+
+import tensorflow as tf
+
+
+class UtilsTest(tf.test.TestCase):
+
+ def test_create_global_step(self):
+ step = common.create_global_step()
+ self.assertEqual(step.name, "global_step:0")
+ self.assertEqual(step.dtype, tf.int64)
+ self.assertEqual(step, 0)
+ step.assign_add(1)
+ self.assertEqual(step, 1)
+
+
+if __name__ == "__main__":
+ tf.test.main()
diff --git a/orbit/utils/epoch_helper.py b/orbit/utils/epoch_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..10c11324ae8371b290c9973ae008902cd76fb9eb
--- /dev/null
+++ b/orbit/utils/epoch_helper.py
@@ -0,0 +1,65 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides a utility class for training in epochs."""
+
+import tensorflow as tf
+
+
+class EpochHelper:
+ """A helper class handle bookkeeping of epochs in custom training loops."""
+
+ def __init__(self, epoch_steps: int, global_step: tf.Variable):
+ """Initializes the `EpochHelper` instance.
+
+ Args:
+ epoch_steps: An integer indicating how many steps are in an epoch.
+ global_step: A `tf.Variable` providing the current global step.
+ """
+ self._epoch_steps = epoch_steps
+ self._global_step = global_step
+ self._current_epoch = None
+ self._epoch_start_step = None
+ self._in_epoch = False
+
+ def epoch_begin(self):
+ """Returns whether a new epoch should begin."""
+ if self._in_epoch:
+ return False
+ current_step = self._global_step.numpy()
+ self._epoch_start_step = current_step
+ self._current_epoch = current_step // self._epoch_steps
+ self._in_epoch = True
+ return True
+
+ def epoch_end(self):
+ """Returns whether the current epoch should end."""
+ if not self._in_epoch:
+ raise ValueError("`epoch_end` can only be called inside an epoch.")
+ current_step = self._global_step.numpy()
+ epoch = current_step // self._epoch_steps
+
+ if epoch > self._current_epoch:
+ self._in_epoch = False
+ return True
+ return False
+
+ @property
+ def batch_index(self):
+ """Index of the next batch within the current epoch."""
+ return self._global_step.numpy() - self._epoch_start_step
+
+ @property
+ def current_epoch(self):
+ return self._current_epoch
diff --git a/orbit/utils/loop_fns.py b/orbit/utils/loop_fns.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e326246942261941ecd5ec8d830b050fa00aed1
--- /dev/null
+++ b/orbit/utils/loop_fns.py
@@ -0,0 +1,192 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for creating loop functions."""
+
+from orbit.utils import tpu_summaries
+
+import tensorflow as tf
+
+
+def create_loop_fn(step_fn):
+ """Creates a loop function driven by a Python `while` loop.
+
+ Args:
+ step_fn: A function taking a nested structure of `tf.data.Iterator` or
+ `DistributedIterator`. There are no constraints on the return value of the
+ function (except that it must be compatible with any `reduce_fn` provided
+ to the returned `loop_fn`).
+
+ Returns:
+ A loop function taking required `iterator` and `num_steps` parameters, as
+ well as optional `state` and `reduce_fn` parameters for accumulating state
+ over multiple iterations of the loop. See the `loop_fn` definition below for
+ additional details.
+ """
+
+ def loop_fn(iterator, num_steps, state=None, reduce_fn=None):
+ """Makes `num_steps` calls to `step_fn(iterator)`.
+
+ Additionally, state may be accumulated across iterations of the loop.
+ Conceptually, state accumulation is handled roughly as follows:
+
+ for _ in range(num_steps):
+ step_outputs = step_fn(iterator)
+ state = reduce_fn(state, step_outputs)
+ return state
+
+ However, the implementation is slightly more complicated in order to support
+ looping until the iterator is exhausted (when `num_steps == -1`) and to
+ properly catch exceptions when running under async remote eager (as is the
+ case in TPU training setups involving separate coordinator/worker machines).
+
+ Args:
+ iterator: A nested structure of `tf.data.Iterator` or
+ `DistributedIterator`.
+ num_steps: The number of steps in the loop. If `num_steps == -1`, will
+ iterate until exausting the iterator.
+ state: An optional initial state before running the loop.
+ reduce_fn: A callable taking two inputs, `state` and `value`, where
+ `state` is the previous output from `reduce_fn`, and `value` is the
+ output from `step_fn`.
+
+ Returns:
+ The final state returned by `reduce_fn`, or `None` if `state` and
+ `reduce_fn` are not provided.
+ """
+ try:
+ step = 0
+ # To make sure the OutOfRangeError exception can be handled well under
+ # async remote eager, we need to wrap the loop body in `async_scope`.
+ with tf.experimental.async_scope():
+ while num_steps == -1 or step < num_steps:
+ outputs = step_fn(iterator)
+ if reduce_fn is not None:
+ state = reduce_fn(state, outputs)
+ step += 1
+ return state
+ except (StopIteration, tf.errors.OutOfRangeError):
+ tf.experimental.async_clear_error()
+ return state
+
+ return loop_fn
+
+
+def create_tf_while_loop_fn(step_fn):
+ """Creates a loop function compatible with TF's AutoGraph loop conversion.
+
+ Args:
+ step_fn: A function taking a nested structure of `tf.data.Iterator` or
+ `DistributedIterator`. Currently, any return values are ignored.
+
+ Returns:
+ A loop function taking required `iterator` and `num_steps` parameters. If
+ called inside a `tf.function`, the loop will be converted by AutoGraph into
+ a `tf.while_loop` construct. See the `loop_fn` definition below for
+ additional details.
+ """
+
+ def loop_fn(iterator, num_steps):
+ """Makes `num_steps` calls to `step_fn(iterator)`.
+
+ Args:
+ iterator: A nested structure of `tf.data.Iterator` or
+ `DistributedIterator`.
+ num_steps: The number of steps in the loop. Should be passed as a
+ `tf.Tensor`. Iterating until iterator exhaustion is not supported.
+ """
+ if not isinstance(num_steps, tf.Tensor):
+ raise ValueError(
+ "`num_steps` should be a `tf.Tensor`. Passing a Python value can "
+ "cause unnecessary retracing when wrapped by `tf.function`.")
+
+ for _ in tf.range(num_steps):
+ # Clear out the outer name scope so the ops created inside `tf.while_loop`
+ # don't get "while/" as name prefix.
+ with tf.name_scope(""):
+ step_fn(iterator)
+
+ return loop_fn
+
+
+def create_tf_while_loop_fn_with_state(step_fn):
+ """Creates a TF while loop function with state.
+
+ This function is similar to `create_tf_while_loop_fn`, but allowing a `state`
+ to be accumulated over multiple iterations of the loop. Note that the
+ structure of the `state` cannot be changed across iterations.
+
+ Args:
+ step_fn: A function taking a nested structure of `tf.data.Iterator` or
+ `DistributedIterator`. Currently, any return values are ignored.
+
+ Returns:
+ A loop function taking required `iterator`, `num_steps`, `state` and
+ `reduce_fn` parameters. If called inside a `tf.function`, the loop will be
+ converted by AutoGraph into a `tf.while_loop` construct. See the `loop_fn`
+ definition below for additional details.
+ """
+
+ def loop_fn_with_state(iterator, num_steps, state, reduce_fn):
+ """Makes `num_steps` calls to `step_fn(iterator)`.
+
+ Args:
+ iterator: A nested structure of `tf.data.Iterator` or
+ `DistributedIterator`.
+ num_steps: The number of steps in the loop. Should be passed as a
+ `tf.Tensor`. Iterating until iterator exhaustion is not supported.
+ state: An initial state before running the loop.
+ reduce_fn: A callable taking two inputs, `state` and `value`, where
+ `state` is the previous output from `reduce_fn`, and `value` is the
+ output from `step_fn`.
+
+ Returns:
+ The final state returned by `reduce_fn`.
+ """
+ if not isinstance(num_steps, tf.Tensor):
+ raise ValueError(
+ "`num_steps` should be a `tf.Tensor`. Passing a Python value can "
+ "cause unnecessary retracing when wrapped by `tf.function`.")
+
+ for _ in tf.range(num_steps):
+ # Clear out the outer name scope so the ops created inside `tf.while_loop`
+ # don't get "while/" as name prefix.
+ with tf.name_scope(""):
+ # Relax the shapes within the loop, so the shape of `state` can change
+ # across iterations. This is useful to aggregate outputs from each step
+ # and concat to `state`.
+ tf.autograph.experimental.set_loop_options(
+ shape_invariants=[(t, tf.TensorShape([None] * t.shape.rank))
+ for t in tf.nest.flatten(state)
+ if tf.is_tensor(t)])
+ outputs = step_fn(iterator)
+ state = reduce_fn(state, outputs)
+ return state
+
+ return loop_fn_with_state
+
+
+class LoopFnWithSummaries(tpu_summaries.OptionalSummariesFunction):
+ """Implements a two-program approach for optimizing summaries on TPU.
+
+ This version works with the result of `create_tf_while_loop_fn`.
+ """
+
+ def __call__(self, iterator, num_steps):
+ if tf.summary.should_record_summaries():
+ output = self.with_summaries(iterator, tf.constant(1))
+ num_steps -= 1
+ if num_steps >= 1:
+ output = self.without_summaries(iterator, num_steps)
+ return output
diff --git a/orbit/utils/summary_manager.py b/orbit/utils/summary_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..63a44940f533a33f39d78edaf936dcd1c8354648
--- /dev/null
+++ b/orbit/utils/summary_manager.py
@@ -0,0 +1,110 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides a utility class for managing summary writing."""
+
+import os
+
+import tensorflow as tf
+
+
+class SummaryManager:
+ """A utility class for managing summary writing."""
+
+ def __init__(self, summary_dir, summary_fn, global_step=None):
+ """Initializes the `SummaryManager` instance.
+
+ Args:
+ summary_dir: The directory in which to write summaries. If `None`, all
+ summary writing operations provided by this class are no-ops.
+ summary_fn: A callable defined accepting `name`, `value`, and `step`
+ parameters, making calls to `tf.summary` functions to write summaries.
+ global_step: A `tf.Variable` containing the global step value.
+ """
+ self._enabled = summary_dir is not None
+ self._summary_dir = summary_dir
+ self._summary_fn = summary_fn
+ self._summary_writers = {}
+
+ if global_step is None:
+ self._global_step = tf.summary.experimental.get_step()
+ else:
+ self._global_step = global_step
+
+ def summary_writer(self, relative_path=""):
+ """Returns the underlying summary writer for a specific subdirectory.
+
+ Args:
+ relative_path: The current path in which to write summaries, relative to
+ the summary directory. By default it is empty, which corresponds to the
+ root directory.
+ """
+ if self._summary_writers and relative_path in self._summary_writers:
+ return self._summary_writers[relative_path]
+ if self._enabled:
+ self._summary_writers[relative_path] = tf.summary.create_file_writer(
+ os.path.join(self._summary_dir, relative_path))
+ else:
+ self._summary_writers[relative_path] = tf.summary.create_noop_writer()
+ return self._summary_writers[relative_path]
+
+ def flush(self):
+ """Flushes the underlying summary writers."""
+ if self._enabled:
+ tf.nest.map_structure(tf.summary.flush, self._summary_writers)
+
+ def write_summaries(self, summary_dict):
+ """Writes summaries for the given dictionary of values.
+
+ This recursively creates subdirectories for any nested dictionaries
+ provided in `summary_dict`, yielding a hierarchy of directories which will
+ then be reflected in the TensorBoard UI as different colored curves.
+
+ For example, users may evaluate on multiple datasets and return
+ `summary_dict` as a nested dictionary:
+
+ {
+ "dataset1": {
+ "loss": loss1,
+ "accuracy": accuracy1
+ },
+ "dataset2": {
+ "loss": loss2,
+ "accuracy": accuracy2
+ },
+ }
+
+ This will create two subdirectories, "dataset1" and "dataset2", inside the
+ summary root directory. Each directory will contain event files including
+ both "loss" and "accuracy" summaries.
+
+ Args:
+ summary_dict: A dictionary of values. If any value in `summary_dict` is
+ itself a dictionary, then the function will create a subdirectory with
+ name given by the corresponding key. This is performed recursively. Leaf
+ values are then summarized using the summary writer instance specific to
+ the parent relative path.
+ """
+ if not self._enabled:
+ return
+ self._write_summaries(summary_dict)
+
+ def _write_summaries(self, summary_dict, relative_path=""):
+ for name, value in summary_dict.items():
+ if isinstance(value, dict):
+ self._write_summaries(
+ value, relative_path=os.path.join(relative_path, name))
+ else:
+ with self.summary_writer(relative_path).as_default():
+ self._summary_fn(name, value, step=self._global_step)
diff --git a/orbit/utils/tpu_summaries.py b/orbit/utils/tpu_summaries.py
new file mode 100644
index 0000000000000000000000000000000000000000..3501c7aa8041f977082e9d224d4105810b78f64c
--- /dev/null
+++ b/orbit/utils/tpu_summaries.py
@@ -0,0 +1,145 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains utilities for TPU summary optimization."""
+
+import contextlib
+import functools
+
+import tensorflow as tf
+
+
+@contextlib.contextmanager
+def _soft_device_placement():
+ """Context manager for soft device placement, allowing summaries on CPU."""
+ original_setting = tf.config.get_soft_device_placement()
+ try:
+ tf.config.set_soft_device_placement(True)
+ yield
+ finally:
+ tf.config.set_soft_device_placement(original_setting)
+
+
+class OptionalSummariesFunction:
+ """Wrapper that provides versions of a function with and without summaries.
+
+ This is a utility class for implementing optimized summary recording via a
+ two-function approach, specifically important for TPUs. Two `tf.function`
+ versions of a given `function` are created: one with soft device placement
+ enabled (for use on steps that require summary writing), and one with summary
+ writing and soft device placement entirely disabled (for use on all other
+ steps). This removes any performance impact of summaries on steps where they
+ aren't recorded (b/148418718).
+
+ This class can be used as a base class to implement summary optimizations for
+ a function with a specific signature. For example, to implement efficient TPU
+ summaries for a standard `train()` method (as in `orbit.AbstractTrainer`):
+
+ class TrainFunctionWithSummaries(orbit.utils.OptionalSummariesFunction):
+ '''Implements a two-program approach for summaries on TPU.'''
+
+ def __call__(self, num_steps):
+ if tf.summary.should_record_summaries():
+ output = self.with_summaries(tf.constant(1))
+ num_steps -= 1
+ if num_steps >= 1:
+ output = self.without_summaries(num_steps)
+ return output
+
+ This can be used directly or to implement a decorator:
+
+ def train_function_with_summaries(function=None, **kwargs):
+ if function is not None:
+ return TrainFunctionWithSummaries(function, **kwargs)
+ return functools.partial(TrainFunctionWithSummaries, **kwargs)
+
+ The decorator can be applied directly to `train()` methods:
+
+ @train_function_with_summaries
+ def train(self, num_steps):
+ ...
+
+ A similar approach approach can be implemented for functions with different
+ signatures.
+
+ Note: The above approach assumes that the frequency of summary writing is
+ based on a step interval that is divisible by the number of steps executed
+ in each call to the `train()` function. This is enforced by the
+ `orbit.Controller`.
+
+ This wrapper properly handles instance methods (see `__get__`).
+
+ Attributes:
+ with_summaries: A wrapped version of the underlying function with summaries
+ enabled (using whatever the active predicate is for
+ `tf.summary.record_if`), and placed inside a "soft device placement"
+ context to enable summary recording on TPU.
+ without_summaries: A wrapped version of the underlying function with all
+ summary recording disabled.
+ """
+
+ def __init__(self, function, **tf_function_kwargs):
+ """Constructs an instance wrapping the given `function`.
+
+ The given `function` is wrapped twice: Once in a "soft device placement"
+ context (allowing summaries to also run on TPU), and once with summary
+ recording entirely disabled.
+
+ Both of these versions are compiled via `tf.function` (optionally using any
+ supplied `tf.function` settings), and made available as attributes.
+
+ Args:
+ function: The underlying function to wrap.
+ **tf_function_kwargs: Additional arguments to pass to `tf.function`.
+ """
+
+ @tf.function(**tf_function_kwargs)
+ @functools.wraps(function)
+ def with_summaries(*args, **kwargs):
+ with _soft_device_placement():
+ return function(*args, **kwargs)
+
+ @tf.function(**tf_function_kwargs)
+ @functools.wraps(function)
+ def without_summaries(*args, **kwargs):
+ with tf.summary.record_if(False):
+ return function(*args, **kwargs)
+
+ self.with_summaries = with_summaries
+ self.without_summaries = without_summaries
+
+ def __get__(self, instance, owner):
+ """Allows this class to be used to wrap methods as well as free functions.
+
+ For `tf.function` to work properly in all cases (e.g., when an
+ input_signature is specified), any `tf.function`-converted methods must be
+ properly bound to an instance if they are called as an instance method.
+
+ This is done by implementing this `__get__` method of the descriptor
+ protocol, and forwarding to the `__get__` method on the underlying
+ `tf.function`s.
+
+ Args:
+ instance: The instance to bind to.
+ owner: The class type of the instance.
+
+ Returns:
+ A new bound instance of `TpuDiscretionarySummariesFunctions`.
+ """
+ new = object.__new__(self.__class__)
+ # pytype: disable=attribute-error # See b/162476201.
+ new.with_summaries = self.with_summaries.__get__(instance, owner)
+ new.without_summaries = self.without_summaries.__get__(instance, owner)
+ # pytype: enable=attribute-error
+ return new
diff --git a/orbit/utils/tpu_summaries_test.py b/orbit/utils/tpu_summaries_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..4aa0d0820fa8c501d7db339b568d56fd7dc1bf28
--- /dev/null
+++ b/orbit/utils/tpu_summaries_test.py
@@ -0,0 +1,120 @@
+# Copyright 2021 The Orbit Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for orbit.utils.tpu_summaries."""
+
+import functools
+import os
+
+from orbit.utils import common
+from orbit.utils import tpu_summaries
+
+import tensorflow as tf
+
+
+class TrainFunctionWithSummaries(tpu_summaries.OptionalSummariesFunction):
+ """Implements a two-program approach for summaries on TPU."""
+
+ def __call__(self, num_steps):
+ if tf.summary.should_record_summaries():
+ output = self.with_summaries(tf.constant(1))
+ num_steps -= 1
+ if num_steps >= 1:
+ output = self.without_summaries(num_steps)
+ return output
+
+
+def train_function_with_summaries(function=None, **kwargs):
+ if function is not None:
+ return TrainFunctionWithSummaries(function, **kwargs)
+ return functools.partial(TrainFunctionWithSummaries, **kwargs)
+
+
+class DummyTrainer(tf.Module):
+
+ def __init__(self):
+ self.step_counter = common.create_global_step()
+
+ @train_function_with_summaries
+ def train_with_tpu_summary_optimization(self, num_steps):
+ for _ in tf.range(num_steps):
+ tf.summary.scalar("step", self.step_counter, step=self.step_counter)
+ self.step_counter.assign_add(1)
+ return self.step_counter
+
+ @train_function_with_summaries(
+ input_signature=[tf.TensorSpec((), dtype=tf.int32)])
+ def train_with_tpu_summary_optimization_and_input_signature(self, num_steps):
+ for _ in tf.range(num_steps):
+ tf.summary.scalar("step", self.step_counter, step=self.step_counter)
+ self.step_counter.assign_add(1)
+ return self.step_counter
+
+ def train_with_tpu_summary_optimization_no_decorator(self, num_steps):
+ for _ in tf.range(num_steps):
+ tf.summary.scalar("step", self.step_counter, step=self.step_counter)
+ self.step_counter.assign_add(1)
+ return self.step_counter
+
+
+class TpuSummariesTest(tf.test.TestCase):
+
+ def setUp(self):
+ super().setUp()
+ self.trainer = DummyTrainer()
+
+ def _get_events_from_logdir(self, logdir):
+ event_files = tf.io.gfile.listdir(logdir)
+ self.assertLen(event_files, 1)
+ path = os.path.join(logdir, event_files[0])
+ events = list(tf.compat.v1.train.summary_iterator(path))
+ return [event for event in events if event.WhichOneof("what") == "summary"]
+
+ def _validate_tpu_summary_optimization(self, function, *args, **kwargs):
+ logdir = self.get_temp_dir()
+ with tf.summary.create_file_writer(logdir).as_default():
+ with tf.summary.record_if(lambda: self.trainer.step_counter % 20 == 0):
+ for _ in range(4):
+ output = function(tf.constant(10), *args, **kwargs)
+ events = self._get_events_from_logdir(logdir)
+ self.assertLen(events, 2)
+ self.assertEqual(events[0].step, 0)
+ self.assertEqual(events[1].step, 20)
+ return output
+
+ def test_train_with_tpu_summary_optimization(self):
+ output = self._validate_tpu_summary_optimization(
+ self.trainer.train_with_tpu_summary_optimization)
+ self.assertEqual(output, self.trainer.step_counter.numpy())
+
+ def test_train_with_tpu_summary_optimization_no_decorator(self):
+ optimized = train_function_with_summaries(
+ self.trainer.train_with_tpu_summary_optimization_no_decorator)
+ output = self._validate_tpu_summary_optimization(optimized)
+ self.assertEqual(output, self.trainer.step_counter.numpy())
+
+ def test_train_with_tpu_summary_optimization_and_input_signature(self):
+ output = self._validate_tpu_summary_optimization(
+ self.trainer.train_with_tpu_summary_optimization_and_input_signature)
+ self.assertEqual(output, self.trainer.step_counter.numpy())
+ function = self.trainer.train_with_tpu_summary_optimization_and_input_signature
+ expected = (tf.TensorSpec((), dtype=tf.int32),)
+ input_signature = function.with_summaries.input_signature
+ self.assertEqual(input_signature, expected)
+ input_signature = function.without_summaries.input_signature
+ self.assertEqual(input_signature, expected)
+
+
+if __name__ == "__main__":
+ tf.test.main()
diff --git a/research/README.md b/research/README.md
index f9e84fb86f44b687c6a9c221fa72cd461e84c01e..204955b3e082469533885a9b9860b134ee51f46f 100644
--- a/research/README.md
+++ b/research/README.md
@@ -7,14 +7,17 @@ This directory contains code implementations and pre-trained models of published
The research models are maintained by their respective authors.
## Table of Contents
-- [Modeling Libraries and Models](#modeling-libraries-and-models)
-- [Models and Implementations](#models-and-implementations)
- * [Computer Vision](#computer-vision)
- * [Natural Language Processing](#natural-language-processing)
- * [Audio and Speech](#audio-and-speech)
- * [Reinforcement Learning](#reinforcement-learning)
- * [Others](#others)
-- [Archived Models and Implementations](#warning-archived-models-and-implementations) (:no_entry_sign: No longer maintained)
+- [TensorFlow Research Models](#tensorflow-research-models)
+ - [Table of Contents](#table-of-contents)
+ - [Modeling Libraries and Models](#modeling-libraries-and-models)
+ - [Models and Implementations](#models-and-implementations)
+ - [Computer Vision](#computer-vision)
+ - [Natural Language Processing](#natural-language-processing)
+ - [Audio and Speech](#audio-and-speech)
+ - [Reinforcement Learning](#reinforcement-learning)
+ - [Others](#others)
+ - [Old Models and Implementations in TensorFlow 1](#old-models-and-implementations-in-tensorflow-1)
+ - [Contributions](#contributions)
## Modeling Libraries and Models
@@ -49,6 +52,7 @@ The research models are maintained by their respective authors.
| Directory | Paper(s) | Conference | Maintainer(s) |
|-----------|----------|------------|---------------|
| [audioset](audioset) | [1] [Audio Set: An ontology and human-labeled dataset for audio events](https://research.google/pubs/pub45857/) [2] [CNN Architectures for Large-Scale Audio Classification](https://research.google/pubs/pub45611/) | ICASSP 2017 | plakal, dpwe |
+| [deep_speech](deep_speech) | [Deep Speech 2](https://arxiv.org/abs/1512.02595) | ICLR 2016 | yhliang2018 |
### Reinforcement Learning
@@ -64,58 +68,9 @@ The research models are maintained by their respective authors.
| [lfads](lfads) | [LFADS - Latent Factor Analysis via Dynamical Systems](https://arxiv.org/abs/1608.06315) | | jazcollins, sussillo |
| [rebar](rebar) | [REBAR: Low-variance, unbiased gradient estimates for discrete latent variable models](https://arxiv.org/abs/1703.07370) | NIPS 2017 | gjtucker |
----
-
-## :warning: Archived Models and Implementations
-
-The following research models are no longer maintained.
+### Old Models and Implementations in TensorFlow 1
-**Note**: We will remove archived models from the master branch in June, 2020.
-After removal, you will still be able to access archived models in the archive branch.
-
-| Directory | Paper(s) | Conference | Maintainer(s) |
-|-----------|----------|------------|---------------|
-| [adv_imagenet_models](adv_imagenet_models) | [1] [Adversarial Machine Learning at Scale](https://arxiv.org/abs/1611.01236) [2] [Ensemble Adversarial Training: Attacks and Defenses](https://arxiv.org/abs/1705.07204) | [1] ICLR 2017 [2] ICLR 2018 | alexeykurakin |
-| [adversarial_crypto](adversarial_crypto) | [Learning to Protect Communications with Adversarial Neural Cryptography](https://arxiv.org/abs/1610.06918) | | dave-andersen |
-| [adversarial_logit_pairing](adversarial_logit_pairing) | [Adversarial Logit Pairing](https://arxiv.org/abs/1803.06373) | | alexeykurakin |
-| [autoencoder](autoencoder) | Various autoencoders | | snurkabill |
-| [brain_coder](brain_coder) | [Neural Program Synthesis with Priority Queue Training](https://arxiv.org/abs/1801.03526) | | danabo, mnorouzi |
-| [cognitive_mapping_and_planning](cognitive_mapping_and_planning) | [Cognitive Mapping and Planning for Visual Navigation](https://arxiv.org/abs/1702.03920) | CVPR 2017 | s-gupta |
-| [compression](compression) | [Full Resolution Image Compression with Recurrent Neural Networks](https://arxiv.org/abs/1608.05148) | CVPR 2017 | nmjohn |
-| [deep_contextual_bandits](deep_contextual_bandits) | [Deep Bayesian Bandits Showdown: An Empirical Comparison of Bayesian Deep Networks for Thompson Sampling](https://arxiv.org/abs/1802.09127) | ICLR 2018 | rikel |
-| [deep_speech](deep_speech) | [Deep Speech 2](https://arxiv.org/abs/1512.02595) | ICLR 2016 | yhliang2018 |
-| [domain_adaptation](domain_adaptation) | [1] [Domain Separation Networks](https://arxiv.org/abs/1608.06019) [2] [Unsupervised Pixel-Level Domain Adaptation with Generative Adversarial Networks](https://arxiv.org/abs/1612.05424) | NIPS 2016 | bousmalis, dmrd |
-| [feelvos](feelvos)| [FEELVOS](https://arxiv.org/abs/1902.09513) | CVPR 2019 | pvoigtlaender, yuningchai, aquariusjay |
-| [fivo](fivo)| [Filtering variational objectives for training generative sequence models](https://arxiv.org/abs/1705.09279) | NIPS 2017 | dieterichlawson |
-| [global_objectives](global_objectives) | [Scalable Learning of Non-Decomposable Objectives](https://arxiv.org/abs/1608.04802) | AISTATS 2017 | mackeya-google |
-| [im2txt](im2txt) | [Show and Tell: Lessons learned from the 2015 MSCOCO Image Captioning Challenge](https://arxiv.org/abs/1609.06647) | TPAMI 2016 | cshallue |
-| [inception](inception) | [Rethinking the Inception Architecture for Computer Vision](https://arxiv.org/abs/1512.00567) | CVPR 2016 | shlens, vincentvanhoucke |
-| [keypointnet](keypointnet) | [KeypointNet](https://arxiv.org/abs/1807.03146) | | mnorouzi |
-| [learned_optimizer](learned_optimizer) | [Learned Optimizers that Scale and Generalize](https://arxiv.org/abs/1703.04813) | ICML 2017 | olganw, nirum |
-| [learning_to_remember_rare_events](learning_to_remember_rare_events) | [Learning to Remember Rare Events](https://arxiv.org/abs/1703.03129) | ICLR 2017| lukaszkaiser, ofirnachum |
-| [learning_unsupervised_learning](learning_unsupervised_learning) | [Meta-Learning Update Rules for Unsupervised Representation Learning](https://arxiv.org/abs/1804.00222) | ICLR 2019 | lukemetz, nirum |
-| [lexnet_nc](lexnet_nc) | [Olive Oil is Made of Olives, Baby Oil is Made for Babies: Interpreting Noun Compounds using Paraphrases in a Neural Model](https://arxiv.org/abs/1803.08073) | NAACL 2018 | vered1986, waterson |
-| [lm_1b](lm_1b) | [Exploring the Limits of Language Modeling](https://arxiv.org/abs/1602.02410) | | oriolvinyals, panyx0718 |
-| [lm_commonsense](lm_commonsense) | [A Simple Method for Commonsense Reasoning](https://arxiv.org/abs/1806.02847) | | thtrieu |
-| [maskgan](maskgan)| [MaskGAN: Better Text Generation via Filling in the](https://arxiv.org/abs/1801.07736) | ICLR 2018 | liamb315, a-dai |
-| [namignizer](namignizer)| Namignizer | | knathanieltucker |
-| [neural_gpu](neural_gpu)| [Neural GPUs Learn Algorithms](https://arxiv.org/abs/1511.08228) | | lukaszkaiser |
-| [neural_programmer](neural_programmer) | [Learning a Natural Language Interface with Neural Programmer](https://arxiv.org/abs/1611.08945) | ICLR 2017 | arvind2505 |
-| [next_frame_prediction](next_frame_prediction) | [Visual Dynamics: Probabilistic Future Frame Synthesis via Cross Convolutional Networks](https://arxiv.org/abs/1607.02586) | NIPS 2016 | panyx0718 |
-| [ptn](ptn) | [Perspective Transformer Nets: Learning Single-View 3D Object Reconstruction without 3D Supervision](https://arxiv.org/abs/1612.00814) | NIPS 2016 | xcyan, arkanath, hellojas, honglaklee |
-| [qa_kg](qa_kg) | [Learning to Reason: End-to-End Module Networks for Visual Question Answering](https://arxiv.org/abs/1704.05526) | ICCV 2017 | yuyuz |
-| [real_nvp](real_nvp) | [Density estimation using Real NVP](https://arxiv.org/abs/1605.08803) | ICLR 2017 | laurent-dinh |
-| [sentiment_analysis](sentiment_analysis)| [Effective Use of Word Order for Text Categorization with Convolutional Neural Networks](https://arxiv.org/abs/1412.1058) | NAACL HLT 2015 | sculd |
-| [seq2species](seq2species) | [Seq2Species: A deep learning approach to pattern recognition for short DNA sequences](https://doi.org/10.1101/353474) | | apbusia, depristo |
-| [skip_thoughts](skip_thoughts) | [Skip-Thought Vectors](https://arxiv.org/abs/1506.06726) | | cshallue |
-| [steve](steve) | [Sample-Efficient Reinforcement Learning with Stochastic Ensemble Value Expansion](https://arxiv.org/abs/1807.01675) | NeurIPS 2018 | buckman-google |
-| [street](street) | [End-to-End Interpretation of the French Street Name Signs Dataset](https://arxiv.org/abs/1702.03970) | ECCV 2016 | theraysmith |
-| [struct2depth](struct2depth)| [Depth Prediction Without the Sensors: Leveraging Structure for Unsupervised Learning from Monocular Videos](https://arxiv.org/abs/1811.06152) | AAAI 2019 | aneliaangelova |
-| [swivel](swivel) | [Swivel: Improving Embeddings by Noticing What's Missing](https://arxiv.org/abs/1602.02215) | | waterson |
-| [tcn](tcn) | [Time-Contrastive Networks: Self-Supervised Learning from Video](https://arxiv.org/abs/1704.06888) | ICRA 2018 | coreylynch, sermanet |
-| [textsum](textsum)| [A Neural Attention Model for Abstractive Sentence Summarization](https://arxiv.org/abs/1509.00685) | EMNLP 2015 | panyx0718, peterjliu |
-| [transformer](transformer) | [Spatial Transformer Network](https://arxiv.org/abs/1506.02025) | NIPS 2015 | daviddao|
-| [video_prediction](video_prediction) | [Unsupervised Learning for Physical Interaction through Video Prediction](https://arxiv.org/abs/1605.07157) | NIPS 2016 | cbfinn |
+:warning: If you are looking for old models, please visit the [Archive branch](https://github.com/tensorflow/models/tree/archive/research).
---
diff --git a/research/a3c_blogpost/README.md b/research/a3c_blogpost/README.md
deleted file mode 100644
index 55e390e703db361fbc4b1d89bb3baff9abb30dac..0000000000000000000000000000000000000000
--- a/research/a3c_blogpost/README.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# A3C Blog Post
-In order to run this code, you will need the following prerequisites:
-
-* [OpenAI Gym](https://github.com/openai/gym) - `pip install gym`
-* [pyglet](https://bitbucket.org/pyglet/pyglet/wiki/Home) - `pip install pyglet`
-* [TensorFlow](https://www.tensorflow.org/install/) - `pip install tensorflow==2.2.0`
diff --git a/research/a3c_blogpost/a3c_cartpole.py b/research/a3c_blogpost/a3c_cartpole.py
deleted file mode 100644
index 62fdcf84929d76b9ccae564db77320d15e774002..0000000000000000000000000000000000000000
--- a/research/a3c_blogpost/a3c_cartpole.py
+++ /dev/null
@@ -1,366 +0,0 @@
-import os
-os.environ["CUDA_VISIBLE_DEVICES"] = ""
-
-import threading
-import gym
-import multiprocessing
-import numpy as np
-from queue import Queue
-import argparse
-import matplotlib.pyplot as plt
-
-
-import tensorflow as tf
-from tensorflow.python import keras
-from tensorflow.python.keras import layers
-
-parser = argparse.ArgumentParser(description='Run A3C algorithm on the game '
- 'Cartpole.')
-parser.add_argument('--algorithm', default='a3c', type=str,
- help='Choose between \'a3c\' and \'random\'.')
-parser.add_argument('--train', dest='train', action='store_true',
- help='Train our model.')
-parser.add_argument('--lr', default=0.001,
- help='Learning rate for the shared optimizer.')
-parser.add_argument('--update-freq', default=20, type=int,
- help='How often to update the global model.')
-parser.add_argument('--max-eps', default=1000, type=int,
- help='Global maximum number of episodes to run.')
-parser.add_argument('--gamma', default=0.99,
- help='Discount factor of rewards.')
-parser.add_argument('--save-dir', default='/tmp/', type=str,
- help='Directory in which you desire to save the model.')
-args = parser.parse_args()
-
-class ActorCriticModel(keras.Model):
- def __init__(self, state_size, action_size):
- super(ActorCriticModel, self).__init__()
- self.state_size = state_size
- self.action_size = action_size
- self.dense1 = layers.Dense(100, activation='relu')
- self.policy_logits = layers.Dense(action_size)
- self.dense2 = layers.Dense(100, activation='relu')
- self.values = layers.Dense(1)
-
- def call(self, inputs):
- # Forward pass
- x = self.dense1(inputs)
- logits = self.policy_logits(x)
- v1 = self.dense2(inputs)
- values = self.values(v1)
- return logits, values
-
-def record(episode,
- episode_reward,
- worker_idx,
- global_ep_reward,
- result_queue,
- total_loss,
- num_steps):
- """Helper function to store score and print statistics.
-
- Arguments:
- episode: Current episode
- episode_reward: Reward accumulated over the current episode
- worker_idx: Which thread (worker)
- global_ep_reward: The moving average of the global reward
- result_queue: Queue storing the moving average of the scores
- total_loss: The total loss accumualted over the current episode
- num_steps: The number of steps the episode took to complete
- """
- if global_ep_reward == 0:
- global_ep_reward = episode_reward
- else:
- global_ep_reward = global_ep_reward * 0.99 + episode_reward * 0.01
- print(
- f"Episode: {episode} | "
- f"Moving Average Reward: {int(global_ep_reward)} | "
- f"Episode Reward: {int(episode_reward)} | "
- f"Loss: {int(total_loss / float(num_steps) * 1000) / 1000} | "
- f"Steps: {num_steps} | "
- f"Worker: {worker_idx}"
- )
- result_queue.put(global_ep_reward)
- return global_ep_reward
-
-
-class RandomAgent:
- """Random Agent that will play the specified game
-
- Arguments:
- env_name: Name of the environment to be played
- max_eps: Maximum number of episodes to run agent for.
- """
- def __init__(self, env_name, max_eps):
- self.env = gym.make(env_name)
- self.max_episodes = max_eps
- self.global_moving_average_reward = 0
- self.res_queue = Queue()
-
- def run(self):
- reward_avg = 0
- for episode in range(self.max_episodes):
- done = False
- self.env.reset()
- reward_sum = 0.0
- steps = 0
- while not done:
- # Sample randomly from the action space and step
- _, reward, done, _ = self.env.step(self.env.action_space.sample())
- steps += 1
- reward_sum += reward
- # Record statistics
- self.global_moving_average_reward = record(episode,
- reward_sum,
- 0,
- self.global_moving_average_reward,
- self.res_queue, 0, steps)
-
- reward_avg += reward_sum
- final_avg = reward_avg / float(self.max_episodes)
- print("Average score across {} episodes: {}".format(self.max_episodes, final_avg))
- return final_avg
-
-
-class MasterAgent():
- def __init__(self):
- self.game_name = 'CartPole-v0'
- save_dir = args.save_dir
- self.save_dir = save_dir
- if not os.path.exists(save_dir):
- os.makedirs(save_dir)
-
- env = gym.make(self.game_name)
- self.state_size = env.observation_space.shape[0]
- self.action_size = env.action_space.n
- self.opt = tf.compat.v1.train.AdamOptimizer(args.lr, use_locking=True)
- print(self.state_size, self.action_size)
-
- self.global_model = ActorCriticModel(self.state_size, self.action_size) # global network
- self.global_model(tf.convert_to_tensor(np.random.random((1, self.state_size)), dtype=tf.float32))
-
- def train(self):
- if args.algorithm == 'random':
- random_agent = RandomAgent(self.game_name, args.max_eps)
- random_agent.run()
- return
-
- res_queue = Queue()
-
- workers = [Worker(self.state_size,
- self.action_size,
- self.global_model,
- self.opt, res_queue,
- i, game_name=self.game_name,
- save_dir=self.save_dir) for i in range(multiprocessing.cpu_count())]
-
- for i, worker in enumerate(workers):
- print("Starting worker {}".format(i))
- worker.start()
-
- moving_average_rewards = [] # record episode reward to plot
- while True:
- reward = res_queue.get()
- if reward is not None:
- moving_average_rewards.append(reward)
- else:
- break
- [w.join() for w in workers]
-
- plt.plot(moving_average_rewards)
- plt.ylabel('Moving average ep reward')
- plt.xlabel('Step')
- plt.savefig(os.path.join(self.save_dir,
- '{} Moving Average.png'.format(self.game_name)))
- plt.show()
-
- def play(self):
- env = gym.make(self.game_name).unwrapped
- state = env.reset()
- model = self.global_model
- model_path = os.path.join(self.save_dir, 'model_{}.h5'.format(self.game_name))
- print('Loading model from: {}'.format(model_path))
- model.load_weights(model_path)
- done = False
- step_counter = 0
- reward_sum = 0
-
- try:
- while not done:
- env.render(mode='rgb_array')
- policy, value = model(tf.convert_to_tensor(state[None, :], dtype=tf.float32))
- policy = tf.nn.softmax(policy)
- action = np.argmax(policy)
- state, reward, done, _ = env.step(action)
- reward_sum += reward
- print("{}. Reward: {}, action: {}".format(step_counter, reward_sum, action))
- step_counter += 1
- except KeyboardInterrupt:
- print("Received Keyboard Interrupt. Shutting down.")
- finally:
- env.close()
-
-
-class Memory:
- def __init__(self):
- self.states = []
- self.actions = []
- self.rewards = []
-
- def store(self, state, action, reward):
- self.states.append(state)
- self.actions.append(action)
- self.rewards.append(reward)
-
- def clear(self):
- self.states = []
- self.actions = []
- self.rewards = []
-
-
-class Worker(threading.Thread):
- # Set up global variables across different threads
- global_episode = 0
- # Moving average reward
- global_moving_average_reward = 0
- best_score = 0
- save_lock = threading.Lock()
-
- def __init__(self,
- state_size,
- action_size,
- global_model,
- opt,
- result_queue,
- idx,
- game_name='CartPole-v0',
- save_dir='/tmp'):
- super(Worker, self).__init__()
- self.state_size = state_size
- self.action_size = action_size
- self.result_queue = result_queue
- self.global_model = global_model
- self.opt = opt
- self.local_model = ActorCriticModel(self.state_size, self.action_size)
- self.worker_idx = idx
- self.game_name = game_name
- self.env = gym.make(self.game_name).unwrapped
- self.save_dir = save_dir
- self.ep_loss = 0.0
-
- def run(self):
- total_step = 1
- mem = Memory()
- while Worker.global_episode < args.max_eps:
- current_state = self.env.reset()
- mem.clear()
- ep_reward = 0.
- ep_steps = 0
- self.ep_loss = 0
-
- time_count = 0
- done = False
- while not done:
- logits, _ = self.local_model(
- tf.convert_to_tensor(current_state[None, :],
- dtype=tf.float32))
- probs = tf.nn.softmax(logits)
-
- action = np.random.choice(self.action_size, p=probs.numpy()[0])
- new_state, reward, done, _ = self.env.step(action)
- if done:
- reward = -1
- ep_reward += reward
- mem.store(current_state, action, reward)
-
- if time_count == args.update_freq or done:
- # Calculate gradient wrt to local model. We do so by tracking the
- # variables involved in computing the loss by using tf.GradientTape
- with tf.GradientTape() as tape:
- total_loss = self.compute_loss(done,
- new_state,
- mem,
- args.gamma)
- self.ep_loss += total_loss
- # Calculate local gradients
- grads = tape.gradient(total_loss, self.local_model.trainable_weights)
- # Push local gradients to global model
- self.opt.apply_gradients(zip(grads,
- self.global_model.trainable_weights))
- # Update local model with new weights
- self.local_model.set_weights(self.global_model.get_weights())
-
- mem.clear()
- time_count = 0
-
- if done: # done and print information
- Worker.global_moving_average_reward = \
- record(Worker.global_episode, ep_reward, self.worker_idx,
- Worker.global_moving_average_reward, self.result_queue,
- self.ep_loss, ep_steps)
- # We must use a lock to save our model and to print to prevent data races.
- if ep_reward > Worker.best_score:
- with Worker.save_lock:
- print("Saving best model to {}, "
- "episode score: {}".format(self.save_dir, ep_reward))
- self.global_model.save_weights(
- os.path.join(self.save_dir,
- 'model_{}.h5'.format(self.game_name))
- )
- Worker.best_score = ep_reward
- Worker.global_episode += 1
- ep_steps += 1
-
- time_count += 1
- current_state = new_state
- total_step += 1
- self.result_queue.put(None)
-
- def compute_loss(self,
- done,
- new_state,
- memory,
- gamma=0.99):
- if done:
- reward_sum = 0. # terminal
- else:
- reward_sum = self.local_model(
- tf.convert_to_tensor(new_state[None, :],
- dtype=tf.float32))[-1].numpy()[0]
-
- # Get discounted rewards
- discounted_rewards = []
- for reward in memory.rewards[::-1]: # reverse buffer r
- reward_sum = reward + gamma * reward_sum
- discounted_rewards.append(reward_sum)
- discounted_rewards.reverse()
-
- logits, values = self.local_model(
- tf.convert_to_tensor(np.vstack(memory.states),
- dtype=tf.float32))
- # Get our advantages
- advantage = tf.convert_to_tensor(np.array(discounted_rewards)[:, None],
- dtype=tf.float32) - values
- # Value loss
- value_loss = advantage ** 2
-
- # Calculate our policy loss
- policy = tf.nn.softmax(logits)
- entropy = tf.nn.softmax_cross_entropy_with_logits(labels=policy, logits=logits)
-
- policy_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=memory.actions,
- logits=logits)
- policy_loss *= tf.stop_gradient(advantage)
- policy_loss -= 0.01 * entropy
- total_loss = tf.reduce_mean((0.5 * value_loss + policy_loss))
- return total_loss
-
-
-if __name__ == '__main__':
- print(args)
- master = MasterAgent()
- if args.train:
- master.train()
- else:
- master.play()
-
diff --git a/research/adv_imagenet_models/README.md b/research/adv_imagenet_models/README.md
deleted file mode 100644
index 6129f7347effe09ef0272de9ac42d4872726fcd1..0000000000000000000000000000000000000000
--- a/research/adv_imagenet_models/README.md
+++ /dev/null
@@ -1,91 +0,0 @@
-
-
-
-
-# Adversarially trained ImageNet models
-
-Pre-trained ImageNet models from the following papers:
-
-* [Adversarial Machine Learning at Scale](https://arxiv.org/abs/1611.01236)
-* [Ensemble Adversarial Training: Attacks and Defenses](https://arxiv.org/abs/1705.07204)
-
-## Contact
-
-Author: Alexey Kurakin,
-github: [AlexeyKurakin](https://github.com/AlexeyKurakin)
-
-## Pre-requesites and installation
-
-Ensure that you have installed TensorFlow 1.1 or greater
-([instructions](https://www.tensorflow.org/install/)).
-
-You also need copy of ImageNet dataset if you want to run provided example.
-Follow
-[Preparing the dataset](https://github.com/tensorflow/models/tree/master/research/slim#Data)
-instructions in TF-Slim library to get and preprocess ImageNet data.
-
-## Available models
-
-Following pre-trained models are available:
-
-Network Architecture | Adversarial training | Checkpoint
----------------------|----------------------|----------------
-Inception v3 | Step L.L. | [adv_inception_v3_2017_08_18.tar.gz](http://download.tensorflow.org/models/adv_inception_v3_2017_08_18.tar.gz)
-Inception v3 | Step L.L. on ensemble of 3 models | [ens3_adv_inception_v3_2017_08_18.tar.gz](http://download.tensorflow.org/models/ens3_adv_inception_v3_2017_08_18.tar.gz)
-Inception v3 | Step L.L. on ensemble of 4 models| [ens4_adv_inception_v3_2017_08_18.tar.gz](http://download.tensorflow.org/models/ens4_adv_inception_v3_2017_08_18.tar.gz)
-Inception ResNet v2 | Step L.L. | [adv_inception_resnet_v2_2017_12_18.tar.gz](http://download.tensorflow.org/models/adv_inception_resnet_v2_2017_12_18.tar.gz)
-Inception ResNet v2 | Step L.L. on ensemble of 3 models | [ens_adv_inception_resnet_v2_2017_08_18.tar.gz](http://download.tensorflow.org/models/ens_adv_inception_resnet_v2_2017_08_18.tar.gz)
-
-All checkpoints are compatible with
-[TF-Slim](https://github.com/tensorflow/models/tree/master/research/slim)
-implementation of Inception v3 and Inception Resnet v2.
-
-## How to evaluate models on ImageNet test data
-
-Python script `eval_on_adversarial.py` allow you to evaluate provided models
-on white-box adversarial examples generated from ImageNet test set.
-
-Usage is following:
-
-```bash
-# ${MODEL_NAME} - type of network architecture,
-# either "inception_v3" or "inception_resnet_v2"
-# ${CHECKPOINT_PATH} - path to model checkpoint
-# ${DATASET_DIR} - directory with ImageNet test set
-# ${ADV_METHOD} - which method to use to generate adversarial images,
-# supported method:
-# "none" - use clean images from the dataset
-# "stepll" - one step towards least likely class method (StepLL),
-# see https://arxiv.org/abs/1611.01236 for details
-# "stepllnoise" - RAND+StepLL method from https://arxiv.org/abs/1705.07204
-# ${ADV_EPS} - size of adversarial perturbation, ignored when method is none
-python eval_on_adversarial.py \
- --model_name=${MODEL_NAME} \
- --checkpoint_path=${CHECKPOINT_PATH} \
- --dataset_dir=${DATASET_DIR} \
- --batch_size=50 \
- --adversarial_method=${ADV_METHOD} \
- --adversarial_eps=${ADV_EPS}
-```
-
-Below is an example how to evaluate one of the models on RAND+StepLL adversarial
-examples:
-
-```bash
-# Download checkpoint
-CHECKPOINT_DIR=/tmp/checkpoints
-mkdir ${CHECKPOINT_DIR}
-wget http://download.tensorflow.org/models/ens_adv_inception_resnet_v2_2017_08_18.tar.gz
-tar -xvf ens_adv_inception_resnet_v2_2017_08_18.tar.gz
-mv ens_adv_inception_resnet_v2.ckpt* ${CHECKPOINT_DIR}
-rm ens_adv_inception_resnet_v2_2017_08_18.tar.gz
-
-# Run evaluation
-python eval_on_adversarial.py \
- --model_name=inception_v3 \
- --checkpoint_path=${CHECKPOINT_DIR}/ens_adv_inception_resnet_v2.ckpt \
- --dataset_dir=${DATASET_DIR} \
- --batch_size=50 \
- --adversarial_method=stepllnoise \
- --adversarial_eps=16
-```
diff --git a/research/adv_imagenet_models/eval_on_adversarial.py b/research/adv_imagenet_models/eval_on_adversarial.py
deleted file mode 100644
index f9188845c6c4e10484f9b24797d9ece3b730ffb0..0000000000000000000000000000000000000000
--- a/research/adv_imagenet_models/eval_on_adversarial.py
+++ /dev/null
@@ -1,331 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Script which evaluates model on adversarial examples."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-import imagenet
-import inception_resnet_v2
-
-import tensorflow as tf
-from tensorflow.contrib.slim.nets import inception
-
-
-slim = tf.contrib.slim
-
-tf.app.flags.DEFINE_integer(
- 'batch_size', 50, 'The number of samples in each batch.')
-
-tf.app.flags.DEFINE_integer(
- 'max_num_batches', None,
- 'Max number of batches to evaluate by default use all.')
-
-tf.app.flags.DEFINE_string(
- 'master', '', 'The address of the TensorFlow master to use.')
-
-tf.app.flags.DEFINE_string(
- 'checkpoint_path', '/tmp/tfmodel/',
- 'The directory where the model was written to or an absolute path to a '
- 'checkpoint file.')
-
-tf.app.flags.DEFINE_integer(
- 'num_preprocessing_threads', 4,
- 'The number of threads used to create the batches.')
-
-tf.app.flags.DEFINE_string(
- 'split_name', 'validation', 'The name of the train/test split.')
-
-tf.app.flags.DEFINE_string(
- 'dataset_dir', None, 'The directory where the dataset files are stored.')
-
-tf.app.flags.DEFINE_string(
- 'model_name', 'inception_v3',
- 'Name of the model to use, either "inception_v3" or "inception_resnet_v2"')
-
-tf.app.flags.DEFINE_float(
- 'moving_average_decay', None,
- 'The decay to use for the moving average.'
- 'If left as None, then moving averages are not used.')
-
-tf.app.flags.DEFINE_string(
- 'adversarial_method', 'none',
- 'What kind of adversarial examples to use for evaluation. '
- 'Could be one of: "none", "stepll", "stepllnoise".')
-
-tf.app.flags.DEFINE_float(
- 'adversarial_eps', 0.0,
- 'Size of adversarial perturbation in range [0, 255].')
-
-
-FLAGS = tf.app.flags.FLAGS
-
-
-IMAGE_SIZE = 299
-NUM_CLASSES = 1001
-
-
-def preprocess_for_eval(image, height, width,
- central_fraction=0.875, scope=None):
- """Prepare one image for evaluation.
-
- If height and width are specified it would output an image with that size by
- applying resize_bilinear.
- If central_fraction is specified it would crop the central fraction of the
- input image.
-
- Args:
- image: 3-D Tensor of image. If dtype is tf.float32 then the range should be
- [0, 1], otherwise it would converted to tf.float32 assuming that the range
- is [0, MAX], where MAX is largest positive representable number for
- int(8/16/32) data type (see `tf.image.convert_image_dtype` for details)
- height: integer
- width: integer
- central_fraction: Optional Float, fraction of the image to crop.
- scope: Optional scope for name_scope.
- Returns:
- 3-D float Tensor of prepared image.
- """
- with tf.name_scope(scope, 'eval_image', [image, height, width]):
- if image.dtype != tf.float32:
- image = tf.image.convert_image_dtype(image, dtype=tf.float32)
- # Crop the central region of the image with an area containing 87.5% of
- # the original image.
- if central_fraction:
- image = tf.image.central_crop(image, central_fraction=central_fraction)
-
- if height and width:
- # Resize the image to the specified height and width.
- image = tf.expand_dims(image, 0)
- image = tf.image.resize_bilinear(image, [height, width],
- align_corners=False)
- image = tf.squeeze(image, [0])
- image = tf.subtract(image, 0.5)
- image = tf.multiply(image, 2.0)
- return image
-
-
-def create_model(x, reuse=None):
- """Create model graph.
-
- Args:
- x: input images
- reuse: reuse parameter which will be passed to underlying variable scopes.
- Should be None first call and True every subsequent call.
-
- Returns:
- (logits, end_points) - tuple of model logits and enpoints
-
- Raises:
- ValueError: if model type specified by --model_name flag is invalid.
- """
- if FLAGS.model_name == 'inception_v3':
- with slim.arg_scope(inception.inception_v3_arg_scope()):
- return inception.inception_v3(
- x, num_classes=NUM_CLASSES, is_training=False, reuse=reuse)
- elif FLAGS.model_name == 'inception_resnet_v2':
- with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
- return inception_resnet_v2.inception_resnet_v2(
- x, num_classes=NUM_CLASSES, is_training=False, reuse=reuse)
- else:
- raise ValueError('Invalid model name: %s' % (FLAGS.model_name))
-
-
-def step_target_class_adversarial_images(x, eps, one_hot_target_class):
- """Base code for one step towards target class methods.
-
- Args:
- x: source images
- eps: size of adversarial perturbation
- one_hot_target_class: one hot encoded target classes for all images
-
- Returns:
- tensor with adversarial images
- """
- logits, end_points = create_model(x, reuse=True)
- cross_entropy = tf.losses.softmax_cross_entropy(one_hot_target_class,
- logits,
- label_smoothing=0.1,
- weights=1.0)
- cross_entropy += tf.losses.softmax_cross_entropy(one_hot_target_class,
- end_points['AuxLogits'],
- label_smoothing=0.1,
- weights=0.4)
- x_adv = x - eps * tf.sign(tf.gradients(cross_entropy, x)[0])
- x_adv = tf.clip_by_value(x_adv, -1.0, 1.0)
- return tf.stop_gradient(x_adv)
-
-
-def stepll_adversarial_images(x, eps):
- """One step towards least likely class (Step L.L.) adversarial examples.
-
- This method is an alternative to FGSM which does not use true classes.
- Method is described in the "Adversarial Machine Learning at Scale" paper,
- https://arxiv.org/abs/1611.01236
-
- Args:
- x: source images
- eps: size of adversarial perturbation
-
- Returns:
- adversarial images
- """
- logits, _ = create_model(x, reuse=True)
- least_likely_class = tf.argmin(logits, 1)
- one_hot_ll_class = tf.one_hot(least_likely_class, NUM_CLASSES)
- return step_target_class_adversarial_images(x, eps, one_hot_ll_class)
-
-
-def stepllnoise_adversarial_images(x, eps):
- """Step L.L. with noise method.
-
- This is an imporvement of Step L.L. method. This method is better against
- adversarially trained models which learn to mask gradient.
- Method is described in the section "New randomized one shot attack" of
- "Ensemble Adversarial Training: Attacks and Defenses" paper,
- https://arxiv.org/abs/1705.07204
-
- Args:
- x: source images
- eps: size of adversarial perturbation
-
- Returns:
- adversarial images
- """
- logits, _ = create_model(x, reuse=True)
- least_likely_class = tf.argmin(logits, 1)
- one_hot_ll_class = tf.one_hot(least_likely_class, NUM_CLASSES)
- x_noise = x + eps / 2 * tf.sign(tf.random_normal(x.shape))
- return step_target_class_adversarial_images(x_noise, eps / 2,
- one_hot_ll_class)
-
-
-def get_input_images(dataset_images):
- """Gets input images for the evaluation.
-
- Args:
- dataset_images: tensor with dataset images
-
- Returns:
- tensor with input images, which is either dataset images or adversarial
- images.
-
- Raises:
- ValueError: if adversarial method specified by --adversarial_method flag
- is invalid.
- """
- # adversarial_eps defines max difference of values of pixels if
- # pixels are in range [0, 255]. However values of dataset pixels are
- # in range [-1, 1], so converting epsilon.
- eps = FLAGS.adversarial_eps / 255 * 2.0
-
- if FLAGS.adversarial_method == 'stepll':
- return stepll_adversarial_images(dataset_images, eps)
- elif FLAGS.adversarial_method == 'stepllnoise':
- return stepllnoise_adversarial_images(dataset_images, eps)
- elif FLAGS.adversarial_method == 'none':
- return dataset_images
- else:
- raise ValueError('Invalid adversarial method: %s'
- % (FLAGS.adversarial_method))
-
-
-def main(_):
- if not FLAGS.dataset_dir:
- raise ValueError('You must supply the dataset directory with --dataset_dir')
-
- tf.logging.set_verbosity(tf.logging.INFO)
- with tf.Graph().as_default():
- tf_global_step = tf.train.get_or_create_global_step()
-
- ###################
- # Prepare dataset #
- ###################
- dataset = imagenet.get_split(FLAGS.split_name, FLAGS.dataset_dir)
- provider = slim.dataset_data_provider.DatasetDataProvider(
- dataset,
- shuffle=False,
- common_queue_capacity=2 * FLAGS.batch_size,
- common_queue_min=FLAGS.batch_size)
- [dataset_image, label] = provider.get(['image', 'label'])
- dataset_image = preprocess_for_eval(dataset_image, IMAGE_SIZE, IMAGE_SIZE)
- dataset_images, labels = tf.train.batch(
- [dataset_image, label],
- batch_size=FLAGS.batch_size,
- num_threads=FLAGS.num_preprocessing_threads,
- capacity=5 * FLAGS.batch_size)
-
- ########################################
- # Define the model and input exampeles #
- ########################################
- create_model(tf.placeholder(tf.float32, shape=dataset_images.shape))
- input_images = get_input_images(dataset_images)
- logits, _ = create_model(input_images, reuse=True)
-
- if FLAGS.moving_average_decay > 0:
- variable_averages = tf.train.ExponentialMovingAverage(
- FLAGS.moving_average_decay, tf_global_step)
- variables_to_restore = variable_averages.variables_to_restore(
- slim.get_model_variables())
- variables_to_restore[tf_global_step.op.name] = tf_global_step
- else:
- variables_to_restore = slim.get_variables_to_restore()
-
- ######################
- # Define the metrics #
- ######################
- predictions = tf.argmax(logits, 1)
- labels = tf.squeeze(labels)
- names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
- 'Accuracy': slim.metrics.streaming_accuracy(predictions, labels),
- 'Recall_5': slim.metrics.streaming_sparse_recall_at_k(
- logits, tf.reshape(labels, [-1, 1]), 5),
- })
-
- ######################
- # Run evaluation #
- ######################
- if FLAGS.max_num_batches:
- num_batches = FLAGS.max_num_batches
- else:
- # This ensures that we make a single pass over all of the data.
- num_batches = math.ceil(dataset.num_samples / float(FLAGS.batch_size))
-
- if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
- checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
- else:
- checkpoint_path = FLAGS.checkpoint_path
-
- tf.logging.info('Evaluating %s' % checkpoint_path)
-
- top1_accuracy, top5_accuracy = slim.evaluation.evaluate_once(
- master=FLAGS.master,
- checkpoint_path=checkpoint_path,
- logdir=None,
- summary_op=None,
- num_evals=num_batches,
- eval_op=list(names_to_updates.values()),
- final_op=[names_to_values['Accuracy'], names_to_values['Recall_5']],
- variables_to_restore=variables_to_restore)
-
- print('Top1 Accuracy: ', top1_accuracy)
- print('Top5 Accuracy: ', top5_accuracy)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/adv_imagenet_models/imagenet.py b/research/adv_imagenet_models/imagenet.py
deleted file mode 100644
index 26c4c7a388a234f647e446951a0765d1c53184cb..0000000000000000000000000000000000000000
--- a/research/adv_imagenet_models/imagenet.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Provides data for the ImageNet ILSVRC 2012 Dataset plus some bounding boxes.
-
-Some images have one or more bounding boxes associated with the label of the
-image. See details here: http://image-net.org/download-bboxes
-
-WARNING: Don't use for object detection, in this case all the bounding boxes
-of the image belong to just one class.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-_FILE_PATTERN = '%s-*'
-
-_SPLITS_TO_SIZES = {
- 'train': 1281167,
- 'validation': 50000,
-}
-
-_ITEMS_TO_DESCRIPTIONS = {
- 'image': 'A color image of varying height and width.',
- 'label': 'The label id of the image, integer between 0 and 999',
- 'label_text': 'The text of the label.',
- 'object/bbox': 'A list of bounding boxes.',
- 'object/label': 'A list of labels, one per each object.',
-}
-
-_NUM_CLASSES = 1001
-
-
-def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
- """Gets a dataset tuple with instructions for reading ImageNet.
-
- Args:
- split_name: A train/test split name.
- dataset_dir: The base directory of the dataset sources.
- file_pattern: The file pattern to use when matching the dataset sources.
- It is assumed that the pattern contains a '%s' string so that the split
- name can be inserted.
- reader: The TensorFlow reader type.
-
- Returns:
- A `Dataset` namedtuple.
-
- Raises:
- ValueError: if `split_name` is not a valid train/test split.
- """
- if split_name not in _SPLITS_TO_SIZES:
- raise ValueError('split name %s was not recognized.' % split_name)
-
- if not file_pattern:
- file_pattern = _FILE_PATTERN
- file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
-
- # Allowing None in the signature so that dataset_factory can use the default.
- if reader is None:
- reader = tf.TFRecordReader
-
- keys_to_features = {
- 'image/encoded': tf.FixedLenFeature(
- (), tf.string, default_value=''),
- 'image/format': tf.FixedLenFeature(
- (), tf.string, default_value='jpeg'),
- 'image/class/label': tf.FixedLenFeature(
- [], dtype=tf.int64, default_value=-1),
- 'image/class/text': tf.FixedLenFeature(
- [], dtype=tf.string, default_value=''),
- 'image/object/bbox/xmin': tf.VarLenFeature(
- dtype=tf.float32),
- 'image/object/bbox/ymin': tf.VarLenFeature(
- dtype=tf.float32),
- 'image/object/bbox/xmax': tf.VarLenFeature(
- dtype=tf.float32),
- 'image/object/bbox/ymax': tf.VarLenFeature(
- dtype=tf.float32),
- 'image/object/class/label': tf.VarLenFeature(
- dtype=tf.int64),
- }
-
- items_to_handlers = {
- 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
- 'label': slim.tfexample_decoder.Tensor('image/class/label'),
- 'label_text': slim.tfexample_decoder.Tensor('image/class/text'),
- 'object/bbox': slim.tfexample_decoder.BoundingBox(
- ['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'),
- 'object/label': slim.tfexample_decoder.Tensor('image/object/class/label'),
- }
-
- decoder = slim.tfexample_decoder.TFExampleDecoder(
- keys_to_features, items_to_handlers)
-
- return slim.dataset.Dataset(
- data_sources=file_pattern,
- reader=reader,
- decoder=decoder,
- num_samples=_SPLITS_TO_SIZES[split_name],
- items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
- num_classes=_NUM_CLASSES)
diff --git a/research/adv_imagenet_models/inception_resnet_v2.py b/research/adv_imagenet_models/inception_resnet_v2.py
deleted file mode 100644
index 2f690e8d2f70ecde9a55f40375a7f74cd25651c7..0000000000000000000000000000000000000000
--- a/research/adv_imagenet_models/inception_resnet_v2.py
+++ /dev/null
@@ -1,358 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contains the definition of the Inception Resnet V2 architecture.
-
-As described in http://arxiv.org/abs/1602.07261.
-
- Inception-v4, Inception-ResNet and the Impact of Residual Connections
- on Learning
- Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-
-def block35(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):
- """Builds the 35x35 resnet block."""
- with tf.variable_scope(scope, 'Block35', [net], reuse=reuse):
- with tf.variable_scope('Branch_0'):
- tower_conv = slim.conv2d(net, 32, 1, scope='Conv2d_1x1')
- with tf.variable_scope('Branch_1'):
- tower_conv1_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')
- tower_conv1_1 = slim.conv2d(tower_conv1_0, 32, 3, scope='Conv2d_0b_3x3')
- with tf.variable_scope('Branch_2'):
- tower_conv2_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')
- tower_conv2_1 = slim.conv2d(tower_conv2_0, 48, 3, scope='Conv2d_0b_3x3')
- tower_conv2_2 = slim.conv2d(tower_conv2_1, 64, 3, scope='Conv2d_0c_3x3')
- mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_1, tower_conv2_2])
- up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,
- activation_fn=None, scope='Conv2d_1x1')
- net += scale * up
- if activation_fn:
- net = activation_fn(net)
- return net
-
-
-def block17(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):
- """Builds the 17x17 resnet block."""
- with tf.variable_scope(scope, 'Block17', [net], reuse=reuse):
- with tf.variable_scope('Branch_0'):
- tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1')
- with tf.variable_scope('Branch_1'):
- tower_conv1_0 = slim.conv2d(net, 128, 1, scope='Conv2d_0a_1x1')
- tower_conv1_1 = slim.conv2d(tower_conv1_0, 160, [1, 7],
- scope='Conv2d_0b_1x7')
- tower_conv1_2 = slim.conv2d(tower_conv1_1, 192, [7, 1],
- scope='Conv2d_0c_7x1')
- mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_2])
- up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,
- activation_fn=None, scope='Conv2d_1x1')
- net += scale * up
- if activation_fn:
- net = activation_fn(net)
- return net
-
-
-def block8(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):
- """Builds the 8x8 resnet block."""
- with tf.variable_scope(scope, 'Block8', [net], reuse=reuse):
- with tf.variable_scope('Branch_0'):
- tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1')
- with tf.variable_scope('Branch_1'):
- tower_conv1_0 = slim.conv2d(net, 192, 1, scope='Conv2d_0a_1x1')
- tower_conv1_1 = slim.conv2d(tower_conv1_0, 224, [1, 3],
- scope='Conv2d_0b_1x3')
- tower_conv1_2 = slim.conv2d(tower_conv1_1, 256, [3, 1],
- scope='Conv2d_0c_3x1')
- mixed = tf.concat(axis=3, values=[tower_conv, tower_conv1_2])
- up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,
- activation_fn=None, scope='Conv2d_1x1')
- net += scale * up
- if activation_fn:
- net = activation_fn(net)
- return net
-
-
-def inception_resnet_v2_base(inputs,
- final_endpoint='Conv2d_7b_1x1',
- output_stride=16,
- align_feature_maps=False,
- scope=None):
- """Inception model from http://arxiv.org/abs/1602.07261.
-
- Constructs an Inception Resnet v2 network from inputs to the given final
- endpoint. This method can construct the network up to the final inception
- block Conv2d_7b_1x1.
-
- Args:
- inputs: a tensor of size [batch_size, height, width, channels].
- final_endpoint: specifies the endpoint to construct the network up to. It
- can be one of ['Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3',
- 'MaxPool_3a_3x3', 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3',
- 'Mixed_5b', 'Mixed_6a', 'PreAuxLogits', 'Mixed_7a', 'Conv2d_7b_1x1']
- output_stride: A scalar that specifies the requested ratio of input to
- output spatial resolution. Only supports 8 and 16.
- align_feature_maps: When true, changes all the VALID paddings in the network
- to SAME padding so that the feature maps are aligned.
- scope: Optional variable_scope.
-
- Returns:
- tensor_out: output tensor corresponding to the final_endpoint.
- end_points: a set of activations for external use, for example summaries or
- losses.
-
- Raises:
- ValueError: if final_endpoint is not set to one of the predefined values,
- or if the output_stride is not 8 or 16, or if the output_stride is 8 and
- we request an end point after 'PreAuxLogits'.
- """
- if output_stride != 8 and output_stride != 16:
- raise ValueError('output_stride must be 8 or 16.')
-
- padding = 'SAME' if align_feature_maps else 'VALID'
-
- end_points = {}
-
- def add_and_check_final(name, net):
- end_points[name] = net
- return name == final_endpoint
-
- with tf.variable_scope(scope, 'InceptionResnetV2', [inputs]):
- with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d],
- stride=1, padding='SAME'):
- # 149 x 149 x 32
- net = slim.conv2d(inputs, 32, 3, stride=2, padding=padding,
- scope='Conv2d_1a_3x3')
- if add_and_check_final('Conv2d_1a_3x3', net): return net, end_points
-
- # 147 x 147 x 32
- net = slim.conv2d(net, 32, 3, padding=padding,
- scope='Conv2d_2a_3x3')
- if add_and_check_final('Conv2d_2a_3x3', net): return net, end_points
- # 147 x 147 x 64
- net = slim.conv2d(net, 64, 3, scope='Conv2d_2b_3x3')
- if add_and_check_final('Conv2d_2b_3x3', net): return net, end_points
- # 73 x 73 x 64
- net = slim.max_pool2d(net, 3, stride=2, padding=padding,
- scope='MaxPool_3a_3x3')
- if add_and_check_final('MaxPool_3a_3x3', net): return net, end_points
- # 73 x 73 x 80
- net = slim.conv2d(net, 80, 1, padding=padding,
- scope='Conv2d_3b_1x1')
- if add_and_check_final('Conv2d_3b_1x1', net): return net, end_points
- # 71 x 71 x 192
- net = slim.conv2d(net, 192, 3, padding=padding,
- scope='Conv2d_4a_3x3')
- if add_and_check_final('Conv2d_4a_3x3', net): return net, end_points
- # 35 x 35 x 192
- net = slim.max_pool2d(net, 3, stride=2, padding=padding,
- scope='MaxPool_5a_3x3')
- if add_and_check_final('MaxPool_5a_3x3', net): return net, end_points
-
- # 35 x 35 x 320
- with tf.variable_scope('Mixed_5b'):
- with tf.variable_scope('Branch_0'):
- tower_conv = slim.conv2d(net, 96, 1, scope='Conv2d_1x1')
- with tf.variable_scope('Branch_1'):
- tower_conv1_0 = slim.conv2d(net, 48, 1, scope='Conv2d_0a_1x1')
- tower_conv1_1 = slim.conv2d(tower_conv1_0, 64, 5,
- scope='Conv2d_0b_5x5')
- with tf.variable_scope('Branch_2'):
- tower_conv2_0 = slim.conv2d(net, 64, 1, scope='Conv2d_0a_1x1')
- tower_conv2_1 = slim.conv2d(tower_conv2_0, 96, 3,
- scope='Conv2d_0b_3x3')
- tower_conv2_2 = slim.conv2d(tower_conv2_1, 96, 3,
- scope='Conv2d_0c_3x3')
- with tf.variable_scope('Branch_3'):
- tower_pool = slim.avg_pool2d(net, 3, stride=1, padding='SAME',
- scope='AvgPool_0a_3x3')
- tower_pool_1 = slim.conv2d(tower_pool, 64, 1,
- scope='Conv2d_0b_1x1')
- net = tf.concat(
- [tower_conv, tower_conv1_1, tower_conv2_2, tower_pool_1], 3)
-
- if add_and_check_final('Mixed_5b', net): return net, end_points
- # TODO(alemi): Register intermediate endpoints
- net = slim.repeat(net, 10, block35, scale=0.17)
-
- # 17 x 17 x 1088 if output_stride == 8,
- # 33 x 33 x 1088 if output_stride == 16
- use_atrous = output_stride == 8
-
- with tf.variable_scope('Mixed_6a'):
- with tf.variable_scope('Branch_0'):
- tower_conv = slim.conv2d(net, 384, 3, stride=1 if use_atrous else 2,
- padding=padding,
- scope='Conv2d_1a_3x3')
- with tf.variable_scope('Branch_1'):
- tower_conv1_0 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1')
- tower_conv1_1 = slim.conv2d(tower_conv1_0, 256, 3,
- scope='Conv2d_0b_3x3')
- tower_conv1_2 = slim.conv2d(tower_conv1_1, 384, 3,
- stride=1 if use_atrous else 2,
- padding=padding,
- scope='Conv2d_1a_3x3')
- with tf.variable_scope('Branch_2'):
- tower_pool = slim.max_pool2d(net, 3, stride=1 if use_atrous else 2,
- padding=padding,
- scope='MaxPool_1a_3x3')
- net = tf.concat([tower_conv, tower_conv1_2, tower_pool], 3)
-
- if add_and_check_final('Mixed_6a', net): return net, end_points
-
- # TODO(alemi): register intermediate endpoints
- with slim.arg_scope([slim.conv2d], rate=2 if use_atrous else 1):
- net = slim.repeat(net, 20, block17, scale=0.10)
- if add_and_check_final('PreAuxLogits', net): return net, end_points
-
- if output_stride == 8:
- # TODO(gpapan): Properly support output_stride for the rest of the net.
- raise ValueError('output_stride==8 is only supported up to the '
- 'PreAuxlogits end_point for now.')
-
- # 8 x 8 x 2080
- with tf.variable_scope('Mixed_7a'):
- with tf.variable_scope('Branch_0'):
- tower_conv = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1')
- tower_conv_1 = slim.conv2d(tower_conv, 384, 3, stride=2,
- padding=padding,
- scope='Conv2d_1a_3x3')
- with tf.variable_scope('Branch_1'):
- tower_conv1 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1')
- tower_conv1_1 = slim.conv2d(tower_conv1, 288, 3, stride=2,
- padding=padding,
- scope='Conv2d_1a_3x3')
- with tf.variable_scope('Branch_2'):
- tower_conv2 = slim.conv2d(net, 256, 1, scope='Conv2d_0a_1x1')
- tower_conv2_1 = slim.conv2d(tower_conv2, 288, 3,
- scope='Conv2d_0b_3x3')
- tower_conv2_2 = slim.conv2d(tower_conv2_1, 320, 3, stride=2,
- padding=padding,
- scope='Conv2d_1a_3x3')
- with tf.variable_scope('Branch_3'):
- tower_pool = slim.max_pool2d(net, 3, stride=2,
- padding=padding,
- scope='MaxPool_1a_3x3')
- net = tf.concat(
- [tower_conv_1, tower_conv1_1, tower_conv2_2, tower_pool], 3)
-
- if add_and_check_final('Mixed_7a', net): return net, end_points
-
- # TODO(alemi): register intermediate endpoints
- net = slim.repeat(net, 9, block8, scale=0.20)
- net = block8(net, activation_fn=None)
-
- # 8 x 8 x 1536
- net = slim.conv2d(net, 1536, 1, scope='Conv2d_7b_1x1')
- if add_and_check_final('Conv2d_7b_1x1', net): return net, end_points
-
- raise ValueError('final_endpoint (%s) not recognized', final_endpoint)
-
-
-def inception_resnet_v2(inputs, num_classes=1001, is_training=True,
- dropout_keep_prob=0.8,
- reuse=None,
- scope='InceptionResnetV2',
- create_aux_logits=True):
- """Creates the Inception Resnet V2 model.
-
- Args:
- inputs: a 4-D tensor of size [batch_size, height, width, 3].
- num_classes: number of predicted classes.
- is_training: whether is training or not.
- dropout_keep_prob: float, the fraction to keep before final layer.
- reuse: whether or not the network and its variables should be reused. To be
- able to reuse 'scope' must be given.
- scope: Optional variable_scope.
- create_aux_logits: Whether to include the auxilliary logits.
-
- Returns:
- logits: the logits outputs of the model.
- end_points: the set of end_points from the inception model.
- """
- end_points = {}
-
- with tf.variable_scope(scope, 'InceptionResnetV2', [inputs, num_classes],
- reuse=reuse) as scope:
- with slim.arg_scope([slim.batch_norm, slim.dropout],
- is_training=is_training):
-
- net, end_points = inception_resnet_v2_base(inputs, scope=scope)
-
- if create_aux_logits:
- with tf.variable_scope('AuxLogits'):
- aux = end_points['PreAuxLogits']
- aux = slim.avg_pool2d(aux, 5, stride=3, padding='VALID',
- scope='Conv2d_1a_3x3')
- aux = slim.conv2d(aux, 128, 1, scope='Conv2d_1b_1x1')
- aux = slim.conv2d(aux, 768, aux.get_shape()[1:3],
- padding='VALID', scope='Conv2d_2a_5x5')
- aux = slim.flatten(aux)
- aux = slim.fully_connected(aux, num_classes, activation_fn=None,
- scope='Logits')
- end_points['AuxLogits'] = aux
-
- with tf.variable_scope('Logits'):
- net = slim.avg_pool2d(net, net.get_shape()[1:3], padding='VALID',
- scope='AvgPool_1a_8x8')
- net = slim.flatten(net)
-
- net = slim.dropout(net, dropout_keep_prob, is_training=is_training,
- scope='Dropout')
-
- end_points['PreLogitsFlatten'] = net
- logits = slim.fully_connected(net, num_classes, activation_fn=None,
- scope='Logits')
- end_points['Logits'] = logits
- end_points['Predictions'] = tf.nn.softmax(logits, name='Predictions')
-
- return logits, end_points
-inception_resnet_v2.default_image_size = 299
-
-
-def inception_resnet_v2_arg_scope(weight_decay=0.00004,
- batch_norm_decay=0.9997,
- batch_norm_epsilon=0.001):
- """Returns the scope with the default parameters for inception_resnet_v2.
-
- Args:
- weight_decay: the weight decay for weights variables.
- batch_norm_decay: decay for the moving average of batch_norm momentums.
- batch_norm_epsilon: small float added to variance to avoid dividing by zero.
-
- Returns:
- a arg_scope with the parameters needed for inception_resnet_v2.
- """
- # Set weight_decay for weights in conv2d and fully_connected layers.
- with slim.arg_scope([slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- biases_regularizer=slim.l2_regularizer(weight_decay)):
-
- batch_norm_params = {
- 'decay': batch_norm_decay,
- 'epsilon': batch_norm_epsilon,
- }
- # Set activation_fn and parameters for batch_norm.
- with slim.arg_scope([slim.conv2d], activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_params) as scope:
- return scope
diff --git a/research/adversarial_crypto/README.md b/research/adversarial_crypto/README.md
deleted file mode 100644
index 3822def1325b8d4eb1fd31335f2f8ce053ff747a..0000000000000000000000000000000000000000
--- a/research/adversarial_crypto/README.md
+++ /dev/null
@@ -1,62 +0,0 @@
-
-
-
-
-# Learning to Protect Communications with Adversarial Neural Cryptography
-
-This is a slightly-updated model used for the paper
-["Learning to Protect Communications with Adversarial Neural
-Cryptography"](https://arxiv.org/abs/1610.06918).
-
-> We ask whether neural networks can learn to use secret keys to protect
-> information from other neural networks. Specifically, we focus on ensuring
-> confidentiality properties in a multiagent system, and we specify those
-> properties in terms of an adversary. Thus, a system may consist of neural
-> networks named Alice and Bob, and we aim to limit what a third neural
-> network named Eve learns from eavesdropping on the communication between
-> Alice and Bob. We do not prescribe specific cryptographic algorithms to
-> these neural networks; instead, we train end-to-end, adversarially.
-> We demonstrate that the neural networks can learn how to perform forms of
-> encryption and decryption, and also how to apply these operations
-> selectively in order to meet confidentiality goals.
-
-This code allows you to train encoder/decoder/adversary network triplets
-and evaluate their effectiveness on randomly generated input and key
-pairs.
-
-## Prerequisites
-
-The only software requirements for running the encoder and decoder is having
-TensorFlow installed.
-
-Requires TensorFlow r0.12 or later.
-
-## Training and evaluating
-
-After installing TensorFlow and ensuring that your paths are configured
-appropriately:
-
-```
-python train_eval.py
-```
-
-This will begin training a fresh model. If and when the model becomes
-sufficiently well-trained, it will reset the Eve model multiple times
-and retrain it from scratch, outputting the accuracy thus obtained
-in each run.
-
-## Model differences from the paper
-
-The model has been simplified slightly from the one described in
-the paper - the convolutional layer width was reduced by a factor
-of two. In the version in the paper, there was a nonlinear unit
-after the fully-connected layer; that nonlinear has been removed
-here. These changes improve the robustness of training. The
-initializer for the convolution layers has switched to the
-`tf.contrib.layers default` of `xavier_initializer` instead of
-a simpler `truncated_normal`.
-
-## Contact information
-
-This model repository is maintained by David G. Andersen
-([dave-andersen](https://github.com/dave-andersen)).
diff --git a/research/adversarial_crypto/train_eval.py b/research/adversarial_crypto/train_eval.py
deleted file mode 100644
index df7a00ad50f2ec01b37d8c162309a928207088d6..0000000000000000000000000000000000000000
--- a/research/adversarial_crypto/train_eval.py
+++ /dev/null
@@ -1,276 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Adversarial training to learn trivial encryption functions,
-from the paper "Learning to Protect Communications with
-Adversarial Neural Cryptography", Abadi & Andersen, 2016.
-
-https://arxiv.org/abs/1610.06918
-
-This program creates and trains three neural networks,
-termed Alice, Bob, and Eve. Alice takes inputs
-in_m (message), in_k (key) and outputs 'ciphertext'.
-
-Bob takes inputs in_k, ciphertext and tries to reconstruct
-the message.
-
-Eve is an adversarial network that takes input ciphertext
-and also tries to reconstruct the message.
-
-The main function attempts to train these networks and then
-evaluates them, all on random plaintext and key values.
-
-"""
-
-# TensorFlow Python 3 compatibility
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-import signal
-import sys
-from six.moves import xrange # pylint: disable=redefined-builtin
-import tensorflow as tf
-
-flags = tf.app.flags
-
-flags.DEFINE_float('learning_rate', 0.0008, 'Constant learning rate')
-flags.DEFINE_integer('batch_size', 4096, 'Batch size')
-
-FLAGS = flags.FLAGS
-
-# Input and output configuration.
-TEXT_SIZE = 16
-KEY_SIZE = 16
-
-# Training parameters.
-ITERS_PER_ACTOR = 1
-EVE_MULTIPLIER = 2 # Train Eve 2x for every step of Alice/Bob
-# Train until either max loops or Alice/Bob "good enough":
-MAX_TRAINING_LOOPS = 850000
-BOB_LOSS_THRESH = 0.02 # Exit when Bob loss < 0.02 and Eve > 7.7 bits
-EVE_LOSS_THRESH = 7.7
-
-# Logging and evaluation.
-PRINT_EVERY = 200 # In training, log every 200 steps.
-EVE_EXTRA_ROUNDS = 2000 # At end, train eve a bit more.
-RETRAIN_EVE_ITERS = 10000 # Retrain eve up to ITERS*LOOPS times.
-RETRAIN_EVE_LOOPS = 25 # With an evaluation each loop
-NUMBER_OF_EVE_RESETS = 5 # And do this up to 5 times with a fresh eve.
-# Use EVAL_BATCHES samples each time we check accuracy.
-EVAL_BATCHES = 1
-
-
-def batch_of_random_bools(batch_size, n):
- """Return a batch of random "boolean" numbers.
-
- Args:
- batch_size: Batch size dimension of returned tensor.
- n: number of entries per batch.
-
- Returns:
- A [batch_size, n] tensor of "boolean" numbers, where each number is
- preresented as -1 or 1.
- """
-
- as_int = tf.random.uniform(
- [batch_size, n], minval=0, maxval=2, dtype=tf.int32)
- expanded_range = (as_int * 2) - 1
- return tf.cast(expanded_range, tf.float32)
-
-
-class AdversarialCrypto(object):
- """Primary model implementation class for Adversarial Neural Crypto.
-
- This class contains the code for the model itself,
- and when created, plumbs the pathways from Alice to Bob and
- Eve, creates the optimizers and loss functions, etc.
-
- Attributes:
- eve_loss: Eve's loss function.
- bob_loss: Bob's loss function. Different units from eve_loss.
- eve_optimizer: A tf op that runs Eve's optimizer.
- bob_optimizer: A tf op that runs Bob's optimizer.
- bob_reconstruction_loss: Bob's message reconstruction loss,
- which is comparable to eve_loss.
- reset_eve_vars: Execute this op to completely reset Eve.
- """
-
- def get_message_and_key(self):
- """Generate random pseudo-boolean key and message values."""
-
- batch_size = tf.compat.v1.placeholder_with_default(FLAGS.batch_size, shape=[])
-
- in_m = batch_of_random_bools(batch_size, TEXT_SIZE)
- in_k = batch_of_random_bools(batch_size, KEY_SIZE)
- return in_m, in_k
-
- def model(self, collection, message, key=None):
- """The model for Alice, Bob, and Eve. If key=None, the first fully connected layer
- takes only the message as inputs. Otherwise, it uses both the key
- and the message.
-
- Args:
- collection: The graph keys collection to add new vars to.
- message: The input message to process.
- key: The input key (if any) to use.
- """
-
- if key is not None:
- combined_message = tf.concat(axis=1, values=[message, key])
- else:
- combined_message = message
-
- # Ensure that all variables created are in the specified collection.
- with tf.contrib.framework.arg_scope(
- [tf.contrib.layers.fully_connected, tf.contrib.layers.conv2d],
- variables_collections=[collection]):
-
- fc = tf.contrib.layers.fully_connected(
- combined_message,
- TEXT_SIZE + KEY_SIZE,
- biases_initializer=tf.constant_initializer(0.0),
- activation_fn=None)
-
- # Perform a sequence of 1D convolutions (by expanding the message out to 2D
- # and then squeezing it back down).
- fc = tf.expand_dims(fc, 2) # 2D
- fc = tf.expand_dims(fc, 3) # 3D -- conv2d needs a depth
- # 2,1 -> 1,2
- conv = tf.contrib.layers.conv2d(
- fc, 2, 2, 2, 'SAME', activation_fn=tf.nn.sigmoid)
- # 1,2 -> 1, 2
- conv = tf.contrib.layers.conv2d(
- conv, 2, 1, 1, 'SAME', activation_fn=tf.nn.sigmoid)
- # 1,2 -> 1, 1
- conv = tf.contrib.layers.conv2d(
- conv, 1, 1, 1, 'SAME', activation_fn=tf.nn.tanh)
- conv = tf.squeeze(conv, 3)
- conv = tf.squeeze(conv, 2)
- return conv
-
- def __init__(self):
- in_m, in_k = self.get_message_and_key()
- encrypted = self.model('alice', in_m, in_k)
- decrypted = self.model('bob', encrypted, in_k)
- eve_out = self.model('eve', encrypted, None)
-
- self.reset_eve_vars = tf.group(
- *[w.initializer for w in tf.compat.v1.get_collection('eve')])
-
- optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)
-
- # Eve's goal is to decrypt the entire message:
- eve_bits_wrong = tf.reduce_sum(
- tf.abs((eve_out + 1.0) / 2.0 - (in_m + 1.0) / 2.0), [1])
- self.eve_loss = tf.reduce_sum(eve_bits_wrong)
- self.eve_optimizer = optimizer.minimize(
- self.eve_loss, var_list=tf.compat.v1.get_collection('eve'))
-
- # Alice and Bob want to be accurate...
- self.bob_bits_wrong = tf.reduce_sum(
- tf.abs((decrypted + 1.0) / 2.0 - (in_m + 1.0) / 2.0), [1])
- # ... and to not let Eve do better than guessing.
- self.bob_reconstruction_loss = tf.reduce_sum(self.bob_bits_wrong)
- bob_eve_error_deviation = tf.abs(float(TEXT_SIZE) / 2.0 - eve_bits_wrong)
- # 7-9 bits wrong is OK too, so we squish the error function a bit.
- # Without doing this, we often tend to hang out at 0.25 / 7.5 error,
- # and it seems bad to have continued, high communication error.
- bob_eve_loss = tf.reduce_sum(
- tf.square(bob_eve_error_deviation) / (TEXT_SIZE / 2)**2)
-
- # Rescale the losses to [0, 1] per example and combine.
- self.bob_loss = (self.bob_reconstruction_loss / TEXT_SIZE + bob_eve_loss)
-
- self.bob_optimizer = optimizer.minimize(
- self.bob_loss,
- var_list=(tf.compat.v1.get_collection('alice') + tf.compat.v1.get_collection('bob')))
-
-
-def doeval(s, ac, n, itercount):
- """Evaluate the current network on n batches of random examples.
-
- Args:
- s: The current TensorFlow session
- ac: an instance of the AdversarialCrypto class
- n: The number of iterations to run.
- itercount: Iteration count label for logging.
-
- Returns:
- Bob and Eve's loss, as a percent of bits incorrect.
- """
-
- bob_loss_accum = 0
- eve_loss_accum = 0
- for _ in xrange(n):
- bl, el = s.run([ac.bob_reconstruction_loss, ac.eve_loss])
- bob_loss_accum += bl
- eve_loss_accum += el
- bob_loss_percent = bob_loss_accum / (n * FLAGS.batch_size)
- eve_loss_percent = eve_loss_accum / (n * FLAGS.batch_size)
- print('%10d\t%20.2f\t%20.2f'%(itercount, bob_loss_percent, eve_loss_percent))
- sys.stdout.flush()
- return bob_loss_percent, eve_loss_percent
-
-
-def train_until_thresh(s, ac):
- for j in xrange(MAX_TRAINING_LOOPS):
- for _ in xrange(ITERS_PER_ACTOR):
- s.run(ac.bob_optimizer)
- for _ in xrange(ITERS_PER_ACTOR * EVE_MULTIPLIER):
- s.run(ac.eve_optimizer)
- if j % PRINT_EVERY == 0:
- bob_avg_loss, eve_avg_loss = doeval(s, ac, EVAL_BATCHES, j)
- if (bob_avg_loss < BOB_LOSS_THRESH and eve_avg_loss > EVE_LOSS_THRESH):
- print('Target losses achieved.')
- return True
- return False
-
-
-def train_and_evaluate():
- """Run the full training and evaluation loop."""
-
- ac = AdversarialCrypto()
- init = tf.compat.v1.global_variables_initializer()
-
- with tf.compat.v1.Session() as s:
- s.run(init)
- print('# Batch size: ', FLAGS.batch_size)
- print('# %10s\t%20s\t%20s'%("Iter","Bob_Recon_Error","Eve_Recon_Error"))
-
- if train_until_thresh(s, ac):
- for _ in xrange(EVE_EXTRA_ROUNDS):
- s.run(ac.eve_optimizer)
- print('Loss after eve extra training:')
- doeval(s, ac, EVAL_BATCHES * 2, 0)
- for _ in xrange(NUMBER_OF_EVE_RESETS):
- print('Resetting Eve')
- s.run(ac.reset_eve_vars)
- eve_counter = 0
- for _ in xrange(RETRAIN_EVE_LOOPS):
- for _ in xrange(RETRAIN_EVE_ITERS):
- eve_counter += 1
- s.run(ac.eve_optimizer)
- doeval(s, ac, EVAL_BATCHES, eve_counter)
- doeval(s, ac, EVAL_BATCHES, eve_counter)
-
-
-def main(unused_argv):
- # Exit more quietly with Ctrl-C.
- signal.signal(signal.SIGINT, signal.SIG_DFL)
- train_and_evaluate()
-
-
-if __name__ == '__main__':
- tf.compat.v1.app.run()
diff --git a/research/adversarial_logit_pairing/README.md b/research/adversarial_logit_pairing/README.md
deleted file mode 100644
index d3f576836c4e0fb28eee9882906b18d88a90c564..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/README.md
+++ /dev/null
@@ -1,281 +0,0 @@
-
-
-
-
-# Adversarial logit pairing
-
-This directory contains implementation of
-[Adversarial logit pairing](https://arxiv.org/abs/1803.06373) paper as well as
-few models pre-trained on ImageNet and Tiny ImageNet.
-
-Please contact [Alexey Kurakin](https://github.com/AlexeyKurakin) regarding
-this code.
-
-## Pre-requesites
-
-Code dependencies:
-
-* TensorFlow 1.8 and Python 2.7 (other versions may work, but were not tested)
-* [Abseil Python](https://github.com/abseil/abseil-py).
-* Script which converts Tiny Imagenet dataset into TFRecord format also
- depends on [Pandas](https://pandas.pydata.org/).
-
-## Datasets
-
-To use this code you need to download datasets. You only need to download
-those datasets which you're going to use. Following list of datasets is
-supported:
-
-* [ImageNet](http://www.image-net.org/). Follow
- [Preparing the datasets](https://github.com/tensorflow/models/tree/master/research/slim#Data)
- instructions in TF-Slim documentation to download and convert ImageNet dataset
- to TFRecord format.
-
-* [Tiny ImageNet](https://tiny-imagenet.herokuapp.com/).
- To obtain Tiny ImageNet dataset do following:
-
- ```
- # Download zip archive with TinyImagenet
- curl -O http://cs231n.stanford.edu/tiny-imagenet-200.zip
-
- # Extract archive
- unzip tiny-imagenet-200.zip
-
- # Convert dataset to TFRecord format
- mkdir tiny-imagenet-tfrecord
- python tiny_imagenet_converter/converter.py \
- --input_dir=tiny-imagenet-200 \
- --output_dir=tiny-imagenet-tfrecord
- ```
-
-## Running the code
-
-NOTE: Provided code supports distributed training on multiple machines,
-and all provided checkpoints were trained in a distributed way. However it is
-beyond the scope of this document to describe how to do distributed training.
-Readed should refer to
-[other material](https://www.tensorflow.org/deploy/distributed) to learn
-about it.
-
-### Training
-
-Following command runs training:
-
-```
-# Following arguments has to be specified for training:
-# - MAX_NUMBER_OF_TRAINING_STEPS - maximum number of training steps,
-# omit this flag or set it to -1 to have unlimited number of training steps.
-# - MODEL_NAME - name of the model, now only "resnet_v2_50" is supported.
-# - MOVING_AVG_DECAY - decay rate for exponential moving average of the
-# trainable variables. Training with exponential moving average usually
-# leads to better accuracy. Default of 0.9999. -1 disable exponential moving
-# average. Default works well, so typically you set it only if you want
-# to disable this feature.
-# - HYPERPARAMETERS - string with hyperparameters,
-# see model_lib.py for full list of hyperparameters.
-# - DATASET - dataset, either "imagenet" or "tiny_imagenet".
-# - IMAGE_SIZE - size of the image (single number).
-# - OUTPUT_DIRECTORY - directory where to write results.
-# - IMAGENET_DIR - directory with ImageNet dataset in TFRecord format.
-# - TINY_IMAGENET_DIR - directory with Tiny ImageNet dataset in TFRecord format.
-#
-# Note that only one of IMAGENET_DIR or TINY_IMAGENET_DIR has to be provided
-# depending on which dataset you use.
-#
-python train.py \
- --max_steps="${MAX_NUMBER_OF_TRAINING_STEPS}" \
- --model_name="${MODEL_NAME}" \
- --moving_average_decay="${MOVING_AVG_DECAY}" \
- --hparams="${HYPERPARAMETERS}" \
- --dataset="${DATASET}" \
- --dataset_image_size="${IMAGE_SIZE}" \
- --output_dir="${OUTPUT_DIRECTORY}" \
- --imagenet_data_dir="${IMAGENET_DIR}" \
- --tiny_imagenet_data_dir="${TINY_IMAGENET_DIR}"
-```
-
-Full list of training hyperparameters could be found in `model_lib.py`.
-These hyperparameters control learning rate schedule, optimizer, weight decay,
-label smoothing and adversarial training.
-
-Adversarial training is controlled by following hyperparameters:
-
-* `train_adv_method` - method which is used to craft adversarial examples during
- training. Could be one of the following:
-
- * `clean` - perform regular training with clean examples;
- * `pgd_EPS_STEP_NITER` - use non targeted PGD with maximum size of
- perturbation equal to `EPS`, step size equal to `STEP`
- and number of iterations equal to `NITER`. Size of perturbation and step
- size are expected to be integers between 1 and 255.
- * `pgdll_EPS_STEP_NITER` - use targeted PGD, where target class is least
- likely prediction of the network.
- * `pgdrnd_EPS_STEP_NITER` - use targeted PGD, where target class is chosen
- randomly.
-
-* `train_lp_weight` - weight of adversarial logit pairing loss. If zero or
- negarive, then no logit pairing is performed and training is done using
- mixed minibatch PGD. If positive then adversarial logit pairing term is added
- to the loss.
-
-Below is example of how to run training with adversarial logit pairing on
-ImageNet 64x64:
-
-```
-python train.py \
- --model_name="resnet_v2_50" \
- --hparams="train_adv_method=pgdll_16_2_10,train_lp_weight=0.5" \
- --dataset="imagenet" \
- --dataset_image_size=64 \
- --output_dir="/tmp/adv_train" \
- --imagenet_data_dir="${IMAGENET_DIR}"
-```
-
-### Fine tuning
-
-Provided trainin script could be used to fine tune pre-trained checkpoint.
-Following command does this:
-
-```
-# Fine tuning adds following additional arguments:
-# - SCOPES_DO_NOT_LOAD_FROM_CHECKPOINT - comma separates list of scopes of
-# variables, which should not be loadeded from checkpoint (and default
-# initialization should be used instead).
-# SCOPES_DO_NOT_LOAD_FROM_CHECKPOINT should be either same or a subset of
-# LIST_OF_SCOPES_OF_TRAINABLE_VARS.
-# - LIST_OF_SCOPES_OF_TRAINABLE_VARS - comma separated list of scopes of
-# trainable variables. Only variables which are prefixed with these scopes
-# will be trained.
-# - PATH_TO_PRETRAINED_CHECKPOINT - directory with pretrained checkpoint which
-# is used as initialization for fine tuning.
-#
-python train.py \
- --max_steps="${MAX_NUMBER_OF_TRAINING_STEPS}" \
- --model_name="${MODEL_NAME}" \
- --moving_average_decay="${MOVING_AVG_DECAY}" \
- --hparams="${HYPERPARAMETERS}" \
- --dataset="${DATASET}" \
- --dataset_image_size="${IMAGE_SIZE}" \
- --output_dir="${OUTPUT_DIRECTORY}" \
- --imagenet_data_dir="${IMAGENET_DIR}" \
- --tiny_imagenet_data_dir="${TINY_IMAGENET_DIR}" \
- --finetune_exclude_pretrained_scopes="${SCOPES_DO_NOT_LOAD_FROM_CHECKPOINT}" \
- --finetune_trainable_scopes="${LIST_OF_SCOPES_OF_TRAINABLE_VARS}" \
- --finetune_checkpoint_path="${PATH_TO_PRETRAINED_CHECKPOINT}"
-```
-
-Below is an example of how to fine tune last few layers of the model on
-Tiny Imagenet dataset:
-
-```
-python train.py \
- --model_name="resnet_v2_50" \
- --hparams="train_adv_method=pgdll_16_2_10,train_lp_weight=0.5,learning_rate=0.02" \
- --dataset="tiny_imagenet" \
- --dataset_image_size=64 \
- --output_dir="/tmp/adv_finetune" \
- --tiny_imagenet_data_dir="${TINY_IMAGENET_DIR}" \
- --finetune_exclude_pretrained_scopes="resnet_v2_50/logits" \
- --finetune_trainable_scopes="resnet_v2_50/logits,resnet_v2_50/postnorm" \
- --finetune_checkpoint_path="/tmp/adv_train"
-```
-
-### Evaluation
-
-Following command runs evaluation:
-
-```
-# Following arguments should be provided for eval:
-# - TRAINING_DIRECTORY - directory where training checkpoints are saved.
-# - TRAINABLE_SCOPES - when loading checkpoint which was obtained by fine tuning
-# this argument should be the same as LIST_OF_SCOPES_OF_TRAINABLE_VARS
-# during training. Otherwise it should be empty.
-# This is needed to properly load exponential moving average variables.
-# If exponential moving averages are disabled then this flag could be
-# omitted.
-# - EVAL_SUBDIR_NAME - name of the subdirectory inside TRAINING_DIRECTORY
-# where evaluation code will be saving event files.
-# - DATASET - name of the dataset.
-# - IMAGE_SIZE - size of the image in the dataset.
-# - DATSET_SPLIT_NAME - name of the split in the dataset,
-# either 'train' or 'validation'. Default is 'validation'.
-# - MODEL_NAME - name of the model.
-# - MOVING_AVG_DECAY - decay rate for exponential moving average.
-# - ADV_METHOD_FOR_EVAL - should be "clean" to evaluate on clean example or
-# description of the adversarial method to evaluate on adversarial examples.
-# - HYPERPARAMETERS - hyperparameters, only "eval_batch_size" matters for eval
-# - NUMBER_OF_EXAMPLES - how many examples from the dataset use for evaluation,
-# specify -1 to use all examples.
-# - EVAL_ONCE - if True then evaluate only once, otherwise keep evaluation
-# running repeatedly on new checkpoints. Repeated evaluation might be useful
-# when running concurrent with training.
-# - IMAGENET_DIR - directory with ImageNet dataset in TFRecord format.
-# - TINY_IMAGENET_DIR - directory with Tiny ImageNet dataset in TFRecord format.
-#
-python eval.py \
- --train_dir="${TRAINING_DIRECTORY} \
- --trainable_scopes="${TRAINABLE_SCOPES}" \
- --eval_name="${EVAL_SUBDIR_NAME}" \
- --dataset="${DATASET}" \
- --dataset_image_size="${IMAGE_SIZE}" \
- --split_name="${DATSET_SPLIT_NAME}" \
- --model_name="${MODEL_NAME}" \
- --moving_average_decay="${MOVING_AVG_DECAY}" \
- --adv_method="${ADV_METHOD_FOR_EVAL}" \
- --hparams="${HYPERPARAMETERS}" \
- --num_examples="${NUMBER_OF_EXAMPLES}" \
- --eval_once="${EVAL_ONCE}" \
- --imagenet_data_dir="${IMAGENET_DIR}" \
- --tiny_imagenet_data_dir="${TINY_IMAGENET_DIR}"
-```
-
-Example of running evaluation on 10000 of clean examples from ImageNet
-training set:
-
-```
-python eval.py \
- --train_dir=/tmp/adv_train \
- --dataset=imagenet \
- --dataset_image_size=64 \
- --split_name=train \
- --adv_method=clean \
- --hparams="eval_batch_size=50" \
- --num_examples=10000 \
- --eval_once=True \
- --imagenet_data_dir="${IMAGENET_DIR}"
-```
-
-Example of running evaluatin on adversarial images generated from Tiny ImageNet
-validation set using fine-tuned checkpoint:
-
-```
-python eval.py \
- --train_dir=tmp/adv_finetune \
- --trainable_scopes="resnet_v2_50/logits,resnet_v2_50/postnorm" \
- --dataset=tiny_imagenet \
- --dataset_image_size=64 \
- --adv_method=pgdrnd_16_2_10 \
- --hparams="eval_batch_size=50" \
- --eval_once=True \
- --tiny_imagenet_data_dir="${TINY_IMAGENET_DIR}"
-```
-
-### Pre-trained models
-
-Following set of pre-trained checkpoints released with this code:
-
-| Model | Dataset | Accuracy on clean images | Accuracy on `pgdll_16_1_20` | Accuracy on `pgdll_16_2_10` |
-| ----------- | ------------ | --------------- | --------------------------- | -------------- |
-| [Baseline ResNet-v2-50](http://download.tensorflow.org/models/adversarial_logit_pairing/imagenet64_base_2018_06_26.ckpt.tar.gz) | ImageNet 64x64 | 60.5% | 1.8% | 3.5% |
-| [ALP-trained ResNet-v2-50](http://download.tensorflow.org/models/adversarial_logit_pairing/imagenet64_alp025_2018_06_26.ckpt.tar.gz) | ImageNet 64x64 | 55.7% | 27.5% | 27.8% |
-| [Baseline ResNet-v2-50](http://download.tensorflow.org/models/adversarial_logit_pairing/tiny_imagenet_base_2018_06_26.ckpt.tar.gz) | Tiny ImageNet | 69.2% | 0.1% | 0.3% |
-| [ALP-trained ResNet-v2-50](http://download.tensorflow.org/models/adversarial_logit_pairing/tiny_imagenet_alp05_2018_06_26.ckpt.tar.gz) | Tiny ImageNet | 72.0% | 41.3% | 40.8% |
-
-
-* All provided checkpoints were initially trained with exponential moving
- average. However for ease of use they were re-saved without it.
- So to load and use provided checkpoints you need to specify
- `--moving_average_decay=-1` flag.
-* All ALP models were trained with `pgdll_16_2_10` adversarial examples.
-* All Tiny Imagenet models were obtained by fine tuning corresponding
- ImageNet 64x64 models. ALP-trained models were fine tuned with ALP.
diff --git a/research/adversarial_logit_pairing/adversarial_attack.py b/research/adversarial_logit_pairing/adversarial_attack.py
deleted file mode 100644
index 804bd64bcf4444007638f9802a83973ee68eb3cf..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/adversarial_attack.py
+++ /dev/null
@@ -1,219 +0,0 @@
-# Copyright 2018 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Library with adversarial attacks.
-
-This library designed to be self-contained and have no dependencies other
-than TensorFlow. It only contains PGD / Iterative FGSM attacks,
-see https://arxiv.org/abs/1706.06083 and https://arxiv.org/abs/1607.02533
-for details.
-
-For wider set of adversarial attacks refer to Cleverhans library:
-https://github.com/tensorflow/cleverhans
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-
-def generate_pgd_common(x,
- bounds,
- model_fn,
- attack_params,
- one_hot_labels,
- perturbation_multiplier):
- """Common code for generating PGD adversarial examples.
-
- Args:
- x: original examples.
- bounds: tuple with bounds of image values, bounds[0] < bounds[1].
- model_fn: model function with signature model_fn(images).
- attack_params: parameters of the attack.
- one_hot_labels: one hot label vector to use in the loss.
- perturbation_multiplier: multiplier of adversarial perturbation,
- either +1.0 or -1.0.
-
- Returns:
- Tensor with adversarial examples.
-
- Raises:
- ValueError: if attack parameters are invalid.
- """
- # parse attack_params
- # Format of attack_params: 'EPS_STEP_NITER'
- # where EPS - epsilon, STEP - step size, NITER - number of iterations
- params_list = attack_params.split('_')
- if len(params_list) != 3:
- raise ValueError('Invalid parameters of PGD attack: %s' % attack_params)
- epsilon = int(params_list[0])
- step_size = int(params_list[1])
- niter = int(params_list[2])
-
- # rescale epsilon and step size to image bounds
- epsilon = float(epsilon) / 255.0 * (bounds[1] - bounds[0])
- step_size = float(step_size) / 255.0 * (bounds[1] - bounds[0])
-
- # clipping boundaries
- clip_min = tf.maximum(x - epsilon, bounds[0])
- clip_max = tf.minimum(x + epsilon, bounds[1])
-
- # compute starting point
- start_x = x + tf.random_uniform(tf.shape(x), -epsilon, epsilon)
- start_x = tf.clip_by_value(start_x, clip_min, clip_max)
-
- # main iteration of PGD
- loop_vars = [0, start_x]
-
- def loop_cond(index, _):
- return index < niter
-
- def loop_body(index, adv_images):
- logits = model_fn(adv_images)
- loss = tf.reduce_sum(
- tf.nn.softmax_cross_entropy_with_logits_v2(
- labels=one_hot_labels,
- logits=logits))
- perturbation = step_size * tf.sign(tf.gradients(loss, adv_images)[0])
- new_adv_images = adv_images + perturbation_multiplier * perturbation
- new_adv_images = tf.clip_by_value(new_adv_images, clip_min, clip_max)
- return index + 1, new_adv_images
-
- with tf.control_dependencies([start_x]):
- _, result = tf.while_loop(
- loop_cond,
- loop_body,
- loop_vars,
- back_prop=False,
- parallel_iterations=1)
- return result
-
-
-def generate_pgd_ll(x, bounds, model_fn, attack_params):
- # pylint: disable=g-doc-args
- """Generats targeted PGD adversarial examples with least likely target class.
-
- See generate_pgd_common for description of arguments.
-
- Returns:
- Tensor with adversarial examples.
- """
- # pylint: enable=g-doc-args
-
- # compute one hot least likely class
- logits = model_fn(x)
- num_classes = tf.shape(logits)[1]
- one_hot_labels = tf.one_hot(tf.argmin(model_fn(x), axis=1), num_classes)
-
- return generate_pgd_common(x, bounds, model_fn, attack_params,
- one_hot_labels=one_hot_labels,
- perturbation_multiplier=-1.0)
-
-
-def generate_pgd_rand(x, bounds, model_fn, attack_params):
- # pylint: disable=g-doc-args
- """Generats targeted PGD adversarial examples with random target class.
-
- See generate_pgd_common for description of arguments.
-
- Returns:
- Tensor with adversarial examples.
- """
- # pylint: enable=g-doc-args
-
- # compute one hot random class
- logits = model_fn(x)
- batch_size = tf.shape(logits)[0]
- num_classes = tf.shape(logits)[1]
- random_labels = tf.random_uniform(shape=[batch_size],
- minval=0,
- maxval=num_classes,
- dtype=tf.int32)
- one_hot_labels = tf.one_hot(random_labels, num_classes)
-
- return generate_pgd_common(x, bounds, model_fn, attack_params,
- one_hot_labels=one_hot_labels,
- perturbation_multiplier=-1.0)
-
-
-def generate_pgd(x, bounds, model_fn, attack_params):
- # pylint: disable=g-doc-args
- """Generats non-targeted PGD adversarial examples.
-
- See generate_pgd_common for description of arguments.
-
- Returns:
- tensor with adversarial examples.
- """
- # pylint: enable=g-doc-args
-
- # compute one hot predicted class
- logits = model_fn(x)
- num_classes = tf.shape(logits)[1]
- one_hot_labels = tf.one_hot(tf.argmax(model_fn(x), axis=1), num_classes)
-
- return generate_pgd_common(x, bounds, model_fn, attack_params,
- one_hot_labels=one_hot_labels,
- perturbation_multiplier=1.0)
-
-
-def generate_adversarial_examples(x, bounds, model_fn, attack_description):
- """Generates adversarial examples.
-
- Args:
- x: original examples.
- bounds: tuple with bounds of image values, bounds[0] < bounds[1]
- model_fn: model function with signature model_fn(images).
- attack_description: string which describes an attack, see notes below for
- details.
-
- Returns:
- Tensor with adversarial examples.
-
- Raises:
- ValueError: if attack description is invalid.
-
-
- Attack description could be one of the following strings:
- - "clean" - no attack, return original images.
- - "pgd_EPS_STEP_NITER" - non-targeted PGD attack.
- - "pgdll_EPS_STEP_NITER" - tageted PGD attack with least likely target class.
- - "pgdrnd_EPS_STEP_NITER" - targetd PGD attack with random target class.
-
- Meaning of attack parameters is following:
- - EPS - maximum size of adversarial perturbation, between 0 and 255.
- - STEP - step size of one iteration of PGD, between 0 and 255.
- - NITER - number of iterations.
- """
- if attack_description == 'clean':
- return x
- idx = attack_description.find('_')
- if idx < 0:
- raise ValueError('Invalid value of attack description %s'
- % attack_description)
- attack_name = attack_description[:idx]
- attack_params = attack_description[idx+1:]
- if attack_name == 'pgdll':
- return generate_pgd_ll(x, bounds, model_fn, attack_params)
- elif attack_name == 'pgdrnd':
- return generate_pgd_rand(x, bounds, model_fn, attack_params)
- elif attack_name == 'pgd':
- return generate_pgd(x, bounds, model_fn, attack_params)
- else:
- raise ValueError('Invalid value of attack description %s'
- % attack_description)
-
diff --git a/research/adversarial_logit_pairing/datasets/__init__.py b/research/adversarial_logit_pairing/datasets/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/adversarial_logit_pairing/datasets/dataset_factory.py b/research/adversarial_logit_pairing/datasets/dataset_factory.py
deleted file mode 100644
index 01c36d4ff4710e1742e989b20a3daef75a6922e1..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/datasets/dataset_factory.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright 2018 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Library which creates datasets."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from datasets import imagenet_input
-from datasets import tiny_imagenet_input
-
-
-def get_dataset(dataset_name, split, batch_size, image_size, is_training):
- """Returns dataset.
-
- Args:
- dataset_name: name of the dataset, "imagenet" or "tiny_imagenet".
- split: name of the split, "train" or "validation".
- batch_size: size of the minibatch.
- image_size: size of the one side of the image. Output images will be
- resized to square shape image_size*image_size.
- is_training: if True then training preprocessing is done, otherwise eval
- preprocessing is done.
-
- Raises:
- ValueError: if dataset_name is invalid.
-
- Returns:
- dataset: instance of tf.data.Dataset with the dataset.
- num_examples: number of examples in given split of the dataset.
- num_classes: number of classes in the dataset.
- bounds: tuple with bounds of image values. All returned image pixels
- are between bounds[0] and bounds[1].
- """
- if dataset_name == 'tiny_imagenet':
- dataset = tiny_imagenet_input.tiny_imagenet_input(
- split, batch_size, image_size, is_training)
- num_examples = tiny_imagenet_input.num_examples_per_epoch(split)
- num_classes = 200
- bounds = (-1, 1)
- elif dataset_name == 'imagenet':
- dataset = imagenet_input.imagenet_input(
- split, batch_size, image_size, is_training)
- num_examples = imagenet_input.num_examples_per_epoch(split)
- num_classes = 1001
- bounds = (-1, 1)
- else:
- raise ValueError('Invalid dataset %s' % dataset_name)
- return dataset, num_examples, num_classes, bounds
diff --git a/research/adversarial_logit_pairing/datasets/imagenet_input.py b/research/adversarial_logit_pairing/datasets/imagenet_input.py
deleted file mode 100644
index 0b210b8ce11f3dbf1f14482b1b4f3a95da02a48a..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/datasets/imagenet_input.py
+++ /dev/null
@@ -1,255 +0,0 @@
-# Copyright 2018 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Imagenet input."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-from absl import flags
-import tensorflow as tf
-
-FLAGS = flags.FLAGS
-
-
-flags.DEFINE_string('imagenet_data_dir', None,
- 'Directory with Imagenet dataset in TFRecord format.')
-
-
-def _decode_and_random_crop(image_buffer, bbox, image_size):
- """Randomly crops image and then scales to target size."""
- with tf.name_scope('distorted_bounding_box_crop',
- values=[image_buffer, bbox]):
- sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
- tf.image.extract_jpeg_shape(image_buffer),
- bounding_boxes=bbox,
- min_object_covered=0.1,
- aspect_ratio_range=[0.75, 1.33],
- area_range=[0.08, 1.0],
- max_attempts=10,
- use_image_if_no_bounding_boxes=True)
- bbox_begin, bbox_size, _ = sample_distorted_bounding_box
-
- # Crop the image to the specified bounding box.
- offset_y, offset_x, _ = tf.unstack(bbox_begin)
- target_height, target_width, _ = tf.unstack(bbox_size)
- crop_window = tf.stack([offset_y, offset_x, target_height, target_width])
- image = tf.image.decode_and_crop_jpeg(image_buffer, crop_window, channels=3)
- image = tf.image.convert_image_dtype(
- image, dtype=tf.float32)
-
- image = tf.image.resize_bicubic([image],
- [image_size, image_size])[0]
-
- return image
-
-
-def _decode_and_center_crop(image_buffer, image_size):
- """Crops to center of image with padding then scales to target size."""
- shape = tf.image.extract_jpeg_shape(image_buffer)
- image_height = shape[0]
- image_width = shape[1]
-
- padded_center_crop_size = tf.cast(
- 0.875 * tf.cast(tf.minimum(image_height, image_width), tf.float32),
- tf.int32)
-
- offset_height = ((image_height - padded_center_crop_size) + 1) // 2
- offset_width = ((image_width - padded_center_crop_size) + 1) // 2
- crop_window = tf.stack([offset_height, offset_width,
- padded_center_crop_size, padded_center_crop_size])
- image = tf.image.decode_and_crop_jpeg(image_buffer, crop_window, channels=3)
- image = tf.image.convert_image_dtype(
- image, dtype=tf.float32)
-
- image = tf.image.resize_bicubic([image],
- [image_size, image_size])[0]
-
- return image
-
-
-def _normalize(image):
- """Rescale image to [-1, 1] range."""
- return tf.multiply(tf.subtract(image, 0.5), 2.0)
-
-
-def image_preprocessing(image_buffer, bbox, image_size, is_training):
- """Does image decoding and preprocessing.
-
- Args:
- image_buffer: string tensor with encoded image.
- bbox: bounding box of the object at the image.
- image_size: image size.
- is_training: whether to do training or eval preprocessing.
-
- Returns:
- Tensor with the image.
- """
- if is_training:
- image = _decode_and_random_crop(image_buffer, bbox, image_size)
- image = _normalize(image)
- image = tf.image.random_flip_left_right(image)
- else:
- image = _decode_and_center_crop(image_buffer, image_size)
- image = _normalize(image)
- image = tf.reshape(image, [image_size, image_size, 3])
- return image
-
-
-def imagenet_parser(value, image_size, is_training):
- """Parse an ImageNet record from a serialized string Tensor.
-
- Args:
- value: encoded example.
- image_size: size of the output image.
- is_training: if True then do training preprocessing,
- otherwise do eval preprocessing.
-
- Returns:
- image: tensor with the image.
- label: true label of the image.
- """
- keys_to_features = {
- 'image/encoded':
- tf.FixedLenFeature((), tf.string, ''),
- 'image/format':
- tf.FixedLenFeature((), tf.string, 'jpeg'),
- 'image/class/label':
- tf.FixedLenFeature([], tf.int64, -1),
- 'image/class/text':
- tf.FixedLenFeature([], tf.string, ''),
- 'image/object/bbox/xmin':
- tf.VarLenFeature(dtype=tf.float32),
- 'image/object/bbox/ymin':
- tf.VarLenFeature(dtype=tf.float32),
- 'image/object/bbox/xmax':
- tf.VarLenFeature(dtype=tf.float32),
- 'image/object/bbox/ymax':
- tf.VarLenFeature(dtype=tf.float32),
- 'image/object/class/label':
- tf.VarLenFeature(dtype=tf.int64),
- }
-
- parsed = tf.parse_single_example(value, keys_to_features)
-
- image_buffer = tf.reshape(parsed['image/encoded'], shape=[])
-
- xmin = tf.expand_dims(parsed['image/object/bbox/xmin'].values, 0)
- ymin = tf.expand_dims(parsed['image/object/bbox/ymin'].values, 0)
- xmax = tf.expand_dims(parsed['image/object/bbox/xmax'].values, 0)
- ymax = tf.expand_dims(parsed['image/object/bbox/ymax'].values, 0)
- # Note that ordering is (y, x)
- bbox = tf.concat([ymin, xmin, ymax, xmax], 0)
- # Force the variable number of bounding boxes into the shape
- # [1, num_boxes, coords].
- bbox = tf.expand_dims(bbox, 0)
- bbox = tf.transpose(bbox, [0, 2, 1])
-
- image = image_preprocessing(
- image_buffer=image_buffer,
- bbox=bbox,
- image_size=image_size,
- is_training=is_training
- )
-
- # Labels are in [1, 1000] range
- label = tf.cast(
- tf.reshape(parsed['image/class/label'], shape=[]), dtype=tf.int32)
-
- return image, label
-
-
-def imagenet_input(split, batch_size, image_size, is_training):
- """Returns ImageNet dataset.
-
- Args:
- split: name of the split, "train" or "validation".
- batch_size: size of the minibatch.
- image_size: size of the one side of the image. Output images will be
- resized to square shape image_size*image_size.
- is_training: if True then training preprocessing is done, otherwise eval
- preprocessing is done.
-
- Raises:
- ValueError: if name of the split is incorrect.
-
- Returns:
- Instance of tf.data.Dataset with the dataset.
- """
- if split.lower().startswith('train'):
- file_pattern = os.path.join(FLAGS.imagenet_data_dir, 'train-*')
- elif split.lower().startswith('validation'):
- file_pattern = os.path.join(FLAGS.imagenet_data_dir, 'validation-*')
- else:
- raise ValueError('Invalid split: %s' % split)
-
- dataset = tf.data.Dataset.list_files(file_pattern, shuffle=is_training)
-
- if is_training:
- dataset = dataset.repeat()
-
- def fetch_dataset(filename):
- return tf.data.TFRecordDataset(filename, buffer_size=8*1024*1024)
-
- # Read the data from disk in parallel
- dataset = dataset.apply(
- tf.data.experimental.parallel_interleave(
- fetch_dataset, cycle_length=4, sloppy=True))
- dataset = dataset.shuffle(1024)
-
- # Parse, preprocess, and batch the data in parallel
- dataset = dataset.apply(
- tf.data.experimental.map_and_batch(
- lambda value: imagenet_parser(value, image_size, is_training),
- batch_size=batch_size,
- num_parallel_batches=4,
- drop_remainder=True))
-
- def set_shapes(images, labels):
- """Statically set the batch_size dimension."""
- images.set_shape(images.get_shape().merge_with(
- tf.TensorShape([batch_size, None, None, None])))
- labels.set_shape(labels.get_shape().merge_with(
- tf.TensorShape([batch_size])))
- return images, labels
-
- # Assign static batch size dimension
- dataset = dataset.map(set_shapes)
-
- # Prefetch overlaps in-feed with training
- dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
- return dataset
-
-
-def num_examples_per_epoch(split):
- """Returns the number of examples in the data set.
-
- Args:
- split: name of the split, "train" or "validation".
-
- Raises:
- ValueError: if split name is incorrect.
-
- Returns:
- Number of example in the split.
- """
- if split.lower().startswith('train'):
- return 1281167
- elif split.lower().startswith('validation'):
- return 50000
- else:
- raise ValueError('Invalid split: %s' % split)
diff --git a/research/adversarial_logit_pairing/datasets/tiny_imagenet_input.py b/research/adversarial_logit_pairing/datasets/tiny_imagenet_input.py
deleted file mode 100644
index 6d216d53ed0bd9f6e7a5770510cedc7f3d9f0a42..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/datasets/tiny_imagenet_input.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# Copyright 2018 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tiny imagenet input."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-from absl import flags
-import tensorflow as tf
-
-FLAGS = flags.FLAGS
-
-
-flags.DEFINE_string('tiny_imagenet_data_dir', None,
- 'Directory with Tiny Imagenet dataset in TFRecord format.')
-
-
-def tiny_imagenet_parser(value, image_size, is_training):
- """Parses tiny imagenet example.
-
- Args:
- value: encoded example.
- image_size: size of the image.
- is_training: if True then do training preprocessing (which includes
- random cropping), otherwise do eval preprocessing.
-
- Returns:
- image: tensor with the image.
- label: true label of the image.
- """
- keys_to_features = {
- 'image/encoded': tf.FixedLenFeature((), tf.string, ''),
- 'label/tiny_imagenet': tf.FixedLenFeature([], tf.int64, -1),
- }
-
- parsed = tf.parse_single_example(value, keys_to_features)
-
- image_buffer = tf.reshape(parsed['image/encoded'], shape=[])
- image = tf.image.decode_image(image_buffer, channels=3)
- image = tf.image.convert_image_dtype(
- image, dtype=tf.float32)
-
- # Crop image
- if is_training:
- bbox_begin, bbox_size, _ = tf.image.sample_distorted_bounding_box(
- tf.shape(image),
- bounding_boxes=tf.constant([0.0, 0.0, 1.0, 1.0],
- dtype=tf.float32,
- shape=[1, 1, 4]),
- min_object_covered=0.5,
- aspect_ratio_range=[0.75, 1.33],
- area_range=[0.5, 1.0],
- max_attempts=20,
- use_image_if_no_bounding_boxes=True)
- image = tf.slice(image, bbox_begin, bbox_size)
-
- # resize image
- image = tf.image.resize_bicubic([image], [image_size, image_size])[0]
-
- # Rescale image to [-1, 1] range.
- image = tf.multiply(tf.subtract(image, 0.5), 2.0)
-
- image = tf.reshape(image, [image_size, image_size, 3])
-
- # Labels are in [0, 199] range
- label = tf.cast(
- tf.reshape(parsed['label/tiny_imagenet'], shape=[]), dtype=tf.int32)
-
- return image, label
-
-
-def tiny_imagenet_input(split, batch_size, image_size, is_training):
- """Returns Tiny Imagenet Dataset.
-
- Args:
- split: name of the split, "train" or "validation".
- batch_size: size of the minibatch.
- image_size: size of the one side of the image. Output images will be
- resized to square shape image_size*image_size.
- is_training: if True then training preprocessing is done, otherwise eval
- preprocessing is done.instance of tf.data.Dataset with the dataset.
-
- Raises:
- ValueError: if name of the split is incorrect.
-
- Returns:
- Instance of tf.data.Dataset with the dataset.
- """
- if split.lower().startswith('train'):
- filepath = os.path.join(FLAGS.tiny_imagenet_data_dir, 'train.tfrecord')
- elif split.lower().startswith('validation'):
- filepath = os.path.join(FLAGS.tiny_imagenet_data_dir, 'validation.tfrecord')
- else:
- raise ValueError('Invalid split: %s' % split)
-
- dataset = tf.data.TFRecordDataset(filepath, buffer_size=8*1024*1024)
-
- if is_training:
- dataset = dataset.shuffle(10000)
- dataset = dataset.repeat()
-
- dataset = dataset.apply(
- tf.data.experimental.map_and_batch(
- lambda value: tiny_imagenet_parser(value, image_size, is_training),
- batch_size=batch_size,
- num_parallel_batches=4,
- drop_remainder=True))
-
- def set_shapes(images, labels):
- """Statically set the batch_size dimension."""
- images.set_shape(images.get_shape().merge_with(
- tf.TensorShape([batch_size, None, None, None])))
- labels.set_shape(labels.get_shape().merge_with(
- tf.TensorShape([batch_size])))
- return images, labels
-
- # Assign static batch size dimension
- dataset = dataset.map(set_shapes)
-
- dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
-
- return dataset
-
-
-def num_examples_per_epoch(split):
- """Returns the number of examples in the data set.
-
- Args:
- split: name of the split, "train" or "validation".
-
- Raises:
- ValueError: if split name is incorrect.
-
- Returns:
- Number of example in the split.
- """
- if split.lower().startswith('train'):
- return 100000
- elif split.lower().startswith('validation'):
- return 10000
- else:
- raise ValueError('Invalid split: %s' % split)
diff --git a/research/adversarial_logit_pairing/eval.py b/research/adversarial_logit_pairing/eval.py
deleted file mode 100644
index 504cc0b0bcf52edff9e7aaa2c0d051079ba521aa..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/eval.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright 2018 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Program which runs evaluation of Imagenet 64x64 and TinyImagenet models."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-from absl import app
-from absl import flags
-
-import tensorflow as tf
-
-import adversarial_attack
-import model_lib
-from datasets import dataset_factory
-
-FLAGS = flags.FLAGS
-
-
-flags.DEFINE_string('train_dir', None,
- 'Training directory. If specified then this program '
- 'runs in continuous evaluation mode.')
-
-flags.DEFINE_string('checkpoint_path', None,
- 'Path to the file with checkpoint. If specified then '
- 'this program evaluates only provided checkpoint one time.')
-
-flags.DEFINE_string('output_file', None,
- 'Name of output file. Used only in single evaluation mode.')
-
-flags.DEFINE_string('eval_name', 'default', 'Name for eval subdirectory.')
-
-flags.DEFINE_string('master', '', 'Tensorflow master.')
-
-flags.DEFINE_string('model_name', 'resnet_v2_50', 'Name of the model.')
-
-flags.DEFINE_string('adv_method', 'clean',
- 'Method which is used to generate adversarial examples.')
-
-flags.DEFINE_string('dataset', 'imagenet',
- 'Dataset: "tiny_imagenet" or "imagenet".')
-
-flags.DEFINE_integer('dataset_image_size', 64,
- 'Size of the images in the dataset.')
-
-flags.DEFINE_string('hparams', '', 'Hyper parameters.')
-
-flags.DEFINE_string('split_name', 'validation', 'Name of the split.')
-
-flags.DEFINE_float('moving_average_decay', 0.9999,
- 'The decay to use for the moving average.')
-
-flags.DEFINE_integer('eval_interval_secs', 120,
- 'The frequency, in seconds, with which evaluation is run.')
-
-flags.DEFINE_integer(
- 'num_examples', -1,
- 'If positive - maximum number of example to use for evaluation.')
-
-flags.DEFINE_bool('eval_once', False,
- 'If true then evaluate model only once.')
-
-flags.DEFINE_string('trainable_scopes', None,
- 'If set then it defines list of variable scopes for '
- 'trainable variables.')
-
-
-def main(_):
- if not FLAGS.train_dir and not FLAGS.checkpoint_path:
- print('Either --train_dir or --checkpoint_path flags has to be provided.')
- if FLAGS.train_dir and FLAGS.checkpoint_path:
- print('Only one of --train_dir or --checkpoint_path should be provided.')
- params = model_lib.default_hparams()
- params.parse(FLAGS.hparams)
- tf.logging.info('User provided hparams: %s', FLAGS.hparams)
- tf.logging.info('All hyper parameters: %s', params)
- batch_size = params.eval_batch_size
- graph = tf.Graph()
- with graph.as_default():
- # dataset
- dataset, num_examples, num_classes, bounds = dataset_factory.get_dataset(
- FLAGS.dataset,
- FLAGS.split_name,
- batch_size,
- FLAGS.dataset_image_size,
- is_training=False)
- dataset_iterator = dataset.make_one_shot_iterator()
- images, labels = dataset_iterator.get_next()
- if FLAGS.num_examples > 0:
- num_examples = min(num_examples, FLAGS.num_examples)
-
- # setup model
- global_step = tf.train.get_or_create_global_step()
- model_fn_two_args = model_lib.get_model(FLAGS.model_name, num_classes)
- model_fn = lambda x: model_fn_two_args(x, is_training=False)
- if not FLAGS.adv_method or FLAGS.adv_method == 'clean':
- logits = model_fn(images)
- else:
- adv_examples = adversarial_attack.generate_adversarial_examples(
- images, bounds, model_fn, FLAGS.adv_method)
- logits = model_fn(adv_examples)
-
- # update trainable variables if fine tuning is used
- model_lib.filter_trainable_variables(FLAGS.trainable_scopes)
-
- # Setup the moving averages
- if FLAGS.moving_average_decay and (FLAGS.moving_average_decay > 0):
- variable_averages = tf.train.ExponentialMovingAverage(
- FLAGS.moving_average_decay, global_step)
- variables_to_restore = variable_averages.variables_to_restore(
- tf.contrib.framework.get_model_variables())
- variables_to_restore[global_step.op.name] = global_step
- else:
- variables_to_restore = tf.contrib.framework.get_variables_to_restore()
-
- # Setup evaluation metric
- with tf.name_scope('Eval'):
- names_to_values, names_to_updates = (
- tf.contrib.metrics.aggregate_metric_map({
- 'Accuracy': tf.metrics.accuracy(labels, tf.argmax(logits, 1)),
- 'Top5': tf.metrics.recall_at_k(tf.to_int64(labels), logits, 5)
- }))
-
- for name, value in names_to_values.iteritems():
- tf.summary.scalar(name, value)
-
- # Run evaluation
- num_batches = int(num_examples / batch_size)
- if FLAGS.train_dir:
- output_dir = os.path.join(FLAGS.train_dir, FLAGS.eval_name)
- if not tf.gfile.Exists(output_dir):
- tf.gfile.MakeDirs(output_dir)
- tf.contrib.training.evaluate_repeatedly(
- FLAGS.train_dir,
- master=FLAGS.master,
- scaffold=tf.train.Scaffold(
- saver=tf.train.Saver(variables_to_restore)),
- eval_ops=names_to_updates.values(),
- eval_interval_secs=FLAGS.eval_interval_secs,
- hooks=[
- tf.contrib.training.StopAfterNEvalsHook(num_batches),
- tf.contrib.training.SummaryAtEndHook(output_dir),
- tf.train.LoggingTensorHook(names_to_values, at_end=True),
- ],
- max_number_of_evaluations=1 if FLAGS.eval_once else None)
- else:
- result = tf.contrib.training.evaluate_once(
- FLAGS.checkpoint_path,
- master=FLAGS.master,
- scaffold=tf.train.Scaffold(
- saver=tf.train.Saver(variables_to_restore)),
- eval_ops=names_to_updates.values(),
- final_ops=names_to_values,
- hooks=[
- tf.contrib.training.StopAfterNEvalsHook(num_batches),
- tf.train.LoggingTensorHook(names_to_values, at_end=True),
- ])
- if FLAGS.output_file:
- with tf.gfile.Open(FLAGS.output_file, 'a') as f:
- f.write('%s,%.3f,%.3f\n'
- % (FLAGS.eval_name, result['Accuracy'], result['Top5']))
-
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/adversarial_logit_pairing/model_lib.py b/research/adversarial_logit_pairing/model_lib.py
deleted file mode 100644
index 1499a378ea1ba6511122ebe54ceed1226d38d649..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/model_lib.py
+++ /dev/null
@@ -1,189 +0,0 @@
-# Copyright 2018 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Library with common functions for training and eval."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import six
-
-import tensorflow as tf
-
-from tensorflow.contrib.slim.nets import resnet_v2
-
-
-def default_hparams():
- """Returns default hyperparameters."""
- return tf.contrib.training.HParams(
- # Batch size for training and evaluation.
- batch_size=32,
- eval_batch_size=50,
-
- # General training parameters.
- weight_decay=0.0001,
- label_smoothing=0.1,
-
- # Parameters of the adversarial training.
- train_adv_method='clean', # adversarial training method
- train_lp_weight=0.0, # Weight of adversarial logit pairing loss
-
- # Parameters of the optimizer.
- optimizer='rms', # possible values are: 'rms', 'momentum', 'adam'
- momentum=0.9, # momentum
- rmsprop_decay=0.9, # Decay term for RMSProp
- rmsprop_epsilon=1.0, # Epsilon term for RMSProp
-
- # Parameters of learning rate schedule.
- lr_schedule='exp_decay', # Possible values: 'exp_decay', 'step', 'fixed'
- learning_rate=0.045,
- lr_decay_factor=0.94, # Learning exponential decay
- lr_num_epochs_per_decay=2.0, # Number of epochs per lr decay
- lr_list=[1.0 / 6, 2.0 / 6, 3.0 / 6,
- 4.0 / 6, 5.0 / 6, 1.0, 0.1, 0.01,
- 0.001, 0.0001],
- lr_decay_epochs=[1, 2, 3, 4, 5, 30, 60, 80,
- 90])
-
-
-def get_lr_schedule(hparams, examples_per_epoch, replicas_to_aggregate=1):
- """Returns TensorFlow op which compute learning rate.
-
- Args:
- hparams: hyper parameters.
- examples_per_epoch: number of training examples per epoch.
- replicas_to_aggregate: number of training replicas running in parallel.
-
- Raises:
- ValueError: if learning rate schedule specified in hparams is incorrect.
-
- Returns:
- learning_rate: tensor with learning rate.
- steps_per_epoch: number of training steps per epoch.
- """
- global_step = tf.train.get_or_create_global_step()
- steps_per_epoch = float(examples_per_epoch) / float(hparams.batch_size)
- if replicas_to_aggregate > 0:
- steps_per_epoch /= replicas_to_aggregate
-
- if hparams.lr_schedule == 'exp_decay':
- decay_steps = long(steps_per_epoch * hparams.lr_num_epochs_per_decay)
- learning_rate = tf.train.exponential_decay(
- hparams.learning_rate,
- global_step,
- decay_steps,
- hparams.lr_decay_factor,
- staircase=True)
- elif hparams.lr_schedule == 'step':
- lr_decay_steps = [long(epoch * steps_per_epoch)
- for epoch in hparams.lr_decay_epochs]
- learning_rate = tf.train.piecewise_constant(
- global_step, lr_decay_steps, hparams.lr_list)
- elif hparams.lr_schedule == 'fixed':
- learning_rate = hparams.learning_rate
- else:
- raise ValueError('Invalid value of lr_schedule: %s' % hparams.lr_schedule)
-
- if replicas_to_aggregate > 0:
- learning_rate *= replicas_to_aggregate
-
- return learning_rate, steps_per_epoch
-
-
-def get_optimizer(hparams, learning_rate):
- """Returns optimizer.
-
- Args:
- hparams: hyper parameters.
- learning_rate: learning rate tensor.
-
- Raises:
- ValueError: if type of optimizer specified in hparams is incorrect.
-
- Returns:
- Instance of optimizer class.
- """
- if hparams.optimizer == 'rms':
- optimizer = tf.train.RMSPropOptimizer(learning_rate,
- hparams.rmsprop_decay,
- hparams.momentum,
- hparams.rmsprop_epsilon)
- elif hparams.optimizer == 'momentum':
- optimizer = tf.train.MomentumOptimizer(learning_rate,
- hparams.momentum)
- elif hparams.optimizer == 'adam':
- optimizer = tf.train.AdamOptimizer(learning_rate)
- else:
- raise ValueError('Invalid value of optimizer: %s' % hparams.optimizer)
- return optimizer
-
-
-RESNET_MODELS = {'resnet_v2_50': resnet_v2.resnet_v2_50}
-
-
-def get_model(model_name, num_classes):
- """Returns function which creates model.
-
- Args:
- model_name: Name of the model.
- num_classes: Number of classes.
-
- Raises:
- ValueError: If model_name is invalid.
-
- Returns:
- Function, which creates model when called.
- """
- if model_name.startswith('resnet'):
- def resnet_model(images, is_training, reuse=tf.AUTO_REUSE):
- with tf.contrib.framework.arg_scope(resnet_v2.resnet_arg_scope()):
- resnet_fn = RESNET_MODELS[model_name]
- logits, _ = resnet_fn(images, num_classes, is_training=is_training,
- reuse=reuse)
- logits = tf.reshape(logits, [-1, num_classes])
- return logits
- return resnet_model
- else:
- raise ValueError('Invalid model: %s' % model_name)
-
-
-def filter_trainable_variables(trainable_scopes):
- """Keep only trainable variables which are prefixed with given scopes.
-
- Args:
- trainable_scopes: either list of trainable scopes or string with comma
- separated list of trainable scopes.
-
- This function removes all variables which are not prefixed with given
- trainable_scopes from collection of trainable variables.
- Useful during network fine tuning, when you only need to train subset of
- variables.
- """
- if not trainable_scopes:
- return
- if isinstance(trainable_scopes, six.string_types):
- trainable_scopes = [scope.strip() for scope in trainable_scopes.split(',')]
- trainable_scopes = {scope for scope in trainable_scopes if scope}
- if not trainable_scopes:
- return
- trainable_collection = tf.get_collection_ref(
- tf.GraphKeys.TRAINABLE_VARIABLES)
- non_trainable_vars = [
- v for v in trainable_collection
- if not any([v.op.name.startswith(s) for s in trainable_scopes])
- ]
- for v in non_trainable_vars:
- trainable_collection.remove(v)
diff --git a/research/adversarial_logit_pairing/tiny_imagenet_converter/converter.py b/research/adversarial_logit_pairing/tiny_imagenet_converter/converter.py
deleted file mode 100644
index 4fdccc32071f8c677bb1395e324c6b94aa7e85af..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/tiny_imagenet_converter/converter.py
+++ /dev/null
@@ -1,241 +0,0 @@
-# Copyright 2018 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Converts Tiny Imagenet dataset into TFRecord format.
-
-As an output this program generates following files in TFRecord format:
-- train.tfrecord
-- validation.tfrecord
-- test.tfrecord
-
-Generated train and validation files will contain tf.Example entries with
-following features:
-- image/encoded - encoded image
-- image/format - image format
-- label/wnid - label WordNet ID
-- label/imagenet - imagenet label [1 ... 1000]
-- label/tiny_imagenet - tiny imagenet label [0 ... 199]
-- bbox/xmin
-- bbox/ymin
-- bbox/xmax
-- bbox/ymax
-
-Test file will contain entries with 'image/encoded' and 'image/format' features.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import namedtuple
-import os
-import random
-
-from absl import app
-from absl import flags
-from absl import logging
-
-import pandas as pd
-
-import tensorflow as tf
-
-
-FLAGS = flags.FLAGS
-
-flags.DEFINE_string('input_dir', '', 'Input directory')
-flags.DEFINE_string('output_dir', '', 'Output directory')
-
-flags.DEFINE_string('imagenet_synsets_path', '',
- 'Optional path to /imagenet_lsvrc_2015_synsets.txt')
-
-
-ImageMetadata = namedtuple('ImageMetadata', ['label', 'x1', 'y1', 'x2', 'y2'])
-
-
-class WnIdToNodeIdConverter(object):
- """Converts WordNet IDs to numerical labels."""
-
- def __init__(self, wnids_path, background_class):
- self._wnid_to_node_id = {}
- self._node_id_to_wnid = {}
- with tf.gfile.Open(wnids_path) as f:
- wnids_sequence = [wnid.strip() for wnid in f.readlines() if wnid.strip()]
- node_id_offset = 1 if background_class else 0
- for i, label in enumerate(wnids_sequence):
- self._wnid_to_node_id[label] = i + node_id_offset
- self._node_id_to_wnid[i + node_id_offset] = label
-
- def to_node_id(self, wnid):
- return self._wnid_to_node_id[wnid]
-
- def to_wnid(self, node_id):
- return self._node_id_to_wnid[node_id]
-
- def all_wnids(self):
- return self._wnid_to_node_id.keys()
-
-
-def read_tiny_imagenet_annotations(annotations_filename,
- images_dir,
- one_label=None):
- """Reads one file with Tiny Imagenet annotations."""
- result = []
- if one_label:
- column_names = ['filename', 'x1', 'y1', 'x2', 'y2']
- else:
- column_names = ['filename', 'label', 'x1', 'y1', 'x2', 'y2']
- with tf.gfile.Open(annotations_filename) as f:
- data = pd.read_csv(f, sep='\t', names=column_names)
- for row in data.itertuples():
- label = one_label if one_label else getattr(row, 'label')
- full_filename = os.path.join(images_dir, getattr(row, 'filename'))
- result.append((full_filename,
- ImageMetadata(label=label,
- x1=getattr(row, 'x1'),
- y1=getattr(row, 'y1'),
- x2=getattr(row, 'x2'),
- y2=getattr(row, 'y2'))))
- return result
-
-
-def read_validation_annotations(validation_dir):
- """Reads validation data annotations."""
- return read_tiny_imagenet_annotations(
- os.path.join(validation_dir, 'val_annotations.txt'),
- os.path.join(validation_dir, 'images'))
-
-
-def read_training_annotations(training_dir):
- """Reads training data annotations."""
- result = []
- sub_dirs = tf.gfile.ListDirectory(training_dir)
- for sub_dir in sub_dirs:
- if not sub_dir.startswith('n'):
- logging.warning('Found non-class directory in training dir: %s', sub_dir)
- continue
- sub_dir_results = read_tiny_imagenet_annotations(
- os.path.join(training_dir, sub_dir, sub_dir + '_boxes.txt'),
- os.path.join(training_dir, sub_dir, 'images'),
- one_label=sub_dir)
- result.extend(sub_dir_results)
- return result
-
-
-def read_test_annotations(test_dir):
- """Reads test data annotations."""
- files = tf.gfile.ListDirectory(os.path.join(test_dir, 'images'))
- return [(os.path.join(test_dir, 'images', f), None)
- for f in files if f.endswith('.JPEG')]
-
-
-def get_image_format(filename):
- """Returns image format from filename."""
- filename = filename.lower()
- if filename.endswith('jpeg') or filename.endswith('jpg'):
- return 'jpeg'
- elif filename.endswith('png'):
- return 'png'
- else:
- raise ValueError('Unrecognized file format: %s' % filename)
-
-
-class TinyImagenetWriter(object):
- """Helper class which writes Tiny Imagenet dataset into TFRecord file."""
-
- def __init__(self, tiny_imagenet_wnid_conveter, imagenet_wnid_converter):
- self.tiny_imagenet_wnid_conveter = tiny_imagenet_wnid_conveter
- self.imagenet_wnid_converter = imagenet_wnid_converter
-
- def write_tf_record(self,
- annotations,
- output_file):
- """Generates TFRecord file from given list of annotations."""
- with tf.python_io.TFRecordWriter(output_file) as writer:
- for image_filename, image_metadata in annotations:
- with tf.gfile.Open(image_filename) as f:
- image_buffer = f.read()
- image_format = get_image_format(image_filename)
- features = {
- 'image/encoded': tf.train.Feature(
- bytes_list=tf.train.BytesList(value=[image_buffer])),
- 'image/format': tf.train.Feature(
- bytes_list=tf.train.BytesList(value=[image_format]))
- }
- if image_metadata:
- # bounding box features
- features['bbox/xmin'] = tf.train.Feature(
- int64_list=tf.train.Int64List(value=[image_metadata.x1]))
- features['bbox/ymin'] = tf.train.Feature(
- int64_list=tf.train.Int64List(value=[image_metadata.y1]))
- features['bbox/xmax'] = tf.train.Feature(
- int64_list=tf.train.Int64List(value=[image_metadata.x2]))
- features['bbox/ymax'] = tf.train.Feature(
- int64_list=tf.train.Int64List(value=[image_metadata.y2]))
- # tiny imagenet label, from [0, 200) iterval
- tiny_imagenet_label = self.tiny_imagenet_wnid_conveter.to_node_id(
- image_metadata.label)
- features['label/wnid'] = tf.train.Feature(
- bytes_list=tf.train.BytesList(value=image_metadata.label))
- features['label/tiny_imagenet'] = tf.train.Feature(
- int64_list=tf.train.Int64List(value=[tiny_imagenet_label]))
- # full imagenet label, from [1, 1001) interval
- if self.imagenet_wnid_converter:
- imagenet_label = self.imagenet_wnid_converter.to_node_id(
- image_metadata.label)
- features['label/imagenet'] = tf.train.Feature(
- int64_list=tf.train.Int64List(value=[imagenet_label]))
- example = tf.train.Example(features=tf.train.Features(feature=features))
- writer.write(example.SerializeToString())
-
-
-def main(_):
- assert FLAGS.input_dir, 'Input directory must be provided'
- assert FLAGS.output_dir, 'Output directory must be provided'
-
- # Create WordNet ID conveters for tiny imagenet and possibly for imagenet
- tiny_imagenet_wnid_conveter = WnIdToNodeIdConverter(
- os.path.join(FLAGS.input_dir, 'wnids.txt'),
- background_class=False)
- if FLAGS.imagenet_synsets_path:
- imagenet_wnid_converter = WnIdToNodeIdConverter(FLAGS.imagenet_synsets_path,
- background_class=True)
- else:
- imagenet_wnid_converter = None
-
- # read tiny imagenet annotations
- train_annotations = read_training_annotations(
- os.path.join(FLAGS.input_dir, 'train'))
- random.shuffle(train_annotations)
- val_annotations = read_validation_annotations(
- os.path.join(FLAGS.input_dir, 'val'))
- test_filenames = read_test_annotations(os.path.join(FLAGS.input_dir, 'test'))
-
- # Generate TFRecord files
- writer = TinyImagenetWriter(tiny_imagenet_wnid_conveter,
- imagenet_wnid_converter)
- tf.logging.info('Converting %d training images', len(train_annotations))
- writer.write_tf_record(train_annotations,
- os.path.join(FLAGS.output_dir, 'train.tfrecord'))
- tf.logging.info('Converting %d validation images ', len(val_annotations))
- writer.write_tf_record(val_annotations,
- os.path.join(FLAGS.output_dir, 'validation.tfrecord'))
- tf.logging.info('Converting %d test images', len(test_filenames))
- writer.write_tf_record(test_filenames,
- os.path.join(FLAGS.output_dir, 'test.tfrecord'))
- tf.logging.info('All files are converted')
-
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/adversarial_logit_pairing/train.py b/research/adversarial_logit_pairing/train.py
deleted file mode 100644
index dd20969f8d09c59f7d294ee34a9e41bd44f86b39..0000000000000000000000000000000000000000
--- a/research/adversarial_logit_pairing/train.py
+++ /dev/null
@@ -1,288 +0,0 @@
-# Copyright 2018 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Program which train models."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from absl import app
-from absl import flags
-
-import tensorflow as tf
-
-import adversarial_attack
-import model_lib
-from datasets import dataset_factory
-
-FLAGS = flags.FLAGS
-
-
-flags.DEFINE_integer('max_steps', -1, 'Number of steps to stop at.')
-
-flags.DEFINE_string('output_dir', None,
- 'Training directory where checkpoints will be saved.')
-
-flags.DEFINE_integer('ps_tasks', 0, 'Number of parameter servers.')
-
-flags.DEFINE_integer('task', 0, 'Task ID for running distributed training.')
-
-flags.DEFINE_string('master', '', 'Tensorflow master.')
-
-flags.DEFINE_string('model_name', 'resnet_v2_50', 'Name of the model.')
-
-flags.DEFINE_string('dataset', 'imagenet',
- 'Dataset: "tiny_imagenet" or "imagenet".')
-
-flags.DEFINE_integer('dataset_image_size', 64,
- 'Size of the images in the dataset.')
-
-flags.DEFINE_integer('num_summary_images', 3,
- 'Number of images to display in Tensorboard.')
-
-flags.DEFINE_integer(
- 'save_summaries_steps', 100,
- 'The frequency with which summaries are saved, in steps.')
-
-flags.DEFINE_integer(
- 'save_summaries_secs', None,
- 'The frequency with which summaries are saved, in seconds.')
-
-flags.DEFINE_integer(
- 'save_model_steps', 500,
- 'The frequency with which the model is saved, in steps.')
-
-flags.DEFINE_string('hparams', '', 'Hyper parameters.')
-
-flags.DEFINE_integer('replicas_to_aggregate', 1,
- 'Number of gradients to collect before param updates.')
-
-flags.DEFINE_integer('worker_replicas', 1, 'Number of worker replicas.')
-
-flags.DEFINE_float('moving_average_decay', 0.9999,
- 'The decay to use for the moving average.')
-
-# Flags to control fine tuning
-
-flags.DEFINE_string('finetune_checkpoint_path', None,
- 'Path to checkpoint for fine tuning. '
- 'If None then no fine tuning is done.')
-
-flags.DEFINE_string('finetune_exclude_pretrained_scopes', '',
- 'Variable scopes to exclude when loading checkpoint for '
- 'fine tuning.')
-
-flags.DEFINE_string('finetune_trainable_scopes', None,
- 'If set then it defines list of variable scopes for '
- 'trainable variables.')
-
-
-def _get_finetuning_init_fn(variable_averages):
- """Returns an init functions, used for fine tuning."""
- if not FLAGS.finetune_checkpoint_path:
- return None
-
- if tf.train.latest_checkpoint(FLAGS.output_dir):
- return None
-
- if tf.gfile.IsDirectory(FLAGS.finetune_checkpoint_path):
- checkpoint_path = tf.train.latest_checkpoint(FLAGS.finetune_checkpoint_path)
- else:
- checkpoint_path = FLAGS.finetune_checkpoint_path
-
- if not checkpoint_path:
- tf.logging.warning('Not doing fine tuning, can not find checkpoint in %s',
- FLAGS.finetune_checkpoint_path)
- return None
-
- tf.logging.info('Fine-tuning from %s', checkpoint_path)
-
- if FLAGS.finetune_exclude_pretrained_scopes:
- exclusions = {
- scope.strip()
- for scope in FLAGS.finetune_exclude_pretrained_scopes.split(',')
- }
- else:
- exclusions = set()
-
- filtered_model_variables = [
- v for v in tf.contrib.framework.get_model_variables()
- if not any([v.op.name.startswith(e) for e in exclusions])
- ]
-
- if variable_averages:
- variables_to_restore = {}
- for v in filtered_model_variables:
- # variables_to_restore[variable_averages.average_name(v)] = v
- if v in tf.trainable_variables():
- variables_to_restore[variable_averages.average_name(v)] = v
- else:
- variables_to_restore[v.op.name] = v
- else:
- variables_to_restore = {v.op.name: v for v in filtered_model_variables}
-
- assign_fn = tf.contrib.framework.assign_from_checkpoint_fn(
- checkpoint_path,
- variables_to_restore)
- if assign_fn:
- return lambda _, sess: assign_fn(sess)
- else:
- return None
-
-
-def main(_):
- assert FLAGS.output_dir, '--output_dir has to be provided'
- if not tf.gfile.Exists(FLAGS.output_dir):
- tf.gfile.MakeDirs(FLAGS.output_dir)
- params = model_lib.default_hparams()
- params.parse(FLAGS.hparams)
- tf.logging.info('User provided hparams: %s', FLAGS.hparams)
- tf.logging.info('All hyper parameters: %s', params)
- batch_size = params.batch_size
- graph = tf.Graph()
- with graph.as_default():
- with tf.device(tf.train.replica_device_setter(ps_tasks=FLAGS.ps_tasks)):
- # dataset
- dataset, examples_per_epoch, num_classes, bounds = (
- dataset_factory.get_dataset(
- FLAGS.dataset,
- 'train',
- batch_size,
- FLAGS.dataset_image_size,
- is_training=True))
- dataset_iterator = dataset.make_one_shot_iterator()
- images, labels = dataset_iterator.get_next()
- one_hot_labels = tf.one_hot(labels, num_classes)
-
- # set up model
- global_step = tf.train.get_or_create_global_step()
- model_fn = model_lib.get_model(FLAGS.model_name, num_classes)
- if params.train_adv_method == 'clean':
- logits = model_fn(images, is_training=True)
- adv_examples = None
- else:
- model_fn_eval_mode = lambda x: model_fn(x, is_training=False)
- adv_examples = adversarial_attack.generate_adversarial_examples(
- images, bounds, model_fn_eval_mode, params.train_adv_method)
- all_examples = tf.concat([images, adv_examples], axis=0)
- logits = model_fn(all_examples, is_training=True)
- one_hot_labels = tf.concat([one_hot_labels, one_hot_labels], axis=0)
-
- # update trainable variables if fine tuning is used
- model_lib.filter_trainable_variables(
- FLAGS.finetune_trainable_scopes)
-
- # set up losses
- total_loss = tf.losses.softmax_cross_entropy(
- onehot_labels=one_hot_labels,
- logits=logits,
- label_smoothing=params.label_smoothing)
- tf.summary.scalar('loss_xent', total_loss)
-
- if params.train_lp_weight > 0:
- images1, images2 = tf.split(logits, 2)
- loss_lp = tf.losses.mean_squared_error(
- images1, images2, weights=params.train_lp_weight)
- tf.summary.scalar('loss_lp', loss_lp)
- total_loss += loss_lp
-
- if params.weight_decay > 0:
- loss_wd = (
- params.weight_decay
- * tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()])
- )
- tf.summary.scalar('loss_wd', loss_wd)
- total_loss += loss_wd
-
- # Setup the moving averages:
- if FLAGS.moving_average_decay and (FLAGS.moving_average_decay > 0):
- with tf.name_scope('moving_average'):
- moving_average_variables = tf.contrib.framework.get_model_variables()
- variable_averages = tf.train.ExponentialMovingAverage(
- FLAGS.moving_average_decay, global_step)
- else:
- moving_average_variables = None
- variable_averages = None
-
- # set up optimizer and training op
- learning_rate, steps_per_epoch = model_lib.get_lr_schedule(
- params, examples_per_epoch, FLAGS.replicas_to_aggregate)
-
- optimizer = model_lib.get_optimizer(params, learning_rate)
-
- optimizer = tf.train.SyncReplicasOptimizer(
- opt=optimizer,
- replicas_to_aggregate=FLAGS.replicas_to_aggregate,
- total_num_replicas=FLAGS.worker_replicas,
- variable_averages=variable_averages,
- variables_to_average=moving_average_variables)
-
- train_op = tf.contrib.training.create_train_op(
- total_loss, optimizer,
- update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS))
-
- tf.summary.image('images', images[0:FLAGS.num_summary_images])
- if adv_examples is not None:
- tf.summary.image('adv_images', adv_examples[0:FLAGS.num_summary_images])
- tf.summary.scalar('total_loss', total_loss)
- tf.summary.scalar('learning_rate', learning_rate)
- tf.summary.scalar('current_epoch',
- tf.to_double(global_step) / steps_per_epoch)
-
- # Training
- is_chief = FLAGS.task == 0
-
- scaffold = tf.train.Scaffold(
- init_fn=_get_finetuning_init_fn(variable_averages))
- hooks = [
- tf.train.LoggingTensorHook({'total_loss': total_loss,
- 'global_step': global_step},
- every_n_iter=1),
- tf.train.NanTensorHook(total_loss),
- ]
- chief_only_hooks = [
- tf.train.SummarySaverHook(save_steps=FLAGS.save_summaries_steps,
- save_secs=FLAGS.save_summaries_secs,
- output_dir=FLAGS.output_dir,
- scaffold=scaffold),
- tf.train.CheckpointSaverHook(FLAGS.output_dir,
- save_steps=FLAGS.save_model_steps,
- scaffold=scaffold),
- ]
-
- if FLAGS.max_steps > 0:
- hooks.append(
- tf.train.StopAtStepHook(last_step=FLAGS.max_steps))
-
- # hook for sync replica training
- hooks.append(optimizer.make_session_run_hook(is_chief))
-
- with tf.train.MonitoredTrainingSession(
- master=FLAGS.master,
- is_chief=is_chief,
- checkpoint_dir=FLAGS.output_dir,
- scaffold=scaffold,
- hooks=hooks,
- chief_only_hooks=chief_only_hooks,
- save_checkpoint_secs=None,
- save_summaries_steps=None,
- save_summaries_secs=None) as session:
- while not session.should_stop():
- session.run([train_op])
-
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/adversarial_text/README.md b/research/adversarial_text/README.md
index 36b5d657615882908719c32ecdabcd75c2668382..643ed8b556fc03a99116163e2d19509057aa34a8 100644
--- a/research/adversarial_text/README.md
+++ b/research/adversarial_text/README.md
@@ -54,7 +54,7 @@ $ PRETRAIN_DIR=/tmp/models/imdb_pretrain
$ python pretrain.py \
--train_dir=$PRETRAIN_DIR \
--data_dir=$IMDB_DATA_DIR \
- --vocab_size=86934 \
+ --vocab_size=87007 \
--embedding_dims=256 \
--rnn_cell_size=1024 \
--num_candidate_samples=1024 \
@@ -83,7 +83,7 @@ $ python train_classifier.py \
--train_dir=$TRAIN_DIR \
--pretrained_model_dir=$PRETRAIN_DIR \
--data_dir=$IMDB_DATA_DIR \
- --vocab_size=86934 \
+ --vocab_size=87007 \
--embedding_dims=256 \
--rnn_cell_size=1024 \
--cl_num_layers=1 \
@@ -111,7 +111,7 @@ $ python evaluate.py \
--run_once \
--num_examples=25000 \
--data_dir=$IMDB_DATA_DIR \
- --vocab_size=86934 \
+ --vocab_size=87007 \
--embedding_dims=256 \
--rnn_cell_size=1024 \
--batch_size=256 \
diff --git a/research/attention_ocr/README.md b/research/attention_ocr/README.md
index b7b2b5eb9ec2de79d547033446cf7155070c27c2..f2042e573fa5a1cecc3cd620c38e45be89c78dc1 100644
--- a/research/attention_ocr/README.md
+++ b/research/attention_ocr/README.md
@@ -1,4 +1,4 @@
-## Attention-based Extraction of Structured Information from Street View Imagery
+# Attention-based Extraction of Structured Information from Street View Imagery
[](https://paperswithcode.com/sota/optical-character-recognition-on-fsns-test?p=attention-based-extraction-of-structured)
[](https://arxiv.org/abs/1704.03549)
@@ -7,14 +7,20 @@
*A TensorFlow model for real-world image text extraction problems.*
This folder contains the code needed to train a new Attention OCR model on the
-[FSNS dataset][FSNS] dataset to transcribe street names in France. You can
-also use it to train it on your own data.
+[FSNS dataset][FSNS] to transcribe street names in France. You can also train the code on your own data.
More details can be found in our paper:
["Attention-based Extraction of Structured Information from Street View
Imagery"](https://arxiv.org/abs/1704.03549)
+## Description
+
+* Paper presents a model based on ConvNets, RNN's and a novel attention mechanism.
+Achieves **84.2%** on FSNS beating the previous benchmark (**72.46%**). Also studies
+the speed/accuracy tradeoff that results from using CNN feature extractors of
+different depths.
+
## Contacts
Authors
@@ -22,7 +28,18 @@ Authors
* Zbigniew Wojna (zbigniewwojna@gmail.com)
* Alexander Gorban (gorban@google.com)
-Maintainer: Xavier Gibert [@xavigibert](https://github.com/xavigibert)
+Maintainer
+
+* Xavier Gibert ([@xavigibert](https://github.com/xavigibert))
+
+## Table of Contents
+
+* [Requirements](https://github.com/tensorflow/models/blob/master/research/attention_ocr/README.md#requirements)
+* [Dataset](https://github.com/tensorflow/models/blob/master/research/attention_ocr/README.md#dataset)
+* [How to use this code](https://github.com/tensorflow/models/blob/master/research/attention_ocr/README.md#how-to-use-this-code)
+* [Using your own image data](https://github.com/tensorflow/models/blob/master/research/attention_ocr/README.md#using-your-own-image-data)
+* [How to use a pre-trained model](https://github.com/tensorflow/models/blob/master/research/attention_ocr/README.md#how-to-use-a-pre-trained-model)
+* [Disclaimer](https://github.com/tensorflow/models/blob/master/research/attention_ocr/README.md#disclaimer)
## Requirements
@@ -49,6 +66,42 @@ cd ..
[TF]: https://www.tensorflow.org/install/
[FSNS]: https://github.com/tensorflow/models/tree/master/research/street
+## Dataset
+
+The French Street Name Signs (FSNS) dataset is split into subsets,
+each of which is composed of multiple files. Note that these datasets
+are very large. The approximate sizes are:
+
+* Train: 512 files of 300MB each.
+* Validation: 64 files of 40MB each.
+* Test: 64 files of 50MB each.
+* The datasets download includes a directory `testdata` that contains
+some small datasets that are big enough to test that models can
+actually learn something.
+* Total: around 158GB
+
+The download paths are in the following list:
+
+```
+https://download.tensorflow.org/data/fsns-20160927/charset_size=134.txt
+https://download.tensorflow.org/data/fsns-20160927/test/test-00000-of-00064
+...
+https://download.tensorflow.org/data/fsns-20160927/test/test-00063-of-00064
+https://download.tensorflow.org/data/fsns-20160927/testdata/arial-32-00000-of-00001
+https://download.tensorflow.org/data/fsns-20160927/testdata/fsns-00000-of-00001
+https://download.tensorflow.org/data/fsns-20160927/testdata/mnist-sample-00000-of-00001
+https://download.tensorflow.org/data/fsns-20160927/testdata/numbers-16-00000-of-00001
+https://download.tensorflow.org/data/fsns-20160927/train/train-00000-of-00512
+...
+https://download.tensorflow.org/data/fsns-20160927/train/train-00511-of-00512
+https://download.tensorflow.org/data/fsns-20160927/validation/validation-00000-of-00064
+...
+https://download.tensorflow.org/data/fsns-20160927/validation/validation-00063-of-00064
+```
+
+All URLs are stored in the [research/street](https://github.com/tensorflow/models/tree/master/research/street)
+repository in the text file `python/fsns_urls.txt`.
+
## How to use this code
To run all unit tests:
@@ -80,7 +133,7 @@ tar xf attention_ocr_2017_08_09.tar.gz
python train.py --checkpoint=model.ckpt-399731
```
-## How to use your own image data to train the model
+## Using your own image data
You need to define a new dataset. There are two options:
diff --git a/research/attention_ocr/python/common_flags.py b/research/attention_ocr/python/common_flags.py
index 86eb355ed85ceff81c10751119bc3c46ffef59b1..60aa49ffea84e6f7cc893715309755a3ca3e4b6f 100644
--- a/research/attention_ocr/python/common_flags.py
+++ b/research/attention_ocr/python/common_flags.py
@@ -17,7 +17,7 @@
import logging
import sys
-from tensorflow.python.platform import flags
+from tensorflow.compat.v1 import flags
import datasets
import model
diff --git a/research/attention_ocr/python/data_provider.py b/research/attention_ocr/python/data_provider.py
index fb7feed7c4018deb2da9d53223c8045fba88ffc3..7a5a2d40cee1d431380b8958439e1d47b7f4e509 100644
--- a/research/attention_ocr/python/data_provider.py
+++ b/research/attention_ocr/python/data_provider.py
@@ -56,14 +56,14 @@ def augment_image(image):
Returns:
Distorted Tensor image of the same shape.
"""
- with tf.variable_scope('AugmentImage'):
+ with tf.compat.v1.variable_scope('AugmentImage'):
height = image.get_shape().dims[0].value
width = image.get_shape().dims[1].value
# Random crop cut from the street sign image, resized to the same size.
# Assures that the crop is covers at least 0.8 area of the input image.
bbox_begin, bbox_size, _ = tf.image.sample_distorted_bounding_box(
- tf.shape(image),
+ image_size=tf.shape(input=image),
bounding_boxes=tf.zeros([0, 0, 4]),
min_object_covered=0.8,
aspect_ratio_range=[0.8, 1.2],
@@ -74,7 +74,7 @@ def augment_image(image):
# Randomly chooses one of the 4 interpolation methods
distorted_image = inception_preprocessing.apply_with_random_selector(
distorted_image,
- lambda x, method: tf.image.resize_images(x, [height, width], method),
+ lambda x, method: tf.image.resize(x, [height, width], method),
num_cases=4)
distorted_image.set_shape([height, width, 3])
@@ -99,9 +99,10 @@ def central_crop(image, crop_size):
Returns:
A tensor of shape [crop_height, crop_width, channels].
"""
- with tf.variable_scope('CentralCrop'):
+ with tf.compat.v1.variable_scope('CentralCrop'):
target_width, target_height = crop_size
- image_height, image_width = tf.shape(image)[0], tf.shape(image)[1]
+ image_height, image_width = tf.shape(
+ input=image)[0], tf.shape(input=image)[1]
assert_op1 = tf.Assert(
tf.greater_equal(image_height, target_height),
['image_height < target_height', image_height, target_height])
@@ -129,7 +130,7 @@ def preprocess_image(image, augment=False, central_crop_size=None,
A float32 tensor of shape [H x W x 3] with RGB values in the required
range.
"""
- with tf.variable_scope('PreprocessImage'):
+ with tf.compat.v1.variable_scope('PreprocessImage'):
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
if augment or central_crop_size:
if num_towers == 1:
@@ -182,7 +183,7 @@ def get_data(dataset,
image_orig, augment, central_crop_size, num_towers=dataset.num_of_views)
label_one_hot = slim.one_hot_encoding(label, dataset.num_char_classes)
- images, images_orig, labels, labels_one_hot = (tf.train.shuffle_batch(
+ images, images_orig, labels, labels_one_hot = (tf.compat.v1.train.shuffle_batch(
[image, image_orig, label, label_one_hot],
batch_size=batch_size,
num_threads=shuffle_config.num_batching_threads,
diff --git a/research/attention_ocr/python/datasets/fsns.py b/research/attention_ocr/python/datasets/fsns.py
index 99ea684212c5e0c24737957c558fd76ba306f6e3..ab6d0f28b1369a8e5945d57c2f102733638058d0 100644
--- a/research/attention_ocr/python/datasets/fsns.py
+++ b/research/attention_ocr/python/datasets/fsns.py
@@ -72,7 +72,7 @@ def read_charset(filename, null_character=u'\u2591'):
"""
pattern = re.compile(r'(\d+)\t(.+)')
charset = {}
- with tf.gfile.GFile(filename) as f:
+ with tf.io.gfile.GFile(filename) as f:
for i, line in enumerate(f):
m = pattern.match(line)
if m is None:
@@ -96,9 +96,9 @@ class _NumOfViewsHandler(slim.tfexample_decoder.ItemHandler):
self._num_of_views = num_of_views
def tensors_to_item(self, keys_to_tensors):
- return tf.to_int64(
+ return tf.cast(
self._num_of_views * keys_to_tensors[self._original_width_key] /
- keys_to_tensors[self._width_key])
+ keys_to_tensors[self._width_key], dtype=tf.int64)
def get_split(split_name, dataset_dir=None, config=None):
@@ -133,19 +133,19 @@ def get_split(split_name, dataset_dir=None, config=None):
zero = tf.zeros([1], dtype=tf.int64)
keys_to_features = {
'image/encoded':
- tf.FixedLenFeature((), tf.string, default_value=''),
+ tf.io.FixedLenFeature((), tf.string, default_value=''),
'image/format':
- tf.FixedLenFeature((), tf.string, default_value='png'),
+ tf.io.FixedLenFeature((), tf.string, default_value='png'),
'image/width':
- tf.FixedLenFeature([1], tf.int64, default_value=zero),
+ tf.io.FixedLenFeature([1], tf.int64, default_value=zero),
'image/orig_width':
- tf.FixedLenFeature([1], tf.int64, default_value=zero),
+ tf.io.FixedLenFeature([1], tf.int64, default_value=zero),
'image/class':
- tf.FixedLenFeature([config['max_sequence_length']], tf.int64),
+ tf.io.FixedLenFeature([config['max_sequence_length']], tf.int64),
'image/unpadded_class':
- tf.VarLenFeature(tf.int64),
+ tf.io.VarLenFeature(tf.int64),
'image/text':
- tf.FixedLenFeature([1], tf.string, default_value=''),
+ tf.io.FixedLenFeature([1], tf.string, default_value=''),
}
items_to_handlers = {
'image':
@@ -171,7 +171,7 @@ def get_split(split_name, dataset_dir=None, config=None):
config['splits'][split_name]['pattern'])
return slim.dataset.Dataset(
data_sources=file_pattern,
- reader=tf.TFRecordReader,
+ reader=tf.compat.v1.TFRecordReader,
decoder=decoder,
num_samples=config['splits'][split_name]['size'],
items_to_descriptions=config['items_to_descriptions'],
diff --git a/research/attention_ocr/python/datasets/fsns_test.py b/research/attention_ocr/python/datasets/fsns_test.py
index 4daedfbd12a58b6635cefed2bdc02bc84fc2c9ef..2f5f3afc78e9f73ec8ce8a295b9664bff8eeda67 100644
--- a/research/attention_ocr/python/datasets/fsns_test.py
+++ b/research/attention_ocr/python/datasets/fsns_test.py
@@ -22,8 +22,9 @@ from tensorflow.contrib import slim
from datasets import fsns
from datasets import unittest_utils
+from tensorflow.compat.v1 import flags
-FLAGS = tf.flags.FLAGS
+FLAGS = flags.FLAGS
def get_test_split():
@@ -91,7 +92,7 @@ class FsnsTest(tf.test.TestCase):
image_tf, label_tf = provider.get(['image', 'label'])
with self.test_session() as sess:
- sess.run(tf.global_variables_initializer())
+ sess.run(tf.compat.v1.global_variables_initializer())
with slim.queues.QueueRunners(sess):
image_np, label_np = sess.run([image_tf, label_tf])
diff --git a/research/attention_ocr/python/datasets/testdata/fsns/download_data.py b/research/attention_ocr/python/datasets/testdata/fsns/download_data.py
index 559e3195f2156af3be97395b5bc8c0d8ea62f174..126ef58060bbfee11d94f756853fd88cdab8f28a 100644
--- a/research/attention_ocr/python/datasets/testdata/fsns/download_data.py
+++ b/research/attention_ocr/python/datasets/testdata/fsns/download_data.py
@@ -10,7 +10,8 @@ KEEP_NUM_RECORDS = 5
print('Downloading %s ...' % URL)
urllib.request.urlretrieve(URL, DST_ORIG)
-print('Writing %d records from %s to %s ...' % (KEEP_NUM_RECORDS, DST_ORIG, DST))
+print('Writing %d records from %s to %s ...' %
+ (KEEP_NUM_RECORDS, DST_ORIG, DST))
with tf.io.TFRecordWriter(DST) as writer:
- for raw_record in itertools.islice(tf.python_io.tf_record_iterator(DST_ORIG), KEEP_NUM_RECORDS):
+ for raw_record in itertools.islice(tf.compat.v1.python_io.tf_record_iterator(DST_ORIG), KEEP_NUM_RECORDS):
writer.write(raw_record)
diff --git a/research/attention_ocr/python/demo_inference.py b/research/attention_ocr/python/demo_inference.py
index d5fcf2515b85412aad272749cc50f5e81752b35d..8c6531d844f8711f1bb5d51ea39678340196cbd3 100644
--- a/research/attention_ocr/python/demo_inference.py
+++ b/research/attention_ocr/python/demo_inference.py
@@ -19,7 +19,7 @@ import numpy as np
import PIL.Image
import tensorflow as tf
-from tensorflow.python.platform import flags
+from tensorflow.compat.v1 import flags
from tensorflow.python.training import monitored_session
import common_flags
@@ -49,7 +49,7 @@ def load_images(file_pattern, batch_size, dataset_name):
for i in range(batch_size):
path = file_pattern % i
print("Reading %s" % path)
- pil_image = PIL.Image.open(tf.gfile.GFile(path, 'rb'))
+ pil_image = PIL.Image.open(tf.io.gfile.GFile(path, 'rb'))
images_actual_data[i, ...] = np.asarray(pil_image)
return images_actual_data
@@ -58,12 +58,13 @@ def create_model(batch_size, dataset_name):
width, height = get_dataset_image_size(dataset_name)
dataset = common_flags.create_dataset(split_name=FLAGS.split_name)
model = common_flags.create_model(
- num_char_classes=dataset.num_char_classes,
- seq_length=dataset.max_sequence_length,
- num_views=dataset.num_of_views,
- null_code=dataset.null_code,
- charset=dataset.charset)
- raw_images = tf.placeholder(tf.uint8, shape=[batch_size, height, width, 3])
+ num_char_classes=dataset.num_char_classes,
+ seq_length=dataset.max_sequence_length,
+ num_views=dataset.num_of_views,
+ null_code=dataset.null_code,
+ charset=dataset.charset)
+ raw_images = tf.compat.v1.placeholder(
+ tf.uint8, shape=[batch_size, height, width, 3])
images = tf.map_fn(data_provider.preprocess_image, raw_images,
dtype=tf.float32)
endpoints = model.create_base(images, labels_one_hot=None)
@@ -76,9 +77,9 @@ def run(checkpoint, batch_size, dataset_name, image_path_pattern):
images_data = load_images(image_path_pattern, batch_size,
dataset_name)
session_creator = monitored_session.ChiefSessionCreator(
- checkpoint_filename_with_path=checkpoint)
+ checkpoint_filename_with_path=checkpoint)
with monitored_session.MonitoredSession(
- session_creator=session_creator) as sess:
+ session_creator=session_creator) as sess:
predictions = sess.run(endpoints.predicted_text,
feed_dict={images_placeholder: images_data})
return [pr_bytes.decode('utf-8') for pr_bytes in predictions.tolist()]
@@ -87,10 +88,10 @@ def run(checkpoint, batch_size, dataset_name, image_path_pattern):
def main(_):
print("Predicted strings:")
predictions = run(FLAGS.checkpoint, FLAGS.batch_size, FLAGS.dataset_name,
- FLAGS.image_path_pattern)
+ FLAGS.image_path_pattern)
for line in predictions:
print(line)
if __name__ == '__main__':
- tf.app.run()
+ tf.compat.v1.app.run()
diff --git a/research/attention_ocr/python/demo_inference_test.py b/research/attention_ocr/python/demo_inference_test.py
index 457fb5ab9ef5dbcb326585c2dc8281ee23d319d1..91b8603383289092dd1fb4a06243cb59028102be 100644
--- a/research/attention_ocr/python/demo_inference_test.py
+++ b/research/attention_ocr/python/demo_inference_test.py
@@ -4,6 +4,7 @@ import os
import demo_inference
import tensorflow as tf
from tensorflow.python.training import monitored_session
+from tensorflow.compat.v1 import flags
_CHECKPOINT = 'model.ckpt-399731'
_CHECKPOINT_URL = 'http://download.tensorflow.org/models/attention_ocr_2017_08_09.tar.gz'
@@ -14,12 +15,13 @@ class DemoInferenceTest(tf.test.TestCase):
super(DemoInferenceTest, self).setUp()
for suffix in ['.meta', '.index', '.data-00000-of-00001']:
filename = _CHECKPOINT + suffix
- self.assertTrue(tf.gfile.Exists(filename),
+ self.assertTrue(tf.io.gfile.exists(filename),
msg='Missing checkpoint file %s. '
'Please download and extract it from %s' %
(filename, _CHECKPOINT_URL))
self._batch_size = 32
- tf.flags.FLAGS.dataset_dir = os.path.join(os.path.dirname(__file__), 'datasets/testdata/fsns')
+ flags.FLAGS.dataset_dir = os.path.join(
+ os.path.dirname(__file__), 'datasets/testdata/fsns')
def test_moving_variables_properly_loaded_from_a_checkpoint(self):
batch_size = 32
@@ -30,15 +32,15 @@ class DemoInferenceTest(tf.test.TestCase):
images_data = demo_inference.load_images(image_path_pattern, batch_size,
dataset_name)
tensor_name = 'AttentionOcr_v1/conv_tower_fn/INCE/InceptionV3/Conv2d_2a_3x3/BatchNorm/moving_mean'
- moving_mean_tf = tf.get_default_graph().get_tensor_by_name(
- tensor_name + ':0')
- reader = tf.train.NewCheckpointReader(_CHECKPOINT)
+ moving_mean_tf = tf.compat.v1.get_default_graph().get_tensor_by_name(
+ tensor_name + ':0')
+ reader = tf.compat.v1.train.NewCheckpointReader(_CHECKPOINT)
moving_mean_expected = reader.get_tensor(tensor_name)
session_creator = monitored_session.ChiefSessionCreator(
- checkpoint_filename_with_path=_CHECKPOINT)
+ checkpoint_filename_with_path=_CHECKPOINT)
with monitored_session.MonitoredSession(
- session_creator=session_creator) as sess:
+ session_creator=session_creator) as sess:
moving_mean_np = sess.run(moving_mean_tf,
feed_dict={images_placeholder: images_data})
@@ -50,38 +52,38 @@ class DemoInferenceTest(tf.test.TestCase):
'fsns',
image_path_pattern)
self.assertEqual([
- u'Boulevard de Lunel░░░░░░░░░░░░░░░░░░░',
- 'Rue de Provence░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue de Port Maria░░░░░░░░░░░░░░░░░░░░',
- 'Avenue Charles Gounod░░░░░░░░░░░░░░░░',
- 'Rue de l‘Aurore░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue de Beuzeville░░░░░░░░░░░░░░░░░░░░',
- 'Rue d‘Orbey░░░░░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue Victor Schoulcher░░░░░░░░░░░░░░░░',
- 'Rue de la Gare░░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue des Tulipes░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue André Maginot░░░░░░░░░░░░░░░░░░░░',
- 'Route de Pringy░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue des Landelles░░░░░░░░░░░░░░░░░░░░',
- 'Rue des Ilettes░░░░░░░░░░░░░░░░░░░░░░',
- 'Avenue de Maurin░░░░░░░░░░░░░░░░░░░░░',
- 'Rue Théresa░░░░░░░░░░░░░░░░░░░░░░░░░░', # GT='Rue Thérésa'
- 'Route de la Balme░░░░░░░░░░░░░░░░░░░░',
- 'Rue Hélène Roederer░░░░░░░░░░░░░░░░░░',
- 'Rue Emile Bernard░░░░░░░░░░░░░░░░░░░░',
- 'Place de la Mairie░░░░░░░░░░░░░░░░░░░',
- 'Rue des Perrots░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue de la Libération░░░░░░░░░░░░░░░░░',
- 'Impasse du Capcir░░░░░░░░░░░░░░░░░░░░',
- 'Avenue de la Grand Mare░░░░░░░░░░░░░░',
- 'Rue Pierre Brossolette░░░░░░░░░░░░░░░',
- 'Rue de Provence░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue du Docteur Mourre░░░░░░░░░░░░░░░░',
- 'Rue d‘Ortheuil░░░░░░░░░░░░░░░░░░░░░░░',
- 'Rue des Sarments░░░░░░░░░░░░░░░░░░░░░',
- 'Rue du Centre░░░░░░░░░░░░░░░░░░░░░░░░',
- 'Impasse Pierre Mourgues░░░░░░░░░░░░░░',
- 'Rue Marcel Dassault░░░░░░░░░░░░░░░░░░'
+ u'Boulevard de Lunel░░░░░░░░░░░░░░░░░░░',
+ 'Rue de Provence░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue de Port Maria░░░░░░░░░░░░░░░░░░░░',
+ 'Avenue Charles Gounod░░░░░░░░░░░░░░░░',
+ 'Rue de l‘Aurore░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue de Beuzeville░░░░░░░░░░░░░░░░░░░░',
+ 'Rue d‘Orbey░░░░░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue Victor Schoulcher░░░░░░░░░░░░░░░░',
+ 'Rue de la Gare░░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue des Tulipes░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue André Maginot░░░░░░░░░░░░░░░░░░░░',
+ 'Route de Pringy░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue des Landelles░░░░░░░░░░░░░░░░░░░░',
+ 'Rue des Ilettes░░░░░░░░░░░░░░░░░░░░░░',
+ 'Avenue de Maurin░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue Théresa░░░░░░░░░░░░░░░░░░░░░░░░░░', # GT='Rue Thérésa'
+ 'Route de la Balme░░░░░░░░░░░░░░░░░░░░',
+ 'Rue Hélène Roederer░░░░░░░░░░░░░░░░░░',
+ 'Rue Emile Bernard░░░░░░░░░░░░░░░░░░░░',
+ 'Place de la Mairie░░░░░░░░░░░░░░░░░░░',
+ 'Rue des Perrots░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue de la Libération░░░░░░░░░░░░░░░░░',
+ 'Impasse du Capcir░░░░░░░░░░░░░░░░░░░░',
+ 'Avenue de la Grand Mare░░░░░░░░░░░░░░',
+ 'Rue Pierre Brossolette░░░░░░░░░░░░░░░',
+ 'Rue de Provence░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue du Docteur Mourre░░░░░░░░░░░░░░░░',
+ 'Rue d‘Ortheuil░░░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue des Sarments░░░░░░░░░░░░░░░░░░░░░',
+ 'Rue du Centre░░░░░░░░░░░░░░░░░░░░░░░░',
+ 'Impasse Pierre Mourgues░░░░░░░░░░░░░░',
+ 'Rue Marcel Dassault░░░░░░░░░░░░░░░░░░'
], predictions)
diff --git a/research/attention_ocr/python/eval.py b/research/attention_ocr/python/eval.py
index ec68ad50bc25cd8528f4e9fd7976adad72782641..108227ba91cc71313cf5dbec25faa5729b5cee48 100644
--- a/research/attention_ocr/python/eval.py
+++ b/research/attention_ocr/python/eval.py
@@ -21,7 +21,7 @@ python eval.py
import tensorflow as tf
from tensorflow.contrib import slim
from tensorflow import app
-from tensorflow.python.platform import flags
+from tensorflow.compat.v1 import flags
import data_provider
import common_flags
@@ -45,8 +45,8 @@ flags.DEFINE_integer('number_of_steps', None,
def main(_):
- if not tf.gfile.Exists(FLAGS.eval_log_dir):
- tf.gfile.MakeDirs(FLAGS.eval_log_dir)
+ if not tf.io.gfile.exists(FLAGS.eval_log_dir):
+ tf.io.gfile.makedirs(FLAGS.eval_log_dir)
dataset = common_flags.create_dataset(split_name=FLAGS.split_name)
model = common_flags.create_model(dataset.num_char_classes,
@@ -62,7 +62,7 @@ def main(_):
eval_ops = model.create_summaries(
data, endpoints, dataset.charset, is_training=False)
slim.get_or_create_global_step()
- session_config = tf.ConfigProto(device_count={"GPU": 0})
+ session_config = tf.compat.v1.ConfigProto(device_count={"GPU": 0})
slim.evaluation.evaluation_loop(
master=FLAGS.master,
checkpoint_dir=FLAGS.train_log_dir,
diff --git a/research/attention_ocr/python/inception_preprocessing.py b/research/attention_ocr/python/inception_preprocessing.py
index a4827f2cab742340da2d8d4972c41b35c9862a1e..b61b895021c230cb22f327b077255e28914a924e 100644
--- a/research/attention_ocr/python/inception_preprocessing.py
+++ b/research/attention_ocr/python/inception_preprocessing.py
@@ -38,7 +38,7 @@ def apply_with_random_selector(x, func, num_cases):
The result of func(x, sel), where func receives the value of the
selector as a python integer, but sel is sampled dynamically.
"""
- sel = tf.random_uniform([], maxval=num_cases, dtype=tf.int32)
+ sel = tf.random.uniform([], maxval=num_cases, dtype=tf.int32)
# Pass the real x only to one of the func calls.
return control_flow_ops.merge([
func(control_flow_ops.switch(x, tf.equal(sel, case))[1], case)
@@ -64,7 +64,7 @@ def distort_color(image, color_ordering=0, fast_mode=True, scope=None):
Raises:
ValueError: if color_ordering not in [0, 3]
"""
- with tf.name_scope(scope, 'distort_color', [image]):
+ with tf.compat.v1.name_scope(scope, 'distort_color', [image]):
if fast_mode:
if color_ordering == 0:
image = tf.image.random_brightness(image, max_delta=32. / 255.)
@@ -131,7 +131,7 @@ def distorted_bounding_box_crop(image,
Returns:
A tuple, a 3-D Tensor cropped_image and the distorted bbox
"""
- with tf.name_scope(scope, 'distorted_bounding_box_crop', [image, bbox]):
+ with tf.compat.v1.name_scope(scope, 'distorted_bounding_box_crop', [image, bbox]):
# Each bounding box has shape [1, num_boxes, box coords] and
# the coordinates are ordered [ymin, xmin, ymax, xmax].
@@ -143,7 +143,7 @@ def distorted_bounding_box_crop(image,
# bounding box. If no box is supplied, then we assume the bounding box is
# the entire image.
sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
- tf.shape(image),
+ image_size=tf.shape(input=image),
bounding_boxes=bbox,
min_object_covered=min_object_covered,
aspect_ratio_range=aspect_ratio_range,
@@ -188,7 +188,7 @@ def preprocess_for_train(image,
Returns:
3-D float Tensor of distorted image used for training with range [-1, 1].
"""
- with tf.name_scope(scope, 'distort_image', [image, height, width, bbox]):
+ with tf.compat.v1.name_scope(scope, 'distort_image', [image, height, width, bbox]):
if bbox is None:
bbox = tf.constant(
[0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4])
@@ -198,7 +198,7 @@ def preprocess_for_train(image,
# the coordinates are ordered [ymin, xmin, ymax, xmax].
image_with_box = tf.image.draw_bounding_boxes(
tf.expand_dims(image, 0), bbox)
- tf.summary.image('image_with_bounding_boxes', image_with_box)
+ tf.compat.v1.summary.image('image_with_bounding_boxes', image_with_box)
distorted_image, distorted_bbox = distorted_bounding_box_crop(image, bbox)
# Restore the shape since the dynamic slice based upon the bbox_size loses
@@ -206,8 +206,8 @@ def preprocess_for_train(image,
distorted_image.set_shape([None, None, 3])
image_with_distorted_box = tf.image.draw_bounding_boxes(
tf.expand_dims(image, 0), distorted_bbox)
- tf.summary.image('images_with_distorted_bounding_box',
- image_with_distorted_box)
+ tf.compat.v1.summary.image('images_with_distorted_bounding_box',
+ image_with_distorted_box)
# This resizing operation may distort the images because the aspect
# ratio is not respected. We select a resize method in a round robin
@@ -218,11 +218,11 @@ def preprocess_for_train(image,
num_resize_cases = 1 if fast_mode else 4
distorted_image = apply_with_random_selector(
distorted_image,
- lambda x, method: tf.image.resize_images(x, [height, width], method=method),
+ lambda x, method: tf.image.resize(x, [height, width], method=method),
num_cases=num_resize_cases)
- tf.summary.image('cropped_resized_image',
- tf.expand_dims(distorted_image, 0))
+ tf.compat.v1.summary.image('cropped_resized_image',
+ tf.expand_dims(distorted_image, 0))
# Randomly flip the image horizontally.
distorted_image = tf.image.random_flip_left_right(distorted_image)
@@ -233,8 +233,8 @@ def preprocess_for_train(image,
lambda x, ordering: distort_color(x, ordering, fast_mode),
num_cases=4)
- tf.summary.image('final_distorted_image',
- tf.expand_dims(distorted_image, 0))
+ tf.compat.v1.summary.image('final_distorted_image',
+ tf.expand_dims(distorted_image, 0))
distorted_image = tf.subtract(distorted_image, 0.5)
distorted_image = tf.multiply(distorted_image, 2.0)
return distorted_image
@@ -265,7 +265,7 @@ def preprocess_for_eval(image,
Returns:
3-D float Tensor of prepared image.
"""
- with tf.name_scope(scope, 'eval_image', [image, height, width]):
+ with tf.compat.v1.name_scope(scope, 'eval_image', [image, height, width]):
if image.dtype != tf.float32:
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
# Crop the central region of the image with an area containing 87.5% of
@@ -276,8 +276,8 @@ def preprocess_for_eval(image,
if height and width:
# Resize the image to the specified height and width.
image = tf.expand_dims(image, 0)
- image = tf.image.resize_bilinear(
- image, [height, width], align_corners=False)
+ image = tf.image.resize(
+ image, [height, width], method=tf.image.ResizeMethod.BILINEAR)
image = tf.squeeze(image, [0])
image = tf.subtract(image, 0.5)
image = tf.multiply(image, 2.0)
diff --git a/research/attention_ocr/python/metrics.py b/research/attention_ocr/python/metrics.py
index 9e2a6a7579812583dc60546f97976f05befe07ff..0bd6c23d009848877441d97981b6514072979a54 100644
--- a/research/attention_ocr/python/metrics.py
+++ b/research/attention_ocr/python/metrics.py
@@ -34,20 +34,21 @@ def char_accuracy(predictions, targets, rej_char, streaming=False):
a update_ops for execution and value tensor whose value on evaluation
returns the total character accuracy.
"""
- with tf.variable_scope('CharAccuracy'):
+ with tf.compat.v1.variable_scope('CharAccuracy'):
predictions.get_shape().assert_is_compatible_with(targets.get_shape())
- targets = tf.to_int32(targets)
+ targets = tf.cast(targets, dtype=tf.int32)
const_rej_char = tf.constant(rej_char, shape=targets.get_shape())
- weights = tf.to_float(tf.not_equal(targets, const_rej_char))
- correct_chars = tf.to_float(tf.equal(predictions, targets))
- accuracy_per_example = tf.div(
- tf.reduce_sum(tf.multiply(correct_chars, weights), 1),
- tf.reduce_sum(weights, 1))
+ weights = tf.cast(tf.not_equal(targets, const_rej_char), dtype=tf.float32)
+ correct_chars = tf.cast(tf.equal(predictions, targets), dtype=tf.float32)
+ accuracy_per_example = tf.compat.v1.div(
+ tf.reduce_sum(input_tensor=tf.multiply(
+ correct_chars, weights), axis=1),
+ tf.reduce_sum(input_tensor=weights, axis=1))
if streaming:
return tf.contrib.metrics.streaming_mean(accuracy_per_example)
else:
- return tf.reduce_mean(accuracy_per_example)
+ return tf.reduce_mean(input_tensor=accuracy_per_example)
def sequence_accuracy(predictions, targets, rej_char, streaming=False):
@@ -66,25 +67,26 @@ def sequence_accuracy(predictions, targets, rej_char, streaming=False):
returns the total sequence accuracy.
"""
- with tf.variable_scope('SequenceAccuracy'):
+ with tf.compat.v1.variable_scope('SequenceAccuracy'):
predictions.get_shape().assert_is_compatible_with(targets.get_shape())
- targets = tf.to_int32(targets)
+ targets = tf.cast(targets, dtype=tf.int32)
const_rej_char = tf.constant(
rej_char, shape=targets.get_shape(), dtype=tf.int32)
include_mask = tf.not_equal(targets, const_rej_char)
- include_predictions = tf.to_int32(
- tf.where(include_mask, predictions,
- tf.zeros_like(predictions) + rej_char))
- correct_chars = tf.to_float(tf.equal(include_predictions, targets))
+ include_predictions = tf.cast(
+ tf.compat.v1.where(include_mask, predictions,
+ tf.zeros_like(predictions) + rej_char), dtype=tf.int32)
+ correct_chars = tf.cast(
+ tf.equal(include_predictions, targets), dtype=tf.float32)
correct_chars_counts = tf.cast(
- tf.reduce_sum(correct_chars, reduction_indices=[1]), dtype=tf.int32)
+ tf.reduce_sum(input_tensor=correct_chars, axis=[1]), dtype=tf.int32)
target_length = targets.get_shape().dims[1].value
target_chars_counts = tf.constant(
target_length, shape=correct_chars_counts.get_shape())
- accuracy_per_example = tf.to_float(
- tf.equal(correct_chars_counts, target_chars_counts))
+ accuracy_per_example = tf.cast(
+ tf.equal(correct_chars_counts, target_chars_counts), dtype=tf.float32)
if streaming:
return tf.contrib.metrics.streaming_mean(accuracy_per_example)
else:
- return tf.reduce_mean(accuracy_per_example)
+ return tf.reduce_mean(input_tensor=accuracy_per_example)
diff --git a/research/attention_ocr/python/metrics_test.py b/research/attention_ocr/python/metrics_test.py
index 5560ec2c898fe7674715ec54daa08ba9e7471adf..3e83194523eb3eba904e7225e580841e7f6e3a3f 100644
--- a/research/attention_ocr/python/metrics_test.py
+++ b/research/attention_ocr/python/metrics_test.py
@@ -38,8 +38,8 @@ class AccuracyTest(tf.test.TestCase):
A session object that should be used as a context manager.
"""
with self.cached_session() as sess:
- sess.run(tf.global_variables_initializer())
- sess.run(tf.local_variables_initializer())
+ sess.run(tf.compat.v1.global_variables_initializer())
+ sess.run(tf.compat.v1.local_variables_initializer())
yield sess
def _fake_labels(self):
@@ -55,7 +55,7 @@ class AccuracyTest(tf.test.TestCase):
return incorrect
def test_sequence_accuracy_identical_samples(self):
- labels_tf = tf.convert_to_tensor(self._fake_labels())
+ labels_tf = tf.convert_to_tensor(value=self._fake_labels())
accuracy_tf = metrics.sequence_accuracy(labels_tf, labels_tf,
self.rej_char)
@@ -66,9 +66,9 @@ class AccuracyTest(tf.test.TestCase):
def test_sequence_accuracy_one_char_difference(self):
ground_truth_np = self._fake_labels()
- ground_truth_tf = tf.convert_to_tensor(ground_truth_np)
+ ground_truth_tf = tf.convert_to_tensor(value=ground_truth_np)
prediction_tf = tf.convert_to_tensor(
- self._incorrect_copy(ground_truth_np, bad_indexes=((0, 0))))
+ value=self._incorrect_copy(ground_truth_np, bad_indexes=((0, 0))))
accuracy_tf = metrics.sequence_accuracy(prediction_tf, ground_truth_tf,
self.rej_char)
@@ -80,9 +80,9 @@ class AccuracyTest(tf.test.TestCase):
def test_char_accuracy_one_char_difference_with_padding(self):
ground_truth_np = self._fake_labels()
- ground_truth_tf = tf.convert_to_tensor(ground_truth_np)
+ ground_truth_tf = tf.convert_to_tensor(value=ground_truth_np)
prediction_tf = tf.convert_to_tensor(
- self._incorrect_copy(ground_truth_np, bad_indexes=((0, 0))))
+ value=self._incorrect_copy(ground_truth_np, bad_indexes=((0, 0))))
accuracy_tf = metrics.char_accuracy(prediction_tf, ground_truth_tf,
self.rej_char)
diff --git a/research/attention_ocr/python/model.py b/research/attention_ocr/python/model.py
index 48d2fc842934172170e71291b6c72ded96546136..b489f964e9d756c90af901fa00da49553b17052a 100644
--- a/research/attention_ocr/python/model.py
+++ b/research/attention_ocr/python/model.py
@@ -92,8 +92,8 @@ class CharsetMapper(object):
Args:
ids: a tensor with shape [batch_size, max_sequence_length]
"""
- return tf.reduce_join(
- self.table.lookup(tf.to_int64(ids)), reduction_indices=1)
+ return tf.strings.reduce_join(
+ inputs=self.table.lookup(tf.cast(ids, dtype=tf.int64)), axis=1)
def get_softmax_loss_fn(label_smoothing):
@@ -110,7 +110,7 @@ def get_softmax_loss_fn(label_smoothing):
def loss_fn(labels, logits):
return (tf.nn.softmax_cross_entropy_with_logits(
- logits=logits, labels=labels))
+ logits=logits, labels=tf.stop_gradient(labels)))
else:
def loss_fn(labels, logits):
@@ -140,7 +140,7 @@ def get_tensor_dimensions(tensor):
raise ValueError(
'Incompatible shape: len(tensor.get_shape().dims) != 4 (%d != 4)' %
len(tensor.get_shape().dims))
- batch_size = tf.shape(tensor)[0]
+ batch_size = tf.shape(input=tensor)[0]
height = tensor.get_shape().dims[1].value
width = tensor.get_shape().dims[2].value
num_features = tensor.get_shape().dims[3].value
@@ -161,8 +161,8 @@ def lookup_indexed_value(indices, row_vecs):
A tensor of shape (batch, ) formed by row_vecs[i, indices[i]].
"""
gather_indices = tf.stack((tf.range(
- tf.shape(row_vecs)[0], dtype=tf.int32), tf.cast(indices, tf.int32)),
- axis=1)
+ tf.shape(input=row_vecs)[0], dtype=tf.int32), tf.cast(indices, tf.int32)),
+ axis=1)
return tf.gather_nd(row_vecs, gather_indices)
@@ -181,7 +181,7 @@ def max_char_logprob_cumsum(char_log_prob):
so the same function can be used regardless whether use_length_predictions
is true or false.
"""
- max_char_log_prob = tf.reduce_max(char_log_prob, reduction_indices=2)
+ max_char_log_prob = tf.reduce_max(input_tensor=char_log_prob, axis=2)
# For an input array [a, b, c]) tf.cumsum returns [a, a + b, a + b + c] if
# exclusive set to False (default).
return tf.cumsum(max_char_log_prob, axis=1, exclusive=False)
@@ -203,7 +203,7 @@ def find_length_by_null(predicted_chars, null_code):
A [batch, ] tensor which stores the sequence length for each sample.
"""
return tf.reduce_sum(
- tf.cast(tf.not_equal(null_code, predicted_chars), tf.int32), axis=1)
+ input_tensor=tf.cast(tf.not_equal(null_code, predicted_chars), tf.int32), axis=1)
def axis_pad(tensor, axis, before=0, after=0, constant_values=0.0):
@@ -248,7 +248,8 @@ def null_based_length_prediction(chars_log_prob, null_code):
element #seq_length - is the probability of length=seq_length.
predicted_length is a tensor with shape [batch].
"""
- predicted_chars = tf.to_int32(tf.argmax(chars_log_prob, axis=2))
+ predicted_chars = tf.cast(
+ tf.argmax(input=chars_log_prob, axis=2), dtype=tf.int32)
# We do right pad to support sequences with seq_length elements.
text_log_prob = max_char_logprob_cumsum(
axis_pad(chars_log_prob, axis=1, after=1))
@@ -334,9 +335,9 @@ class Model(object):
"""
mparams = self._mparams['conv_tower_fn']
logging.debug('Using final_endpoint=%s', mparams.final_endpoint)
- with tf.variable_scope('conv_tower_fn/INCE'):
+ with tf.compat.v1.variable_scope('conv_tower_fn/INCE'):
if reuse:
- tf.get_variable_scope().reuse_variables()
+ tf.compat.v1.get_variable_scope().reuse_variables()
with slim.arg_scope(inception.inception_v3_arg_scope()):
with slim.arg_scope([slim.batch_norm, slim.dropout],
is_training=is_training):
@@ -372,7 +373,7 @@ class Model(object):
def sequence_logit_fn(self, net, labels_one_hot):
mparams = self._mparams['sequence_logit_fn']
# TODO(gorban): remove /alias suffixes from the scopes.
- with tf.variable_scope('sequence_logit_fn/SQLR'):
+ with tf.compat.v1.variable_scope('sequence_logit_fn/SQLR'):
layer_class = sequence_layers.get_layer_class(mparams.use_attention,
mparams.use_autoregression)
layer = layer_class(net, labels_one_hot, self._params, mparams)
@@ -392,7 +393,7 @@ class Model(object):
]
xy_flat_shape = (batch_size, 1, height * width, num_features)
nets_for_merge = []
- with tf.variable_scope('max_pool_views', values=nets_list):
+ with tf.compat.v1.variable_scope('max_pool_views', values=nets_list):
for net in nets_list:
nets_for_merge.append(tf.reshape(net, xy_flat_shape))
merged_net = tf.concat(nets_for_merge, 1)
@@ -413,10 +414,11 @@ class Model(object):
Returns:
A tensor of shape [batch_size, seq_length, features_size].
"""
- with tf.variable_scope('pool_views_fn/STCK'):
+ with tf.compat.v1.variable_scope('pool_views_fn/STCK'):
net = tf.concat(nets, 1)
- batch_size = tf.shape(net)[0]
- image_size = net.get_shape().dims[1].value * net.get_shape().dims[2].value
+ batch_size = tf.shape(input=net)[0]
+ image_size = net.get_shape().dims[1].value * \
+ net.get_shape().dims[2].value
feature_size = net.get_shape().dims[3].value
return tf.reshape(net, tf.stack([batch_size, image_size, feature_size]))
@@ -438,11 +440,13 @@ class Model(object):
with shape [batch_size x seq_length].
"""
log_prob = utils.logits_to_log_prob(chars_logit)
- ids = tf.to_int32(tf.argmax(log_prob, axis=2), name='predicted_chars')
+ ids = tf.cast(tf.argmax(input=log_prob, axis=2),
+ name='predicted_chars', dtype=tf.int32)
mask = tf.cast(
slim.one_hot_encoding(ids, self._params.num_char_classes), tf.bool)
all_scores = tf.nn.softmax(chars_logit)
- selected_scores = tf.boolean_mask(all_scores, mask, name='char_scores')
+ selected_scores = tf.boolean_mask(
+ tensor=all_scores, mask=mask, name='char_scores')
scores = tf.reshape(
selected_scores,
shape=(-1, self._params.seq_length),
@@ -499,7 +503,7 @@ class Model(object):
images = tf.subtract(images, 0.5)
images = tf.multiply(images, 2.5)
- with tf.variable_scope(scope, reuse=reuse):
+ with tf.compat.v1.variable_scope(scope, reuse=reuse):
views = tf.split(
value=images, num_or_size_splits=self._params.num_views, axis=2)
logging.debug('Views=%d single view: %s', len(views), views[0])
@@ -566,7 +570,7 @@ class Model(object):
# multiple losses including regularization losses.
self.sequence_loss_fn(endpoints.chars_logit, data.labels)
total_loss = slim.losses.get_total_loss()
- tf.summary.scalar('TotalLoss', total_loss)
+ tf.compat.v1.summary.scalar('TotalLoss', total_loss)
return total_loss
def label_smoothing_regularization(self, chars_labels, weight=0.1):
@@ -605,7 +609,7 @@ class Model(object):
A Tensor with shape [batch_size] - the log-perplexity for each sequence.
"""
mparams = self._mparams['sequence_loss_fn']
- with tf.variable_scope('sequence_loss_fn/SLF'):
+ with tf.compat.v1.variable_scope('sequence_loss_fn/SLF'):
if mparams.label_smoothing > 0:
smoothed_one_hot_labels = self.label_smoothing_regularization(
chars_labels, mparams.label_smoothing)
@@ -625,7 +629,7 @@ class Model(object):
shape=(batch_size, seq_length),
dtype=tf.int64)
known_char = tf.not_equal(chars_labels, reject_char)
- weights = tf.to_float(known_char)
+ weights = tf.cast(known_char, dtype=tf.float32)
logits_list = tf.unstack(chars_logits, axis=1)
weights_list = tf.unstack(weights, axis=1)
@@ -635,7 +639,7 @@ class Model(object):
weights_list,
softmax_loss_function=get_softmax_loss_fn(mparams.label_smoothing),
average_across_timesteps=mparams.average_across_timesteps)
- tf.losses.add_loss(loss)
+ tf.compat.v1.losses.add_loss(loss)
return loss
def create_summaries(self, data, endpoints, charset, is_training):
@@ -665,13 +669,14 @@ class Model(object):
# tf.summary.text(sname('text/pr'), pr_text)
# gt_text = charset_mapper.get_text(data.labels[:max_outputs,:])
# tf.summary.text(sname('text/gt'), gt_text)
- tf.summary.image(sname('image'), data.images, max_outputs=max_outputs)
+ tf.compat.v1.summary.image(
+ sname('image'), data.images, max_outputs=max_outputs)
if is_training:
- tf.summary.image(
+ tf.compat.v1.summary.image(
sname('image/orig'), data.images_orig, max_outputs=max_outputs)
- for var in tf.trainable_variables():
- tf.summary.histogram(var.op.name, var)
+ for var in tf.compat.v1.trainable_variables():
+ tf.compat.v1.summary.histogram(var.op.name, var)
return None
else:
@@ -700,7 +705,8 @@ class Model(object):
for name, value in names_to_values.items():
summary_name = 'eval/' + name
- tf.summary.scalar(summary_name, tf.Print(value, [value], summary_name))
+ tf.compat.v1.summary.scalar(
+ summary_name, tf.compat.v1.Print(value, [value], summary_name))
return list(names_to_updates.values())
def create_init_fn_to_restore(self,
@@ -733,9 +739,9 @@ class Model(object):
logging.info('variables_to_restore:\n%s',
utils.variables_to_restore().keys())
logging.info('moving_average_variables:\n%s',
- [v.op.name for v in tf.moving_average_variables()])
+ [v.op.name for v in tf.compat.v1.moving_average_variables()])
logging.info('trainable_variables:\n%s',
- [v.op.name for v in tf.trainable_variables()])
+ [v.op.name for v in tf.compat.v1.trainable_variables()])
if master_checkpoint:
assign_from_checkpoint(utils.variables_to_restore(), master_checkpoint)
diff --git a/research/attention_ocr/python/model_export.py b/research/attention_ocr/python/model_export.py
index 7c5f72e68a0b7a36260f0c0eec81078654952284..c4606003ae6f277ab9ac02f8b54c37146e1ee0dc 100644
--- a/research/attention_ocr/python/model_export.py
+++ b/research/attention_ocr/python/model_export.py
@@ -25,7 +25,7 @@ import os
import tensorflow as tf
from tensorflow import app
from tensorflow.contrib import slim
-from tensorflow.python.platform import flags
+from tensorflow.compat.v1 import flags
import common_flags
import model_export_lib
@@ -42,7 +42,8 @@ flags.DEFINE_integer(
'image_height', None,
'Image height used during training(or crop height if used)'
' If not set, the dataset default is used instead.')
-flags.DEFINE_string('work_dir', '/tmp', 'A directory to store temporary files.')
+flags.DEFINE_string('work_dir', '/tmp',
+ 'A directory to store temporary files.')
flags.DEFINE_integer('version_number', 1, 'Version number of the model')
flags.DEFINE_bool(
'export_for_serving', True,
@@ -116,7 +117,7 @@ def export_model(export_dir,
image_height = crop_image_height or dataset_image_height
if export_for_serving:
- images_orig = tf.placeholder(
+ images_orig = tf.compat.v1.placeholder(
tf.string, shape=[batch_size], name='tf_example')
images_orig_float = model_export_lib.generate_tfexample_image(
images_orig,
@@ -126,22 +127,23 @@ def export_model(export_dir,
name='float_images')
else:
images_shape = (batch_size, image_height, image_width, image_depth)
- images_orig = tf.placeholder(
+ images_orig = tf.compat.v1.placeholder(
tf.uint8, shape=images_shape, name='original_image')
images_orig_float = tf.image.convert_image_dtype(
images_orig, dtype=tf.float32, name='float_images')
endpoints = model.create_base(images_orig_float, labels_one_hot=None)
- sess = tf.Session()
- saver = tf.train.Saver(slim.get_variables_to_restore(), sharded=True)
+ sess = tf.compat.v1.Session()
+ saver = tf.compat.v1.train.Saver(
+ slim.get_variables_to_restore(), sharded=True)
saver.restore(sess, get_checkpoint_path())
- tf.logging.info('Model restored successfully.')
+ tf.compat.v1.logging.info('Model restored successfully.')
# Create model signature.
if export_for_serving:
input_tensors = {
- tf.saved_model.signature_constants.CLASSIFY_INPUTS: images_orig
+ tf.saved_model.CLASSIFY_INPUTS: images_orig
}
else:
input_tensors = {'images': images_orig}
@@ -163,21 +165,21 @@ def export_model(export_dir,
dataset.max_sequence_length)):
output_tensors['attention_mask_%d' % i] = t
signature_outputs = model_export_lib.build_tensor_info(output_tensors)
- signature_def = tf.saved_model.signature_def_utils.build_signature_def(
+ signature_def = tf.compat.v1.saved_model.signature_def_utils.build_signature_def(
signature_inputs, signature_outputs,
- tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME)
+ tf.saved_model.CLASSIFY_METHOD_NAME)
# Save model.
- builder = tf.saved_model.builder.SavedModelBuilder(export_dir)
+ builder = tf.compat.v1.saved_model.builder.SavedModelBuilder(export_dir)
builder.add_meta_graph_and_variables(
- sess, [tf.saved_model.tag_constants.SERVING],
+ sess, [tf.saved_model.SERVING],
signature_def_map={
- tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+ tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
signature_def
},
- main_op=tf.tables_initializer(),
+ main_op=tf.compat.v1.tables_initializer(),
strip_default_attrs=True)
builder.save()
- tf.logging.info('Model has been exported to %s' % export_dir)
+ tf.compat.v1.logging.info('Model has been exported to %s' % export_dir)
return signature_def
diff --git a/research/attention_ocr/python/model_export_lib.py b/research/attention_ocr/python/model_export_lib.py
index 3c68dc1fd45f4a0d887e408eac491cea5be1055e..d5d141be2a88f07dbf7b4c1396dd585e8d4d5490 100644
--- a/research/attention_ocr/python/model_export_lib.py
+++ b/research/attention_ocr/python/model_export_lib.py
@@ -36,7 +36,7 @@ def normalize_image(image, original_minval, original_maxval, target_minval,
Returns:
image: image which is the same shape as input image.
"""
- with tf.name_scope('NormalizeImage', values=[image]):
+ with tf.compat.v1.name_scope('NormalizeImage', values=[image]):
original_minval = float(original_minval)
original_maxval = float(original_maxval)
target_minval = float(target_minval)
@@ -68,16 +68,17 @@ def generate_tfexample_image(input_example_strings,
A tensor with shape [batch_size, height, width, channels] of type float32
with values in the range [0..1]
"""
- batch_size = tf.shape(input_example_strings)[0]
+ batch_size = tf.shape(input=input_example_strings)[0]
images_shape = tf.stack(
[batch_size, image_height, image_width, image_channels])
tf_example_image_key = 'image/encoded'
feature_configs = {
tf_example_image_key:
- tf.FixedLenFeature(
+ tf.io.FixedLenFeature(
image_height * image_width * image_channels, dtype=tf.float32)
}
- feature_tensors = tf.parse_example(input_example_strings, feature_configs)
+ feature_tensors = tf.io.parse_example(
+ serialized=input_example_strings, features=feature_configs)
float_images = tf.reshape(
normalize_image(
feature_tensors[tf_example_image_key],
@@ -97,11 +98,11 @@ def attention_ocr_attention_masks(num_characters):
names = ['%s/Softmax:0' % (prefix)]
for i in range(1, num_characters):
names += ['%s_%d/Softmax:0' % (prefix, i)]
- return [tf.get_default_graph().get_tensor_by_name(n) for n in names]
+ return [tf.compat.v1.get_default_graph().get_tensor_by_name(n) for n in names]
def build_tensor_info(tensor_dict):
return {
- k: tf.saved_model.utils.build_tensor_info(t)
+ k: tf.compat.v1.saved_model.utils.build_tensor_info(t)
for k, t in tensor_dict.items()
}
diff --git a/research/attention_ocr/python/model_export_test.py b/research/attention_ocr/python/model_export_test.py
index 19f73f905bf90e60633f812669b4554ac72e3ba4..4dc6688ca469e5b738983a19028f77a9a7b042b8 100644
--- a/research/attention_ocr/python/model_export_test.py
+++ b/research/attention_ocr/python/model_export_test.py
@@ -19,6 +19,7 @@ import os
import numpy as np
from absl.testing import flagsaver
import tensorflow as tf
+from tensorflow.compat.v1 import flags
import common_flags
import model_export
@@ -29,7 +30,7 @@ _CHECKPOINT_URL = (
def _clean_up():
- tf.gfile.DeleteRecursively(tf.test.get_temp_dir())
+ tf.io.gfile.rmtree(tf.compat.v1.test.get_temp_dir())
def _create_tf_example_string(image):
@@ -47,17 +48,18 @@ class AttentionOcrExportTest(tf.test.TestCase):
for suffix in ['.meta', '.index', '.data-00000-of-00001']:
filename = _CHECKPOINT + suffix
self.assertTrue(
- tf.gfile.Exists(filename),
+ tf.io.gfile.exists(filename),
msg='Missing checkpoint file %s. '
'Please download and extract it from %s' %
(filename, _CHECKPOINT_URL))
- tf.flags.FLAGS.dataset_name = 'fsns'
- tf.flags.FLAGS.checkpoint = _CHECKPOINT
- tf.flags.FLAGS.dataset_dir = os.path.join(
+ flags.FLAGS.dataset_name = 'fsns'
+ flags.FLAGS.checkpoint = _CHECKPOINT
+ flags.FLAGS.dataset_dir = os.path.join(
os.path.dirname(__file__), 'datasets/testdata/fsns')
tf.test.TestCase.setUp(self)
_clean_up()
- self.export_dir = os.path.join(tf.test.get_temp_dir(), 'exported_model')
+ self.export_dir = os.path.join(
+ tf.compat.v1.test.get_temp_dir(), 'exported_model')
self.minimal_output_signature = {
'predictions': 'AttentionOcr_v1/predicted_chars:0',
'scores': 'AttentionOcr_v1/predicted_scores:0',
@@ -93,10 +95,10 @@ class AttentionOcrExportTest(tf.test.TestCase):
size=self.dataset.image_shape).astype('uint8'),
}
signature_def = graph_def.signature_def[
- tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
+ tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
if serving:
input_name = signature_def.inputs[
- tf.saved_model.signature_constants.CLASSIFY_INPUTS].name
+ tf.saved_model.CLASSIFY_INPUTS].name
# Model for serving takes input: inputs['inputs'] = 'tf_example:0'
feed_dict = {
input_name: [
@@ -126,11 +128,11 @@ class AttentionOcrExportTest(tf.test.TestCase):
export_for_serving: True if the model was exported for Serving. This
affects how input is fed into the model.
"""
- tf.reset_default_graph()
- sess = tf.Session()
- graph_def = tf.saved_model.loader.load(
+ tf.compat.v1.reset_default_graph()
+ sess = tf.compat.v1.Session()
+ graph_def = tf.compat.v1.saved_model.loader.load(
sess=sess,
- tags=[tf.saved_model.tag_constants.SERVING],
+ tags=[tf.saved_model.SERVING],
export_dir=self.export_dir)
feed_dict = self.create_input_feed(graph_def, export_for_serving)
results = sess.run(self.minimal_output_signature, feed_dict=feed_dict)
diff --git a/research/attention_ocr/python/model_test.py b/research/attention_ocr/python/model_test.py
index ce9b439ca840c59682f88d82c585509d624f6411..6632a38358ad17d4f37888f86d4881362a402c20 100644
--- a/research/attention_ocr/python/model_test.py
+++ b/research/attention_ocr/python/model_test.py
@@ -52,7 +52,7 @@ class ModelTest(tf.test.TestCase):
self.num_char_classes)
self.length_logit_shape = (self.batch_size, self.seq_length + 1)
# Placeholder knows image dimensions, but not batch size.
- self.input_images = tf.placeholder(
+ self.input_images = tf.compat.v1.placeholder(
tf.float32,
shape=(None, self.image_height, self.image_width, 3),
name='input_node')
@@ -89,8 +89,8 @@ class ModelTest(tf.test.TestCase):
with self.test_session() as sess:
endpoints_tf = ocr_model.create_base(
images=self.input_images, labels_one_hot=None)
- sess.run(tf.global_variables_initializer())
- tf.tables_initializer().run()
+ sess.run(tf.compat.v1.global_variables_initializer())
+ tf.compat.v1.tables_initializer().run()
endpoints = sess.run(
endpoints_tf, feed_dict={self.input_images: self.fake_images})
@@ -127,7 +127,7 @@ class ModelTest(tf.test.TestCase):
ocr_model = self.create_model()
conv_tower = ocr_model.conv_tower_fn(self.input_images)
- sess.run(tf.global_variables_initializer())
+ sess.run(tf.compat.v1.global_variables_initializer())
conv_tower_np = sess.run(
conv_tower, feed_dict={self.input_images: self.fake_images})
@@ -141,9 +141,9 @@ class ModelTest(tf.test.TestCase):
ocr_model = self.create_model()
ocr_model.create_base(images=self.input_images, labels_one_hot=None)
with self.test_session() as sess:
- tfprof_root = tf.profiler.profile(
+ tfprof_root = tf.compat.v1.profiler.profile(
sess.graph,
- options=tf.profiler.ProfileOptionBuilder
+ options=tf.compat.v1.profiler.ProfileOptionBuilder
.trainable_variables_parameter())
model_size_bytes = 4 * tfprof_root.total_parameters
@@ -163,9 +163,9 @@ class ModelTest(tf.test.TestCase):
summaries = ocr_model.create_summaries(
data, endpoints, charset, is_training=False)
with self.test_session() as sess:
- sess.run(tf.global_variables_initializer())
- sess.run(tf.local_variables_initializer())
- tf.tables_initializer().run()
+ sess.run(tf.compat.v1.global_variables_initializer())
+ sess.run(tf.compat.v1.local_variables_initializer())
+ tf.compat.v1.tables_initializer().run()
sess.run(summaries) # just check it is runnable
def test_sequence_loss_function_without_label_smoothing(self):
@@ -188,7 +188,7 @@ class ModelTest(tf.test.TestCase):
Returns:
a list of tensors with encoded image coordinates in them.
"""
- batch_size = tf.shape(net)[0]
+ batch_size = tf.shape(input=net)[0]
_, h, w, _ = net.shape.as_list()
h_loc = [
tf.tile(
@@ -200,7 +200,8 @@ class ModelTest(tf.test.TestCase):
h_loc = tf.concat([tf.expand_dims(t, 2) for t in h_loc], 2)
w_loc = [
tf.tile(
- tf.contrib.layers.one_hot_encoding(tf.constant([i]), num_classes=w),
+ tf.contrib.layers.one_hot_encoding(
+ tf.constant([i]), num_classes=w),
[h, 1]) for i in range(w)
]
w_loc = tf.concat([tf.expand_dims(t, 2) for t in w_loc], 2)
@@ -272,8 +273,8 @@ class ModelTest(tf.test.TestCase):
endpoints_tf = ocr_model.create_base(
images=self.fake_images, labels_one_hot=None)
- sess.run(tf.global_variables_initializer())
- tf.tables_initializer().run()
+ sess.run(tf.compat.v1.global_variables_initializer())
+ tf.compat.v1.tables_initializer().run()
endpoints = sess.run(endpoints_tf)
self.assertEqual(endpoints.predicted_text.shape, (self.batch_size,))
@@ -289,7 +290,7 @@ class CharsetMapperTest(tf.test.TestCase):
charset_mapper = model.CharsetMapper(charset)
with self.test_session() as sess:
- tf.tables_initializer().run()
+ tf.compat.v1.tables_initializer().run()
text = sess.run(charset_mapper.get_text(ids))
self.assertAllEqual(text, [b'hello', b'world'])
diff --git a/research/attention_ocr/python/sequence_layers.py b/research/attention_ocr/python/sequence_layers.py
index 82e0de1b026623e01e100182da72c5995d0de45b..15c4b1c3f9451ed54d92a734dcdb4f58e1162f89 100644
--- a/research/attention_ocr/python/sequence_layers.py
+++ b/research/attention_ocr/python/sequence_layers.py
@@ -111,12 +111,12 @@ class SequenceLayerBase(object):
self._mparams = method_params
self._net = net
self._labels_one_hot = labels_one_hot
- self._batch_size = tf.shape(net)[0]
+ self._batch_size = tf.shape(input=net)[0]
# Initialize parameters for char logits which will be computed on the fly
# inside an LSTM decoder.
self._char_logits = {}
- regularizer = slim.l2_regularizer(self._mparams.weight_decay)
+ regularizer = tf.keras.regularizers.l2(0.5 * (self._mparams.weight_decay))
self._softmax_w = slim.model_variable(
'softmax_w',
[self._mparams.num_lstm_units, self._params.num_char_classes],
@@ -124,7 +124,7 @@ class SequenceLayerBase(object):
regularizer=regularizer)
self._softmax_b = slim.model_variable(
'softmax_b', [self._params.num_char_classes],
- initializer=tf.zeros_initializer(),
+ initializer=tf.compat.v1.zeros_initializer(),
regularizer=regularizer)
@abc.abstractmethod
@@ -203,8 +203,8 @@ class SequenceLayerBase(object):
A tensor with shape [batch_size, num_char_classes]
"""
if char_index not in self._char_logits:
- self._char_logits[char_index] = tf.nn.xw_plus_b(inputs, self._softmax_w,
- self._softmax_b)
+ self._char_logits[char_index] = tf.compat.v1.nn.xw_plus_b(inputs, self._softmax_w,
+ self._softmax_b)
return self._char_logits[char_index]
def char_one_hot(self, logit):
@@ -216,7 +216,7 @@ class SequenceLayerBase(object):
Returns:
A tensor with shape [batch_size, num_char_classes]
"""
- prediction = tf.argmax(logit, axis=1)
+ prediction = tf.argmax(input=logit, axis=1)
return slim.one_hot_encoding(prediction, self._params.num_char_classes)
def get_input(self, prev, i):
@@ -244,10 +244,10 @@ class SequenceLayerBase(object):
Returns:
A tensor with shape [batch_size, seq_length, num_char_classes].
"""
- with tf.variable_scope('LSTM'):
+ with tf.compat.v1.variable_scope('LSTM'):
first_label = self.get_input(prev=None, i=0)
decoder_inputs = [first_label] + [None] * (self._params.seq_length - 1)
- lstm_cell = tf.contrib.rnn.LSTMCell(
+ lstm_cell = tf.compat.v1.nn.rnn_cell.LSTMCell(
self._mparams.num_lstm_units,
use_peepholes=False,
cell_clip=self._mparams.lstm_state_clip_value,
@@ -259,9 +259,9 @@ class SequenceLayerBase(object):
loop_function=self.get_input,
cell=lstm_cell)
- with tf.variable_scope('logits'):
+ with tf.compat.v1.variable_scope('logits'):
logits_list = [
- tf.expand_dims(self.char_logit(logit, i), dim=1)
+ tf.expand_dims(self.char_logit(logit, i), axis=1)
for i, logit in enumerate(lstm_outputs)
]
diff --git a/research/attention_ocr/python/sequence_layers_test.py b/research/attention_ocr/python/sequence_layers_test.py
index fd41e2d824c014084129707631d45de334ec741b..29be1875b2aff0bd35da5c16628543e14051a04e 100644
--- a/research/attention_ocr/python/sequence_layers_test.py
+++ b/research/attention_ocr/python/sequence_layers_test.py
@@ -29,13 +29,13 @@ import sequence_layers
def fake_net(batch_size, num_features, feature_size):
return tf.convert_to_tensor(
- np.random.uniform(size=(batch_size, num_features, feature_size)),
+ value=np.random.uniform(size=(batch_size, num_features, feature_size)),
dtype=tf.float32)
def fake_labels(batch_size, seq_length, num_char_classes):
labels_np = tf.convert_to_tensor(
- np.random.randint(
+ value=np.random.randint(
low=0, high=num_char_classes, size=(batch_size, seq_length)))
return slim.one_hot_encoding(labels_np, num_classes=num_char_classes)
diff --git a/research/attention_ocr/python/train.py b/research/attention_ocr/python/train.py
index fa91fb73b412287889f05d0af5875e269f1ce367..9b59fd380601b33d901f35a3784c3c02b0fe6f14 100644
--- a/research/attention_ocr/python/train.py
+++ b/research/attention_ocr/python/train.py
@@ -23,7 +23,7 @@ import logging
import tensorflow as tf
from tensorflow.contrib import slim
from tensorflow import app
-from tensorflow.python.platform import flags
+from tensorflow.compat.v1 import flags
from tensorflow.contrib.tfprof import model_analyzer
import data_provider
@@ -96,16 +96,16 @@ def get_training_hparams():
def create_optimizer(hparams):
"""Creates optimized based on the specified flags."""
if hparams.optimizer == 'momentum':
- optimizer = tf.train.MomentumOptimizer(
+ optimizer = tf.compat.v1.train.MomentumOptimizer(
hparams.learning_rate, momentum=hparams.momentum)
elif hparams.optimizer == 'adam':
- optimizer = tf.train.AdamOptimizer(hparams.learning_rate)
+ optimizer = tf.compat.v1.train.AdamOptimizer(hparams.learning_rate)
elif hparams.optimizer == 'adadelta':
- optimizer = tf.train.AdadeltaOptimizer(hparams.learning_rate)
+ optimizer = tf.compat.v1.train.AdadeltaOptimizer(hparams.learning_rate)
elif hparams.optimizer == 'adagrad':
- optimizer = tf.train.AdagradOptimizer(hparams.learning_rate)
+ optimizer = tf.compat.v1.train.AdagradOptimizer(hparams.learning_rate)
elif hparams.optimizer == 'rmsprop':
- optimizer = tf.train.RMSPropOptimizer(
+ optimizer = tf.compat.v1.train.RMSPropOptimizer(
hparams.learning_rate, momentum=hparams.momentum)
return optimizer
@@ -154,14 +154,14 @@ def train(loss, init_fn, hparams):
def prepare_training_dir():
- if not tf.gfile.Exists(FLAGS.train_log_dir):
+ if not tf.io.gfile.exists(FLAGS.train_log_dir):
logging.info('Create a new training directory %s', FLAGS.train_log_dir)
- tf.gfile.MakeDirs(FLAGS.train_log_dir)
+ tf.io.gfile.makedirs(FLAGS.train_log_dir)
else:
if FLAGS.reset_train_dir:
logging.info('Reset the training directory %s', FLAGS.train_log_dir)
- tf.gfile.DeleteRecursively(FLAGS.train_log_dir)
- tf.gfile.MakeDirs(FLAGS.train_log_dir)
+ tf.io.gfile.rmtree(FLAGS.train_log_dir)
+ tf.io.gfile.makedirs(FLAGS.train_log_dir)
else:
logging.info('Use already existing training directory %s',
FLAGS.train_log_dir)
@@ -169,7 +169,7 @@ def prepare_training_dir():
def calculate_graph_metrics():
param_stats = model_analyzer.print_model_analysis(
- tf.get_default_graph(),
+ tf.compat.v1.get_default_graph(),
tfprof_options=model_analyzer.TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
return param_stats.total_parameters
@@ -186,7 +186,7 @@ def main(_):
# If ps_tasks is zero, the local device is used. When using multiple
# (non-local) replicas, the ReplicaDeviceSetter distributes the variables
# across the different devices.
- device_setter = tf.train.replica_device_setter(
+ device_setter = tf.compat.v1.train.replica_device_setter(
FLAGS.ps_tasks, merge_devices=True)
with tf.device(device_setter):
data = data_provider.get_data(
diff --git a/research/attention_ocr/python/utils.py b/research/attention_ocr/python/utils.py
index aa71f91b7ffe9629dd762f81510e2a9e95d48ce7..5d282f72874856ecdac6dd3932badbbe9d915a9a 100644
--- a/research/attention_ocr/python/utils.py
+++ b/research/attention_ocr/python/utils.py
@@ -37,16 +37,16 @@ def logits_to_log_prob(logits):
probabilities.
"""
- with tf.variable_scope('log_probabilities'):
+ with tf.compat.v1.variable_scope('log_probabilities'):
reduction_indices = len(logits.shape.as_list()) - 1
max_logits = tf.reduce_max(
- logits, reduction_indices=reduction_indices, keep_dims=True)
+ input_tensor=logits, axis=reduction_indices, keepdims=True)
safe_logits = tf.subtract(logits, max_logits)
sum_exp = tf.reduce_sum(
- tf.exp(safe_logits),
- reduction_indices=reduction_indices,
- keep_dims=True)
- log_probs = tf.subtract(safe_logits, tf.log(sum_exp))
+ input_tensor=tf.exp(safe_logits),
+ axis=reduction_indices,
+ keepdims=True)
+ log_probs = tf.subtract(safe_logits, tf.math.log(sum_exp))
return log_probs
@@ -91,7 +91,7 @@ def ConvertAllInputsToTensors(func):
"""
def FuncWrapper(*args):
- tensors = [tf.convert_to_tensor(a) for a in args]
+ tensors = [tf.convert_to_tensor(value=a) for a in args]
return func(*tensors)
return FuncWrapper
diff --git a/research/audioset/vggish/README.md b/research/audioset/vggish/README.md
index 0be0ae86687b81f2074d9011f4c16b5402a138bf..d20e5587af44de4bc029ad1820cd625b6648f0cd 100644
--- a/research/audioset/vggish/README.md
+++ b/research/audioset/vggish/README.md
@@ -16,17 +16,14 @@ VGGish depends on the following Python packages:
* [`numpy`](http://www.numpy.org/)
* [`resampy`](http://resampy.readthedocs.io/en/latest/)
-* [`tensorflow`](http://www.tensorflow.org/) (currently, only TF v1.x)
+* [`tensorflow`](http://www.tensorflow.org/)
* [`tf_slim`](https://github.com/google-research/tf-slim)
* [`six`](https://pythonhosted.org/six/)
* [`soundfile`](https://pysoundfile.readthedocs.io/)
These are all easily installable via, e.g., `pip install numpy` (as in the
-sample installation session below).
-
-Any reasonably recent version of these packages shold work. Note that we currently only support
-TensorFlow v1.x due to a [`tf_slim` limitation](https://github.com/google-research/tf-slim/pull/1).
-TensorFlow v1.15 (the latest version as of Jan 2020) has been tested to work.
+sample installation session below). Any reasonably recent version of these
+packages shold work.
VGGish also requires downloading two data files:
@@ -60,7 +57,7 @@ Here's a sample installation and test session:
$ sudo python -m pip install --upgrade pip wheel
# Install all dependences.
-$ sudo pip install numpy resampy tensorflow==1.15 tf_slim six soundfile
+$ sudo pip install numpy resampy tensorflow tf_slim six soundfile
# Clone TensorFlow models repo into a 'models' directory.
$ git clone https://github.com/tensorflow/models.git
@@ -129,7 +126,10 @@ changes we made:
fully connected layer. This acts as a compact embedding layer.
The model definition provided here defines layers up to and including the
-128-wide embedding layer.
+128-wide embedding layer. Note that the embedding layer does not include
+a final non-linear activation, so the embedding value is pre-activation.
+When training a model stacked on top of VGGish, you should send the
+embedding through a non-linearity of your choice before adding more layers.
### Input: Audio Features
@@ -150,14 +150,7 @@ VGGish was trained with audio features computed as follows:
where each example covers 64 mel bands and 96 frames of 10 ms each.
We provide our own NumPy implementation that produces features that are very
-similar to those produced by our internal production code. This results in
-embedding outputs that are closely match the embeddings that we have already
-released. Note that these embeddings will *not* be bit-for-bit identical to the
-released embeddings due to small differences between the feature computation
-code paths, and even between two different installations of VGGish with
-different underlying libraries and hardware. However, we expect that the
-embeddings will be equivalent in the context of a downstream classification
-task.
+similar to those produced by our internal production code.
### Output: Embeddings
diff --git a/research/audioset/vggish/vggish_export_tfhub.py b/research/audioset/vggish/vggish_export_tfhub.py
new file mode 100644
index 0000000000000000000000000000000000000000..c3956f2365aa3d8475a9d001343b4b9260bc4462
--- /dev/null
+++ b/research/audioset/vggish/vggish_export_tfhub.py
@@ -0,0 +1,126 @@
+"""Exports VGGish as a SavedModel for publication to TF Hub.
+
+The exported SavedModel accepts a 1-d float32 Tensor of arbitrary shape
+containing an audio waveform (assumed to be mono 16 kHz samples in the [-1, +1]
+range) and returns a 2-d float32 batch of 128-d VGGish embeddings, one per
+0.96s example generated from the waveform.
+
+Requires pip-installing tensorflow_hub.
+
+Usage:
+ vggish_export_tfhub.py
+"""
+
+import sys
+sys.path.append('..') # Lets us import yamnet modules from sibling directory.
+
+import numpy as np
+import resampy
+import tensorflow as tf
+assert tf.version.VERSION >= '2.0.0', (
+ 'Need at least TF 2.0, you have TF v{}'.format(tf.version.VERSION))
+import tensorflow_hub as tfhub
+
+import vggish_input
+import vggish_params
+import vggish_slim
+from yamnet import features as yamnet_features
+from yamnet import params as yamnet_params
+
+
+def vggish_definer(variables, checkpoint_path):
+ """Defines VGGish with variables tracked and initialized from a checkpoint."""
+ reader = tf.compat.v1.train.NewCheckpointReader(checkpoint_path)
+
+ def var_tracker(next_creator, **kwargs):
+ """Variable creation hook that assigns initial values from a checkpoint."""
+ var_name = kwargs['name']
+ var_value = reader.get_tensor(var_name)
+ kwargs.update({'initial_value': var_value})
+ var = next_creator(**kwargs)
+ variables.append(var)
+ return var
+
+ def waveform_to_features(waveform):
+ """Creates VGGish features using the YAMNet feature extractor."""
+ params = yamnet_params.Params(
+ sample_rate=vggish_params.SAMPLE_RATE,
+ stft_window_seconds=vggish_params.STFT_WINDOW_LENGTH_SECONDS,
+ stft_hop_seconds=vggish_params.STFT_HOP_LENGTH_SECONDS,
+ mel_bands=vggish_params.NUM_MEL_BINS,
+ mel_min_hz=vggish_params.MEL_MIN_HZ,
+ mel_max_hz=vggish_params.MEL_MAX_HZ,
+ log_offset=vggish_params.LOG_OFFSET,
+ patch_window_seconds=vggish_params.EXAMPLE_WINDOW_SECONDS,
+ patch_hop_seconds=vggish_params.EXAMPLE_HOP_SECONDS)
+ log_mel_spectrogram, features = yamnet_features.waveform_to_log_mel_spectrogram_patches(
+ waveform, params)
+ return features
+
+ def define_vggish(waveform):
+ with tf.variable_creator_scope(var_tracker):
+ features = waveform_to_features(waveform)
+ return vggish_slim.define_vggish_slim(features, training=False)
+
+ return define_vggish
+
+
+class VGGish(tf.Module):
+ """A TF2 Module wrapper around VGGish."""
+ def __init__(self, checkpoint_path):
+ super().__init__()
+ self._variables = []
+ self._vggish_fn = tf.compat.v1.wrap_function(
+ vggish_definer(self._variables, checkpoint_path),
+ signature=(tf.TensorSpec(shape=[None], dtype=tf.float32),))
+
+ @tf.function(input_signature=(tf.TensorSpec(shape=[None], dtype=tf.float32),))
+ def __call__(self, waveform):
+ return self._vggish_fn(waveform)
+
+
+def check_model(model_fn):
+ """Applies vggish_smoke_test's sanity check to an instance of VGGish."""
+ num_secs = 3
+ freq = 1000
+ sr = 44100
+ t = np.arange(0, num_secs, 1 / sr)
+ waveform = np.sin(2 * np.pi * freq * t)
+
+ waveform = resampy.resample(waveform, sr, vggish_params.SAMPLE_RATE)
+ embeddings = model_fn(waveform)
+
+ expected_embedding_mean = -0.0333
+ expected_embedding_std = 0.380
+ rel_error = 0.1
+ np.testing.assert_allclose(
+ [np.mean(embeddings), np.std(embeddings)],
+ [expected_embedding_mean, expected_embedding_std],
+ rtol=rel_error)
+
+
+def main(args):
+ # Create a TF2 wrapper around VGGish.
+ vggish_checkpoint_path = args[0]
+ vggish = VGGish(vggish_checkpoint_path)
+ check_model(vggish)
+
+ # Make TF-Hub export.
+ vggish_tfhub_export_path = args[1]
+ tf.saved_model.save(vggish, vggish_tfhub_export_path)
+
+ # Check export in TF2.
+ model = tfhub.load(vggish_tfhub_export_path)
+ check_model(model)
+
+ # Check export in TF1.
+ with tf.compat.v1.Graph().as_default(), tf.compat.v1.Session() as sess:
+ model = tfhub.load(vggish_tfhub_export_path)
+ sess.run(tf.compat.v1.global_variables_initializer())
+ def run_model(waveform):
+ embeddings = model(waveform)
+ return sess.run(embeddings)
+ check_model(run_model)
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
diff --git a/research/audioset/vggish/vggish_inference_demo.py b/research/audioset/vggish/vggish_inference_demo.py
index 6d9d631b36d8eeac68ea23b59bd0938b5dbbd30c..2294698f5611050de271045bc62fc249e632d925 100644
--- a/research/audioset/vggish/vggish_inference_demo.py
+++ b/research/audioset/vggish/vggish_inference_demo.py
@@ -50,7 +50,6 @@ import numpy as np
import six
import soundfile
import tensorflow.compat.v1 as tf
-tf.disable_v2_behavior()
import vggish_input
import vggish_params
@@ -89,7 +88,7 @@ def main(_):
num_secs = 5
freq = 1000
sr = 44100
- t = np.linspace(0, num_secs, int(num_secs * sr))
+ t = np.arange(0, num_secs, 1 / sr)
x = np.sin(2 * np.pi * freq * t)
# Convert to signed 16-bit samples.
samples = np.clip(x * 32768, -32768, 32767).astype(np.int16)
diff --git a/research/audioset/vggish/vggish_slim.py b/research/audioset/vggish/vggish_slim.py
index 0a838c4b8e2619b2573c490f546044b113f3bb55..84a8aac3986b06eb42ee96de16f6ec34c2d3465f 100644
--- a/research/audioset/vggish/vggish_slim.py
+++ b/research/audioset/vggish/vggish_slim.py
@@ -31,28 +31,31 @@ https://github.com/tensorflow/models/blob/master/research/slim/nets/vgg.py
"""
import tensorflow.compat.v1 as tf
-tf.disable_v2_behavior()
import tf_slim as slim
import vggish_params as params
-def define_vggish_slim(training=False):
+def define_vggish_slim(features_tensor=None, training=False):
"""Defines the VGGish TensorFlow model.
All ops are created in the current default graph, under the scope 'vggish/'.
- The input is a placeholder named 'vggish/input_features' of type float32 and
- shape [batch_size, num_frames, num_bands] where batch_size is variable and
- num_frames and num_bands are constants, and [num_frames, num_bands] represents
- a log-mel-scale spectrogram patch covering num_bands frequency bands and
- num_frames time frames (where each frame step is usually 10ms). This is
- produced by computing the stabilized log(mel-spectrogram + params.LOG_OFFSET).
- The output is an op named 'vggish/embedding' which produces the activations of
- a 128-D embedding layer, which is usually the penultimate layer when used as
- part of a full model with a final classifier layer.
+ The input is either a tensor passed in via the optional 'features_tensor'
+ argument or a placeholder created below named 'vggish/input_features'. The
+ input is expected to have dtype float32 and shape [batch_size, num_frames,
+ num_bands] where batch_size is variable and num_frames and num_bands are
+ constants, and [num_frames, num_bands] represents a log-mel-scale spectrogram
+ patch covering num_bands frequency bands and num_frames time frames (where
+ each frame step is usually 10ms). This is produced by computing the stabilized
+ log(mel-spectrogram + params.LOG_OFFSET). The output is a tensor named
+ 'vggish/embedding' which produces the pre-activation values of a 128-D
+ embedding layer, which is usually the penultimate layer when used as part of a
+ full model with a final classifier layer.
Args:
+ features_tensor: If not None, the tensor containing the input features.
+ If None, a placeholder input is created.
training: If true, all parameters are marked trainable.
Returns:
@@ -76,11 +79,13 @@ def define_vggish_slim(training=False):
kernel_size=[2, 2], stride=2, padding='SAME'), \
tf.variable_scope('vggish'):
# Input: a batch of 2-D log-mel-spectrogram patches.
- features = tf.placeholder(
- tf.float32, shape=(None, params.NUM_FRAMES, params.NUM_BANDS),
- name='input_features')
+ if features_tensor is None:
+ features_tensor = tf.placeholder(
+ tf.float32, shape=(None, params.NUM_FRAMES, params.NUM_BANDS),
+ name='input_features')
# Reshape to 4-D so that we can convolve a batch with conv2d().
- net = tf.reshape(features, [-1, params.NUM_FRAMES, params.NUM_BANDS, 1])
+ net = tf.reshape(features_tensor,
+ [-1, params.NUM_FRAMES, params.NUM_BANDS, 1])
# The VGG stack of alternating convolutions and max-pools.
net = slim.conv2d(net, 64, scope='conv1')
@@ -96,7 +101,8 @@ def define_vggish_slim(training=False):
net = slim.flatten(net)
net = slim.repeat(net, 2, slim.fully_connected, 4096, scope='fc1')
# The embedding layer.
- net = slim.fully_connected(net, params.EMBEDDING_SIZE, scope='fc2')
+ net = slim.fully_connected(net, params.EMBEDDING_SIZE, scope='fc2',
+ activation_fn=None)
return tf.identity(net, name='embedding')
diff --git a/research/audioset/vggish/vggish_smoke_test.py b/research/audioset/vggish/vggish_smoke_test.py
index f27e583aee473c6a04a5af20fd101c7a54871e94..82a644a91e3ba8af3f47b71c7de0c3943ffe156f 100644
--- a/research/audioset/vggish/vggish_smoke_test.py
+++ b/research/audioset/vggish/vggish_smoke_test.py
@@ -33,7 +33,6 @@ from __future__ import print_function
import numpy as np
import tensorflow.compat.v1 as tf
-tf.disable_v2_behavior()
import vggish_input
import vggish_params
@@ -54,7 +53,7 @@ rel_error = 0.1 # Up to 10%
num_secs = 3
freq = 1000
sr = 44100
-t = np.linspace(0, num_secs, int(num_secs * sr))
+t = np.arange(0, num_secs, 1 / sr)
x = np.sin(2 * np.pi * freq * t)
# Produce a batch of log mel spectrogram examples.
@@ -77,8 +76,8 @@ with tf.Graph().as_default(), tf.Session() as sess:
[embedding_batch] = sess.run([embedding_tensor],
feed_dict={features_tensor: input_batch})
print('VGGish embedding: ', embedding_batch[0])
- expected_embedding_mean = 0.131
- expected_embedding_std = 0.238
+ expected_embedding_mean = -0.0333
+ expected_embedding_std = 0.380
np.testing.assert_allclose(
[np.mean(embedding_batch), np.std(embedding_batch)],
[expected_embedding_mean, expected_embedding_std],
@@ -88,8 +87,8 @@ with tf.Graph().as_default(), tf.Session() as sess:
pproc = vggish_postprocess.Postprocessor(pca_params_path)
postprocessed_batch = pproc.postprocess(embedding_batch)
print('Postprocessed VGGish embedding: ', postprocessed_batch[0])
-expected_postprocessed_mean = 123.0
-expected_postprocessed_std = 75.0
+expected_postprocessed_mean = 122.0
+expected_postprocessed_std = 93.5
np.testing.assert_allclose(
[np.mean(postprocessed_batch), np.std(postprocessed_batch)],
[expected_postprocessed_mean, expected_postprocessed_std],
diff --git a/research/audioset/vggish/vggish_train_demo.py b/research/audioset/vggish/vggish_train_demo.py
index d8be0f1774549b0b0ec4bdbcf840a16696fa6322..7a968b171106e360861ed794e6aeab512d6abac4 100644
--- a/research/audioset/vggish/vggish_train_demo.py
+++ b/research/audioset/vggish/vggish_train_demo.py
@@ -49,7 +49,6 @@ from random import shuffle
import numpy as np
import tensorflow.compat.v1 as tf
-tf.disable_v2_behavior()
import tf_slim as slim
import vggish_input
@@ -95,7 +94,7 @@ def _get_examples_batch():
# Make a waveform for each class.
num_seconds = 5
sr = 44100 # Sampling rate.
- t = np.linspace(0, num_seconds, int(num_seconds * sr)) # Time axis.
+ t = np.arange(0, num_seconds, 1 / sr) # Time axis
# Random sine wave.
freq = np.random.uniform(100, 1000)
sine = np.sin(2 * np.pi * freq * t)
@@ -129,14 +128,15 @@ def _get_examples_batch():
def main(_):
with tf.Graph().as_default(), tf.Session() as sess:
# Define VGGish.
- embeddings = vggish_slim.define_vggish_slim(FLAGS.train_vggish)
+ embeddings = vggish_slim.define_vggish_slim(training=FLAGS.train_vggish)
# Define a shallow classification model and associated training ops on top
# of VGGish.
with tf.variable_scope('mymodel'):
- # Add a fully connected layer with 100 units.
+ # Add a fully connected layer with 100 units. Add an activation function
+ # to the embeddings since they are pre-activation.
num_units = 100
- fc = slim.fully_connected(embeddings, num_units)
+ fc = slim.fully_connected(tf.nn.relu(embeddings), num_units)
# Add a classifier layer at the end, consisting of parallel logistic
# classifiers, one per class. This allows for multi-class tasks.
@@ -146,19 +146,16 @@ def main(_):
# Add training ops.
with tf.variable_scope('train'):
- global_step = tf.Variable(
- 0, name='global_step', trainable=False,
- collections=[tf.GraphKeys.GLOBAL_VARIABLES,
- tf.GraphKeys.GLOBAL_STEP])
+ global_step = tf.train.create_global_step()
# Labels are assumed to be fed as a batch multi-hot vectors, with
# a 1 in the position of each positive class label, and 0 elsewhere.
- labels = tf.placeholder(
+ labels_input = tf.placeholder(
tf.float32, shape=(None, _NUM_CLASSES), name='labels')
# Cross-entropy label loss.
xent = tf.nn.sigmoid_cross_entropy_with_logits(
- logits=logits, labels=labels, name='xent')
+ logits=logits, labels=labels_input, name='xent')
loss = tf.reduce_mean(xent, name='loss_op')
tf.summary.scalar('loss', loss)
@@ -166,29 +163,22 @@ def main(_):
optimizer = tf.train.AdamOptimizer(
learning_rate=vggish_params.LEARNING_RATE,
epsilon=vggish_params.ADAM_EPSILON)
- optimizer.minimize(loss, global_step=global_step, name='train_op')
+ train_op = optimizer.minimize(loss, global_step=global_step)
# Initialize all variables in the model, and then load the pre-trained
# VGGish checkpoint.
sess.run(tf.global_variables_initializer())
vggish_slim.load_vggish_slim_checkpoint(sess, FLAGS.checkpoint)
- # Locate all the tensors and ops we need for the training loop.
- features_tensor = sess.graph.get_tensor_by_name(
- vggish_params.INPUT_TENSOR_NAME)
- labels_tensor = sess.graph.get_tensor_by_name('mymodel/train/labels:0')
- global_step_tensor = sess.graph.get_tensor_by_name(
- 'mymodel/train/global_step:0')
- loss_tensor = sess.graph.get_tensor_by_name('mymodel/train/loss_op:0')
- train_op = sess.graph.get_operation_by_name('mymodel/train/train_op')
-
# The training loop.
+ features_input = sess.graph.get_tensor_by_name(
+ vggish_params.INPUT_TENSOR_NAME)
for _ in range(FLAGS.num_batches):
(features, labels) = _get_examples_batch()
- [num_steps, loss, _] = sess.run(
- [global_step_tensor, loss_tensor, train_op],
- feed_dict={features_tensor: features, labels_tensor: labels})
- print('Step %d: loss %g' % (num_steps, loss))
+ [num_steps, loss_value, _] = sess.run(
+ [global_step, loss, train_op],
+ feed_dict={features_input: features, labels_input: labels})
+ print('Step %d: loss %g' % (num_steps, loss_value))
if __name__ == '__main__':
tf.app.run()
diff --git a/research/audioset/yamnet/README.md b/research/audioset/yamnet/README.md
index 983724c0d3b526a288f564d5c90b8f8330ac14ee..4f3caddfd0fc258421ec9eaff5c565e4909a9c37 100644
--- a/research/audioset/yamnet/README.md
+++ b/research/audioset/yamnet/README.md
@@ -18,12 +18,8 @@ YAMNet depends on the following Python packages:
* [`pysoundfile`](https://pysoundfile.readthedocs.io/)
These are all easily installable via, e.g., `pip install numpy` (as in the
-example command sequence below).
-
-Any reasonably recent version of these packages should work. TensorFlow should
-be at least version 1.8 to ensure Keras support is included. Note that while
-the code works fine with TensorFlow v1.x or v2.x, we explicitly enable v1.x
-behavior.
+example command sequence below). Any reasonably recent version of these
+packages should work.
YAMNet also requires downloading the following data file:
diff --git a/research/audioset/yamnet/export.py b/research/audioset/yamnet/export.py
new file mode 100644
index 0000000000000000000000000000000000000000..87bb00612648c894deb7a94f161381a482742984
--- /dev/null
+++ b/research/audioset/yamnet/export.py
@@ -0,0 +1,213 @@
+"""Exports YAMNet as: TF2 SavedModel, TF-Lite model, TF-JS model.
+
+The exported models all accept as input:
+- 1-d float32 Tensor of arbitrary shape containing an audio waveform
+ (assumed to be mono 16 kHz samples in the [-1, +1] range)
+and return as output:
+- a 2-d float32 Tensor of shape [num_frames, num_classes] containing
+ predicted class scores for each frame of audio extracted from the input.
+- a 2-d float32 Tensor of shape [num_frames, embedding_size] containing
+ embeddings of each frame of audio.
+- a 2-d float32 Tensor of shape [num_spectrogram_frames, num_mel_bins]
+ containing the log mel spectrogram of the entire waveform.
+The SavedModels will also contain (as an asset) a class map CSV file that maps
+class indices to AudioSet class names and Freebase MIDs. The path to the class
+map is available as the 'class_map_path()' method of the restored model.
+
+Requires pip-installing tensorflow_hub and tensorflowjs.
+
+Usage:
+ export.py
+and the various exports will be created in subdirectories of the output directory.
+Assumes that it will be run in the yamnet source directory from where it loads
+the class map. Skips an export if the corresponding directory already exists.
+"""
+
+import os
+import sys
+import tempfile
+import time
+
+import numpy as np
+import tensorflow as tf
+assert tf.version.VERSION >= '2.0.0', (
+ 'Need at least TF 2.0, you have TF v{}'.format(tf.version.VERSION))
+import tensorflow_hub as tfhub
+from tensorflowjs.converters import tf_saved_model_conversion_v2 as tfjs_saved_model_converter
+
+import params as yamnet_params
+import yamnet
+
+
+def log(msg):
+ print('\n=====\n{} | {}\n=====\n'.format(time.asctime(), msg), flush=True)
+
+
+class YAMNet(tf.Module):
+ "''A TF2 Module wrapper around YAMNet."""
+ def __init__(self, weights_path, params):
+ super().__init__()
+ self._yamnet = yamnet.yamnet_frames_model(params)
+ self._yamnet.load_weights(weights_path)
+ self._class_map_asset = tf.saved_model.Asset('yamnet_class_map.csv')
+
+ @tf.function
+ def class_map_path(self):
+ return self._class_map_asset.asset_path
+
+ @tf.function(input_signature=(tf.TensorSpec(shape=[None], dtype=tf.float32),))
+ def __call__(self, waveform):
+ return self._yamnet(waveform)
+
+
+def check_model(model_fn, class_map_path, params):
+ yamnet_classes = yamnet.class_names(class_map_path)
+
+ """Applies yamnet_test's sanity checks to an instance of YAMNet."""
+ def clip_test(waveform, expected_class_name, top_n=10):
+ predictions, embeddings, log_mel_spectrogram = model_fn(waveform)
+ clip_predictions = np.mean(predictions, axis=0)
+ top_n_indices = np.argsort(clip_predictions)[-top_n:]
+ top_n_scores = clip_predictions[top_n_indices]
+ top_n_class_names = yamnet_classes[top_n_indices]
+ top_n_predictions = list(zip(top_n_class_names, top_n_scores))
+ assert expected_class_name in top_n_class_names, (
+ 'Did not find expected class {} in top {} predictions: {}'.format(
+ expected_class_name, top_n, top_n_predictions))
+
+ clip_test(
+ waveform=np.zeros((int(3 * params.sample_rate),), dtype=np.float32),
+ expected_class_name='Silence')
+
+ np.random.seed(51773) # Ensure repeatability.
+ clip_test(
+ waveform=np.random.uniform(-1.0, +1.0,
+ (int(3 * params.sample_rate),)).astype(np.float32),
+ expected_class_name='White noise')
+
+ clip_test(
+ waveform=np.sin(2 * np.pi * 440 *
+ np.arange(0, 3, 1 / params.sample_rate), dtype=np.float32),
+ expected_class_name='Sine wave')
+
+
+def make_tf2_export(weights_path, export_dir):
+ if os.path.exists(export_dir):
+ log('TF2 export already exists in {}, skipping TF2 export'.format(
+ export_dir))
+ return
+
+ # Create a TF2 Module wrapper around YAMNet.
+ log('Building and checking TF2 Module ...')
+ params = yamnet_params.Params()
+ yamnet = YAMNet(weights_path, params)
+ check_model(yamnet, yamnet.class_map_path(), params)
+ log('Done')
+
+ # Make TF2 SavedModel export.
+ log('Making TF2 SavedModel export ...')
+ tf.saved_model.save(yamnet, export_dir)
+ log('Done')
+
+ # Check export with TF-Hub in TF2.
+ log('Checking TF2 SavedModel export in TF2 ...')
+ model = tfhub.load(export_dir)
+ check_model(model, model.class_map_path(), params)
+ log('Done')
+
+ # Check export with TF-Hub in TF1.
+ log('Checking TF2 SavedModel export in TF1 ...')
+ with tf.compat.v1.Graph().as_default(), tf.compat.v1.Session() as sess:
+ model = tfhub.load(export_dir)
+ sess.run(tf.compat.v1.global_variables_initializer())
+ def run_model(waveform):
+ return sess.run(model(waveform))
+ check_model(run_model, model.class_map_path().eval(), params)
+ log('Done')
+
+
+def make_tflite_export(weights_path, export_dir):
+ if os.path.exists(export_dir):
+ log('TF-Lite export already exists in {}, skipping TF-Lite export'.format(
+ export_dir))
+ return
+
+ # Create a TF-Lite compatible Module wrapper around YAMNet.
+ log('Building and checking TF-Lite Module ...')
+ params = yamnet_params.Params(tflite_compatible=True)
+ yamnet = YAMNet(weights_path, params)
+ check_model(yamnet, yamnet.class_map_path(), params)
+ log('Done')
+
+ # Make TF-Lite SavedModel export.
+ log('Making TF-Lite SavedModel export ...')
+ saved_model_dir = os.path.join(export_dir, 'saved_model')
+ os.makedirs(saved_model_dir)
+ tf.saved_model.save(yamnet, saved_model_dir)
+ log('Done')
+
+ # Check that the export can be loaded and works.
+ log('Checking TF-Lite SavedModel export in TF2 ...')
+ model = tf.saved_model.load(saved_model_dir)
+ check_model(model, model.class_map_path(), params)
+ log('Done')
+
+ # Make a TF-Lite model from the SavedModel.
+ log('Making TF-Lite model ...')
+ tflite_converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir)
+ tflite_model = tflite_converter.convert()
+ tflite_model_path = os.path.join(export_dir, 'yamnet.tflite')
+ with open(tflite_model_path, 'wb') as f:
+ f.write(tflite_model)
+ log('Done')
+
+ # Check the TF-Lite export.
+ log('Checking TF-Lite model ...')
+ interpreter = tf.lite.Interpreter(tflite_model_path)
+ audio_input_index = interpreter.get_input_details()[0]['index']
+ scores_output_index = interpreter.get_output_details()[0]['index']
+ embeddings_output_index = interpreter.get_output_details()[1]['index']
+ spectrogram_output_index = interpreter.get_output_details()[2]['index']
+ def run_model(waveform):
+ interpreter.resize_tensor_input(audio_input_index, [len(waveform)], strict=True)
+ interpreter.allocate_tensors()
+ interpreter.set_tensor(audio_input_index, waveform)
+ interpreter.invoke()
+ return (interpreter.get_tensor(scores_output_index),
+ interpreter.get_tensor(embeddings_output_index),
+ interpreter.get_tensor(spectrogram_output_index))
+ check_model(run_model, 'yamnet_class_map.csv', params)
+ log('Done')
+
+ return saved_model_dir
+
+
+def make_tfjs_export(tflite_saved_model_dir, export_dir):
+ if os.path.exists(export_dir):
+ log('TF-JS export already exists in {}, skipping TF-JS export'.format(
+ export_dir))
+ return
+
+ # Make a TF-JS model from the TF-Lite SavedModel export.
+ log('Making TF-JS model ...')
+ os.makedirs(export_dir)
+ tfjs_saved_model_converter.convert_tf_saved_model(
+ tflite_saved_model_dir, export_dir)
+ log('Done')
+
+
+def main(args):
+ weights_path = args[0]
+ output_dir = args[1]
+
+ tf2_export_dir = os.path.join(output_dir, 'tf2')
+ make_tf2_export(weights_path, tf2_export_dir)
+
+ tflite_export_dir = os.path.join(output_dir, 'tflite')
+ tflite_saved_model_dir = make_tflite_export(weights_path, tflite_export_dir)
+
+ tfjs_export_dir = os.path.join(output_dir, 'tfjs')
+ make_tfjs_export(tflite_saved_model_dir, tfjs_export_dir)
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
diff --git a/research/audioset/yamnet/features.py b/research/audioset/yamnet/features.py
index 98661124787c1b3f672185483c5715edb375cb2a..9b1cf7775db8cd29b154a6703da033566284bced 100644
--- a/research/audioset/yamnet/features.py
+++ b/research/audioset/yamnet/features.py
@@ -19,61 +19,147 @@ import numpy as np
import tensorflow as tf
-def waveform_to_log_mel_spectrogram(waveform, params):
- """Compute log mel spectrogram of a 1-D waveform."""
+def waveform_to_log_mel_spectrogram_patches(waveform, params):
+ """Compute log mel spectrogram patches of a 1-D waveform."""
with tf.name_scope('log_mel_features'):
# waveform has shape [<# samples>]
# Convert waveform into spectrogram using a Short-Time Fourier Transform.
# Note that tf.signal.stft() uses a periodic Hann window by default.
window_length_samples = int(
- round(params.SAMPLE_RATE * params.STFT_WINDOW_SECONDS))
+ round(params.sample_rate * params.stft_window_seconds))
hop_length_samples = int(
- round(params.SAMPLE_RATE * params.STFT_HOP_SECONDS))
+ round(params.sample_rate * params.stft_hop_seconds))
fft_length = 2 ** int(np.ceil(np.log(window_length_samples) / np.log(2.0)))
num_spectrogram_bins = fft_length // 2 + 1
- magnitude_spectrogram = tf.abs(tf.signal.stft(
- signals=waveform,
- frame_length=window_length_samples,
- frame_step=hop_length_samples,
- fft_length=fft_length))
+ if params.tflite_compatible:
+ magnitude_spectrogram = _tflite_stft_magnitude(
+ signal=waveform,
+ frame_length=window_length_samples,
+ frame_step=hop_length_samples,
+ fft_length=fft_length)
+ else:
+ magnitude_spectrogram = tf.abs(tf.signal.stft(
+ signals=waveform,
+ frame_length=window_length_samples,
+ frame_step=hop_length_samples,
+ fft_length=fft_length))
# magnitude_spectrogram has shape [<# STFT frames>, num_spectrogram_bins]
# Convert spectrogram into log mel spectrogram.
linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
- num_mel_bins=params.MEL_BANDS,
+ num_mel_bins=params.mel_bands,
num_spectrogram_bins=num_spectrogram_bins,
- sample_rate=params.SAMPLE_RATE,
- lower_edge_hertz=params.MEL_MIN_HZ,
- upper_edge_hertz=params.MEL_MAX_HZ)
+ sample_rate=params.sample_rate,
+ lower_edge_hertz=params.mel_min_hz,
+ upper_edge_hertz=params.mel_max_hz)
mel_spectrogram = tf.matmul(
magnitude_spectrogram, linear_to_mel_weight_matrix)
- log_mel_spectrogram = tf.math.log(mel_spectrogram + params.LOG_OFFSET)
- # log_mel_spectrogram has shape [<# STFT frames>, MEL_BANDS]
+ log_mel_spectrogram = tf.math.log(mel_spectrogram + params.log_offset)
+ # log_mel_spectrogram has shape [<# STFT frames>, params.mel_bands]
- return log_mel_spectrogram
-
-
-def spectrogram_to_patches(spectrogram, params):
- """Break up a spectrogram into a stack of fixed-size patches."""
- with tf.name_scope('feature_patches'):
- # Frame spectrogram (shape [<# STFT frames>, MEL_BANDS]) into patches
- # (the input examples).
- # Only complete frames are emitted, so if there is less than
- # PATCH_WINDOW_SECONDS of waveform then nothing is emitted
+ # Frame spectrogram (shape [<# STFT frames>, params.mel_bands]) into patches
+ # (the input examples). Only complete frames are emitted, so if there is
+ # less than params.patch_window_seconds of waveform then nothing is emitted
# (to avoid this, zero-pad before processing).
- hop_length_samples = int(
- round(params.SAMPLE_RATE * params.STFT_HOP_SECONDS))
- spectrogram_sr = params.SAMPLE_RATE / hop_length_samples
+ spectrogram_hop_length_samples = int(
+ round(params.sample_rate * params.stft_hop_seconds))
+ spectrogram_sample_rate = params.sample_rate / spectrogram_hop_length_samples
patch_window_length_samples = int(
- round(spectrogram_sr * params.PATCH_WINDOW_SECONDS))
+ round(spectrogram_sample_rate * params.patch_window_seconds))
patch_hop_length_samples = int(
- round(spectrogram_sr * params.PATCH_HOP_SECONDS))
+ round(spectrogram_sample_rate * params.patch_hop_seconds))
features = tf.signal.frame(
- signal=spectrogram,
+ signal=log_mel_spectrogram,
frame_length=patch_window_length_samples,
frame_step=patch_hop_length_samples,
axis=0)
- # features has shape [<# patches>, <# STFT frames in an patch>, MEL_BANDS]
+ # features has shape [<# patches>, <# STFT frames in an patch>, params.mel_bands]
+
+ return log_mel_spectrogram, features
+
+
+def pad_waveform(waveform, params):
+ """Pads waveform with silence if needed to get an integral number of patches."""
+ # In order to produce one patch of log mel spectrogram input to YAMNet, we
+ # need at least one patch window length of waveform plus enough extra samples
+ # to complete the final STFT analysis window.
+ min_waveform_seconds = (
+ params.patch_window_seconds +
+ params.stft_window_seconds - params.stft_hop_seconds)
+ min_num_samples = tf.cast(min_waveform_seconds * params.sample_rate, tf.int32)
+ num_samples = tf.shape(waveform)[0]
+ num_padding_samples = tf.maximum(0, min_num_samples - num_samples)
+
+ # In addition, there might be enough waveform for one or more additional
+ # patches formed by hopping forward. If there are more samples than one patch,
+ # round up to an integral number of hops.
+ num_samples = tf.maximum(num_samples, min_num_samples)
+ num_samples_after_first_patch = num_samples - min_num_samples
+ hop_samples = tf.cast(params.patch_hop_seconds * params.sample_rate, tf.int32)
+ num_hops_after_first_patch = tf.cast(tf.math.ceil(
+ tf.cast(num_samples_after_first_patch, tf.float32) /
+ tf.cast(hop_samples, tf.float32)), tf.int32)
+ num_padding_samples += (
+ hop_samples * num_hops_after_first_patch - num_samples_after_first_patch)
+
+ padded_waveform = tf.pad(waveform, [[0, num_padding_samples]],
+ mode='CONSTANT', constant_values=0.0)
+ return padded_waveform
+
+
+def _tflite_stft_magnitude(signal, frame_length, frame_step, fft_length):
+ """TF-Lite-compatible version of tf.abs(tf.signal.stft())."""
+ def _hann_window():
+ return tf.reshape(
+ tf.constant(
+ (0.5 - 0.5 * np.cos(2 * np.pi * np.arange(0, 1.0, 1.0 / frame_length))
+ ).astype(np.float32),
+ name='hann_window'), [1, frame_length])
+
+ def _dft_matrix(dft_length):
+ """Calculate the full DFT matrix in NumPy."""
+ # See https://en.wikipedia.org/wiki/DFT_matrix
+ omega = (0 + 1j) * 2.0 * np.pi / float(dft_length)
+ # Don't include 1/sqrt(N) scaling, tf.signal.rfft doesn't apply it.
+ return np.exp(omega * np.outer(np.arange(dft_length), np.arange(dft_length)))
+
+ def _rdft(framed_signal, fft_length):
+ """Implement real-input Discrete Fourier Transform by matmul."""
+ # We are right-multiplying by the DFT matrix, and we are keeping only the
+ # first half ("positive frequencies"). So discard the second half of rows,
+ # but transpose the array for right-multiplication. The DFT matrix is
+ # symmetric, so we could have done it more directly, but this reflects our
+ # intention better.
+ complex_dft_matrix_kept_values = _dft_matrix(fft_length)[:(
+ fft_length // 2 + 1), :].transpose()
+ real_dft_matrix = tf.constant(
+ np.real(complex_dft_matrix_kept_values).astype(np.float32),
+ name='real_dft_matrix')
+ imag_dft_matrix = tf.constant(
+ np.imag(complex_dft_matrix_kept_values).astype(np.float32),
+ name='imaginary_dft_matrix')
+ signal_frame_length = tf.shape(framed_signal)[-1]
+ half_pad = (fft_length - signal_frame_length) // 2
+ padded_frames = tf.pad(
+ framed_signal,
+ [
+ # Don't add any padding in the frame dimension.
+ [0, 0],
+ # Pad before and after the signal within each frame.
+ [half_pad, fft_length - signal_frame_length - half_pad]
+ ],
+ mode='CONSTANT',
+ constant_values=0.0)
+ real_stft = tf.matmul(padded_frames, real_dft_matrix)
+ imag_stft = tf.matmul(padded_frames, imag_dft_matrix)
+ return real_stft, imag_stft
+
+ def _complex_abs(real, imag):
+ return tf.sqrt(tf.add(real * real, imag * imag))
- return features
+ framed_signal = tf.signal.frame(signal, frame_length, frame_step)
+ windowed_signal = framed_signal * _hann_window()
+ real_stft, imag_stft = _rdft(windowed_signal, fft_length)
+ stft_magnitude = _complex_abs(real_stft, imag_stft)
+ return stft_magnitude
diff --git a/research/audioset/yamnet/inference.py b/research/audioset/yamnet/inference.py
index 1aa015550933c8696e56f92bdedd4de61ac518cb..88509b0f0e2ead6da92c05ceb4180f5f7d89a409 100644
--- a/research/audioset/yamnet/inference.py
+++ b/research/audioset/yamnet/inference.py
@@ -23,17 +23,16 @@ import resampy
import soundfile as sf
import tensorflow as tf
-import params
+import params as yamnet_params
import yamnet as yamnet_model
def main(argv):
- assert argv
+ assert argv, 'Usage: inference.py ...'
- graph = tf.Graph()
- with graph.as_default():
- yamnet = yamnet_model.yamnet_frames_model(params)
- yamnet.load_weights('yamnet.h5')
+ params = yamnet_params.Params()
+ yamnet = yamnet_model.yamnet_frames_model(params)
+ yamnet.load_weights('yamnet.h5')
yamnet_classes = yamnet_model.class_names('yamnet_class_map.csv')
for file_name in argv:
@@ -41,24 +40,22 @@ def main(argv):
wav_data, sr = sf.read(file_name, dtype=np.int16)
assert wav_data.dtype == np.int16, 'Bad sample type: %r' % wav_data.dtype
waveform = wav_data / 32768.0 # Convert to [-1.0, +1.0]
+ waveform = waveform.astype('float32')
# Convert to mono and the sample rate expected by YAMNet.
if len(waveform.shape) > 1:
waveform = np.mean(waveform, axis=1)
- if sr != params.SAMPLE_RATE:
- waveform = resampy.resample(waveform, sr, params.SAMPLE_RATE)
+ if sr != params.sample_rate:
+ waveform = resampy.resample(waveform, sr, params.sample_rate)
# Predict YAMNet classes.
- # Second output is log-mel-spectrogram array (used for visualizations).
- # (steps=1 is a work around for Keras batching limitations.)
- with graph.as_default():
- scores, _ = yamnet.predict(np.reshape(waveform, [1, -1]), steps=1)
+ scores, embeddings, spectrogram = yamnet(waveform)
# Scores is a matrix of (time_frames, num_classes) classifier scores.
# Average them along time to get an overall classifier output for the clip.
prediction = np.mean(scores, axis=0)
# Report the highest-scoring classes and their scores.
top5_i = np.argsort(prediction)[::-1][:5]
- print(file_name, ':\n' +
+ print(file_name, ':\n' +
'\n'.join(' {:12s}: {:.3f}'.format(yamnet_classes[i], prediction[i])
for i in top5_i))
diff --git a/research/audioset/yamnet/params.py b/research/audioset/yamnet/params.py
index 5d848ad71695f2fdb29eddea5b7c135509fa5fe2..306c94218d942e1232d50e05a975f1cba7ca6ad3 100644
--- a/research/audioset/yamnet/params.py
+++ b/research/audioset/yamnet/params.py
@@ -15,28 +15,37 @@
"""Hyperparameters for YAMNet."""
-# The following hyperparameters (except PATCH_HOP_SECONDS) were used to train YAMNet,
+from dataclasses import dataclass
+
+# The following hyperparameters (except patch_hop_seconds) were used to train YAMNet,
# so expect some variability in performance if you change these. The patch hop can
# be changed arbitrarily: a smaller hop should give you more patches from the same
# clip and possibly better performance at a larger computational cost.
-SAMPLE_RATE = 16000
-STFT_WINDOW_SECONDS = 0.025
-STFT_HOP_SECONDS = 0.010
-MEL_BANDS = 64
-MEL_MIN_HZ = 125
-MEL_MAX_HZ = 7500
-LOG_OFFSET = 0.001
-PATCH_WINDOW_SECONDS = 0.96
-PATCH_HOP_SECONDS = 0.48
+@dataclass(frozen=True) # Instances of this class are immutable.
+class Params:
+ sample_rate: float = 16000.0
+ stft_window_seconds: float = 0.025
+ stft_hop_seconds: float = 0.010
+ mel_bands: int = 64
+ mel_min_hz: float = 125.0
+ mel_max_hz: float = 7500.0
+ log_offset: float = 0.001
+ patch_window_seconds: float = 0.96
+ patch_hop_seconds: float = 0.48
+
+ @property
+ def patch_frames(self):
+ return int(round(self.patch_window_seconds / self.stft_hop_seconds))
+
+ @property
+ def patch_bands(self):
+ return self.mel_bands
-PATCH_FRAMES = int(round(PATCH_WINDOW_SECONDS / STFT_HOP_SECONDS))
-PATCH_BANDS = MEL_BANDS
-NUM_CLASSES = 521
-CONV_PADDING = 'same'
-BATCHNORM_CENTER = True
-BATCHNORM_SCALE = False
-BATCHNORM_EPSILON = 1e-4
-CLASSIFIER_ACTIVATION = 'sigmoid'
+ num_classes: int = 521
+ conv_padding: str = 'same'
+ batchnorm_center: bool = True
+ batchnorm_scale: bool = False
+ batchnorm_epsilon: float = 1e-4
+ classifier_activation: str = 'sigmoid'
-FEATURES_LAYER_NAME = 'features'
-EXAMPLE_PREDICTIONS_LAYER_NAME = 'predictions'
+ tflite_compatible: bool = False
diff --git a/research/audioset/yamnet/yamnet.py b/research/audioset/yamnet/yamnet.py
index ce36ff8cc462bc3a37bcaacd615d7c997d46f6ef..cac7f87d99e1c03991f36d4afde807353e886dfd 100644
--- a/research/audioset/yamnet/yamnet.py
+++ b/research/audioset/yamnet/yamnet.py
@@ -22,53 +22,52 @@ import tensorflow as tf
from tensorflow.keras import Model, layers
import features as features_lib
-import params
-def _batch_norm(name):
+def _batch_norm(name, params):
def _bn_layer(layer_input):
return layers.BatchNormalization(
name=name,
- center=params.BATCHNORM_CENTER,
- scale=params.BATCHNORM_SCALE,
- epsilon=params.BATCHNORM_EPSILON)(layer_input)
+ center=params.batchnorm_center,
+ scale=params.batchnorm_scale,
+ epsilon=params.batchnorm_epsilon)(layer_input)
return _bn_layer
-def _conv(name, kernel, stride, filters):
+def _conv(name, kernel, stride, filters, params):
def _conv_layer(layer_input):
output = layers.Conv2D(name='{}/conv'.format(name),
filters=filters,
kernel_size=kernel,
strides=stride,
- padding=params.CONV_PADDING,
+ padding=params.conv_padding,
use_bias=False,
activation=None)(layer_input)
- output = _batch_norm(name='{}/conv/bn'.format(name))(output)
+ output = _batch_norm('{}/conv/bn'.format(name), params)(output)
output = layers.ReLU(name='{}/relu'.format(name))(output)
return output
return _conv_layer
-def _separable_conv(name, kernel, stride, filters):
+def _separable_conv(name, kernel, stride, filters, params):
def _separable_conv_layer(layer_input):
output = layers.DepthwiseConv2D(name='{}/depthwise_conv'.format(name),
kernel_size=kernel,
strides=stride,
depth_multiplier=1,
- padding=params.CONV_PADDING,
+ padding=params.conv_padding,
use_bias=False,
activation=None)(layer_input)
- output = _batch_norm(name='{}/depthwise_conv/bn'.format(name))(output)
+ output = _batch_norm('{}/depthwise_conv/bn'.format(name), params)(output)
output = layers.ReLU(name='{}/depthwise_conv/relu'.format(name))(output)
output = layers.Conv2D(name='{}/pointwise_conv'.format(name),
filters=filters,
kernel_size=(1, 1),
strides=1,
- padding=params.CONV_PADDING,
+ padding=params.conv_padding,
use_bias=False,
activation=None)(output)
- output = _batch_norm(name='{}/pointwise_conv/bn'.format(name))(output)
+ output = _batch_norm('{}/pointwise_conv/bn'.format(name), params)(output)
output = layers.ReLU(name='{}/pointwise_conv/relu'.format(name))(output)
return output
return _separable_conv_layer
@@ -93,47 +92,46 @@ _YAMNET_LAYER_DEFS = [
]
-def yamnet(features):
+def yamnet(features, params):
"""Define the core YAMNet mode in Keras."""
net = layers.Reshape(
- (params.PATCH_FRAMES, params.PATCH_BANDS, 1),
- input_shape=(params.PATCH_FRAMES, params.PATCH_BANDS))(features)
+ (params.patch_frames, params.patch_bands, 1),
+ input_shape=(params.patch_frames, params.patch_bands))(features)
for (i, (layer_fun, kernel, stride, filters)) in enumerate(_YAMNET_LAYER_DEFS):
- net = layer_fun('layer{}'.format(i + 1), kernel, stride, filters)(net)
- net = layers.GlobalAveragePooling2D()(net)
- logits = layers.Dense(units=params.NUM_CLASSES, use_bias=True)(net)
- predictions = layers.Activation(
- name=params.EXAMPLE_PREDICTIONS_LAYER_NAME,
- activation=params.CLASSIFIER_ACTIVATION)(logits)
- return predictions
+ net = layer_fun('layer{}'.format(i + 1), kernel, stride, filters, params)(net)
+ embeddings = layers.GlobalAveragePooling2D()(net)
+ logits = layers.Dense(units=params.num_classes, use_bias=True)(embeddings)
+ predictions = layers.Activation(activation=params.classifier_activation)(logits)
+ return predictions, embeddings
-def yamnet_frames_model(feature_params):
+def yamnet_frames_model(params):
"""Defines the YAMNet waveform-to-class-scores model.
Args:
- feature_params: An object with parameter fields to control the feature
- calculation.
+ params: An instance of Params containing hyperparameters.
Returns:
- A model accepting (1, num_samples) waveform input and emitting a
- (num_patches, num_classes) matrix of class scores per time frame as
- well as a (num_spectrogram_frames, num_mel_bins) spectrogram feature
- matrix.
+ A model accepting (num_samples,) waveform input and emitting:
+ - predictions: (num_patches, num_classes) matrix of class scores per time frame
+ - embeddings: (num_patches, embedding size) matrix of embeddings per time frame
+ - log_mel_spectrogram: (num_spectrogram_frames, num_mel_bins) spectrogram feature matrix
"""
- waveform = layers.Input(batch_shape=(1, None))
- # Store the intermediate spectrogram features to use in visualization.
- spectrogram = features_lib.waveform_to_log_mel_spectrogram(
- tf.squeeze(waveform, axis=0), feature_params)
- patches = features_lib.spectrogram_to_patches(spectrogram, feature_params)
- predictions = yamnet(patches)
- frames_model = Model(name='yamnet_frames',
- inputs=waveform, outputs=[predictions, spectrogram])
+ waveform = layers.Input(batch_shape=(None,), dtype=tf.float32)
+ waveform_padded = features_lib.pad_waveform(waveform, params)
+ log_mel_spectrogram, features = features_lib.waveform_to_log_mel_spectrogram_patches(
+ waveform_padded, params)
+ predictions, embeddings = yamnet(features, params)
+ frames_model = Model(
+ name='yamnet_frames', inputs=waveform,
+ outputs=[predictions, embeddings, log_mel_spectrogram])
return frames_model
def class_names(class_map_csv):
"""Read the class name definition file and return a list of strings."""
+ if tf.is_tensor(class_map_csv):
+ class_map_csv = class_map_csv.numpy()
with open(class_map_csv) as csv_file:
reader = csv.reader(csv_file)
next(reader) # Skip header
diff --git a/research/audioset/yamnet/yamnet_test.py b/research/audioset/yamnet/yamnet_test.py
index c3f64859949ce4bc7cc83529334a9e29da0d0124..d0d16da8082217a495f8eeb7e68d56956dd09058 100644
--- a/research/audioset/yamnet/yamnet_test.py
+++ b/research/audioset/yamnet/yamnet_test.py
@@ -23,46 +23,46 @@ import yamnet
class YAMNetTest(tf.test.TestCase):
- _yamnet_graph = None
+ _params = None
_yamnet = None
_yamnet_classes = None
@classmethod
def setUpClass(cls):
- super(YAMNetTest, cls).setUpClass()
- cls._yamnet_graph = tf.Graph()
- with cls._yamnet_graph.as_default():
- cls._yamnet = yamnet.yamnet_frames_model(params)
- cls._yamnet.load_weights('yamnet.h5')
- cls._yamnet_classes = yamnet.class_names('yamnet_class_map.csv')
+ super().setUpClass()
+ cls._params = params.Params()
+ cls._yamnet = yamnet.yamnet_frames_model(cls._params)
+ cls._yamnet.load_weights('yamnet.h5')
+ cls._yamnet_classes = yamnet.class_names('yamnet_class_map.csv')
def clip_test(self, waveform, expected_class_name, top_n=10):
"""Run the model on the waveform, check that expected class is in top-n."""
- with YAMNetTest._yamnet_graph.as_default():
- prediction = np.mean(YAMNetTest._yamnet.predict(
- np.reshape(waveform, [1, -1]), steps=1)[0], axis=0)
- top_n_class_names = YAMNetTest._yamnet_classes[
- np.argsort(prediction)[-top_n:]]
- self.assertIn(expected_class_name, top_n_class_names)
+ predictions, embeddings, log_mel_spectrogram = YAMNetTest._yamnet(waveform)
+ clip_predictions = np.mean(predictions, axis=0)
+ top_n_indices = np.argsort(clip_predictions)[-top_n:]
+ top_n_scores = clip_predictions[top_n_indices]
+ top_n_class_names = YAMNetTest._yamnet_classes[top_n_indices]
+ top_n_predictions = list(zip(top_n_class_names, top_n_scores))
+ self.assertIn(expected_class_name, top_n_class_names,
+ 'Did not find expected class {} in top {} predictions: {}'.format(
+ expected_class_name, top_n, top_n_predictions))
def testZeros(self):
self.clip_test(
- waveform=np.zeros((1, int(3 * params.SAMPLE_RATE))),
+ waveform=np.zeros((int(3 * YAMNetTest._params.sample_rate),)),
expected_class_name='Silence')
def testRandom(self):
np.random.seed(51773) # Ensure repeatability.
self.clip_test(
waveform=np.random.uniform(-1.0, +1.0,
- (1, int(3 * params.SAMPLE_RATE))),
+ (int(3 * YAMNetTest._params.sample_rate),)),
expected_class_name='White noise')
def testSine(self):
self.clip_test(
- waveform=np.reshape(
- np.sin(2 * np.pi * 440 * np.linspace(
- 0, 3, int(3 *params.SAMPLE_RATE))),
- [1, -1]),
+ waveform=np.sin(2 * np.pi * 440 *
+ np.arange(0, 3, 1 / YAMNetTest._params.sample_rate)),
expected_class_name='Sine wave')
diff --git a/research/audioset/yamnet/yamnet_visualization.ipynb b/research/audioset/yamnet/yamnet_visualization.ipynb
index 49e2186f2c7df022903f0c74e3937947663dabea..db08acfbc98e13ecb44995a4c95871885117a4de 100644
--- a/research/audioset/yamnet/yamnet_visualization.ipynb
+++ b/research/audioset/yamnet/yamnet_visualization.ipynb
@@ -1,198 +1,274 @@
{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Copyright 2019 The TensorFlow Authors All Rights Reserved.\n",
- "#\n",
- "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
- "# you may not use this file except in compliance with the License.\n",
- "# You may obtain a copy of the License at\n",
- "#\n",
- "# http://www.apache.org/licenses/LICENSE-2.0\n",
- "#\n",
- "# Unless required by applicable law or agreed to in writing, software\n",
- "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
- "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
- "# See the License for the specific language governing permissions and\n",
- "# limitations under the License.\n",
- "# =============================================================================="
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Visualization of the YAMNet audio event classification model.\n",
- "# See https://github.com/tensorflow/models/tree/master/research/audioset/yamnet/"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Imports.\n",
- "import numpy as np\n",
- "import soundfile as sf\n",
- "\n",
- "import matplotlib.pyplot as plt\n",
- "\n",
- "import params\n",
- "import yamnet as yamnet_model\n",
- "import tensorflow as tf"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Sample rate = 16000\n"
- ]
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.6"
+ },
+ "colab": {
+ "name": "yamnet_visualization.ipynb",
+ "provenance": []
}
- ],
- "source": [
- "# Read in the audio.\n",
- "# You can get this example waveform via:\n",
- "# curl -O https://storage.googleapis.com/audioset/speech_whistling2.wav\n",
- "\n",
- "wav_file_name = 'speech_whistling2.wav'\n",
- "\n",
- "wav_data, sr = sf.read(wav_file_name, dtype=np.int16)\n",
- "waveform = wav_data / 32768.0\n",
- "# The graph is designed for a sampling rate of 16 kHz, but higher rates \n",
- "# should work too.\n",
- "params.SAMPLE_RATE = sr\n",
- "print(\"Sample rate =\", params.SAMPLE_RATE)"
- ]
},
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
+ "cells": [
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "WARNING:tensorflow:From /Users/dpwe/google/vggish/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1635: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
- "Instructions for updating:\n",
- "If using Keras pass *_constraint arguments to layers.\n"
- ]
- }
- ],
- "source": [
- "# Set up the YAMNet model.\n",
- "class_names = yamnet_model.class_names('yamnet_class_map.csv')\n",
- "params.PATCH_HOP_SECONDS = 0.1 # 10 Hz scores frame rate.\n",
- "graph = tf.Graph()\n",
- "with graph.as_default():\n",
- " yamnet = yamnet_model.yamnet_frames_model(params)\n",
- " yamnet.load_weights('yamnet.h5')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
+ "cell_type": "code",
+ "metadata": {
+ "id": "xcZmKVHusxQT"
+ },
+ "source": [
+ "# Copyright 2019 The TensorFlow Authors All Rights Reserved.\n",
+ "#\n",
+ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# http://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License.\n",
+ "# =============================================================================="
+ ],
+ "execution_count": 1,
+ "outputs": []
+ },
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "WARNING:tensorflow:When passing input data as arrays, do not specify `steps_per_epoch`/`steps` argument. Please use `batch_size` instead.\n"
- ]
- }
- ],
- "source": [
- "# Run the model.\n",
- "with graph.as_default():\n",
- " scores, spectrogram = yamnet.predict(np.reshape(waveform, [1, -1]), steps=1)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
+ "cell_type": "code",
+ "metadata": {
+ "id": "0sDpjNbksxQa"
+ },
+ "source": [
+ "# Visualization of the YAMNet audio event classification model.\n",
+ "# See https://github.com/tensorflow/models/tree/master/research/audioset/yamnet/\n",
+ "#\n",
+ "# This notebook can be run in Google Colab at https://colab.research.google.com\n",
+ "# by either downloading this ipynb and uploading it, or by looking up the\n",
+ "# notebook directly on GitHub in Colab's \"Open notebook\" dialog."
+ ],
+ "execution_count": 2,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "hnI3jFRHs-N7",
+ "outputId": "24b5696f-e4cb-4d49-bddc-40ab3ef211b9",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ }
+ },
+ "source": [
+ "# Install required packages.\n",
+ "!pip install soundfile\n",
+ "!git clone https://github.com/tensorflow/models.git\n",
+ "%cd models/research/audioset/yamnet\n",
+ "\n",
+ "# Download YAMNet data\n",
+ "!curl -O https://storage.googleapis.com/audioset/yamnet.h5\n",
+ "\n",
+ "# Download audio for testing\n",
+ "!curl -O https://storage.googleapis.com/audioset/speech_whistling2.wav\n",
+ "\n",
+ "!ls -l"
+ ],
+ "execution_count": 3,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "text": [
+ "Collecting soundfile\n",
+ " Downloading https://files.pythonhosted.org/packages/eb/f2/3cbbbf3b96fb9fa91582c438b574cff3f45b29c772f94c400e2c99ef5db9/SoundFile-0.10.3.post1-py2.py3-none-any.whl\n",
+ "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.6/dist-packages (from soundfile) (1.14.3)\n",
+ "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi>=1.0->soundfile) (2.20)\n",
+ "Installing collected packages: soundfile\n",
+ "Successfully installed soundfile-0.10.3.post1\n",
+ "Cloning into 'models'...\n",
+ "remote: Enumerating objects: 67, done.\u001b[K\n",
+ "remote: Counting objects: 100% (67/67), done.\u001b[K\n",
+ "remote: Compressing objects: 100% (65/65), done.\u001b[K\n",
+ "remote: Total 46144 (delta 26), reused 43 (delta 2), pack-reused 46077\u001b[K\n",
+ "Receiving objects: 100% (46144/46144), 551.17 MiB | 32.01 MiB/s, done.\n",
+ "Resolving deltas: 100% (31621/31621), done.\n",
+ "/content/models/research/audioset/yamnet\n",
+ " % Total % Received % Xferd Average Speed Time Time Time Current\n",
+ " Dload Upload Total Spent Left Speed\n",
+ "100 14.5M 100 14.5M 0 0 17.0M 0 --:--:-- --:--:-- --:--:-- 17.0M\n",
+ " % Total % Received % Xferd Average Speed Time Time Time Current\n",
+ " Dload Upload Total Spent Left Speed\n",
+ "100 153k 100 153k 0 0 1314k 0 --:--:-- --:--:-- --:--:-- 1314k\n",
+ "total 15296\n",
+ "-rw-r--r-- 1 root root 7816 Oct 22 17:31 export.py\n",
+ "-rw-r--r-- 1 root root 7490 Oct 22 17:31 features.py\n",
+ "-rw-r--r-- 1 root root 2307 Oct 22 17:31 inference.py\n",
+ "-rw-r--r-- 1 root root 1847 Oct 22 17:31 params.py\n",
+ "-rw-r--r-- 1 root root 5012 Oct 22 17:31 README.md\n",
+ "-rw-r--r-- 1 root root 157484 Oct 22 17:31 speech_whistling2.wav\n",
+ "-rw-r--r-- 1 root root 14096 Oct 22 17:31 yamnet_class_map.csv\n",
+ "-rw-r--r-- 1 root root 15296092 Oct 22 17:31 yamnet.h5\n",
+ "-rw-r--r-- 1 root root 5549 Oct 22 17:31 yamnet.py\n",
+ "-rw-r--r-- 1 root root 2564 Oct 22 17:31 yamnet_test.py\n",
+ "-rw-r--r-- 1 root root 140923 Oct 22 17:31 yamnet_visualization.ipynb\n"
+ ],
+ "name": "stdout"
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "M0woGtbhsxQg"
+ },
+ "source": [
+ "# Imports.\n",
+ "import numpy as np\n",
+ "import soundfile as sf\n",
+ "\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import params as yamnet_params\n",
+ "import yamnet as yamnet_model\n",
+ "import tensorflow as tf"
+ ],
+ "execution_count": 4,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "jt2v3i94sxQl"
+ },
+ "source": [
+ "# Read in the audio.\n",
+ "wav_file_name = 'speech_whistling2.wav'\n",
+ "wav_data, sr = sf.read(wav_file_name, dtype=np.int16)\n",
+ "waveform = wav_data / 32768.0"
+ ],
+ "execution_count": 5,
+ "outputs": []
+ },
{
- "data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAArgAAAHSCAYAAAAHR7iOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd5wcdd3A8c93Ztv1VNIhgYQSCBAIvUqHKIgFwQKoyOOjqDw8iqGKSLWL5REEUVARBAsYivQeILSQAqT3cmnXt838nj9mbrN3ubK7t+3uvu/X65Ldqb/9zc7sd37zK2KMQSmllFJKqYHCKnUClFJKKaWUyicNcJVSSiml1ICiAa5SSimllBpQNMBVSimllFIDiga4SimllFJqQNEAVymllFJKDSiBUidAlY8RI0aYiRMnljoZSimllFK9evPNNzcbY0Z2NU8DXJUyceJE5s6dW+pkKKWUUkr1SkRWdjdPqygopZRSSqkBRQNcpZRSSik1oGiAq5RSSimlBhQNcJXKQkssycRZs3ngjdWlTopSSimluqEBrlJZ2NAYBeC3zy8tcUqUUkop1R0NcJVSSiml1ICiAW6ZE5HTROQDEVkiIrO6mH+hiNSLyDv+30Vp8y4QkcX+3wXFTblSSimlVGloP7hlTERs4NfAycAa4A0RedgYs7DTovcbYy7ptO4w4HvADMAAb/rrbitC0pVSSimlSkZLcMvbocASY8wyY0wc+CtwVobrngo8aYzZ6ge1TwKnFSidg9ai9Y1cePfrxJJOqZOilFJKKZ8GuOVtHJDeXH+NP62zT4rIPBF5UEQmZLOuiFwsInNFZG59fX2+0j1gGdPx/ayH5vHcB/W8vWp7aRKklFJKqZ1ogNv/PQJMNMbsj1dK+8dsVjbG3GGMmWGMmTFyZJfDOSvgoj/OZcYNT+6YIN5/765pAOB27VVBKaUGBGMMD7yxmpZYstRJUX2gAW55WwtMSHs/3p+WYozZYoyJ+W/vBA7OdF2VuacWbWRzc5xVW1sAWFbf0mH+ss0tXa2mlFKqH3l16RbO//3rXP7QPC74/eulTo7qAw1wy9sbwBQRmSQiIeBc4OH0BURkTNrbM4FF/usngFNEZKiIDAVO8aepPuiuKsLKLa1FTolSSql8O+93c3hx8WYA5q7UNtn9mfaiUMaMMUkRuQQvMLWB3xtjFojI9cBcY8zDwDdF5EwgCWwFLvTX3SoiP8ALkgGuN8ZsLfqHUEoppZQqMg1wy5wx5lHg0U7Trk17fQVwRTfr/h74fUETOMi8sULvEZQaDFZuaeHOF5dz3Zn7YltS6uSoEvnH22s4e/r4UidD5UCrKCiVhc3N8VInQSlVBF//y1vcO2clC9Y1lDopqoT+5/53S50ElSMNcJVSSqluNLQlSp2Eotjnmse55bH3AWiNa+8Bqv/TAFepXvzzbe18QqnBZv7aRgC+cNfgaEnflnD47fNLeXz+eqZe+wTz1gy+vr3//taaUidB5ZEGuEr14tL730m9XrKpuYQpUUoVw+qtg7dXlK/+6S0A5q3JvmqG6xpM59Fw+pE/v7aq1ElQeaQBrlJKKZXmTe0eiljSzXjZR95dx8RZs9n9yke57eklBUxVYXV33B23/wbtg5kGuEoppVSa9Kc2g0FXAdxtTy/OeP1v3Pd26vXPnvowL2kqJ399Q0t2+yMNcJXqwZJNTaVOglKqiAZjad321p17hxksjesyccuj75c6CSoHGuAq1YPP3zk4GpgopTyvLd+y07R9rnm8BCkpnu8+NC+v27vv9f5X4tnTjU1TTHuV6I80wFWqB04/bjChlMpBF6d8W8IpfjqK6KlFm/K6vSv+/l5et1cMC9c1ljoJKs80wFVKKaXUoLZYq6MNODpUr1I9qG+KlToJSqkiWra5pcvpq7e2MmFYZZFTU/7i3fS2MHHWbIZWBtnWmmDOFScyui5S5JRl57IHdMSygUZLcJXqxuvLt5Y6CUqpIrv6n/O7nH7MD58tckpK77VlO9dH7mzPqx/rdt62Vq+h2jurtdu1cvX68q28sWJg/tZpgKtUN/I9qMOTCzeScDLvW1IppUrpM3fMyct2ogm97pWrc25/lU//9tVSJ6MgNMBVqhv5HI/9hQ/r+co9c/n5AOwjUimlenLp/e9wXp6CZVUY0QwbUt47Z2WHfo/LmQa4SnXj509l3tF5b7a2eP1MLqvvun6fUkqVgttLv7+rtnQ/bHE2w/K+mkF1B1Vc6cf+3dXbe13+l08v5pp/zueRd9exsTGamr6+oa0g6esrDXCV6kZLHktw2z02fwMvLq7P+3aVUioXu1/5aI/zo8nuS/ba69hmqi1ent2tZRKor9wy8AonrvzHju7cPnPHnC5vdra1xDn4B0/y2d/N4SdP7ngC+YN/L2T11la+eu+bHHHzM7y0eDMAtzz2Pjc9uqjwic+ABrhKdSNfXeA2tCY6DP3ZfiFQqr97+N11PLlwY1YleeVsS/Pg6jWlux4Q0iWd7o/tj57IboSvfa4tzwEzMhm87ot/eKPwCfG1xJKs3tp9yXm+/PWN1R3eX/fIgp2WeWFxPVta4ryytGMJ/L/nreeYHz7L4ws2APD5u15j4qzZ/Pb5pdzxwrLCJToLGuCWORE5TUQ+EJElIjKri/mXichCEZknIk+LyG5p8xwRecf/e7i4KR/4Mn0ss2IA3vmrwaehLdGhhKehLcE373ubr9wzl7/NXVPClOXPwTc81eP85ABrJNpTDwjtnv+w+ydO972+utt53Yn1UCJcKm+u7L2Xh2JWL9v3e09wzA+f5ebHilsSes+rKzvcrG5pjvGtv77Twxrdu+/1Vby8ZDNzSlg1RQPcMiYiNvBr4HRgKnCeiEzttNjbwAxjzP7Ag8AP0+a1GWMO9P/OLEqiB5F/vr2u1ElQqiga2hIc8P3/cOsT77NkUzN7XPkoB3z/P6n5l+d5qNdy9ci8wXfO3/r4+3kdenevq8uvFDeT+qdAUXrBSb+Juv35ZazZVviS3HTpNy1f+/NbOW/nir+/x+fufI1z75jDs+/nd6S8TGmAW94OBZYYY5YZY+LAX4Gz0hcwxjxrjGk/A+YA44ucxkHr1sezezzXzgCPvLuOibNms701nt9EKVUA7QHO7c8v46SfPo+TyTPdAeh/7h+cgwF0NfTu3+ZmX3pbrjLt3WZbS+Gv1+d26m1i3pqGgu8zXXq93Nfy1Bd8Mat3pNMAt7yNA9KvImv8ad35MpD+zCkiInNFZI6IfLyrFUTkYn+ZufX12vipGIwx3PnScgCWdzNqklLlYmNjlFsey+1mrj8ZKPWIM5Xt5+1cJ/Q7Dw6cUvuWDBu/da6zWghzO1WX6Espal9kUm0jGxNnzebZD4pbkqsB7gAhIp8HZgA/Spu8mzFmBvBZ4Ociskfn9YwxdxhjZhhjZowcObJIqVV5a8GmVAHd/8YqDrvp6VInoyj+/Fr+HsP3B9mWwn/2Tu3H9qdPFqYf80XrG5k4azZH3/pMQbafrcZogk/+3yt53+4X736jqE9/NMAtb2uBCWnvx/vTOhCRk4CrgDONMalmwMaYtf7/y4DngOmFTOxAYYzJa2vq15Z3rGT/5MKNqVa7IpK3/SiVT45r+O5DOz+a7s6f5qwsYGoKr7shegeqbPulXb01f32dPvP+xrxtayA4/RcvArBmW/H6k+0p0Nz/uv90O6+v/l3Eeuwa4Ja3N4ApIjJJRELAuUCH3hBEZDpwO15wuylt+lARCfuvRwBHAQuLlvJ+7IG5q3ttTZ2phONy06MdH++u2NLKe2uLW69KqWzdMDu7y0UxA8RVW1oHXZWCfMulIK2hLbt+b7vz5EINcEutMU/HMlu59sqQCw1wy5gxJglcAjwBLAIeMMYsEJHrRaS9V4QfAdXA3zp1B7YPMFdE3gWeBW4xxmiAm4GXl+SvWxNXf4RVP3X3yyuyXqcYQec/317LsT96lv99YHA2+MqXtTmUFv7Fr8bx1qq+1c/MpXuxwez9DY153+Yvns7fSJ3lSgPcMmeMedQYs6cxZg9jzI3+tGuNMQ/7r08yxozq3B2YMeYVY8w0Y8wB/v93lfJzlKMF6xp45N3Sdvvz8V+/POg6l1d9k3RcJs6azT2vrijYPlpzHMXv1sc/yHNKdvZ/zy0F4O9vry3JuZPJ4Aj9wfMfZt/gp73nmE/8Jv/1M1X3Yon8f+f+8MqKvG+z3GiAqwatmbe9xDfue3un6fmsFiv0vrHFm5rzt0M1oD25cCOTr/I6Srn2XzuPOpQv67bnVhfwt88vzXNKdvbBxqbU60I0hOnND3PsHrDcNLbldhOjXRvm197X9D7YxkB7Eji/SFX0NMBVqpN8Nvsy9H5h2tAQ5ehbn+lyaEbXNWxsjOYxRSpTry7dwsRZs5k4azYNraWpr5auJZbkK/fM7TBt5m0vFmRf5TKISUNbgleWbCaedLn50UXsf90THeav2FLcTvCBVBd//V22jczaHXj9k3lOSWmVui53NIPS2YdL/KQx3z76y5dYuK6Rx+ev7zA6Yr5pgKtUAWVy7fz722tZs62NY374LBsadgSzq7a0ctjNT3PYTU/nXKKmcnfe73Z0i3TA9YVrVZyJv7+1hn2/98RO0xesy3/dPIBfPbukINvN1sX3zOWzd77GH15Zzu0vLKMxmlupo1LdSX8iUGyZjoyWS334cnfGbS/y1T+9xe5XPlqwfWiAqwa9znfwxe66K71lcvroaJ++/RXqm7w6hn+as7LkJQ2DSbnVs7yshwZVxhiMMRzzw2e4+bFFBS0RKbb2kZQ690SiVL58sKF0Ae73Hi5cNaP+pDFamCdkGuCqQe/+tNFp/jZ3Nau6qCqQia6qGGQifRz0f7y9NtWIZlvLjpP+N88t5ZF563Pavsre7V3UJb3/jY4DAcxZtqXDsJal8rMnP+SG2YtYvbWN259fVtASkUz0NcDe1Bjln2+vpSWWWWmtPt1QffFKlr3mvLR4c972/ZcsBhfZ48pHmbdme+8LZqC94KRc7H/dfzIuzc5GIO9bVKofSC81fe6Deuau3May+mbeWpX7BeR/H3iXB756RIdpx//ouay3c+vj7+O4LvFOJ3y5XZQGsp90MWLRdx96j6OnjOSoW57hpH124alFXiv0C46YyF6jawqWlt5G/vnL66vZ3Kk3gZeXbOaoySMKlqaeXPiHN3jhw3oW33g6QTv7MpRD/ZHTpo2ry2j5Xz+7hN2GV/KloyYRyGF/qjS2t8YZUhkqdTKybsC1tL6Zo6cU/9xyXMOZv3qZFbfM7PO2nv+wPg8pyq8pVz2Wl8+WTq8GalBK70Pz8QUbePDNNX0KbgFeX7F1p2kbcmwg9uP/7Bxg/fGVFTQV6FGO2qGnEpqjbvGG0mwPbgHWNRS2BLG3ngI6B7cAn7vztUIlp1cv+D+e/3qnbw1jMh0M5c+vreKmR9/nvtdzH2o3XwMYqMy9sjR//Y33RWvCyWr5Gx9dVKCUFE8hSkvLkQa4alBaX+CgpBBWbW3VOltF8Pm7sgsOb32scPVDk47LO6vz81iy2L79t+wHYljShy7z5q3Jveuh/ng96O++9ue3Sp0EAGZnWfWr3Orn5+LnT+1cgFIOHnsvv9XwNMBVKo/a4k7qR7oQjcK2NGsflOXm/QI1UokmnFSft4PFST99Pud1//bmmpzXXV7fktXyA6khnyqNQjWsysTGxvKs7nbp/fkdxlfr4KpBKdMGLNn6xn1vpR5ff+rg8Xnf/kDr8LvclFMn9g/38RF/rgZjbx0PvbU2q+Vnv7eejx0wtiBpeXFxPf94ay0/OeeAgvXoMhiPcbm5/pGFpU5C2YnluXRcS3DVoFSoDuJfTmuR+2AfSpS68+Lizazckl1pk8rMqi2tOXdiX4jSmMsfmpf3bWZiWx4HtegvdcafWrQxq+Ufn7+hIOlIOi5fuOt1/v72Wj72q5cKln9PL8p+mN5C6K+Bdm8NPzOR6+9Df82zUtAAVw06W1sKV0rXlmWDhVwcl0PPDKp3x/7o2ZzX3f+60g4E0ZW3Vm0rdRJ4MY9dKpWT2XmuK9juhcU7WrfPX9vItAJ9r5ZvLo+b5LV+F28X/fGNflXX/PkPS3eDkI/gupzls/qPBrhq0HktxyEq1cD13AflUaKVT7lWcXh/Q2FGR+vNsvrcG5gNFF/6w9ydpq0qwNOmP766Iu/bzMXdL69g+eYWnlq0iY//+uVSJydjXR2nYlnfMLCHbs9nPVwNcJVSg96Fd79R6iTk3b1zVua03i157BXi0SxKOk/4Se4NzNo9Pr//DoayuJshY4/90bPEkk6X3cHlqlwaq9710nI+8uPnSp2MnJSqoeFzZdiHbT49/G7+2h5ogKtUP3Tmr14qdRJUJ9vyWPUlH48hc91GX7rb6uzlJcWtovDVP2Xe9ZQxhtZ4YRqb5uLkn73Q7by9rn6cGTc8RXOeGscWoyrVQLfXNaXp4eT/nl1Skv0W07/eWZuqvtIXGuCqkmmJJdmSx1KJwWTemgYd2azMXHD360ycNZuJs2Zz76sruPIf7/H68p0H/8hErsNFdzZx1uySNkrZ1ppg4qzZ/LSLkeFKbcpVjzH12idyrpP/k/98wMRZs/Ocqp7t970nUq9/9cxi3lxZ+nrWg1XCMSUpxd04CK773/rrO6lBdfpCA1zVrRWbW3bq1Pr15Vs71GF9c+U2bnp0ERf9cW6HH9LGaIIr//Fet62AF61vZN/vPcHBNzxFWzyz0oTZ89Zz82N9H0VmoNRhOuHHzw2aEWn6g/SSz2v+tYC/vLaKc25/FfBGypo4a3aPAe+HG5t4YO5qAO55dUXe0vXRX77ExFmzSWbwXVmyqTB9+t729GJ++/zSgmy7s4mzZjPtuid6XS7pByd/zrEqxy+fWZLa37o+ljZlG2RPnDWbH//nw15Huetvin3DEO1jSfbuVz6ap5RkznFNn3rx6E+j9vX15lwD3DInIqeJyAciskREZnUxPywi9/vzXxORiWnzrvCnfyAip2az318/u4Tjf/wce179GK8s2cyTCzcycdZszrn9VT5zxxyuf2Qh/563jk/+3yvc8cIynlq0kScWeF3tOK7huw/O4y+vreJ3Lywj6bjc+vj7nHfHHE766fO8uLie03/xYmpf+1z7OG+v2sYdLywllnT8E3g9Nz26iA0NURatb2RbS5yv/+Utbn9+GXtf8xhf+sMb/OudtTmNKnP9vwdG/4NNsST3vprbj7PaoRDduaX79bNLOOD7Xmv4c25/lQfeWL3TMsYYTvnZC1z+4DyMMdz98oq87X/BOq/RWG8jtN329GJO+mn3j8n7Kr1u7+9eWMZGfxjrx+dvyHtg0xRN8q931qZK1Hv6ofxJHkqXj7zlmdS+cilVvT2L4D/XG4VowslLMF5o7fmYyQ1ZX/Vl5Lx2xhieWLCBWDLzYLmvN5Jf/dObOa/7jfve7tO+i2nSFY8ycdbsnBuGi/apVr5ExAY+BE4G1gBvAOcZYxamLfM1YH9jzFdF5FzgbGPMZ0RkKnAfcCgwFngK2NMY0+1ZOGPGDDN37ty8/9iMqg0XdOSUV2adQChgUR0OEAnaO81vr7dWHQ6wvqGNI27u+6OPcjKmLsLZ08cxc/8x7D6imoqQjTGmYJ3EF1MxPkexS40Anvv28UwcUZV6/1/3zk3dIBbD0pvOwLY65msp8mHpTWewRwlKwRbfeDpTCjxK3HmH7sq0cXV8fPpYKkMdx1R6b00DH/Pr0f/14sM59445Oe9n79E1PPrNY7Csns+TUhzfvvrZZw7gxH1GEQ5YvLp0C4dOGrZTXvZFIfLkmCkjuPfLh7F2exujayMdzrNtLXGm/yC3vra78sZVJ/Hu6u28u2Y7mxpjrNneyjdOmMJ3H5pHXUWQhy85eqd1+uP3AOCpy45l5m0vEUu6PP+d49l1WCUigoi8aYyZ0dU6GuCWMRE5ArjOGHOq//4KAGPMzWnLPOEv86qIBIANwEhgVvqy6ct1t7/d99nfXPf7h7n2XwsK9ZFUidVEAjRFd26ostvwSlamdUf0+cN35YMNTQRti+2tCSJBi8WbmlPrjqgOYQxs6fRoddyQig6NA0ZUhzlwwhDeWb2drS0xdh9ZTUXQZuWWFuoqg6zeumPZ/cbVMn9t111URYIWgrDv2Fp2HVbJ0s0tfLChkbOnj2PPUTW0xh2qwwHqm2I4xlARtFnf0Ma7qxs4YMIQltY3M3F4JXNXbmNZfQv7j69j3fZoXlumK1Wuzp4+jn+8nd1obf3F6NoIk3ep5rBJw5i3tiF1UzyiOsSUXWpYt72N3UZUEUs4VIRsEkmXWNJl/rpGHslji31VGitv/agGuP2RiHwKOM0Yc5H//gvAYcaYS9KWme8vs8Z/vxQ4DLgOmGOM+ZM//S7gMWPMg532cTFwMUBo9OSDx1zw84J/LqWUUkqpvuopwM1fWb/ql4wxdwB3ABww/WDzzNUncfANT5U4VbkZWRPmkIlDSTqGYVUhXl++lS0t8X5VqT4fRtdGCActDp80nFG1YSIhm7dWbmPPUTUctvtw7n11BdGEy0tLNnPxsbszc9oYkq4hmnBYtL6R8UMrqKsIARAKCI1tSba3xVmwtpHV21o5bs9dGFMXoTGaoDXuUBmyWbOtjYa2BLsOq6QxmqA5msS2hCmjathzVDUv+SNaja6LIAit8STjhlRQ3xxjzbY2wgGL6bsOYdH6Jhasa2Dd9ijhgEXccfn0wRMYXh3Cdb2SmZpIgIAtbG2OM7w6jMEQtC2GVAQJB202NUZxDbjGkHQMCcdFBCJBm3Xb21i9tZW9RtcSTTic//vXS3iklCq833zuIA6cMIQj89AqvVwct+dIDpwwhGjS4bgpIxERRtdFSPrnelM0SWUoQDhgEU062CKEAzahgEU4YBEOWqze2sapPy9cnXNVehrglre1wIS09+P9aV0ts8avolAHbMlw3Q6CtjC8Osyz3z6+T51vd35M/cJ3PtJhGNSLjp7EnS8t73Ld/z5+D1ZsbuExv5XohUdO5A+vrOhxf8tvPiOrepr9tQ5Sd+6+8BCO2GN4l/WPu3LcniO7nXfU5BHdzjt7etZJS9l7dG1Gyx2827Dcd+KrHlnd7bw9R9V0eL/ilpkl+T6suGVmh/cbG6McdtPTBd/v9z42le8/spD3f3DaTt+XYuTDMVNGpIbvHVIZ5J1rTynKfqeOqWXher+x3eG7csPHpxV8vx/ecDqhQNftuF3XpFrg9/U7eOsnpzFt3BCmjs3sHOtPltx4OgG7MG3h9xpdw5F7DOeVpfkb2fI//3Mse4ys3ql+e7r31jTwyLx13PHCsj7vL/060hpPsnJLK1N2qSbpGkK21WW97P76+7filpk0RhNsaowyeZcd13G5tft1tIpCGfMD1g+BE/GC0zeAzxpjFqQt83VgWlojs08YY84RkX2Bv7CjkdnTwJRMGpmB18djezc40PVF+M7zZ3DRPd7yL3znI+w6vDI17+lFG/nyH+futO4/vnYkB04YwhMLNnZoCXrvlw9l3JAKdveDE2MMS+ubmbxLDS8v2cyaba1896H3Oux/j5FVPHzJ0VSFs7tP+8RvXuatVf1n3POe/PScA/jEQeNLnYx+beG6Rs647cXeF8zR45cew2dun5N6ktDdj3b7ObLilpkcdcszeenoPN0vzj2Qsw4c1+38QucDeJ8t6bisb4gyYZh3vWiOJTv075ov737vlFTvFZ1vKKAwP/RHTR7Ony86POv1Hp+/PuNBKl678sQON0PZ3OC/vWob03cdWvZBzm8/fzCn7Te64PvZ3BxjRh+fWHb13epN+g1OMfcL8MunF+el55Bi6umz9tTITEtwy5gxJikilwBPADbwe2PMAhG5HphrjHkYuAu4V0SWAFuBc/11F4jIA8BCIAl8vafgtrP/PWUvLjt5TxKOweDdBP3wk/tz+UPzuHrmPkwdW8uRe+wo7UsPbgFO3GdUh/fPfft4IkGb0XURAE7ddxSTd6lmyaZmZk4bwzFTOpYqikjqLq29VPGEvUcxvCrUa2vh3kwcXjUgAtyLjp6kwW0eFLrka+/Rtbx9zck8+8Gmnc6LdItvPD01+ti+Y2vzFuCef8Ru3PPqSs6YNqbH5aaOrS1oifbb15wMQMC2UsEteL2b5HO/VSGbaz46lbqKYEZBwI8+tT/feXBezvs7fq+R/OycAxlaFcp5G4fvPjzjZUfVRth7dA3vb/C6msrm6dX0XYdmnbZiuvDIiVx35r5F29+I6nAf18/tmPf1N+zuCw/Jed0vHzOp3wS4uQbx7TTALXPGmEeBRztNuzbtdRT4dDfr3gjcmOu+RYRQYMeJeM4hEzj7oHEEM3xkFLItzjxwLECHLpHat/3zzxzIR3/5EkdP6f6xeLqRNX27GBXLmLpIUQaTuPqjUwu+D5W7v1x0GIdO8qpcWJb0GNwCBG2L9loDZx44lv8s7Hu3YecduivXn7Uf15+1X5+3lasvHL4bP/h48fa/4PrTMlrutvOm87MnP+Ts6eNyCnAfv/QYfvzEB/zu/Bl97spuSGV2gdLjlx7LhoYodRXBPu233BQzuM2HuVefXPR9njFtNB/Ze5ec148EMqvKNhBogKuy0jm4fXnWCbR0Mz76hzee3uO29htXx5wrTmRUbXED16lja/l7AbvMeeQbR/f5sZfqf5bffAZX/3M+Xzp6Env0UA84Ex/dfyyX/KXvHbLf/Ilpfd5GX2Ua3I6ti7CuiKMMnnnAWM48YGzO6+89upY7L8i9JK2zV684odc+ul+edULqdfvTMFUavzj3wFInISd9LT0uhr6W3LbTkcxUn4wbUrFTw51sjK6LFH1Agi8dNalg277kI5MZmmVpjCq9SZ2eMORCRLjx7Gl9Dm7zJddS0wPG1+UtDdncvH7jxCl93t871xa/RC1fxtRVdDn9xL134bUrT+Tf3ziacUO6XkYVX0/12Qvp6MndNxJWHWmAqwadQt7BfvvUvbAt4U9fPoxnv318wfbzzP8eV7BtD0aPX3pMqZOQd5+YntsP8HdO3TtvacjmHMjHk5xsH/WXm+ouGsz+4rzpjKqNsN+4/N14fDMPNxP5cMyUEbx33SmlTkbWPn1w6c8KhToAACAASURBVNo+7DmqPG6gC+Wpy/L326YBrlIFcPSUEQzrQ6OTnnxi+rhUbxMqP8J9rJf2xKXH5ikl+ZNt7yLt8lmvM5thVT+yV+71CgeKv3/tyJ2mdRX09tWuwyp7X6gIzp4+jpqI1xgwX4+li+GHn9q/ZPvef/yQku27GCbvkr/fNg1wlSqQIte8UH3Ul5KkvUbnXk2nO9eUqBFhT314FlKxqyqlO+vA7Orinp1j6XhvpnT6cX/qssLcOB20a3kESf21F5hSfle761tZ7UxzSqkCKdQlMJsuhVTmaiLBsqqq8KWjJpZkv4V68lDOeuvhorMzswyIMyUifPuUPQGvGlJ6h/b5NLyqf/RIM5D99JwDSp2EAU8DXKXyZPzQjg1ACnGXP7QyyKdn9M9Sj/5gchlV/RARLj9tr5zWHd6HILWUrfP3LkBJeCY+tn/PfQR31rmkNZ8uOWEKK26ZWdBqSHWVA6t7sf6oc9/vCs7J82+bBrhK5UnnlviFKMG96expJX08NtDlMizotwrYYOdrx0/Oab37Ls5+NK1y0NMw0r35yjG5946S7Tk1fmh51GFVfbd/lr2G7DGy7z2uQGn7dZ+WxwaL+XTVzPxWy9IAV6kCKUQcqrFt4c3YrfvRnt66ZuduqC4+dvdCJof7cwhW+9J1X76k99maqf89JfsS60MneoNpHDk5swFjlEr3ySzrAZdrcJiNLxyxW6mT0KV8D1yiAz0olSfGdHwvBSjD1ZKjwnvwv4/sctjYf3/jaIZVhVKtvTc2Rlm0vjHn3goydVgvda4fvuQobpi9iNeXbwXgr2VQevvQfx+ZU5+tuTSguf+/Dqct4WTVY4Mqva56jCiF06eN5nsPL8h4+Qkl6oEiYAlvXHVSXrZ16tTRXE7uw1MXwvF75b/KhpbgqkHpl+dNL/g+8l3a+revHpHXvjBVdjrn/ajaCMcXqWurq2fu0+28/cbW8bvzZwBw31cOL3kjxGU3ncHBPZSC96a9RDZTIqLBbT900K65f0fyyvS+SLqLjinsE5uuhAIWS246g6F5agBaW1F+58sfvnho3repAa4alD7WhyE6u1MRys8Y358+eDz3fGnnkz0fo22pzLx7bXl1Pv/lo7uvX2pZQl2F15foEXuUvoeNvg6kcsikMgl81KBQE8nusXg+H6Pf6d+Y9iaQ5677yq0dx4c3nF6Q7WqAq1SavnTgfcsnpvV5/7vUhPnRpw/g2LTGNrWR8rvbHujKrZW5iLD85jN2mh7KoVFcufvWiXvyk08fwILvn1rqpKgufP0je5Q6CXmVr4KJXBw5ObMb0sMmZfdUoz+45RPT2Ht0DctvPqNgffsOvKujUjk668CxfRqCcXh17q1iT99vNAAn7L3jkfcfvngIP/zk/vzPyV6/mDUa6BZVe3+kI2vCvHl1fuq+9YWIVwfvof8+IjVtwfUDLwgMBSw+efD4gtdtVrnJ51DOg12mVWvOmTGhwCkprqAtnHvorjx+6bEFLU3WK4hSvukThuT1ZMtmRKjqcIBXZp3QoeuY9PqdXzwq9y6QVG4uOWEKl5xQuC7AcjGyJszImnDBhzX9r+N25/bnlxV0H5n42vF78JvnlpY6GR383+cOKnUS8qIyZNMad3Ja95Spo/jPwo15TpHqTl+r/ZSbFy7/SFH2oyW4SvmOy3ODoaBt8c0s+kgdO6SC4AB85Kz6n0+VyRCq3z5lL9665mSuP2vfLuf/+xtHFzlFsM+Y2qLvsxD60ufwbwZIkF8OXsmgO70jy6BufT6Nqcu+h5Vc6K+pUsDuI6sK0ogr0xGPLizRsKxKdSXXJxn/fXx+62daljCsKsTnDuu6385S9CpSqm6i8u28Q3fNed1cBkRJd9I+xel9pD8Ym0F3etk2hMvE2dPH5X2b5UYD3DIlIsNE5EkRWez/v1PTYhE5UEReFZEFIjJPRD6TNu8PIrJcRN7x/w4s7idQmfrmiVPYd6x2/6XKR66jNRVqVDfbEu6+8JAO0849JL/1Emednlnd0myqHpWzyTkMN9xdSXq2tLvD0juzAD0JZeLuLx7S+0J5ogFu+ZoFPG2MmQI87b/vrBU43xizL3Aa8HMRGZI2/zvGmAP9v3cKn+T+K/0n65sn5DY8ajb2Hl36kaaU6k6uJbiRYOFapH8krQFmTTjABUdOzOv2B1pDnt5kUnLY2flHTMzLvncfmX1wXQ5mThtT6iTkzW7DS/Mkopg9QmiAW77OAv7ov/4j8PHOCxhjPjTGLPZfrwM2AfkfDmSQueyUvTLqomjfsbnXxbv2o1O59CS/tKvzEGhKqW6FbIv3vn9q3uvCDoxy2cK5/LSOwyjPueLEnLfVl4FASml4dX4GWugs308jMtFTlbxClu4Wc1AWDXDL1yhjzHr/9QZgVE8Li8ihQAhIb3J8o1914Wci0mUfViJysYjMFZG59fX1eUl4f/GXrxzGD7p55JZJF0WHTepDxX/xOu//1MHjuejY4o+Mo1Rvrjwju+6gRtXm3k1ephZdfxrzriuvQTgGi+kTOgalo+siOW8rl2GcC+kjGQ4T+5kCBaI3nd33PtSz1dNTmtuKMNJnMWiAW0Ii8pSIzO/i76z05Ywxhh4GFBSRMcC9wBeNMa4/+Qpgb+AQYBjw3a7WNcbcYYyZYYyZMXLk4Cr8PXKPEalhTXN5JFsR6tvpUxMJ8uNPH0BtARoQKNVXFx+7BytumclVZ3Q/THC6H5y1X4FT5HXKX6hqEPkcoaq/+Oj+mT1yP3z3YRy++8AbbKBdpj1KFKqthGUJK26Z2WX3f7/6bGmCzXw/IXnqsuMK3r1hZxrglpAx5iRjzH5d/P0L2OgHru0B7KautiEitcBs4CpjzJy0ba83nhhwN5D/gZ6VUgPeVzJ8wnDy1B4fMpW9TPoarSrhqFeFcEWGNy9/vfiIvPURPr8MR6gbVdt7afSFea7z3Z3Ojf/aC2GK5Ut+n+uPfjN/XfAdOGFITo0a+0oD3PL1MHCB//oC4F+dFxCREPAP4B5jzIOd5rUHx4JXf3d+QVPbT2ntV6V69/Y1J/e6TLmNb5+L3kqYMg0I+4tMqgp0rnvbF/913O5Ul+EIdafuO7rXZb58dHEG2+ncfdfQysLU++3Od0/3jnc+z+d/fv2ovG0rGxrglq9bgJNFZDFwkv8eEZkhInf6y5wDHAtc2EV3YH8WkfeA94ARwA3FTX7/sOuwSiYMq+Daj07Net1DJg7cR3ZKpRtateNH9qBdh+w0f0QfhqnuTz53WO59x5arP190WI/zJ+exx4PvnJK/YDmfMim9Tx9lspC+lDZq5RWn713UbuleveIEwoEdTyke+9Yxfdre5aftxfs/OK2vycqZBrhlyhizxRhzojFmil+VYas/fa4x5iL/9Z+MMcG0rsBS3YEZY04wxkzzqzx83hjTXMrPU64iQZsXLz+BY7uog3XMlBE9rtuXOkqibbZVP9NeR/CSTt3oRYIWT112bIlSVVwDoZS6s6Mm93ydO76HER6/ckx2pZp9HSCilArZBV66ipDNm1efxMxpY/iv4/I7cEpnd10wo8P7ziOM7TOmtsenGvuP775O8tnTx/HVY/coWr51pfyeFShVJk7aZxQvLt7c7fxM7qyrI3qKqYHloF2HErItLj9tLybvUt1jAKT6v1Cg+6D0nBkT+N2Ly3vdxndP25u9x2jf35kaXh3m10UYDnm34X0bvfPhS47mZ09+yC+eXgzAGdNG85vPHZyPpOWF/voqlaNMHssev+dIhlQG2d6aKEKKlCq8IZUhPrzx9FInQ5WBKaMyC1pPnjqqJI2MVM9GZNiv74pbZhJPurQlHA74/n9S08Abnnv+2gaumrlP2Q3g0X+fFyjVD4jIoBshSan+7L6vHF7qJAwoj33rGA1uy9SQtAZsnasrdBYKWNRVBHnyf47tMMhHJGhz14WHlF1wC1qCq1TBDbxae0oNPmP7MLBBf3Xdx3pvfLv/+DrmrWnYafq73zuFpOMyfJA0QOzvTtwns27+Mi21LwdagqtUN4ZU5qnj9y4i3Bqtm6tUWequHdk9Xx58XYlfeFTvjcge+K8jupxeVxHsV8HtlEFayvzM/x7HU5cdV+pkFIQGuEp1Y+a0MZySh87rO/eYcN6hu7LfuMKMiKOU6pvKLgZzOPOAsUzepf+UXBVTKVvJ59Md5/f8iH6g2n1k9YCtQqIBrlLdCNhWXi56nUuETtm3f4/4pNRAZnVRhHvzJ6aVICWqmCaN6L5HgSJ2RavySANcpQpMr41K9R9dlWZVleHoW/n0vS7q2s7cf0zO2/vOqeU5oEOuPrxBew3pjzTAVarABmDf8EoNWJGgXdTRo8rBFw7fbadp++dYjerDG07n6x+Z3PuC/Uh/HqBiMNOjplSBdfXIUylVvgZqo5vudBXAXXDkxIzX/+JR3rJXnrF3jwNDKFVMA/u5i1J5dtI+u/DUok1ZraPhrVL9S3p9zHsHWe8Jd3zhYE7Zd3RW61z70alceOTEPo+MVWqfPGg8D721ptTJUHmit1pK9WLckB3jc08dm/1ju5Ondvyx0IBXKVVuHvzqEVSHAxyxx/Cs1xWRfh/cAlx/1r6lToLKIw1wlerFy7NO6NP608bXce4hOpqZUv3RxAEQuGVixsRhzP/+qdRE8tT/dz800BsTDjZ6NJXKgpa+KjU4/ONrRzJuSAW71A6+EcyUGgg0wFVKKaU6mb7r0FInQZWBWafvXeokqBxpFQWllFJKqS589bg9Sp0ElSMNcJUqgk8cNL7USVBKKdWL939wWqmToPJEA9wyJSLDRORJEVns/9/l8zIRcUTkHf/v4bTpk0TkNRFZIiL3i0ioeKlXnR06aRjHTBkBeC2OlVJKlZ9I0E4FuecfsfMAGKr/0AC3fM0CnjbGTAGe9t93pc0Yc6D/d2ba9FuBnxljJgPbgC8XNrmqN+2tsesqBm8rZaWUKneRoM2KW2Zy/Vn7lTopqg80wC1fZwF/9F//Efh4piuKV0R4AvBgLuurwrhq5j78/sIZHDhhSKmTopRSSg1oGuCWr1HGmPX+6w3AqG6Wi4jIXBGZIyLtQexwYLsxJum/XwOM62plEbnYX39ufX193hI/UFX3oZ/ESNDmhL27O4xKKaWUyhcNcEtIRJ4Skfld/J2VvpwxxgCmm83sZoyZAXwW+LmIZNXk0xhzhzFmhjFmxsiRI3P7IIPIyVO9AHWXmjAAe46qBjqOdqaUUkqp0tJ+cEvIGHNSd/NEZKOIjDHGrBeRMcCmbrax1v9/mYg8B0wHHgKGiEjAL8UdD6zN+wcYhNrbh0WCNgAPX3I0H//1y9zzpcE1Xr1SSilVzrQEt3w9DFzgv74A+FfnBURkqIiE/dcjgKOAhX6J77PAp3paX/VdJGjz+KXH6mhHSimlVBnRALd83QKcLCKLgZP894jIDBG5019mH2CuiLyLF9DeYoxZ6M/7LnCZiCzBq5N7V1FTr5RSSilVIlpFoUwZY7YAJ3YxfS5wkf/6FWBaN+svA/S5eZ789eLD2dgYLXUylFJKKZUBDXCVysDhuw8HYOWWlhKnRCmllFK90QBXqSxMGFrJ5w7blfOPmFjqpCillFKqGxrgKpUFyxJuPLvLWiFKKaWUKhPayEwppZRSSg0oGuAqpZRSSqkBRQNcpZRSSik1oGiAq5RSSimlBhTxBr1SCkSkCfig1OkoUyOAzaVORJnSvOme5k33NG+6p3nTPc2b7g3GvNnNGDOyqxnai4JK94ExZkapE1GORGSu5k3XNG+6p3nTPc2b7mnedE/zpnuaNx1pFQWllFJKKTWgaICrlFJKKaUGFA1wVbo7Sp2AMqZ50z3Nm+5p3nRP86Z7mjfd07zpnuZNGm1kppRSSimlBhQtwVVKKaWUUgOKBrgKABE5TUQ+EJElIjKr1OkpFBH5vYhsEpH5adOGiciTIrLY/3+oP11E5DY/T+aJyEFp61zgL79YRC5Im36wiLznr3ObiEhxP2FuRGSCiDwrIgtFZIGIfMufrnkjEhGR10XkXT9vvu9PnyQir/mf534RCfnTw/77Jf78iWnbusKf/oGInJo2vV+ffyJii8jbIvJv/73mDSAiK/zv/DsiMtefNujPKQARGSIiD4rI+yKySESO0LwBEdnL/760/zWKyKWaNzkwxujfIP8DbGApsDsQAt4FppY6XQX6rMcCBwHz06b9EJjlv54F3Oq/PgN4DBDgcOA1f/owYJn//1D/9VB/3uv+suKve3qpP3OG+TIGOMh/XQN8CEzVvDH46a32XweB1/zP8QBwrj/9t8B/+6+/BvzWf30ucL//eqp/boWBSf45Zw+E8w+4DPgL8G//veaN97lWACM6TRv055Sf9j8CF/mvQ8AQzZud8sgGNgC7ad5k/6cluArgUGCJMWaZMSYO/BU4q8RpKghjzAvA1k6Tz8K72OL///G06fcYzxxgiIiMAU4FnjTGbDXGbAOeBE7z59UaY+YY7ypyT9q2ypoxZr0x5i3/dROwCBiH5g3+Z2z23wb9PwOcADzoT++cN+159iBwol9CchbwV2NMzBizHFiCd+716/NPRMYDM4E7/feC5k1PBv05JSJ1eIUNdwEYY+LGmO1o3nR2IrDUGLMSzZusaYCrwAtkVqe9X+NPGyxGGWPW+683AKP8193lS0/T13QxvV/xHxtPxyup1Lwh9Qj+HWAT3g/FUmC7MSbpL5L+eVJ54M9vAIaTfZ71Fz8HLgdc//1wNG/aGeA/IvKmiFzsT9Nzyiulrwfu9qu23CkiVWjedHYucJ//WvMmSxrgKpXGv6MdtF2LiEg18BBwqTGmMX3eYM4bY4xjjDkQGI9Xqrh3iZNUFkTko8AmY8ybpU5LmTraGHMQcDrwdRE5Nn3mID6nAnhVxf7PGDMdaMF77J4yiPMGAL/e+pnA3zrPG+x5kykNcBXAWmBC2vvx/rTBYqP/2Ab//03+9O7ypafp47uY3i+ISBAvuP2zMebv/mTNmzT+Y9RngSPwHgW2D3ee/nlSeeDPrwO2kH2e9QdHAWeKyAq86gMnAL9A8wYAY8xa//9NwD/wbo70nPJKDdcYY17z3z+IF/Bq3uxwOvCWMWaj/17zJksa4CqAN4Ap4rV8DuE9Fnm4xGkqpoeB9hamFwD/Spt+vt9K9XCgwX9E9ARwiogM9VuyngI84c9rFJHD/XqF56dtq6z56b0LWGSM+WnaLM0bkZEiMsR/XQGcjFdH+VngU/5infOmPc8+BTzjl7g8DJwrXk8Ck4ApeI09+u35Z4y5whgz3hgzES/dzxhjPofmDSJSJSI17a/xzoX56DmFMWYDsFpE9vInnQgsRPMm3XnsqJ4AmjfZ66rlmf4Nvj+8lpgf4tUtvKrU6Sng57wPWA8k8EoRvoxXB/BpYDHwFDDMX1aAX/t58h4wI207X8JrCLME+GLa9Bl4P2JLgV/hD6ZS7n/A0XiPvOYB7/h/Z2jeGID9gbf9vJkPXOtP3x0vCFuC9xgx7E+P+O+X+PN3T9vWVf7n/4C0lssD4fwDjmdHLwqDPm/8PHjX/1vQnnY9p1JpPxCY659X/8Rr6a9546W9Cu/JRl3aNM2bLP90JDOllFJKKTWgaBUFpZRSSik1oGiAq5RSSimlBhQNcJVSSiml1ICiAa5SSimllBpQNMBVSimllFIDiga4SimllFJqQNEAVymllFJKDSga4CqllFJKqQFFA1yllFJKKTWgBEqdAFU+QlbEhGtHeAO2+qyE671wXbAscF1MwEaSjjddBIzp9D/e4IE+Y3lvxDUYSxADuGbHMsZ4227nuhjb8pYDjHjrem/8fywLI+3bdUEEI5L22k9b2jpuyEql2Q2AG/K23b5MqCpBdSBGWJIExCEsSVw/kRExRI0QEUPMQMwEqZQ4jj+/yY3Q5oRIGBtjIJbodGq54t1OumAFXNyEBY7syCYDJuD9Lw6I6/9v6HA8MF5eiJuWx277wfLXS7r+5xJ/BX9BS0hGBDfkb9sBK2FS+SyOQRzj56v/2pYOx8FfEnHcDsc4tQ92LOul3ez4XvjHFpHUd6nzegDGtsAY7/tiW96+UnmQ9l1r3wad8siSjvtMW9X733jLtH+nDbhhm0SVt7wJ4iU+7buBZbxjmNqH/yVvz3u3m33hHRMrSWpZcb1jaCXb89vbnRuwcCq87TghvO+LGLBM6qOIfzBcV7CsHTsRAceRHV9546VJkl4araR/vJPeOpJ0/Tzw3xtIVNk4Fd76gXASyzIELQdjwMVCMCQcG9f/ItiWi2sEWwyuEVwjmB1fEi8NjoAjWEmwEv53N3UcwIomcMNBnIjgVLuEg0ksST+YYIvBAEnXwvL31b6Ma4Ska+G6lre/pIWVoMM5Yce873b7dSl984kqwY0Y7ICLJW4q/bblbSDo/59wvWtHwrFT+Q/e/sQFK95+nL1jKgkHY9v+8d7xPW+/LrlBwYl409yQSR1ry/L+nLTvU/vX2XW97yr+sbUS/rFPguWA+Psm/XqJf/31zwc3ZJGoAoKufxq6/uc1JB0L23IxCELHY+C4HeeJQCJpp9Y3RgjYLq7rrRm0XeJJO/UdtcXFcS0c10LE4LrS4fQ0RpCY95m865+XXiuedu6L96USx/996HR98jeU+qwdfmtI2wZ+HhoDtoUT8Y5TMgImaMA2HX/OnPbrp38N8L/Xdsw/5gmDtI8I61+fd+zT/9//fTO2pI6hGEOixsYJgwS9fEw/vwFs2+A4aRnlChL3zic7bnZcU/z9GxHE7PidNfjpSb8eGsACN2h5nzliEDGY9n0GXO+9/52zLO94tc8Xy5uXzrZd7/tJ+6XZYInBaZ+WtNKuv35eOuIdD8sgjnedEv/zWEn/czmdrpMJP+6wrdTvRFPr+s3GmJF0QQNclVJh13DAMd/ESuz49obXNYMtWE1tuNURpC2OM7wae0uzt0AwAIkkhIIQi3vvnfYgxr+AV4UBsNoSuJEAknSRtjjYfsCZdDBVEdrPGmmN4dZWeBdrvIuC1Zbw5jsuknRwaysxQe/CZLXGMUEbE7SxGtswwYD3Puz/wDgGXJfWCVVeemyhbYRF027+jwsgjrDrwWs5YsRyJoXrGRloZEpwM00mCMBewSRLEjaTgw7LExZLEyM5KLyOJuOdQs+07M27TROoj1YTcwKs2Di8Q94mW4JI2MHELSqHttFaX0WgwU6dyBhIDHUhKYQahECLEN5usBI7TnZjgR0zBNtcAq0ujh+w21HvpHfDFoEWh+C2NoxlQcCCpItb4aXRDdlsmRqhZYIhtF0INRqq1zmpoCe0PYHd4uUlSRerJYpbFUEcxz8ONtji/fA0tHrHuv2Gov0GxSL1XoxB4knv++HPl2jMOz6VYe87EArS4WopglMTRpIuVlsCpyqE3RL3tgHesv53zlSEvG3AjhsuYzCVESQax4SDfposSDo7vl+JJCYURFqjmErve9c2aSgbZ4QAiI5xMCEXCTtgBBOzsGsSOM3BVDLtqiSuI5i4d6NitdqpeSZgsKIWiEFcwW4TIlsg0OLtP9hmCLa6ROrj2E1RjG0jrkt8RCVb9/HOleZdDU6Viwm4WJVJQhHv89u29wsQjQaJRLwIx7ZcLDE0NVdgB7x8cBwLpzVAsD5IoEWIbDaEGwyRrd52wluiOBVB7Na498NiDPUz6th6gLf94ZO2URuJMrKiGdcIrckQAXFZ21RHPOl91ppIjLZEgJpwnJZ4iHjSpi26I49cx8ZpChLYbhPZLFSvdQm2uanriyQNle9vJDp5F7ZPDrH9mCiTx9QTDiRT2wiIQ2UgQdJYbI9VUBmI05IIUxHwPnvUCVDfUkVTSwQnYcPmMJF6i0CLf95VwpAlDuEGB0ka3KBgpwVNGw6P0LpPlLohrf7n8dI/vNLbwIiI9399WzUihrUNdQC0NnvHyWwLEWi2qF4FwVao2JIk2JgkuKEBd4h3vbFa45iA/0MfsHDDAaIjw2yZ6p2XrbslkYokYhkqquJUR2I0tkZSeRAOJr28bQl7gUJSkDabynXeNiNbDZGtLqGGJKGGOG7Awm5LpNZ3KoJYCQc3aNMyPsLGwwTGRgkGHSrC3vlTVxFlc3MVdRVRHP+mpf1mAqAxGqY2EsMxQtBysS2X9dtqqaqIefns2IyobqEpFsZxhTE1TazcNpSq9u2Ho2xtq6ShOUIkkqC1JYIdcFKnfjIeILgqTNUaCDUbQs0uri1UrotiJb3vpBuwsOJJrIZW3JpK7/pUW+FdYwAsC4klUr9HbjiIuG7qGuF9ofzzNOkg8QTukGqa9qgBYOveNm3jk9i1CQLBJIl4ANt2STR51wW7KokTtSFuEWiwqVsMgZihckMC2y8IstqSWLHkjn22B56V3vfFqQ6Da7CSLhJPsv7YoTTu6RDcpc1LXsBBBNpavH3W1rbR1FyRuitz2gJULAsR3gZ1KxLYUdf/XfD274Qt7JhLstLGjrkYSwg0x5GEixvxvm9W0sUN2bSODrN1qk10SpRwRYKYv8+6oS1EQgniSZvWaJhIKEFrNETcnx+sjOM6dioYN0BtdRstbWH/WNpUVMWpCCVobIlgXCG5NZIKxq2Y4FS5BBpsTMC7zgUbbOxWIeBlA5HN3o1wZLvjXSebY5iAhb1xu7fPmkqcmgjiuDw59/sr6YYGuGqHYJCtewWJbDFUbPV+KJun1FGxybuIIYKpDHsX7Arvy47llbA5VSGsoJ26w3IrQ6kAVeJJJJbERIJYbQnvYh8MIM2t3jZsG+OXxEnSReIJcHdcuKy2GCZgewGNazDBAG4kgNXqX0QSSUyll57k8KpUiVx7yXGwMYYkvR8ASXgn99Z9wpiJrakSiGQ0SFUwTlAcqqwYw+1mQuJi+xeoVtdhrO0ANuMDSdY6ccYHKgiKd8F8J9pMlR1ncbQCxwhDaltpag2TiPsXlYqk/+PklTYRcknWQnCbH6Q7IHHBx+sy2AAAIABJREFUjgnBZiHQ7JXMhJpcLD92c0KCuBCrsUlUWBgbAlGD8S/aRgRJGuyKIMa2sOIOBCysmJ+PcQesiBfUt5dwOKR+7MXxLrrSGsNUhv38djGhHQGyJF3sZu/7YII2Ejd+aat/9YomIBjAVIaQ1oR3oxMM7LjQR8I7Sl9DwR3H0fF/xGoiuAELsS0v2K0MYrfEdwSw8YT3edt/qIzxbo5aoqnti+NAPIG4Lm5tpVdaGdgRgBJPINGYd4NlCU5NBS1jghh/ESsmmCFewGEcwRjbK6EIpYpLsCwXEcEB6upaicaDRLd7QYldlcCN29AcwG4R3KAh0AbBVi8PAlFDsNHBijt+SaqFUxUiOixAbIi3i2RdkmBtHNcIlZUxjBGCtoPtl4hVhuMYI9iWS0NLBRXhOMFQklDQO9auEVqSFslqF0laJCuFQBTidQH/WHs/Rm4w4j3ZMH7J31DvnKoOx3Bci42tNdSEYoSsJNXBGNRAyPb2sS1WScByqQ7FqA7F2NhcQ3MyQmWV9/1oaQxgN1tU1AvBJoPlQGh7kuBm77yPj6oiMXaYVzLjeMF7SyJETcg7llEnyLZ4JVujwi6VTSRcm22xSra0VDKyuiV1OKPxIBWRBI1NYSRovO+0X1gV2WoINToEt8cI1DcSHz8MN2SlnkYEm8CqD9EcdGmLBYk1h5GAy+Y13oHYOKYxld/rNw4hEE5iXAu31cvH6tU24e2GuqVxL29dSFQHkF1qU+lzgxGMbRFoinklj0mX8PYElRvav5MB3GCA6NgkLS1B4kNsktEgps2b3xZxsAKudzOVtAhstwk2C1Xr2r9PLpGtCay4iyQcLNfdUdIF3jXMGOykS7A5RGh7kOgIG4KOFzwB8WSAeNxmfUsddbWtxBIBKsMJ2uLB9tOMuGPT7AcxgYBDMhGg2c/IRFswVcpriWF9kxc0bve3v7WxEhEIh5MkkzZuwsKJ2Yh/YxhotKhaC5WbXYLNLpENLbSOryZZFSDQ5n8W15CsDRMA3FAAcYJINJm69pigeDfhoQA4Bqs1hgkHMCHvM4jrIlHv++0Mq8Zq8wpC7Lh/49kCbcY7v23bJWEg0RBGKpKpPJBWGxM0qZJGJ+QV5DhB/wY+bpGsCWOHAljN0VQpPu2/hY7rl+5bJIdVULXRpW20Rdz2rh1xQKqTRCrjRFtDbK+vRmI2xvbWt1ot7Dg4FZCosojUR8EY3PbCnoAgrsGOuditCe+JiX+DIM6OGwVJOASirpe+piCJgEv1EO+8TLoWW7ZXe8e1NUg8HMCN2YifBidpe088K5LE4wEwQmNzBTVV3nnb7ERIJi22NFVDU5DgyDZM+7UTcIICQZfkUENwWwA3IRgL3LCBaPtTDQg1GkLbk5iAEB9RiROyCEW8Y2k3Rf3P1KkouROtg6uUUkoppQYULcEdAERkCHAnsB9eedyXgA+A+4GJwArgHGPMtp62YwTsKCDQMtq/I4yDscPYsRBWwqsHE9kcTZXIWW0J3MowknD9+rkWYGFFk16VBUg9qsMYknUVWPEk4ECVd3dvQoHUIzxxvMdGJmzj2t7dnIX3aM+Eg97jcxHcoI2p8UvMLMsreYwmMHUVYAlW3Csh8zMIEwogSQO2kKjxSzyNEAz4Jbhi2NpWybbqSqImSIsbZrUJUGl5pVHLkpU0uRWp93NbdsdmKXG/2G9VfASbYtXUN3iPMmNNYa+eVlN7SSOYCoMVE+LRKgi7Xv1I/xYzUi8EWoVgM4S3ud6jr3VRTNDCbvEeN8aHhAk1xElWB706XEnXq6IZS6Y+p1MZ8OvSOrhBy6/LZvnH0qFqvYOxbEKNfrWEhkSqbrI4xqvm0dLmVU3wj1mqDnXSTdXnMpVhjC2YCq9UPlnnHUu71d5xjCIhrPZHhjsqkeJWRbDiSdxQACua9B/R+3fibnsdY++7ZrcmvOoF7aU0lRFv+VgcUx3BjQS95YN+KXNlECsmUFsFiaRXWuKX6KTqqaWX5jpeiX6iWjABv4RiWIKg7RKOJIhFg7i2wXEsxK8eYAe9x4i4gCs0NVfgNAS94wm48TASE+yoEGwSKjYbgs0mVUIcaHVIVtpYTgBpS+BUescz2OpSUe8di2RlAKfVxq12iIrBdS3axKSqEIYjCVxXiMcDWGJodsJghDbHryJgBNMawEoIoUYh2GyorE8SaPXOiUBzAjfglWDbMQe7IUrz2KG4fjWM+spqwsEEFcEk9a1VJB2bgO3QGgtR6T92dlyLXWu3sS1WiWsEEUMg6KQe39McxFh4TxravCo2bsjCqYv430cXN2xjOYZA1JDYHqaxIk7S9UpPm6NhLMtlSEWULdEqmmJhLDEkEgHWbfOqCoSCSeKxIHZlDIlZBBsswtv9Ulwg3OBibCG6SwUVjvfo1liSKhUcujhOvDZEMlpBbEwCCbhePUe/1C4aDxKPBWiNehX2k9EgsjVIMOYf64BXx7dlTNB/nNrmlab51d/BeyRskt5TLowh0BBFEi7V6/wvhASJDvfrP7ZYJOwwJAWrza/WkPCfAjTaWAkIb/WevISa/cfiCYMkXJyIjRW3vXMl6WD80i6JJ3FqIiSrQwSbEtSsChAfEiQas5CIlw8tsQi2X8q2bbNX+upUW951DLDCDvFYkFA4QSwaIpnwqp84Ie8zhKriRNtCOA1BiPj1lysSmNWV3rbCBlPpELf+n703+ZVly9K8fms31nhzutu9iIwmI0uZRSGVEAimCKn+AGZMASExqzH8CUxrWkJCDJggJsyYpMS0RAIlRBVQCWRF896L257GG2t2x2BtM7+RiizEgCzlky/p6b57jh8/7ma2t1/71rd+X6Oqbd2W7Enfoz8KZi7072cwut/7Y8R/ORNv6j4fkqqQGdzLWD8TDKnT77uPB+Q0kN/cgSmk2x4zJ4qv/udsMVXFNKdJ97rnM81G946bXwnzrWPKwnQv5NkiSSizvke7DXDW2RA7qVLaPVUbwLmqvFb34OIMsnHYc4Sc9fMQtavEm45S9+b2S2DzfQvo7wi3qtTP1mNsQTaRGM3a3ncnwY5w8+uImQtp4y5+c9R+h9cugYRM8doJK96sCq7kAkn3dTtA6RM5GIZztVFEs86JqGKdEZe1AwmUUfe6CXBNYj41iM08P9bPeVNIwVBGC5tE/G6j14S9qK3mxSEZ3FGwg8WO6sLoP1TP9qQdH/XOF+wcKXL552ratUgupP5iifp9dVVwfxj1D4D/rpTyrwD/GvC/Af8p8KellD8G/rT+/VrXuta1rnWta13rB19XBfdveInILfBvA/8BQCllBmYR+XeBf6c+7L8E/nvgP/kXPldM3PwqkhtRn0ytIuCPUSe/h7ia/QGICTN9ZeK3FlIi7zdI9QUtKprMEXua1in6Um+vcmMp1StaliGm47wOkeVNo1+rd7+p3hkvd6TxtirI2wYK2CHoMNz95vKyDhPkTG4d7pTYfmsprmfe1mGOpjC9cnyadvxjfsbHds8790xXR5XvzJl/Pr/mkDse7IkpO/705V9dn/8ldozJk5PB+QRRsC+O5qmq0BFOv4jYaGgeBSmmemz1jrX7kkit4M+F/v3E+KZhvmtoHi8DKjZk4s5jT2E9XmnTML2qilgomJB1Gn/j1kldU4fQimR2vzxh5w0UcEPChLSqp8UKeAvGYI4DpW3g6ynlxWe7Pr76r3ft7xAVcuswox634i2iI9f699ZhTuM6YBhvVM1dfHTmcMYcRf25jVM/bUqUvg71WEvpHJKzDhT2jVIf6muyx0kH3kSvo2JtnZYWOFUvuXfqAZ4DZFWl3/xPJ57+RK+Xz3uPfWo532aKhfZZSJ27TPZmVpiDq96x/tGsvjx30glnE1Rlaw6JIkJZdltRBdEMkbzxev1XP/T+14tf2jG+MgzvBD47xEDcJ502Bs60mFEH2MJdxp4NxRZM0O/HXaZ5MTTPOmBmIutQIugEtZ108CpuHWa07L4LlP9BFZGnv7PntE9Il3R6OokSCraBuRJCQrA8HXpSsIiBNBvMs8cdqho56aT59vtM/0n9xiYWiqtrYojqX+wc7XNm82vHoew5bvUY+C4yH3uOrse1kTg7rMvEwVWKBczekUfL8VOLZKF/L7ihXAZUjZA6o17JH2/xh0huDLYqatN9T3EwfxPYvzpRitC4iK9q/eenHc4n7vdnhtbr0NXtwOk3qnLGjVCcHudiBInd2vEwi+8y6rpUP2kkbRukwMsf6rE+/BzCNzOui6TO0m5mcjLETe00JYPvA/2PZl6+bJnfCc17h2T9/uZDBmkqmcFgpkwjsvo+49tdvaYScecJW0gPke3dwOvqZf5y7mld4jQ2uN1I6xKti6S9+jJfzh1vb46UIhyawLYJDHu/enTnyeGbyPbHI9ZkxtljbSb90UGPQbD0vXrKp62j6wIhuJUQMP9qi5lFFdDWMN947JQJD5c9PO4b7BBV4fVW173I2qkrnb/sB40jbfyFIAAUb8i7ltw4bO2cuJDIdS9qvwTu/pnw8UbITw10CbZx3drSbGGXaT4b/EEVR3dO2ClfXoM1uJP6RklFPytDWbuZYi32FIi3LUWE6cEzvIGwW/bUAgbSwYO7SLOudgK1G1LW/due4zpkDWAmu64pCQkzzvpZG+pmpc9C7hynt5bxTcY0CefTOsDatOpBniZPaRK+0YG7XDtcdjcTg8U3EWsz/m5gHD3e1/mbLOrT3UWMyaQm0fhEGC5qq9zMlPcd06tM+0XnSbKBsNOjPd0JzXMh/ayle0qYUPAvMzLVIdqtr3vm1YP7Q69fAB+B/0JE/mcR+c9FZAu8K6V8Xx/zW+Dd7/thEfmPReTPROTP5jT8Nb3ka13rWte61rWuda3//+qq4P7NLwf8G8DfL6X8IxH5B/wlO0IppYjI773VKaX8Q+AfAtz6NwWB4ZVh86F69U6J1NmVYQtVFawYmtI3SjsoRb20jcOcJiSl9e4q9/Vuy9Rp7VSQki/Ypq+4hmnbqqqI+imhchwNkLMqsUaq0lvvz1LBvkzkjdIDyFlfV50klpjJm0ZVq9NMqriUsC2k2yrLucJDf8abxIdxz8bMvHPP/GZW3NcXs6Mzgc4EDhUW+u1wx12jNwVDUnUnnh0Rh3SJBIxdVV8HnX6WAsXB5ttC9tB/rn61Y2ZGfVnj64bNt8NlCnfB90ZH2nrMFAmbnmKEuHXkqogVVzBzIrcWSQV7DuudPUC6aXBPI81T0EnvrApD3Olxbj+eMceJ9PpG1e7G/Q5n0j1O5N6TbntVZkNSJcUI7qneHM0BY8yKaMvW6+TwdNlqxOVVlTZjVD9upWCwbS/nLWdK6/X8VeSXmQJyGCneqTqREmw7SvXgLiqFPJ+U1zyHC0HBX15DbhxmQYeJMN/6lXLw8L8IsQc7GnKjPkv/DN1n/f74qnJdIzz874Hx3mLnrOxalPcaK89WkqKpkpdVAU4t+EPCPg9QeZVh32DmTGor+ukpMz1YbBWd066oarnRJxFTSNaTXyd49uS2UGxZFV6ykLoCT8LNL2cQ7cTkZdp78eCNMy4rTzN7Q7jRn+9/K5zagtlFmnZmnjzOTziXV0Znni0ZSxmtelZHS7kLzO0yGW8pTtnLqRXinad5TviD7h1x3+CfRlzMxL7yQHeRvlIYQKfKmz7QNBHpAjEZfBMZPldgr8+YLpGjwb4Yui8Zfy7rmnBDRqJ6oO2gHk5JhXCjJyt7IfaFZj+zqWikh82w4rHcwwupiCqSJpOiw9sLoQBbLvxOgdQbzFSn2KvalL3BhISvfuBS2avzbWUe9xeWrG0SpQjh7OlvK03i3BAOjbJibaYUVbxSbT6FrZCdoTllzJSRrF2UFUmc1NNu5kRurHaNiqLkDlOz/u4pWlofmYIjZ8NxaHEVO2dt5sPLTnm3LvHxpcX7uFJonNfXPVQlb9dPxGTWa2W7mZijJWdD0yTGoSFOTj3PgNhCboTzu4buMeIPcb1OF7VdhkDptPtSUIyg7gtV/Wy9dpky2I/PkG60A7QgDHNWLvJhIt202u3qmrX7lDrL+RtDkUxpE7ZT9XKqymOZLLjMfCuYWQiTMLx2NKfM5lgV2qRIxFwMubGYGNeOJaD7lNHZgmINh58apm8CVP/z5mbEmEIIlpyM8qWDodzUeZb3HfONAawiwvYNTUj6eQjayTTgpkS86+rnwIyERKzdSnPW9ZcbyK8D3iecS3SNfr1xiSk4SnPB9XX9TFuvhV078eW04XxucS7TuMiIJ4bqx24D1ub12hlH7Wwu59p57QqFu4CcHPOtos6ksO4/qlBD+6Jc9uaLetuXj3wTdR2viLi/oq4K7t/8+g3wm1LKP6p//2/Qf/C+F5EfAdQ/P/xLen3Xuta1rnWta13rWn+tdVVw/4ZXKeW3IvJrEfnbpZT/A/h7wD+t//37wH9W//xv/1+fzFimW0PYCPNO732OP9IwgunO4MaC6y4AaYD5rqH9NKoalYqqo5W7t0yNS9Q8sLRRNc6MARIX+Lm9pLy4l1Gnjb290AFS0WnQjIZOnIMmsTQXcLWUsia3FG/Va1tVxbRrVxUy7VqmB8fwTkj7eEmgmoXHsWffjPzdm+94cCdu7IitP3hIHXNx/KH/yHcADXyc93x31mnujPDhtOPm9YnTuSWd3SW9DJ0idmchNwV5Ec7vFoVJH5AaYbw32LngzgV519PnjMRMeKiEglPAf/9C2bQ0H47k3us08eK/GqNOT3tH3vj1eCyBG/akXFr/6Qjs8F/OqoqP6m+VkCht3RKM0eM6JORY1dnGr77bYlVtNseJ3PpVgRVnwOi0sEyRvGv1TrtyatOrHbl+vzQXBq7U8IDilIgRXm31eniekHHGzLVj0LVITBrW0Hpy12CfT18B1bsaAlF5uyFSjEfCV3f6C7f3qw6CpELYLKSH+jCjFBE76Z+rArspzI0qtZs//4J/u2d806rPs5ZkCBtla5q56H81UGPxj+ebHvN8RoZAMwTSbUfqq+I/53VC3Q6aPleswez1OKTBrSlApc2QhLKJxMX3PhlyUxjfFA4/bfBD+SqJDrrHgj1H5YoeA2nrmfcXvSNuoTSZHIVuH9Y0o2n0SFXtms1M00QOjxtKqGt4Nuu0tBsEM4GdNeGrfYy4U1w933ZMqizd9sy3ltTr14dKYdjsJ+5ea6BMzIZNOxOSYZw90lfP46JYW31/807wp4sH18wZNybmG78merVfJsKu8oBzoXk2nA4NHwaPa+PvpBCKwDzbauku6o21BlvXrUQ915sPmf7jrGQIb7DncOFFVwKMmWaKteTekVrL7td1nzSGefCEaiITp77I8VzV1dFizpby7DCAPwjt46X70z4mciP4Q8K/6IxDETBLSl3MFBHMHPGlwI88JCEGy5fPSqzoX59JydA0cVUQjSnrZL2xiZINcbJsb0f1WSZDSpdrxrlMCJ6UCuepwdvENOp7SD6uam6KjnD2MBtK/ZBwSZRJXMM4UqM0mJVKAciuWTuHadusZJal41MwyKRdJeWzL/zcqhIbQxENwHHPum8WZ9YgicNPG17+JCKbiPN5DTszixfWZeViR11b843QfYHu/XRREjNAwk5gRf3ZCz8eNHTDDkquKVYpBu7ZEd+oQjtNfgVwlKTpYdZ/xZDdR4prkARxY3HnxPyqx78sczD6mUEq2BPgDMUobWKZUzBzJHaOzcfM8xdPNJ55H5kqkaBkfY8lGCVH1DrWNf7FbilJcG1kHBpG8ZQs6lEGJShMVj//MuAK5mQp95UGdHJIMJiglBl/FGJXkw6/6jObuWBSof08YT8+Y7b9JTSqqvpp6fz9FXX9B+4Po/4+8F+JSAP838B/iP7T6r8Wkf8I+CXw7/1LfH3Xuta1rnWta13rWn9tdf0H7g+gSin/GPg3f8+3/t7/pyeyhse/I8Q+c/ij+qVR1SsQ2kdD98XQf05kv9ABCtPrjvizDe6sHjOzseTGaI440H4YiHet+v9ywWw9RWT1G+oTgTtZTOfXCfxc1cTiDdka3DkQth1mUi/ZorS13x/IvSduPSbkylY1q28TwH54Qu52pE2DO2WaZ0P5paN9qgrtz+H2j0d+sf28khO+DfeEqjDc2TMf454/C3pgjqnlEFtNdwLO0ROiJSbD/c2ZL3lLng2lVRXBBEfqC7lPzFEn5rtPwnS7KLngT0XJEqIpNekPdzSHtN55S+8wZ526tyGp9y+VdaJ8OYeS88qLlCXmeCkR8qbRKeK9ptItnmQ36oRy3Hpc9ZQR86qUpF2r08IxayJv/Z6ERG7rnXSpFIaXA2XbXyZ8l6jekFTdHWeMCLn3OhlbVRSskHatTrrXiem861aVxpxHVWdDVB/urOe61N9fWospZY3ixBiYZlWdq5JdNurzlVkVbfftF8x4w3z7dfoUtE+Xw+YGTSAD6D5prro7F+UFF/V6Lp2I2AkmFppDJVMI2OErWCXgfvtEenNLfLXDBPUC2+OEaxYOrmX3beH4B4JJ4H4rFGM55cr8nPQacr9WJdcECHuDHeq52hT93Rk2nyIS9e9LpKc7hVUZW9ZN95hWEgTZIMkz3zqenj2lzZqslgT7z5V3Oe8ycTJQfewShe1fOMKCva7r0wT1+5qUCbdeedRovKhUBaZ5TriTx/5Fx3ynL+KMcjbDsUGaxDFucH2NSJ6W2LmisbXf2qpy6/Hf/1K7DrmxzHuPpII/hNWD7I8Lt9TTPhWGo6PYQnryxIcZ+WpN9ZuZxkUOx5549Mhs8HV5dJ9VpW5elEghKSvDu3VrRDYZ9UFOkbJ3Gj0tsu6hd39eCBvD6bnBRDj/OJO2GZnM8hahQO4y/sXQHPRabF4ql3nOSBbcOWA/HyibTrtbdXtdffijUmSaY6b93jO/vqjtw2MPSZittn3swTI/fLVvtEVPaDCc3m/BFcp+ouR6vZ4dk9XHZGOZUSXQvNR5h9f1UEwWc3A0p5pe1VzoAe2zriN/WhIs6591bzDPZyUltF6/Zyu5YnmNMa/e/9L4dX9aOoW5sdoFqY9d97Z6zRdD5fMqx9X4TAxulRXL+w7Juv7tpOl73ZdIuPFKB0L3H0lF96tY6TA1sVHXQlJFvzFIyNz+RcCfHMMbVSbH14W0yZhZyPtKDHhqcHVdd6NSUdxUsLNSSfzLjHk8rqeqbDvyrtHPQbjMJSxJZnX/d0PD7lee6aGQZo+tMdvT64Rso15zHzuloQyyzgM0T/D0dyOxCGWwmMGQ+3wh6QSdNbGjofskzPuCnYUQapcvg38R+vfKv04tnN9pTG/7WLs/U53fsUJuLent3e+k88XbVveU8Ss//O+pqwf3Wte61rWuda1rXetaP6i6KrjXWit7Q9wUyn1YvW3JWsysvspidaL8+ReO7kud+vwQyV7vpIdXqg7YQZXIVAkC85seOyWKqBrrzjPz7cU7U5ys/kU7Cqm1SCnErnpLQ8akQm6tToK3Vl9rvyR0bdXnm4pOgr/eYIZI2jX15xPp4Wb9/+7DGck9cXNJ0JruHM9jxym1DKnhm/YZL4lPQRmSk/d8P98Ss6W3M39+eMvOT8yVRfn+vOdw6ui6wHFoKdFgZoMMy2vUZKt8ksoyFOIG2pottyQvuVG9hM1RFSF3DJcs8ZDBGU3nsYKZAmnXrsqhm6N6WGO+/Mx44UDmjXrY0qZdGZvOGcKN3rm7w4SMATvGmoSjfra439fXqKpH8VbV4blODO+ar/LOC4RA3m1WhVRVkqrCjJqqJF2jNI7DSN536sUFChYJmebjSROMMheeLSAEJSBQvYXGYEKEmjdPUvVWlRtNVFsTfCZVpCSX9ZiRM/lhT+o9219Xr7EA9BSB8UGVNv8546oKe/d/ZlIj2LmQdg3DO+0KLIqlPxXsWK9ZK/hzJLUWV5Wp1FrCHzyQWkvz4QSm+giNwR2r11iE/lPEDXqd+1NWZbh6Hue9UjO6L6rMSlSP9+LzTUfBTgV/ovKQwZ6TJsOhHEl7CuSiilPc19SrKlwef57h7aQ+wMkikyEMHnGZ8LpyaveqdDopzKP68E4/T7hD5Yp+1tSl/nOs0/CJlFk5uIj6iuPWkXpD8wTj20Kp6VqMljA4zGjU2ysF+djjJ0320nMJkgQ3gjmgtIS5rPtLbqQqRoHU1q5PzqqaA80zNHvD9leG84/0QOWjg11VD23hfGw5546SBffkcEdZ1Sr/UuieM/55xswR83wm/8Gd+rpDPRe9Tv7nTauT/70ndRe6ybwzhJ0w3xck6j4rk1FvNSh9JYI/GLIDM+k5XWYcTMy4Q9TO1xyUEb3tSTUBLHVO94zbnrD3NC+R9skx37MEaEEQJAqlK2BVRZQXj5nr58B9qCQbAa+T+uHcrOqnOTjyLiE+w7P+XNoncj2OzhTlF0dRgkNXkCDrmmk/G4pR9Th2ulZy61T5XmY66hqXIZBu2kv6Ye0wmSmoRzNWis4wk3ct5qD+f0OdM3Dqw5XzpB2fei1sfxsp/6NjvnOkBoYfp7o2l+evaZMnve7aJ1VQzZwvKnlGiUIipIetvv7TiMkX9XTxjUpWPnJ7yEheOgswPVhSV2g/e9ygnw3L54Nkvb7b54Q/Ruwp6HuqHl9EtHvQWk0qFMEOug9/zTPXz49M81KYHrRLm5vldwgcPXYQTBQK0H3isre0wuZXjmKckloEcrSkLl9+vqgqH7Z67Ma3CXdciEK6L4SdkHrl3TbP0H/JNM/6RrsPZ00pq3ubzJHSOH1PQLbK8V7mS/6quiq417rWta51rWtd61rX+kHVVcG91qUKNE+GsbVInZjsPhmcht3gT+qZefrbrP7ZYjxuyLhzxp3BnyLZG9zp4ueTUgg75bX6Q9Jc7AKpW1ihekdsgvIr3Un9Vc1TVfWcwT0POnVvhPm+RWKm/1A9YrngXsbV57T4QpdaUmbSRr2GxQi5MTTPl8n6uLNsm5mQLbEYUjEcUoept70bM7OzEwOe99MPzNK7AAAgAElEQVSem2bgGFrGpKrCcWwpWRAplAIl6nTowiW1I8RdUfUHCLeq1C78Qn9URdfMQvelECdNxpICdqjvpdU7ePW8tjqJ36jaDaypP/YUVM1ovSrvi/IREsWoYqZczLzyUPXne9zRrcqtMiXzRQFNGRknMEpGKI2rVIx4IWd4S7rpMHNS221lctqhciLPI6YqCwtZwRzGrxReQ9q3kJSLLClRrEVKZYruOvW15Qy5IKVQ2gYZqkGsNJqSFhO0jR7fkCBEZLnbzxmcVYXXWoox2POsCWigqk0qnH5sSR2kBuKmTmCjftzYq1+s3el5ihtzUZJCxs7KdjVBJ5pN0L/rA4S51aSm+e22enTV87b4BacHhx2VDGJiIWwN52+EtAg1GZKH8zdS/W6CO+nfl9dokiAlY+ZMuHHYUdbXYFLBeIudEvNtg3+ZSb2jGHt5/tmqF9UUSqOe7nK8eDvj5ChRkKbm1NsC4aKojK8K229VpWteZp3sN4J/r+lYy3r1uZB9ixsK2YFU5dL4rIlOryacT6RoybeBMFrMs/5svouYZ0fcqLpmZjDPhdM3+n07q19xvmvIjdA8Vc92PVd2CNjQYGdI+0T7MFBmt3pwBdjuJ46PG2SwxH0ie0P/vipuQ8Efkvrmbztk3yrRJZfV225G7bjYU1BvuDPr+gOYbxxhB3GfcQdD7i7+W72m9Y/hXab4QrGW7pMQtrU7FBwe3efKbqOq5RBW7312SnWgFLw1TPcN2YIJsqpu2qEr+P1MnK2qqdtImqt/s1GCAKbgPnviPoErquai/mCk6GP6RHmIECzGp/Ut2D6uFIJSQAxffUZUgkIruHPBHabLeqz7mzmeSa/2FK/ryhzHNSETID5sK9u5ek+L171poS2cRvAOGWbKplWVNxWm18v8ALQvmaEmfJVGfaX2pOcx9Zoy5k+CO6naXIS/1DFT7nbeaIcq915nAhafr1fFOPULxUO96PM3uu7GN0rckQTTq0yIgjvIOg9QrHqXyfqfhHRRpdH9wz6dkdTpaypFf+dwYaLLMFO8I3V2JcOE20zZLtQJIAl2csS+IBnG16LKLOqbztWaHO4jEoweq3otlCT4T564zwSp1+wukkLtsol2XRDwBwg7Pe52Lhx+psdlfNjTvmT67056fOtcjqlKt/v4op24r+Zsfl9dFdxrXeta17rWta51rWv9oOqq4F7rUgbm+4wZDbZ6r+ykE9rNoawKa+7z6iGb94KdhLgRus+BYlSxKa1ZJ8dz9dHYBO4cVD3pL5Pl2RtMLKTG4M6JcOuxQ8bPqvol15C2LcUZcqP/uVO6KLOtJbze6HR2Vq+ue55WGoM5DOSbjXq/6pTrvDfMe8PTn1RP41t97Ck2vG6PHFPLr4d7XjcqX/+v4cfc+QFnMl4yoRg2LnCOevf/sD0zDJo2BGC6RNy4SqCA5kWIoNnfs6yTsosiN73KlC4jwRD2hrC1uKHQdMLmfVmPkzsGVQhz9VXlsqoHlKLpY95Qiv4pKa884eIt5jzSfZ85/q0bmkqQ0GQckGrqU19XvvhnF+Uzob8zXpLIFqV3UZFJCTsZ9dTmTN5W8kL1vKbbG8xx1DSiTas+uE27+mOJqm6ouqzett9VSFR9WNLq1kS1hbhh7crFLcYg84V3u9AoyFXd3bRKWTCQW0+syXlhb3n+hSVuYXpVFaimYE/1PVQv2Xwn+DN0j4mwNcRuAegK840ntoIbDZ3IqqIC2LGm8JSyTvWrz1iT6ZYKW8O8Vw/pvNd0tbitalZQP7ydIHaFsCs03qze1KlTqoHthfnOrSrg4lv0L7NOJxunZIWsnvdqKcedDGmn3N3i6mVgEzkbclheM5iuqreg6q1A3NTp/rMhtXD6xle+qZJV0vbiQdSUQs2U1+4NpNo1yUFV4TxbTbo6O2Vqtol8W1+CzeQ2w6YgSYkSqTH4YWEcF9xp8QaypnmV5bgb4fzGMLwBfzuRoqVp43rJLEzY3f2ZoW3Voxvb9TgNrw3ZefzO4s6q3JcilF4uxzxl0taTW8VLSMikxqzn3g2F8bU+PlslGKgaapbLifAqYY9WVdeNqmquvkfdc5XxmzftSg4olRNenLmwYo3OAMSt+l7jT1RNdy5Vq7xgbKUHAKlSFkpUcojpI6lLOJdJs8Usqp3LuKZ2WZJBpCC2YO2S0gYlG/rdxCkLZaivrW5dsS/ETth+r2r39G6LOwQlKNQOVrrf11RDp92bYVLm+a5GupWCPQZV0Id53bdk1O5Ovt3W1Lm6N4SEjCPmlXLGx4fKgd8X8k5VeZKQXT2RrhDbzNka2k/auXGVab0c62WfMseReLfRDlPtdAHIlIh37brH5tuG2Bvmej1PrxJY1Icuuhbi3jHfLR1V3ZNT69h9B3Z0pG2jnSLUnxxf7XTWYaMkBTMEZY1P9cNIBBknTFRKy3yb4SbA2a3vs33vyA7yzdLldJhYO313SeklXcK6TKpEk+VakLYQbgz2NmBsgiKE55ZS96YyC+Gm1L0C7Cwcfg6b9+arNaNfP/7hThnaTwH3NF6IQN5RYoL0u3Sav1xXBfda17rWta51rWtd61o/qLoquNdaq4jgfnRmfuwwNZWkGEgtTEbYfFQ6gnsxOL3xx4TCeC+0zzrNnRqDHRPzrbuwcmdNp8lOsKMj9srJW6apTVT1RjPiLanROzn/cvHzqT8QMIIdEqkzSK6K286ROuWC+kMkNwZo17vaZerfzJHiLc9/a8PxJ4bpVSHViW2ZDGN0xGKYsi6Lh+bMj6v56TfTPafYcufPOJP4J5++4ZvtYVVsQ9Kc9XH0vHt44f3nW+wgpGrvGt4Vii2UVzP5sVmnlxcPVNkmTJso0TCLxQSVzeykhAdQRqSkQu4d9hyQKVBaT7irGeOzqtpx11yYh41b8+I1i129r3bY6ePHSFsnV+2YlYH5eCLvek2mq7xBABFBth3xttU0rqBqWG6aVU2XoipCbhySkk6/WktZVJbqnY0PvXp6XU0fqpSEsrG4pwkzBQhR79TNhWlszyNYoylgx1ET1L7yXBdvKF2j/r1KJyjWqnq9wEtBPbkTlNaRt+oZXq7X1BriFs1I96rcymgwX2FB/QFM0rQyO5mVfas/LySvlIXl7xRh8xtdNGnj1WPoLf4xIseBfKv+6WVNSFYCw/mdx8yF1Ovk+TJ1Hm4zpUvkjaX0CTlaYl/Wyfp4k0hHRxp13TXPqSYQ1uut+lD945nSVR+j6VblszglCpQMvo3Mg6ftEuPZrv5U66pa2STSS6OK1z4gXyq9JF28lXFjaZ6UQ7uq/6X68/ee8WFZEFAWJanNbF+dmUZP18/0tyc+P+6wLq0qspiCzEbVoUq9kKjpaQCupmEVVxncc1IvfmWsZq8+xLjLkA03+zM33cQYq4KZDc+HHnxiux2xJvM42dXPmBpV191YuymprBSSpeznA3CjbO7OIijneDkXcSOkVveHtEtKF2gKtnYK0j4j20jyGZkszSfLfAdurB7cWDDRwkOvKmvI0FbeLiDREu573HFWysxW1cfzTyOvt+qDf+jPDNEzRoe3iVyE1l7mGD4etjQu0fpILoK3iZQN5/nigfQ2MUdHSobb7cA4+3XvKUVo/WXm4ThvYb7QKNxZyRhxY2k/T+t+Yg6jqtLotSJzxCb1lebbre4vdW9YfiY3FpGW0lrsYQJ7IRyUzmnXp3pTsd2aehc7w/ha17E5q99bmkLOy2yHEi3cKBQH2QnzbUP3cWB+qHvwzmPGpK81ZexhhBDJd9v1tekDhXTTEDeW0Bvm27ru2gxZVt6zGChuJpl6nD87iiiD20yZ+b5VUsuh+q33DXZKun/Wyr0mUJrz5WsyB6U/GDQFcHCXdMDRMt/nugcAUZTLfLiQP7CFEgxFCq5NxKNfk+r8dibv9PWnQ4fZBcxoyM1iuIbsdb4gO/X0pl1mwK4KbtiKzkbM0Bx0jgGRS5LZixJvVnrEX1FXBfda17rWta51rWtd61o/qLoquNdaS1IhBr2LWqZbh7eZ5sXQv79wJx/+acGf6525WSgIhdTq5HtuDc1LWhUKO2YQo/5dJ+RWsENGLjf0yvIsYKZC8zRfvFbLa8sFEOw5VDVXVoW2eZpJG4c9R+JWPYU6Pf/V81e/W+odx58aTr8IuH3AVEksDZbD0PGd3JK3gjOZh+bMd5NmtX93vuVn2y/8argHYI6WL+OG41Sz1rMhHR1mGxFgsx057lpKv0ym6u9pukgZW1KB4liPkZwttIl2OzPOHdkV7AB2Kiu/UDLYg7IbixWk8mVXhbV6Gu2o6qpUZu2aCGccuXHYw0T7OGHGSG5UWQOwh5G070j3W1WAncEep3UKOVelzx41Ha04gxSrfrJqWlRPrCaQyTBT+gZzni4nIiXKrr8oviHp46tHV45x9dbSOH2f00VJKq7mxz+fwet7IV84lJL1Tn9JNJIxYMaZ0reXx6QC3l2mq6vavaR82Tmz/U7wR0N2Rid9b8C/LBejdi7srH+e35rKw1zOgz7MjeorjL1B0iWZz8zK6TXHGXMaKF2jiW+tV5oCELadJjoVT9wK7qx8yfGhrsNZMJNDklBGUdVnUCUQwD9Z3EmVnuagvsbsZX3PxWXsORBf9epd9xYTCrEK7c2jUIwjGMg+UWa7dity0OOYBwdJkE0El2m+98SdYfP9wn7W39++ZExQr7g/z+SuZt4bMFOm/TRA6ZjuGvU7LgQDl4nRkIJhto4YdW8KT91len9wmALmbJSgkNBUxJqE5I+Z+aHDTkl95eeAfZku10JNsmqeDMPO8Xi45VEKLGpTEkhCOHnssyPdR+Rkid3i8dVjMt0IzbOe2/lOKS9uoZ/c7Qg3SqoAiz1OdEYTmpbrqRjLWSzFFXJblEm7bH8J5EtDaTLmfNGklnPlaypYai3t51GJJlZI1VNuT0HXblBl0Q2q9JvJ8Ok7NX9+2WzJo0XOFnmYa6IXLJtPem6Yny3PP57gsYH7GQ4e9nWTffH19VnMKLzfbOE2YKp/NY1WjyWoStdkihfaj5dOoZkhbAwSG/yxquCblumtemS7354uqYaVJCBTQJZzOUX1Ie87zDiTfAelkG4X73OCVCjeYA6jnvtymWGYb4XTzyNmH9RzXvfd9o2qheOhJd6CCZbSQfdRP7dS91W3MqBdsjEiU9Ium3cXFq/VDkbatbq3nBImFPxhoX7omp4nQ94mzMliz4Krl1JqtHsUO/Xmkwt+SsqEBVKnjPhijHYsRZQlmxYILqsX1x1nui8N850ht5myiKFSsIMlPgQIRvcYW9bUOTMaymTAFXIwZJ+VoFKJGyF069qwZ0OSgtQOC0DzfEnxA30/xWg63PIYOxaaQ6H/HNfOS+4uHFxpXaXrXN7X76urgnuta13rWte61rWuda0fVF0V3Gv9TjX/rNeJ/nplbH5rMJNyXJe0MX8u9O9VlQs7R/M8E7ee1Jk1R5wMqb8oFGbKNFMme0P7aca/f7koaOeRcrsj916NeUaVs8WrZ8eIeRmIr3fMdw2pNbghV6+tcibdIZCXHPBcVK2pE+O5dZhJp0pTY1R1jobNZsJWj9gLG87HlnH0fDlu+NnDI8fQ8rOtRo0Zyfzq9MBvT3v+6PYzw6TEhGFSlaRpovL9TOH9lxuaNlC6RHOrx8n7RIyGMDvKNzMlGMzJrlPxAOWxIb3R6dnmRciNqjSLRxYg3bTY43TJWbcWd9Q7cjMn0rbBPQ3KS2ycKoNrFnl9jn2rWehJj1OuTMZ40+E/HpHjmfTNvaofmwb/vR4D2XSX89N6VYSWhBm7qKFFGbPWQq/+2HTb6/moZcYZCalO/yZkmnXCGTDDRN61YCqdIRVK7y+UBavPbcJA2nbqNYtJ1RjAHA4UZzEhKp0hJuUAzwEzXGgTpW91knqKGNBjNS2T5r6qeoXUC/OtcmXXrHVRtUeS+mvjVlPEFl506lXtlWLw50L7pGl/YafHuXmeCbctpnOUN0r3kFQoTtaEOYDYW5qD+t7DtqYD1UlkO4pyOUuheTL4F+qarWv0RVO93KAMziVJael62HMkdY7UWmyKlE69774SB9xgmUDVy2OD383Ms1IKxNXuzaxUgzI4yOoLliSrn9BMmi8vudT0OFFO8pJQGGv6mwh2yoQtxE1Zux2uiYiA9Xmdxt/tR44CubJJKULeJtyTcn4lqYq2//MDAKc/3JGzJgLal4nSuXUKH+D0k47xjTDfZXAFGQxlm5atibu3B1I2ynF9U5iCY+4c3U90zR0/b+j/omF4I/SfLdO9Q5ImyOXqyY5v+tU7L3nxfqq/Wt8DzDcQbxM33xwYRv87tIpydtBHrM/kLAQvxB2Y+v32JZObyhi3AtaSW0vY6DHq5lRJDg3uGHj5Wct0nzGvJ1VhgfLcKHf7YVbmcDJ0XeB8rP7XPpHqeSn7iLWFtIm4ei3kuxmiIWfBvpvwpmBMYR7reQoGsw34NhLmug52M1Ne2N2GIeueJ8UStwY7FdrP0DxOdV0apeB4ZU6798+121T3+Zsec54xY0SGCWOt8rBrchihzmEUeyHEpLx298xccM+WvA/qP4+GEg3j4gnPAm0GY/HPQvucib0gya+KpHuZyK1DpkTeeErvKz1kiSITZL54tIvAvDe4sa550S6MRJQ/7XXd+EPtAD7r+Y6daBLkmJQRX/f47ldn8rYm2O1bioimqR2H1a+ad73u+QU2HzPzjWV4J8hTTbQ8Ct0X4dBZSpexL1Y7Q/WjqriCfzFkqzSgYgRzsuveRIHS6PxC3uhnrUyyesZTp92u6VWm/WLYfZs5v1PiytL98gH8kLHnWGcsMuY8kd/s6rkAPPDV58rvq6uCe61rXeta17rWta51rR9UXRXca62lKgvqrak3Y9MDSIDUy+qZzdZwMeyAiR47asJP2Kk/1k55nfjOndHJ9FYny3PbEPavVkXMzBl/DBRrSJ1dvbwmLBPdHhczYe+JvSF7oX1KK9NzTQkygpmKqoKVmACsaS9x35Ibo4lKg/7yTaPKYKzTxD/aHziFhpAt59Dwf6XXgE4Ix2x46M+8hI673ZnHw4Z50Lve6djqtOnJ4/cT58ceXGHbX/ynMTaIKRRTsNugvM/FlyboHXIRZKwsxsoFXbyhYW/pfzWQty0S9PXmjV8JAqn3mDGoL2zb6nN2DvtJ1ay835B2DfY4E7s6jXqeVh/TkghUbneYl0ETwlJW7iw6jYsxOsGaEjSe3DWqAlb105xnigjmPGlS0GnEhrSyaPO2pVhL3njs8wBuUfmrF/r1DdOrDjsmVVQFJUYsU+k5K7f2douZo6r/i7IMqthYA41X1cJZpS5sWiQtk9mD0hcySKgKdHNRTv0xMd4Z4kYY3tTkHq/8UVC6R+rUJ5mbUifXoftS/YZVMPLnwu5XZ+xf/Jbyo9fra4x3XY12knVqHCBjcecLr3R85ekeE93HiY//+la5m6eqolmQKJigynzcXLLkQb2oqRXsrClrbsw0T2Fdk6nTRDI7Jc2qHy40Ev0FqrSowVVTy0DV24WiUCrBgG2kRMEePakthBs9V5tvLXEjnF87/FCQvWXz3Ygdl2slqMRiTO1aFKZ7Ie8uXQ1rM42PnE5dPf2CMYW8SDNZ07eCKZA9zYuquOef1Y5ALNgpE3c6Sb6wiJduRvbCdF+wPzmTzh65m9ls5nUiPGXDMHqcy3T9xPHUkc6OY1X17KPX6fozjHeG5phxQ6kT6hefoYkZWVIUW4eEvKpV2aLHwWUK9VjLhSahPmtLmgx2Uo6tHYX2sar5U8G/JJpPJ0rriVtf/c4LokXTz1LvMCET9oJ5Paq/tq3XW/Uae59wLmFMYRwanL90j0KplBSbyVHwXVw9us4CPuFvxrUrdj63q1/b3swYKcRo1zSzIOoFB5BsaA7aKckWmnOhOQRVV+vlYL8cwWz0c2LjyXfb393nJ01ry71H7Ea7SM7oBD56jIuxmmjmVN0tfYOp1+PmU8ZEw3PuCG+i+suT4J9qJ3EQhp9EJKlnOOyE9lmfd7mexreb+jlqcC+jemFzXpPMzByIN50y3Z0wvHaaRFg/JsZXBRN1z5HBUtpM6iEuar4R7KzXuARNkYx3vSbVAWXxt3eOIqryFmMQ73SdUUkz3mCGSDE6+1J8Wf3kcVt4eRtXH3rqMxKE9suSnFd53Bb8wRBuA2U0Sl2oJV2iJEOxCfvkkCjrMSoCYa+dKTsqS9qdiqYv1kZdc8gUEYo3uMOsarg3+Ef1Q+fOkXqP/RcLuFcF91rXuta1rnWta13rWj+suiq411qrOPWiFVsoS0KNNZhJMElov0DqlNsYqwJq58J86zCbqj7N1OQkszI5s9cMeFO9d8XBdGdXBcPW6UsbMvOtI7ZC+5IIXfWYfQlISviXGTeoD8uegjIGQRNttr3erZeifqRksY9qisy7Doze/cXNDjdCyELjEn989xGAf/L5G16qSnTTjjxPHS9Dx5PUr3UT3iZCskzRMUeLMYUyXnzGGCAJ8XOPOxnimxlXbzG3zUwpguknOh8ZZk/ZCrlKauPkmY8N8XOHOxnsKJgIYS8c/0ClueaQGX56g50zfo6Qiqa6jV/hIowh3jf4D0dK61XVbfSuPm2r568my2m8ULkwZBvNac/bFvOSkZQwkyhxYKmclR3b9Ze0MzS5B1AFddtBNjrN3XjiXY/7oucidQ5/mnDfHcgPe1UVtj2pX1iWqjrZc6gqV/0d8eJhM89npXbcbqHZg9WvAZTjCQaD3OxVZRah2EtaGEDZaE67nEby/U7V21yYXrXruew/J07eIkkIt4m4Fcwr9V0OjzWJa9ZJZn9QlWlRbptjVastDO86zKufEzYGGxYFOJEbgxky3fuhJioFyuvdqvSErYNSOL2znF9vsHMh7ETPBxBuCnZSJTc79cWZIISqfk5ZamoTuElJDmSvU/aswhvmPJM7hxkCzcvlOnr/b1nk7cSmmxmOLWIKbV9Zm3kZh47kLLRtJATL3GTKZNf3ML4y2BHMrGzRuDc0zw5TFTV7zMRdhx0jsbekRkibdKEo1GSt4dRRsiCmMH3pkSTr6y99Irw0SFQVOOzBn+DwU71m+49Z31dSGkn3YSTu/OoZj50jdYV4bMAUmjZiTWY467qPzuB9IsyOj4cbGC32fNnb3FBJMpWK4saidJTK3QWQWDBTwn4+Et/eYIZI2vrV73x+o3SI5nvP+fkWXs+Uk8MeK2t3n1QBC4bug8Gf9Dl33+la8c8z/vtHwo/vdd0kpx2VvEjEYMZIqelp46v1MscsfmojlOB0TytC+OWWtM20b3XdzpNHBFyj/twcDHG2axeuBIPfzqSavtY3gdFmTHNRgPt+5uVxg/3sSdtMPrqVVCAJpjtoHwvNMdM8zfjHgdy4dUYg/OgOd5iwjwfMzYa4b3GVFgNKbaHRjlZpdP0Qy7qnkLJSXLpWSQKVpGLPcz0GHalRJrLpdKYi164EaNoaUgg3RRP3vDDvVH1euo22KvN2jOq1zeXSpQKdBQieeevIXrtEYScEtSIT7itdJgtlvciFcLckxhnCWeg+FeJW/d51oej3vdU9tRTSxuFOQbt8+ZI+qZ2TmnYn4I7gDobwUM+V1Q5jyUIenLJ5xTC+0d/hD+qVNhOE+0w5OczDvI4oUCDPlWkbDfxopHzbI3XfiJuCPyiZYXpVCHvtRvUfy5oQmK3BTQU7WczsahdC2d26prLuAfbSJfl9dVVwr3Wta13rWte61rWu9YOqq4J7rbU031wI24wsfppBE3fsoOpttpD3YKuS5EZVZ0qvrL3maaZ5UjVY6hRx2FtlA2b1K+bG0BzyOv1qUsENqmq5c1ZlSgQ7XTw9ufNKD5girvOE+/6SzLLpmN9ect6LN/jjTN6oClMaB9WDO+8tYQfxzUzjIh9Hnco8jQ3Tc8e37pbW6/OmZHRyHDgdOx7uTsRkCMkynBqsT8r/A2S0tB8t09tE89mQOqAIh6oEpSzM0dI4eBk6jBT6JnCqHN04VwZlFOxZE8y6T4WwFdYgncVzXPmNpbX4L8PqQTNj0HSsT2dVY0PCUr2zgB30+4isPmUJUX2qsCq5EpKSELyF+NWd//NZ1QLvMOeATDOlbbD5ouSWtlHv1LbFfnohvdpjDyMyVvbiS51gdlZVy76BUjCjqixmCnS/OSgftvHkTavUhbgklTlovFpYHw+UviXtu0uizR+8U4X6+aTEhZpqZk6j8nZBlZ1eCQvmNEFNxJMqwcatYbw3HH4B4S4qc7VYUvVEupPBDYKZIPWlZtKzkkdMKiQvpEbIzuIHVRvcsSYtecHMdfq8dZTek95sSL1dz3HslRVpohIbzKLWtgt/FRCI+0wRQ+oL7WdZk/Hmu4w7CcUJ2Rb6p0TzeSDcdvVaiOq9jQlzVuKIhMx839RrDHIwnM8bcJluOxOjIc5upRyIzViXGYaGHAy+i8yTBb/QSzKSdS1kX/3Au0taUWq2+BdNN9PHq9qbdCkTRkfymRwNvg/6ooaaWrYof2d9Pnc2um5GPTb9Rz2+7XPCf/ukyukUNWEvF8QsKYr6+BIFt4/EYCmFS1Ja9YnmbDSxaTbktmDGuv+dBX+sPthjWbng7hRXIkbxynJO9/WNUVnIF4EOCthZ8CfhtLO6rywKnssQLGYW5ntVv+zMqiIDMCk7vDiD/3BYPaa6JuuFmQqpppjlzw3cBcpCYthPRJ8JsyNHgdcBBss4XIzdtkmEY4PdBshCAfXhAiEa4uTIPlMKPE89fT8TK8A1Rsvp3GKbTLpNkISyi5R6LQXRvc9EnTko3hBvOv3cWGcMnHaQukb3tiFc1FrQvcQYzNNRuzQvJ90ratLZMhOgx9TCHMCYlZk9vDI6a5JFE/omAz6v1yNOZyaKgAmVVnIquHNezyVG50Hs5wN50yFJExlL7aItaWxSZ1jsCEV0nwcwgyFv6vN9xSFent9MgjsX/JBxp2Hl5egAACAASURBVLju42lf1/VhpDRC9hY7JvXsHkaKtxdeujNIzMSbltQK41t+h+aDFB0JKUAGtw/E50bfP5BHS7yNmJOuPf8wEieH1M/CXFNQTZvIsZIT7gP+qNdS/0FILfijIWyL2s9HmG8uyXaSwHyA8cFi5oxJBYmFtK+zI3PETBEZv4Lp/566KrjXuta1rnWta13rWtf6QdVVwb3WpUqheRTCjaFUcdSNQvMs+BPK68yF1CjfEmD7PmBCZrrzNcXMkv8f9t7c17ZtW+/6tV6Mas65yr332eeee+4rjCwbExGQkAAhWEREIESARIqEkJEzEgIiIEKyRECGRMR/4JDATrGEHw8/33vuKfZexVyzGFUvCFofY+5jvfeEE1vveDbp6qy71yzGHKP3Ptf4+td+nxf8Max3jPVrItZFuRGo9ouSVt7jPCv31gjuFLRz3xpVmKD45zK5csRdw3TnsX1a77xz0ru85I16z8aoKt7iLY3aSZq94fhrw/Ax4OrIj0+3fB81qcx8qrDAua45U2NsJvaWpe3cNIGn5+2qeDAbkvlCzZr0nMy9MH6ISBtgsqtf0QjMs+W8b3GNehePNOvdrn12VEflmdpRu7/rQ6L7nFYl3I6J0FmSCP23O9wpIjGvHjKJGYkBOfXQNcqqjXFVD+xpIoGqSXcNYu3PE76GWdmzhW1LzkoeKHfJkjLxvsP0qt5ijHYQj/HC2h001Sy5SukGIpowNGibsEEJCOHuHjNF7A8vSm04lgFVeWQYiY87ZVqOsybWTOqxpamI21o/y90W83rEfn5FNpp2lJ1V1abyK+sye4s5DysNQs5FUR4nZJwwcsv4cUsqnOdYCdEXNfDJkkqW/cId7X5UJq0/ZuadENpy6IeFLqDJUoh61P0hYse0djqbKSL9xFyYjrE2xNaqIlfGm+szAajmTHKG8UG7/c0XgkWsM/ZsSJVyeLOF7nt9/vmjEheqfSFAFGa0W46hLx3nC13CGGLjOH5d0tYC+N9Va4rRuPGkNmE2M2lYzqMnCqRdwO4dc+1pPllNI0M7/e0Eu98m6ueZ6mVgfNcy3RYO6ltQ4kfnqF4ntr9VFm8IOl7Dxwn5bYvxmTkYZDSYIKvaqedaE7AkCtUB6udM+xRXUgNGGP7wQdMVnyE1qmQtqrHrE90PltNvMikJabT6/uU6SK3+224zEIIlbYR0cpi3Rc1HmcmHTPUWmTdG0/eC8sFBu9ZT55m3DjslzCgc/3BDqMu1HjOb7+D4GyF0meqTI7aZuFt85/o/f1DmcqphqjRpDmCbYPo3v8W/qaKXds3KydbnyzruxjtV+qoXyxxk9cCOACbj25kUHJiMFA8mgFTKwZUqEkeLWH3s9FY4uYOB20SaDeNosVViDnbdAUuTxdWBEAybxzOnl1bnx2ZJQrNkkxnuBTca6s+aSll/OquSC5BZCQSI7kIsPG5g3aHJm5bUesw0K1mh7FCZH5/JD7c63mPS3ZKUyIvxM2uCln8TQqhIbUZGWWVAsxdSrSo9uaixWSk+/qjv7Y76XRYfdyWF0arK3JTzmBIyBvzbhAlKy3h9cMy7spsZRC3sN7P6V6NgR8GWHQM76O5ONkJsLPPO0/xwXv3eufbMu0pJQ89fJEiW6w9Km4htR6wN/TthfIzkNnLzrvSsZMFIJgNz4/A+0AOxqLEBaB97pp0mGYbBIzZhFu98G5SUkTXlMBbWsqy7S3qOzRlI6qeft5nkL5QYk7W3oNknMMLc6t8V7qSfof4cCNtKe1H+kroquNe61rWuda1rXeta1/pF1VXBvdZayalCgNdEH6CwWWF4UEad0hQydipewc5Qv0TaHwZiq52hsTFk61c/je0ToTWYkKleZ+ZdSSwrysJsPNXToGlOVjPazZSYb1UdyFZI5S7ZDkHv/sYv7tycwZ4mjOjdtnnrmb+6We9YszOrt9H2aD52rDGD4OblzrgoVQ/Kg4yCqrNfWJNI+juSILNgJrN6sdxBHxsfAraO+CowvlZMJZf7qfeYzx66RBgqzZrfxIsSdBaqF/V0mRn8OWlG+X7GP+mddbjvME7Vp1ir4i0xr2qUp/jLqp0qqyGRugpbCAZ4p3zbriY2lrBx+DfHfKveqPrHM+ZwVvVjCOrdFVnPHeOgz9805NYXwkYA90UXb+EwkhJMM2LVb5uLgiuNKsZmUv9r7hp9v8Ufm9LaCZ26as2Mz0WhRQR7KuprcuS21mFWGI8cTohz5NstTDO58ZjzCP2AuJ/TILC2dOcK/nXADoXkEDx2zLTPQqiFeae+u8UTLlm7fbMFd850PylrdUkzSlZoniOpUsXBjgkzBsZ3zWXI9g2xVtVWPZuR2Fz0Bjskpp0n1qoGj/eZ1CXMobAsgbhJVJ8s7mwITSa2mdOv9PmxydQvQuigfUogMHzs1t2A9vOBvC2qd+WYHprCr9bnb34H053Qf5XUb+my+gP9xfhpR2F+CEhvMaNootqBlaeZKtj+LtF8VlV/YYTWr3qtY23xU8KOkXnncaOqY9VrmZPnWpUtAWYht5EE5DphP13mDUY7uusXVW/9KSgXm8Jt7Wf17gk/U2/1GIThsXgHRwtBkGBovtfPOd0ZJAmHO6/zPgr2bFYlfd4q/zh6wZ8CEi12TNR/9kS60db4+aHFxEi1V0Z02FW4Pq0q8bwxiqR9gfFe2caLogdgX52OkwT1kxAaXYsXCk0ovloP2H2vfkwRKKqh6WfCbUO2Bn9OdL8X+g+CZMO8XdL9gMkQvUVMJh29slbrCwUhTwa7dySfyW1imuvVI5p9htEivdGAsC4yPFXUz3oeh1/NRGvIJ8f54DGzkLaBWBTiahDqFx3r+j3kCBtL2Oz0XKHqqD1NxG2NOc/EbaXkHKnKnFFygdmfMMsOk3cXikHllX1bO6g9jDOpqy7fMwaGBzVEp0rV29SkVQbMRucBPpOOUpTUzLQz+OMyniy2n0mNV197SeQ0L/oASZnUNSRnCK1e19OvM/G2DKggSBswLmFr9SiHXpm5oCzebLQXZr7RNLK4rdbvw9g46h8OhLtWiTVPJ13D5+mi8pY1un4ZkegxoxAb4bAv60ESGNQTnatEvCk7hIvc30SGU6XEFJchCDlZpDwuzQbjMmkwuCdPtrobMd0t6+dCY9J1KrZJmeKbSHKXeZ2d/u1gR00qNSGtc1dixp1m5b3/JXVVcK91rWtd61rXuta1rvWLqquCe61LiTC+i0gVYSw8w0+amDPdCrZXz5E/q98MVF0lZqa7imxFKQovUdOoSrqV7YOmkxWCwrwxVPuw+gGXMkMAqyk3sXUrZcEOUdWez0dSV5GdXHxTKDkhVVa9WRmkrTQJ7YvXl2nGese7kMm2pf9KPT9rJ7KUJKgoMBloVPUiFDXq7DAnq0zRXeEuznJJa6szpqQB3ezOOJuY4gZ3Lv7WZ4vthdGoWhy7DEe3eqtCl6mfNd88VoI/Juy0SDR6Hv3vX/DGaFdwyszvt7jDSCxZ6xIzsfW4t0HZwEYwOTN/fQOAe+mR4ku1gzJiJWeq1yLbWSHtutVT+892qObaI/2oHldxiGSmxw7/MmAWFm8hKsg5g7XKpnQWub/V1/AOgvJt5eWNfLvDvJ5WCoJ5PpC7BnMeWJK/8qZVEzPAHJSoME4wTGANedutPmJKqlm2qrITM7nxyNystAicRQ5n5eQ2tXJox6C+VIDUIjdelYbKUL8JdkpUz6pQHP6oJYuO/VzajZtPI+evSyfznAkbVV0kgX+bCkWkfISNWeeOmfR8Vc89WYRUVLf5xtN+mgnbpfvZEDq7KoexhnB2a/JPcxaSVWUECh0gsXrlkytUkmXX5Js77DmoIpL1GOebiuZVx9qpsiSnnrjpIVE9WaaHqApT2bWITUZGQ/PJYsvmgR3zqmQ//F8z/TvHdKvkguSE5mlevcipdZAy7mkgfdioevt2mZOhBUwmdRGZDO6TV/VqlpXiUh1g3ujzTFGnzRhX9rEMAXM4aXocUH0+Y54PTH/4Xs+bQPXqGa0QndHPtgv0f1iUy8nQ/OQws8MdheF9YvtPhfGhTApRgobkTPQGdwrEzjF/fUcqPQLaH2DBqMIHFP+8HuN4ozzU4b3uDEgS6hdDaC8+4+wyqdKfY6Oe6/TFt3f9POO/f9U5Nyf1pZaxka3OeVX5PNUpEw/C2wdlnuvFKO/1Wqn/0yd93rGowGW3KruMGQ1xc+EV63koSv8uwGhhNEgQxoWtmlUZlCTYg6ZemaNbmcl2EFyf2fyQcOdI6Cx2SLg+rsQR//sXsrOYyiE5Y4+T9hgUj/yS2rXu5qS0riE6YDWVUWKGedZ1KiT6d6poHv4Y8q/P1PVMnQwpCc5FbGGZD4Nnfqv1mMtx94+G7e8j472+RnLC7v+NmGHWnojzqHPuUHbRjIFtAwJmToTWsfszof+weHSVvGJHUY/9LEiV1+8ZE9Wfn7zgXpQBLPES52XPE9kY7NtI3NW6jo9B15+xeMIbr4+rHO1TYnivnvFwV3Yj97Z4jTP+2TEnwYyXpDKZdNckV2kdN2RIL8uOayZNmriWBapXQ5gvu6F21FRUs7CjP1vMDOnV4EqrhTvrbqau4wnbz0yP7eqtD7taaSHbq4J7rWtd61rXuta1rnWtf4XqquBeay0JmeYHhyS3qk2b3ydiLVSvGX9W760d05qIhBGyCPXTsHpeU2Xxb9OaMGOmSPNJc9jt8xEz3iApY/dFXhLRlBlnCXfKt/ULsxX1krqXoIlkKekdfcqXzv2onZZuP6jf6TiUu/Qid3mnvFfvlJ0YW9xJMJE1OcWE8vNsVo4niZX9J4MhV5kUL6w+hPWuNkVV0ORseb85cZor0iZSP5cO0sVyJ5SkG8EdZGWnulPxWt4Z3DkXH7NgnGBv1BtlzhPz4+aCRnSGsKvV0wx0nw8Yb4m7GpsSMs6kyq00C3MsqmgvWBElTSzEBDTpzJ6UcalKX6EoxKLWb1uoKyUtzBGS4M4z5nBeObWEqO+REjlEpRvMgdxe7rQlJVV2ffFie01QA8g3G+TUr8SD3NXKqiyJNeY8kVNCKn1ubmtVc50qdAubV/ry3zloqhpcvHg567jdduoPtpa89auP2PYz821FaITkhFgLiMG9qrzgz7UmYUmmfg7E1vD8t7qVotA/GJKH5iWTxVA/GcycVvVBUlYP2WHS3HqnqW+588ROP/fw4Ehe/c/JCvU+0z5f1Ki5VWrCvBXsoKlmsYHh8eIp98fL42Ol/MwlccmMUakj51lVJqu7JqEtfu4+c3LCfKPsTzOpz1Jmo8lGFNb1rOPWnSBsYd4Ibgm16wy+Vy+5nZR7baa47qxITMgYdRfh84nh3R2hhfHdwgguiYpJyHVi7qKyou8mUkkdDK2SGpKD468N0QuSKqoXHU/zQ0P2BkTw37+S7rfkm81KaDFzxARPvI1rR//7+wO1LUpRMhx+VROLosfgOQ/NqjI3n4Xz10L3A4z3DhMd7pyU2VpOv//dE/OvH7EvA7b1OueMwZaErmpnmW9kXR/mm6Q7BPUCDNa0vHmTVZHPquQuSvlyPSVE6MeLqlXmTPYWiQl7HHHnWn3ITl9n8+3hMi8lU7uISCZEwxQcU6G8hMmRRovUkRSFejPxZXkfSUloqhkjMAZLzrJSZFIyqoTuRuZ3lhgs+blaySTDV5H6yTLcG/KjwZ8y/pSpfjqtHO/cNcRtrb7qWcdNqjWlCyBuKlVF3yl7mzmQdq0qmADWqIrpLMREeNxi+xl/LvN+sIxnR7DK8k3REiZLLkp7ngzmaHFn3UHwR2Vg+1Nck/FcBkIitQ73+UjuaiU5POgOVqo0PS1VFneakZQZbwxmPZ2qziYPZFEe7CTUr2XeTvqe9T5ix6g8632/sn51TdSf3ac3ZJyVBd7Uq7JtDj3xYUvYeHZ/NtI/toQOfFHrQ5exZyHVkKpM+52j+yHz+jfLOJl17aEMM/equxPL3wwk8Edh3hUiRZeVdFJ2QyUrP7x+EapX3Vna/KREoGX9SU6wU6b5qcf0M6n1uMO0/o1RBhXm+HNSxD9bVwX3Wte61rWuda1rXetav6i6KrjXWstMgeqgLMBFHcgW6r36Ue2YcKeIGePatSlzRE49abchdR677zHBIcOod66od9OESD71kBO2qZEvvVFBWa0yjLhP6rHMbY0Uxt3CZDVHlYbMoIlEpqRjMQdV+SqviS21J4sgi2JXvEepq8gCzWsiNobxPq8Kqu3VM2T64hV2kNuI6fQYkjeYV0+8CUidkGdPbPLqgYxNuZv/eCYhjMHhn5wmTqFJLaGF1CZybyGpQlVCtPAH5ZwmLzSo18qORclayAQi2DGSnMF9eiO+26lvufgqw/sdZL2OaVMTHzeYkLB96bB93Ok5FGUEy1FVzsWn5j+rQin9SG4qVdW/oF3IaSBvGuQcEGOQpwNyu1WVolwbvGjS0DiDNXA4kUNQ5RfU/7qkCYGqwf24HkOqK/BO8+In9bExjOSS1EPKyrAtbF+9eBZz0N2A7KyOvUVJNkY/j3fKBwZNW2tq5DyomlN5VYOK8tF/3THvLOONKhWhVf9YtVcv8/m9xUyaMDbuDHZeEqwKoeBZfdTtU0AihE53RPIiJB9mVfhAWbTOIN6SrcGV7v/61ZAqUZLG28x0X60+VoDQWE2SOmSqU1ZmcTL4S/AUEsANWbvQE/hjWP162SiNIm4qjBXMW6+JQUXhPX+lnrvsM9lmpgeQ0WAHIdRfdN4bmO4T47tM9WxxZwjlUu3/2CoD9xw0ySwo3zo86o6EPc3gDGFTYabIcKdqT+wunkKqpB3dTcS4RIyC+22zztv+60j2Gbe32EEY3ikfVj7qA7qfItNtp4QWd6+qdeeQWd9j3nnq18ypN7TvNN0wZ+E861iYgyVlQSTjXMJ0mdOjw78sahcrH9wEmHYGOySqT6fVp0nKmuaXM6G20DmmnV/X2OFBmHa6ixTqTNxGpI34WudtmC2T9+AyqbJUrwY7QfWm16r5POPeBvKm1V2ft554v1mvtdmfSXcbVbFfR0LbMrxTKseisNY+EJKhcgEBGhfYJ0NdVOS6DrADIxlrEl2l43Tf68V2JjElS0oGYxPO6HufJpX57ndnpmAZZ4cxmWwS6XaGkuQoQTh/nWmehOpNk7rsWZneiwq3zE+zPyvrt6j/y9phj+P6HZG2DWKMMrRN2VncbVTBXYg654nsLd1vlXCw+foWM1eEzhN2pbO/SSsVpHrRI6ledfevec00n/U8+HP5Pox5lQ3jwwYzBFXQyw6VlN6J5ATxVulAfaYcIsnqrs10o3NBku7uLd77+pDVx94H7Fl3ObO1KwuYnDXRriROZqspotkYJJXvU2f1u1UqTr+q8efM5sfMeKOf8+2v6c6kmYSwSYVsAX5f1FWfmduMGXQ9kKhhkEs/iczw8f8c+d2/WzHfZkiq+i7+2rDRJDilBWWqY2K8Kbux5c8CSSXRzhlS4zD9rGP706s+oKl1TR+uCu61rnWta13rWte61rX+FaqrgnutL0rY/D6RnHZpg3Yy2jHR/u5QuuhnTRArahXTrKkww6h3S86u3ZrpXpOasreY46j51l2DOfXkriHcKSfS/7hXla2t1TOZ88/zu0NCjr12zHsHlaaSLdjXdLfBfn4jbRq9kxX5uYLcqG80ixIXmqfAtPWMj+o3AvUaJZ/JXcTUkXzwyGhIiwo8XxiISCbtYmEF612rP6jyi8kYModzrepdyfjW7lj9faoy1YthekhrJ3RyUD+rnzG06uet9qq0TXd6HqqXfDkfMal6C/jPqkCkrlpVROWrZmLrVgUXihoeM+b1UJLKEskuKWBGlYZphtoj40R8f7sq6RJKylnxleW2VgU2y8r8lH5S1aCRVVmR3RZOX/itF//rrOpzbuv1WptTXzzZkdzVetee80UNM8rVpVfFiph0XCxd8yHqtXbqO8zG6M8hXpLMQiSfzlAtfuJAtobhKz0PsTJEr4rrtFO1pv+Q+e3XhbfZq6LrBuWRIqo6zV05huKRc6eoSV2V0W7w9dqIMjuzeoqXayZLuhJQP6mffHhfY8fIcGcxUVMEQb23Zsrq0Zsz01ZZz1LarZVmgCZnzVnVsJjWzn5Qz+JS6bbDDDPho078VEL8sigpoXo22FGVmlgoEDqhwR1lZegmB7kouHZQr3D3hV/eHqeLwtV6pYA0lnnrGB+E+T4iS0e/zYjJmNsJshCPyqKd34WVbiKFTWuCJprFGmIjK6dbE68SZk4rP9qO8UJyqC3+qIlwx08bZd2OZvUTmlkINxFcQlzWJEP5Qo0/q4ru+oykzOaHeR3ncafnN942OhZFyN5oqpwo61ivlWHzfSY0gvyknN7+g2O61/FqT4bmzRDaTNwlzATVPq/UiOHRU/2YdN6J6M7HnJQRC6RtqzsFlSrXsRL1/DvL2eq8nZ4d2WWO7yZy75C5+KvdFzzewRDuAuZkebkJeq6m4k81WXnpkyFtg5JofEYKReYpGsLRQxS63zqmD0lJH+U6VnvBnWD7XaR5XpRGo3O8fJ/QOPVve7cST5JY5VmD8q7HL3Z3ctadqnP5oth2Os73Z1W3+wmZ7WUH65RJz4IUFnLYZGSS9RgX5rmdWP3s062jeZ70+0iPiNhV2PNE7CrMcSLXFokXPrikjDvrmpNqo9708h6hVTaz62F8zDq2o74fwLTRZNHqPJMqi2QIjy3+02k9Z8v4y67sZKa0+pRB17+w3RI6y9wJ/Tv9bjp/XIgIGZmF+U5JGeMHmG8FWxTY2Ol3YHa6HogoISZWeoxVLzz9GzUmQLSZ+T5SPdkLJ7zKmFGYN+APEGoheWHzQ1i97eONLWltDpvyRcVdeOilcl3xl9VVwb3Wta51rWtd61rXutYvqq4K7i+kRMQC/wD4Luf8t0Xkj4D/DXgE/iHwn+acp7/sNUiJ5mlmeOdXgTYb8Ce9ozYvR+14P+hdNADWXpTXYdTfB+VQLp2dyRtS7XBPFz9t6qq1Kz3XXpW5pB49SVnvupdOemeUb+osqfXIEFTpWvh/MRdWKsiUSJsa+SJxCWfJInr37wxmztrxnoTcqHcpWvVO2Tqy2/bI7ZlTX3O77deX+fTjrSqJRw+VpvnEzaIAW5qfLNtu4DhXiJS0mQ+qNEwPBr+3yGRI7yfmVKvXsCgksYO+M6pM/WCwTjj+2uGPmeZFj3F8rImV0DzPpPoBcqb63fN6HsRbZAykribVyh51b+NKo8hCyU/P5F2HHHtVYRel9TzqdbTqXWPxapXBkFqvaT7eYnPWx4qo0rKe70qz143B7I/QtRfaAcDLHu5vVb0NUX1yxQML6OMKD9YcR2g8MkyYqShj3qkim72OM8dKyIDC2U1K5GDsVzUn1/7Cy3x6hZuterjeNOknbmvGW/394TeG6U6z2F2vSqQdLxxHN+i5dEdVcLNRxXxR1KYboX3KhI0lOcH1CQlp5UKvytIU9JhyZr5XZWLx2Fb7iWnn2f+x4/lv7FSxm2X1bCcLblYFZbhTRTA0Zk0qG2+F9pOqyskJtjbKUC0K8Oa7YfXwpdrh9gOp9asXLnQQthnaiDlZ+m8C1En9pr7MO4F81DQmOwrTjXKyF8qImYTpVjh/VeHGRGgspjerfz9VlmzUdxxbx/AucfebVz5sj2VaG24rnX87P3IKFSEZpuR4HfR8HYaafvBMpqEdiuc9XxTW5IT51uAGo/77p6C7Ig+FTBJV0ZQEbjsTZ0NODnev3r4YDUyF63o05C5i3yyxXlR3pUa0P45kr+mC7qDq3aKWS9BdBvd8Im0b0n2N7SO2+IBD41dP8flj8dV+EuxYdhwikGDze+H0jQGj49J80VA+fbUjOaH+3JO3HXZ/ulBkjJDutsgUGL++4fzBYieYQLnfQNzqtWXvkbsZXj3pJiBjWTtMJviIzMpNZtTj8K9lvLSZbFV1x2akK5zckoiZkiC9JW8D800mVQk32UvnfYbmKWPnwkv+/qg7ULsaYxfYOJiXt1XFkzkglXK1AfX8GyVmmJcjuWt0bVjWABFle7/syQ93+hrngfBBCQfNa2S8NZhZd94W3/x6jGXHIsvy76qizxu3snrFWGwfVJWdI/G2ITYWdyge3AyIJrVlIwx3ltDIei3njWivhqf0vej7jXeXBK/shPlekwfNlHAv/UrCyc7ortCpL5QYo+u3lQtpIeguZ6zLurFVBXlhZodtxu8N4Q5kNsgkqrpOZdckCG4STSHrdAdv8dkCzFkT/sJO+b0SdN1adjPnbcYZMEFIldJAkhNe/9ivJBqkKNln/b7BCAkQs/Ctx+IxvpBE/ry6Kri/nPovgX/0xf//74H/Ief8rwEvwH/+L+WornWta13rWte61rX+BddVwf0FlIj8GvgPgP8O+K9ERIB/D/iPy0P+V+C/Bf7nv/SFYsQfZ84f/XpHGVrl3GIMcjwTv36HOfYXX1RdAfbSpT7O5E55tSsFoa7xn47qvfRW77pK4hVo1/7CK0w3LeY0QluvqmBqHHYKpE49kxggZtJGzX7qx1U2a/YWcxrXO1qAtKnVs2WAnOk/eGIlmCBIUXDrbmY4VsSDp/cBYzIxWD79oHf3ZCBKUTAE92YgsSYqKZwRrElMwWFMglFI+3JnveSZV4ntzUDvMt4k5r0qm+5mQraZMDrSs/5bHAVp4Lyk3CyyOjDdemWr3u9+fpta+LaL2ogI9lA6TVdSgVevcqMEBbNf2lsjeZqRulq7U5cksuUUSMqkch3FqgInMSPLa3inirsIWEO66TAvB6CoWTc7HU+Ff5udVZVloWrkrKSGQucgZxgnTWADiIl4t0ViRAbdScjTDHvleYpzmrT2doSmXlV/+93nNREup4jMlTJDvSPtWk3emxbWJPg37Q62I9z8WeTtDy3utKj1qlhoMpWmz2UjK4HADlA/j8w3FWbOuMOsnunX02VMblvSrsGeJpJRv1nybo04cAAAIABJREFUQiwcyPmmY9wZJMD8WOgJov4/YO1AHu6V9BArTaVbu7E9DO+E+kU7s10fNBWoqFHTfYXp3Ko6x01Fat2qEI+PierrE483J6b3jtrpXN74iVAG89vQMN5YTruGCKTeYc4WCct7gGTh8K3BTobd7yLj3YZ6X7ranYA02HNgvPOQhZAMPx3Vu7/fd9TtTO0DtQ+Ms6OrJ+ZosaVL/9u7V576jr1L9HTUP/18vp3fG9rn4r/PYAf1PubiBxzvPCZoYlIG9Z92F2k0B6N+0jqRg6wqrz0WT/GoCuvwoabaB8iqTNsxYoqyKP1MeGx1Z2mcsWdLqi3DY/F0z5loBInqmzazpiou6VUITO8Tw68y7qDrzrQV3FDGa8ycv6rwp6RUjDFgUtJ1ePkczpB9rQlhY2Z8EFJ9WU/cwRCrTGoTnB1yO+vyXBRcM2iSWdhGcpehSkhvmUuqY24S5mxJTYK+ED7e7Jr6mPcVZoaYhPmrCQZL3CT9PECsc+kb0O758eOW+scTZgjrupc6S7q/UVUyJNKmId5U2DedFLFrMJtadxrbWkkLQ1j9pzIp2UVudjr3jcA0Y8tOot0pqQQjq/fa3sykJ71O40Ok/mwZPign1vWWm386qN95ER6zMq5JCfs2ELc1/jyrPxgI77ZkJ9g+loQ5i+8z03b5HlGahqbj6fxX9bNcp6Nw+spRHQz+nIi1wT+ldbzb/Yl0u1Ez+jgh2eu6VxjzAMYZ3E9vyLt3jPdC8pnhMa9499iW3hCbtc9E9HtLgr6JG4XhY5kjNpMThR2vv5+6pPMjAVaJDLHJFx77rGM7eV0Dphv9ufvx4iv354Q/RkxMjPcV9fOoPSaF4GIah/3xFRZV+i+oq4L7y6j/Efg7XFpxHoHXnPPSXfQ74Js/74ki8l+IyD8QkX8w5eHPe8i1rnWta13rWte61l+puiq4f8VLRP428FPO+R+KyL/zz/v8nPPfA/4ewK19l0Pn6X4KuEMhEFjBP5+VVWot9tOrqnNz+f2ixunBoLdzercsxTfp+0m9uaceO87acR+SduujvktN28nYH15U6W1qUunyXnyfUjpnZUnSMkvqSUWunHavnsqd3qxd+HosgewMcVtjpkholoxsCN/rY6aqgk3EnA1T7MhdgN5ixvIeTaJ6sqrQFH9UtZdVUfNHGB5hKgk+/bHGu7wyFLNRhYIonPatkhiyA1dSeJbu7Mkw3Sl3OHlDVcnK2nVnZfiG1lLtZ7ITwm2tzM1SqaswL0ds6caPj7vVp5QaVShkCmCFdNupJ62otfEPvsLue1KrSWAyTqp0FfXUnEbmxw3++UyqHOmmxX7aqydu6dLNGRlmUqdkBJkjabfBfH7RX+82q8or+6BMW7ikjFFU48W3m9LPOme1QzwqySOhxA4RxF/SjiREpG1Wjq96IO06ZuX2hvxWFF8a5Psnun5i+rX68rZ/diZs1G887ywSoX7JtE+qWswbQ/0aOb93xaOe8YeZVBdO5lF3PewYdX4IhG21ek+zMcSbSo8/ZeLGY2JCkhCbomh5JQFko5QAyapumUWMdzDcC5ufEq9/bFX9SoIt96jjY2LzO8N0K/i3TKwN3e/7lb9r+0AqSnuqjPphRWif9To+d4ldN+JNoqpHahuI2ZAQTIFVvt8ccSbx2rbELByGmvO5Jh70WtiDZbxT66J/E4Y7g0RVgEBpFfPGEKsKBG7/BPr9PYdfFzrIi0XeWt6+SqRO/Z9vo1DtDeOjXoufohIBwjYroKOwf79UPwGa54Ad4trt7gpFIRt4+41Tv+dzjZmEZDJh+Wo8OOxgiDvAQDh6JF4UYkTJEfrZdKxIa5CXjFvoJrsGmVXlx1rc60BuvdIUgPpV/dPTrSBByRjzNq/Hnh3Uny/eUH8CO11S6cjgxoTrI6G1SG0xjVs76t1h1B2RMVC1XpVi55h3ghkXnzCIlVWplRdLbLXbHYoPVaB6sYQ2IwdL3ERY09YEexbs4AhdwoxmTYnUYzDUT8LZQjZWz3OT1td3Z/V9S8iEzmJCZnzfgZHVV+4PE+G+VYU6JOWgW8EUWo7MUVO8KlUtzWFApnn15+N0xylvLTLMunYYs35PnL721G+Z/kNJ53uzxMkoNaWcg2zBjII/anJgFik7HyUBcD+RaoMbZ90925ddqLKGute+0DQsWMGfPMOdYd7p77sfE/2jKeq8rh3z7pK4iVGfbraG5inoPO4qzLmo2HdbVYut0TU1RO2VCBH7VvyrcyBtCzO57FYN79PKsaVOultZFHl7NqrOF7HU9eCOSvUwwZYEvnz5riw6W2r1uTkb7FlWxq0ZzZpuNt6X8durmusK+10iHH9V0T7pWjB8qKmf5pV+kssO4fr98RfU9Q/cv/r1bwP/oYj8+0AD3AD/E3AnIq6ouL8GvvuXeIzXuta1rnWta13rWv/C6voH7l/xyjn/XeDvAhQF97/OOf8nIvK/A/8RSlL4z4D/4//Hi+EPE8kb/A+aGKIM2axqXEqqis0zuMtdMTGpl7GRS7LINJM+3ANg3s6q1ha1V449uak1Hxv0LnwIyP6od9mVJ5XMcQDpg6azGO3cJQFz0DtTVHmQOeLmqN7bcodsn1Sly11Dah32ODI9doRG/YkI3P4/+h7Do2F4p2k1boDzR0PcakoSsCqxALFL5Cbh3zzjffGHeeWAhuIPFKsd5f6tdJbeQGw0Hcf+VCFJO4+/7AHNXcS9OpLLuLOqBGZkNRJJonSIC9OtR5IqfP7z5VXMWw91RUa9y+oH1LteexzVi2ctZioEhNpiWvXqmfNMbitVvGunP/fTRZ0Vwb/0MM2YnMnJruo7fulsjvq8WZWDLAK1U5oCrKq7Pq5WisEXqWQSEzln5dkC6aYjbmvcSyFuLGpuQr142atXd1FpvFvv7hcFyzy96Wv7L/iY3kNdkT49Ye5u4XjGDLvy2gn/+UzcKYP2/HVN+xTxBx1vc1eRvFC/ReyQGO8c4b2/+KyNIKmCrGQFe55JlVFFCt0VWXyFsa6pP51X3/KitofWMDwYxgdNyUuen3kmSYI/CeO9ZXxMuLM+7/wrfYw7L3n2aEdyJfQfm4tPuA/rfLR9ILZK3Wh/VEWk+9OW58MDT3XGPfaEp5b6J8vwB9PKNjX3RTU6eNyrJTUZMwjSLl5mTQesn8EfM5sfA+4UL15yA2aMTHc1EhKxtYTGMRV/a7asqpHdOzCZ2CYmA+6kj2k+C7HWdKlYq0fZny+eRjdkqlelrtj9QNzVpFrVagA7J6YbCDdJk9u6iH1xxLbMKZuJD7P6cBNIb0HymlAYGvVcNy8JOyTmrdXOdyOaLFjGmz1rEpOO2ax9Bu913jXPgeQNh994YpcV8WvzSjiQSFGNhfpFcL0m59nSub+k1JEzsXO6XntD6Ip33qrP2VSO7HSM9R+Uo7xwuDWJUTC9qpfJF1b37YWfWr2qAn//jzLPfwvss2Uuv89tZH4ATFbKxDZCE7GfS1JZ2YW6+RMd02GTkbOylZfPGFodE/4YVLVNefUxgzKjl3OXNjXmeVBaTlkrhKJWpkSuK8x5uFBVAMTp7t9x0NTLyhO/uiN2JcHwnSF5VZOHj7pjIBH8cZHSSyJlp3MqWTh/1HRMW+ZVFt0ls4Mj54w5J/3uPC/fi0kpNr7THaJOGB6F8UGf338QskvETcK/qDpavQnzTSEQ3CgvPTbC2x9UbH7U5Dy3+FtHJcosCW8yzcoIt1bZ50B8f0euLcevLW9/I5JtprofuNlcLIoxCXO0iGRCsPRvDakwj8NDxm1nDOBcJCXB2szYl50bl3A+4gFrEyEYjMkYo5/hfKhBoG+r9TveDjouX8uYtaPyvbNx+D7jzol567BFqTYHpV/Y0wS/5S+sqwf3l1v/Ddpw9ieoJ/d/+Zd8PNe61rWuda1rXeta/0LqquD+girn/PeBv19+/lPg3/rnej5gjgNG5JIi9nogDyPS1KoMzrP6Gxd26uEMlSeVdKule57KX7r2N23x/dSq3izdxbHIIKNy+ai88korh8yRuNXh6U6jeoCdJvXkxiPZXVJMFiGj9phB88WTt5jCXg03Dbaflc0aE+O9EOtM7DLuXDyP5aXmXWZ60FQzfCJL8Va92rVT1h8McRayu6TbZFM4s8Dr6wb7Q40/aDoPgJ2E0FiGb2fkZDCj4GBVgrIAJ4c/CqkSbK93sd2niD8uiUcJM6mPMLSW5vuz3qIuyqG1qg4UNXBJ1FkZtSKEXY09Kf9UpvAzhqxMsxr+5oBMmbTr1Dv9Zd63d+pZK15brKafybEorG2NZK/XYNMiPz4hHx4uPu2SqGSGSTmdzpKb6uKlyllpGo83pRsZ/cyFYZu9XTuS9TOLfoaFs5szvL4hm05JHFb0PQqtQQdKIoeINIJ885G4aTCHM1/K6blxxM4xPKpS/vatw4TiV8yQnCF0QnKW0ArVPtO86nUab4VpZ6kOkdNXHr7ybL6fmO4uXFPXJyUbGIGYsU+v5Nst6bEp1zoz3sP47YRvZ4xNNFVQLitgTKbvK+p6xgZLKP9eV/oZp8lx3FdUT5bQCdNRcL3QPC9KU0N26mXLJbWs++mSMtY8aVf0+dvA3HskCDd/mgkbT9hdKAPxzat33Gv6kRsuJ7HaqxoosWTOv0yahpcu7GgzBJrvZ86/2THeGvU4ljkVukz9JOtr68mDtAu4U/FcW5hvVFHNAs2zzkVfkp+SFeadw5+icq4PI2ay65oxt40q41GQ3UwerfKGm6IAvxnitzNp1DlrZmV2Lr54PV7ovh8wU8S/WUxQaoNZ/IK1hZAwv/ukDGhjSDftyhfv31dMW4M/g/8nhukWhq/DSmj5cm2JNdz+adTUvDVtLeHeBmJX0f7jT8y/usceR8xY1NOUsS9nwuOGZGVNlPJH+WJNE6Zbnf79NwEzKDc5dfom7tUy3SfaHw3P/7qOGS6N85iDMojjVr3kVAmz99TPxW/dqBrff4D5Rrv+3Ukuux6OVfkeHiv8MWL7iPnT7y/++rbGTDMyB+w4kxvtvVjOg0Sl6KjfeCJ1DTJO69qSvdPvKGO0v2DTInPi8K2uHeM9ZKu81/Z7x7zJhMfAXPiv4TEo23c0zBtLtRfmrfJjfSGsaKJfIuwqYm1xfYU9zdCWa3EeSfdb5psKd5rZ/zXD/DfOtG3h5Iru6hnJhK8MKYnO+TK/4/ctsdbrdvqVMO88/pCxk54jf87ULzPVp5N+D1hTdlcrTFGysze4H/eEbou5mxBRdXUq61uIprTZWDbtiKsScpuxtqxvo0ckk5JgTCZnISWh2+q67G1kjpaclX8MEGaLL2tTs5mYJ0e4mTX502VCFOZ7u+5a2LOhfjble0PIYmg/zZcEu13DfFOpx/kvqauCe61rXeta17rWta51rV9UXRXca10q59UjSfH0xI+P2P3p4mXq7mCYoC9+nboiO1UoSFn9l2MgNzXmuXhgbzaFX2sgRlLnCduK6rPKmxKTdrQCctSu/vSbD5j54r9CRNVDY4ibGttrFvdS7vOw3q1lK7inI2lXOJCiJAXJmlZWvbWETr204/1FGcoG0lY7e/MuYFwizYvEANmDOwjDh0SuE7a/JEs1T6rqHI8N/p/WuLP+/9CV4zuVTtgg+P3lPRfv5LzJ+KOsfrvtd2ntCF8+2PjgaD5nzBipRiUJSEjqkwVMiOqZxihvU4S48aogAKl2mClipqCKqzEXFaT83p6mlUWrL0phHZcKUf2zzhRFZFKfV/HG5sarV7F4fGW3gX7UMQLk9ovXqjz5UDy4ZTzlx7tCzJjUR9s4zOtJs+NRTy4pYY5nHRPDSA4RbhQUKedBx1lKpMbpa4WE7I/ku+KJnGZNQXs7wLsHPR8l3Qeg/3qDHVSlnDaqqMcGxk5/X73B3P38+k53wvignzHW6ilrPwmnb4TqDeo3S/edfsbUFCV4Stgp6Dn2DqYZV963f++wE8jRkp4docqaPLUMxzpBEMYbwZhEVQXCbL9QeBOxTpjgNOEvKmFgulm8p7KmNNkhq+9x1iRBgPYpYYIhtpbpTrC9+jaTS1Sf9finqiTxmUxyhupV1D9eVFzJ6Hiddf70XzXYITGU89T9qHPYFU5zNoIJrN3cqcqFB5uJ26ipYbcT4a0idIuHOdN8Nsy7jB0KS9YJVdn1mDtlkyYrpK4iebsqyADHbyrGd6ruShZV6BL4l8L0fB/wNpEL5UTTnNSvCnDzTxKStcu7+64HI4StJ3QWiTqH/H7CDUGVyH6EEJi/uV2JGdPGMG+F4V2GrJ5LczarP9YOmvhkBx1bh28c3edE6Mq1DkLY7Kj2gfT5GbtrtYP/7bLTkbaFt30KxNbSfFbVdLrR3/cf9fzFOuOfLXZSPqqE8j3QZrid6bOn2hvcC0x3GSnqZuoiprfIaHRMverBL9dJqTD6mtkoVzX39ovEObA9mEkT75IVXEwwjuSH2/VzYMxKeSEm9eUuOzM5k7d6ztNti/vpDb7c3QlRd2+6Bjmckf0Rebzh+G1Zj//mgboKeBtJydBI5r7riX9Q/N5u5qfjlmHyDNuKoanwb8J0dxmz7ScIjaP9bGg/B53jn99I2+K/b2riFwl3kiAePafp8l2WzxaJQnYZ2QR2Nz1tpWv4i8kMdUv1VBTdWb2rzZM+t3pT9Vb6URmx3qmCXZjnoDti4atbqn3m9LnW+flmePuNfo/YV+Vh520gfN+RnSab9Xfl+zgI/qD++vE+UT8bbA/Hv7ZcB1VgU63jJ1slLJzLzs/uHzvSDdBk7VvZaCqo31vdOUWJMNNNZvOdEkP8KWlCXFHjDeC9Xf9u+IvqquBe61rXuta1rnWta13rF1VXBfdaP6+cIabVd2mPvWZYl872XLy1eVcUsy+UL6y5vIY1pHflzjslBKcJM95hcsbP6ZJU1lXI2GDGGXk7IdZgjuOq9mVXmH5ZM7Xd01EVimPxAnq3HjfO6l29KJkBtGM/bWrl6BpD90mZfy9bVX6gdClbsEeDCWDaonpSJNrSNbuot6BqyuLFM8W+mo5eO5Kj+qSWZClJRfURVT6qV6F+EuaioMw3GX9Srm5yejzVPq6kBID6JawcQHuaNKHr8xvxvZ5nmYLyHaew+pj9S1g78yVnUuOIuxppPabXzm5zKD4m6co5tnptigd2eX72ltiqOiQhKfliEnJdqWIA5K4ulICoqWO3GyU2FJXFvOq1y5tW3+vxTq9vviRqYQQ7R5gD9kUV5aVT2jy9Ed/fYfcnfc2mRqaZ2JVUqLcT+eFWu+QT2JczPO9VhV7u9o360sRt9N9m9SxPt3qt543h9JWOzf6rxUOYV7WerIlg/qi+wukuE24j+JKyU0fSYOk/OqpXPY7zO0usmnIdoPk8EzaOalJCyOpXdxcObrXPhPYLZcfmtZs6W8EdLN3/7Tj+QUJOQmoy1WvpdG4zsku4E9SvqpqZGW7+ybiOBZkuOyS2nxk+blZ/5vCgKUrupP63VMPpm0R+mJFj8RP2FnsyhPdzmROmzJOLHzFZVYNtn/DnQGgszateS9frf2NX4c4RszOY6ZIOaD9pV7tkIYuqheHosSezKov1s+AP6iXefK9UAH9KTJuiur1Eqr1SLMLGl3lrvqBVQPU40NQzbz/skNEwP4aSzARyLqq4ZFINac6rCrlcyyzK8Dx/0+Lfos7x4sEGiK3DDI78Bx+wb6rif7k7lRzMW11fTID+q/Qz6SlVRQnzKE/bCaEWXBFok4rKmDkhv/5IqhzETNwVX+ZLT+gqMJCsYe403W28EWRTzsNG38NMwvSQCNuEPxi63+n4O/2xkiTyXWBODgmqMC/Kpe2VCR0eAzHqc9zpssNlwrKmCanK5KhEnKmogmZWRXC6sZrQ1RrsdyN883ElXsgcSF1JqvMtsuxiuUUG1rV+/e+i3i5JZsW3K/2oJKDi61/6IFISchasyVRuxpvEa9+Qkr7+j9OO4anF7a0uISflTqv3e6GXgDsrM9sNuluWK0/cqYqcyq6bPSub3Z1h+ydeucfod8qy2+KPwvDOcvqx5tCW8zQKBpAguAHqp0yzT9hxeb7RNXSjNIfYemw/Yw7DSi2SY0+6vaP7lJh+a5Wtu81lIAFZiUB58koWMZl5mzGlX0UKC97MSh0iQdiyzkmJQryNIBn32WN7nddz8fiOD5nmk/rZh3fQfSeEja43y3edJF23koNmn7RfwBmGP3ws463syMYvdnn/nLoquNe61rWuda1rXeta1/pF1VXBvdbP66cn7UAvldtaVdq+J4cAXUPuGr1LRP08ch61Q93Z4uvUBCuZVLVLXUWuwMREtlYVm7eBcKOKlnsb1rvsfLNR2sHTK6CZ9OFhgxtmpQSEwhUcxtUbKqceRJO5zGmEBPG2XbuYF0+mxIicEq5vmTaWaq9pLPoiIEsjv4P5uUGyrApGtupDc2dD8kJssqaYlRvI6UbvvO3JYAf1Z1YHVuanCXrn6/YWM+l7hI2+LihjMgv0X2XcUTu1553FH+J6G5qcIDEx3zWY86ykiE27Uimmxw53nLCvZ8KuECuMYIYvkn5EyM5gUlZ/6rEnLZn1IanHdpwwMRLf7VSBMIVN+HJC5vLYpN3hMqtCsXBoU+PU55gzYavXWU79xQdnjP5cfLDxbot9OaxEDDPpuMneIskoz3fTYI5FZa68HpM1yIwqMnNQxaaMH+agXcuuXPcvPcQsPuFUjqVChpl4266eSDtl7CxMNxcKwLzLl652B6HJDB8yqVa1zfRmxW2yd/jx0qGOqJKv6UTqRZScia3jvNlQv85k2SIxM28LYzbCXKm3bvv7yPFjIYgUkcW8avf43KkvHMDvzfp7fxAkG9wJzATVKamPvCiX5hyxZ90FSLVlbhqmG7tmwVdvmVgL1VtmfFSvK0COypvV8aQ+QUajyVV+yZEvBIKTdsZLzJiYIWaq/XTJpO8D41cdyWlnf7JCdcwX/nWEeSuQLdNt4ffuHdl+6UvXLmt/1MfHTnDnRP2ia48dgqql2WpamxNcHxnvL+zlufdMx0p3I6qkSpZc+K757KBJ2IMlA/YsmNLt/fLXheYp43opCWp6bSWxKo+xNshdje0D4a7FTFGVtuLFXnjFJqon30y60xMb/cyhy9hZ1dvlvMQGwheecH9OHL9taBtLbAz+qGq5ngP9rG4/8vbXb3j7I1XLUsU6RiUKYaM+3NRFpNd0vOGxPCAJMprCABbC6nm+jPHhVzMU76h/K2p+GTduMKsfN7vCB88C/x97b/IzS5amef3OYKMPn3/THSIiMyIqK7ursquoQkAjIdESEjuWSPSaP4klvWTHmgWbXkILqWl1Q5FkDZmVU8Sdv8Enm87E4j1ufhOqa0+Wv1Lo3vjcP3M3s2PHrj3neX/PCY/tRcmOhWJcG+pHz3S3wAx+9hLHZSWK5BhkHGuNW5UUT1kV73vUdpJ7SGTuIzkptamRlRqiMNpVNxIWFe17+V7TLxeMQG8hrMI5hTKvSBTPhjJ7xEOTWP464VvF4v2ZKlLsA9olyocOgnBoheKi5/EYK4M5yM2mOMo2ymf5rOkKqsdMB1kK31n6Ms73oeKQx4tPLN959BixXV4hcxH9dCBeL9H7QQgt26PMk6d77LKZr8FyL551MyiKnYwTv4T6k+zPtFL4VtR1dYLQZPVWByAKG3haQ/NGvqNbpUwCMud+E85M+GkjKX22F2KF7RJWIDxyvSPbLA55Lu4j2iW0C/P8FRor512ft/931UXBvdSlLnWpS13qUpe61O9VXRTcS/1uBVFI53Sp7H9MyxZCELVUa8z0mTp6Sk3Z9tBUxCtJnzLHnHS0KDCdhxDQkwOrhWeaPbJhVZGMnhVX8+5hTkEDUZ1SWxFrK4lHtcU+Iwod4F9fC+9VZT/v4PP2cmd+YaEpUbsj8X5D/XFgWrZUDwqbeZfjjXQwxyoR64TpNWHj6X6YlaOjwCiVScJ69KLi1h9z/rYVxSUsItFqUdCC5G2D+Ii6V4bqSeGWCd+IAuKzSqMSVM8KlRTFDuyYqJ49xMR4nfmpUTxP0SrCuhJ/rtXE9qxGxdKg1k32RUsaUKxzwowL2OcOv2nRXc5ir0owJ6qDGI2TbVC7I2Y/yhNzVohTXaIPA6mpiKUQCojxdzi0yZz9jSokwqJCuYDfyKqA3Wnh4H7mn4pXi1mBJSX48ED48VewrDCHEX9VUebVgGQ1qsv+7LKAcSJuVjNJIlUluh+Jixrz4Zl4tSCuG/Fl589TLqB8IG4aYmnwty3jteXwZaYDrMid8ZF0M2FsxKiEd5kTuS1E1VGATpSfbO4WzsqkU9SfRI1p3id0gPIQiTZ3Wr+TcVk+O7QL6MNEWFdMm5LuPn9Gq/At1B8T3Z3Grc5qPyBqSspEh5RTvEbptJffFxW3OCbcQtF+iiSlmNYylkqE5hBKjXZJViVK6G+yB7gS1mn/QtLDUpNBs0486gDh9YQyieK3tSjtZRKm7gnE4hVJK0nOcwbjLLZPcyKcBXyrKQ6B/rbg+IW8f7rJ7NWjdN1rB/HFSPIavbWijma1WA8Kt5Lkv1DJqsh0ZWaSw3RlsV2gfifElrAohSNdZa/xrUI9FRinJGmwCbATWguAXnjiaOS6v0qo0WDfn7u9i14xXitUTAy3muKQJNlqZFZQm08et7SEyhALRXHwTBtLKPN3uBP1dPFbhVtBMcgx9NLmgD0qzCjd6MVBcfwqSaJZZmz7GkIlqVfxVVbhajmuAMN9Q7F3TLcN/a1iuI8sfqNnNi0gbOME/grwCq4cbqHAn05mPg+lfB+3SsQ2Ees8N9gEZQSv0ZNmuguYo57HwvDKi0czSoe8WyWK/Xk8T7cB32ghd1QwPJYUx0TSJXYQmbd6DhRHL6rtfkJNnvKDE84wzD521Y/EdY0uC9TuQPjqPl8zeYnFGmF4lwUqRKqt/Hz9C810pRjuE/aDwKHdOgpdA1h8p5jWmbO8l/mJYa/BAAAgAElEQVS1eYjYPs4reckoUkjo7ZG4bklthZo8xYOcrFgVKKUITYH2EV9nFTzvQnGA4V6uIzMq6k+JUCtufyoq9uMfV/hG1NPl20j1cZB+jNyDoLedpJZtO+KqFu75037ukQDwm4bdDyt8C75R+ObcTwDiI+7vwfYw3p7vdeX2NK/LamYMCt/ItWezFxnAX+XEzkExvBTaUNJnNV85xfGriGs1xRGmK/H83v3FyO6Hsh+3/9eB4UVN+TQRaqH/KB/RY+7FyPSZtDyvNv9ddVFwL3WpS13qUpe61KUu9XtVFwX3UufKaWJ8eoLX9/OPw9VCErKMwrx5kM71rIal3QFlLamtRUmbnChkWjHd5aerBMZH4mZBqC0YUWqKx+yrdKL8YTX6MEBTowY3J2yZY+72H/yZfWvO3hs9iQdT5xSsVFjscyfMVIQuEGuLVgrVT6RFia9V5tDKNo5f5/1pA6oz8Gpk0UwzVzRcK+J3LemHAyoolIb4WIqfCCifFKFBOJoqMW3ElzduZPuxFL+vf5E/xyR8K2oRQPPOoEdR5oxL+EoxfVlIStZBPmO8UpipoTj4zCuN4qfNHlv7cU+4yXSLEAmNpfxwnJVud9NSvt1hugl312IPdvZJA+Aj7qbFPg+oE/dYfSYbOk+yBnfdYA8TjEG8s0YT6zyVpIQ9OkhJPJ6VJZUWk1mnc4qZD6RC/p5KO5M4lAuo5QL7eCS2FbEpKJ6HswJhjXh36wL7/lkIDimJ/xvEm60U5uOzrDgMTpQb99l+KkUq7HzsDv94yeOfgL/NHMjWQ1K07YjVkdFZ+m2NOsg+1h800yaJb7CQFC0Vmb2pKPGg1Z/SuePYJ/R4IiBoVEziT+4n3Ms1KftQTx40kLHgsg84ltB/4VE5D36qAtUHgx3ArU782LPqN13J97JHSdzr7izlMVLuMt/3yqJiQk+JfmOZ1opppdj/OHtXD4ZkEqmQ/0wTCINBFZHpPquXOZnIbWSbJ49myL5K3ypUJd/PjLB4m1Ax4RfnRDgVE+OV4fCVwq0TyaRZHR2/9DBqKBJVpppMvSEt/XzdRAtxlSi3huFefL+hUITi9BmJxXcydvSHJ9KXd9B7hj+RL9n94YRpPMFr6nbCO4PfJBjl99ernr4sCN7QbEYOu4b4pJleZcrKQyEs1CtR4qKR461iYryRc3n1C0f3qhJffatwi5JQKD7+x9nfagK61/gWputEqBObnykOf5BXNTR4r9C9ZnztKFYjfdlSPZyoNcJptp0QV8wkqYfFTr5j/7qm3Iq/e/uPEuZVT3y/wC1ltQpArxxKJ9bLnufnBWnSqNHAQs5xsxrodIMqIkMhaq39WMwe3vJmYDqUrO4PTFeW2ka6Q0XK47Vaj4z7CnSSbU8av4CwlH3cvNqz/e6KaSW0kqTF+5nM2Q/qa0v7QY4vqqR6N+Ukw3ycyoJkDKkWdTS1Faob5tfV4AjXLebQy/y4qlEhzuSQUOXeiWehI/SvEnER5nSt4VZj8y0rlOAXQlOpntLMr3YLS7E7paQF/HXFZzMosS1ICh5/0uRVokT1KMo9wLSOxGWQ61xp+hdybb/9z6p5Tij3zEryeFdTNBazyxx554nrFtVP6G6i8JHUVLjbBfY5L+8oxeEHiv4PJuynglBHhhfndD49iWIbKkVYnxATirE4M+FjI570WEfCCvhgZs94shG1Cky1YXl/5PDYwqhndrTyEBYyjtKjmX3c7/5plRMUYf/tAtcqklKYIRBLIzz3E9ToxH8v/v5/wl4U3Etd6lKXutSlLnWpS/1e1UXBvdS5UkKVJSmmOd1K9ZN0Bbsg3cVWlL1T17sqCukSzV6j1Fbo5wPh9TUxP/EVe4cKgViV6OnEvixmn1syGrMfpOs/RHkCPZEZAL+psYdJVNqcZa8mT8rdseb9M2nZoj4bzn7Tok9P7l2QbVsjSUONQUUY7hLjq8+UPUDtLSqBsYHCBMbcgdy0I/tlpK0c3acW3RtRI/IjolsJMxYFaeXxWJIyMwfXrZR03F85Umdp749YE9k5IUWEWuPW4iNMz8L9DJX6HWVQBVFmVEyYwTPc1VRPafYyp7Yi1FYUTaB87InLEnJyU/HUo4aRuKzo7wv0TcHqr55J2RuqJofpjXT++oAaHSoM8++n9QJ8oHjshHQxOVJTihc3K7DFuy3u5RVKZYVwcOjtkXC3zvuQUAehKijnSdsd6mZz9nzDnKYXayspd4czhUEde1HoP2zzQVGox+3MZUYp8eIpJSsNzhPLAhY1+sOTvKepSfsjOqu4oRRFXR1l/ISsOh22BcppMAl7lK7wU1WPopqVWzlnKkqaHUiXcCiFerH5RaC/1rhGk/JXjKXCHgMqFCirGe5L8VzWavbZ+lYUmukqkqqE3Wr0oIlt9qC1nlAbtj+WBKjpOjBt1Jx+lYpI9dGKctxLN7LtP2OvGjAnS+IQGTdG+JR3IlENqhHfZetJTlNWjr6zUETaW1n2cJPFPYsqR5Gg9ZJRv5dzObyOKKcwnUY7hVsoVNAzqcG9LKgfA+PK0H87oQYjXs5K9nF93RGi5vjcMPUFZeNg5agXE0NW04uDZlg7Dj+SlZP0wc5+ZDkQ4BcF4aaCVwvKp5HptmbIC1SvvnjiMFQcdzXGRO5WR45Twf4gCu+L5YHv/BXXq46UFFNtGV8aylX2fC8c/VNF/d5iuxMPFMJGUexkP7d/0GDHxLhWuTteMdwm2h+ICbU/luhDTWjArQPYxPbHZmbxYhLFQ4FbR7ARYxL+1jGm7LdtlXhyexmHbgmhNIxXsg/GwXhTMVwbFt8+Y1Ri/6IhvRopq/P8F4KitIF2OXLV9rx5c8PmWryjV83AsXI871pWX25JSXH8eE3xUsbL1bLnU19QmEAqFHXhGWyBziq/MRFlI8urnsOugSoSgsI28vrd8oj+YeQpXAsNxCTGG7m+xs2ZXgIGMyVCqfHtFeWzw5x452NPakpCW8x0n7Ro5lWqVFjhEFclaXIko3Gbiu7+rEy6hVArxJ/6GRsWoUTUbwt8IxzzpMVbb6aI7uUzipCkv+HY419cYTqHGh0+r6zpweFXFcOt+JDdreAj3FW+GDWYnUFFhV9E9CSeZJ85uaZX2E5WC7p7zfrXETXF8wraZomKkVQX8z0cpYQ8kd8zXRX0X3ruXuz4xDoTM+J8n0mdJl55Um/QR0NsoqxKXWWJN0riXygjOKFhDN+M8/xZXI24p5rqtmdRTRxoMZ2WOZScigekRWCwCZP5urGAIr+nu1MUXWLcaFa/dvjWEJblvHKsJi+9Fcezt/jvqouCe6lLXepSl7rUpS51qd+ruii4l/rdihE2q5nhmJaZpzpOqBPDNCeVAWA0qa1JWnyFqhvFj1sZ7PGcEY7WqDGIH9RHyo/HmaWbrKShJKVIVy2hMhirOSekqaz4lqiQsB934r3JXyF8cYvuJlQ/ijKYEvbTfk5Co7Dgo/guS4tfGGKRu7TX8gS4Wgw8PayorgeGh4apK6Rr/oM81u43ol5120ZSfGrpnD35h8qdcE9DawgLhR515qXmXZiENWmLgL5xfLHZsR1qVJ/TYaKifJaOWciMSwV2nyi6zJJUolxMK0P1pGf/2SkRzi8KklboPoCWrv6k1cwKjLpA9SX96wXTSmOmRPfNmvqdKHIqp3/NXFvn4dihmpzv3hSgSsy7J1g2pFbGQiwM9lm2Ea4XWfEX7nCsC4iLs8qslDAat0eICbVakj4+kL56nc+Vgcnh7pfi803pd9Td1NbiBS5E3T3xbOcktaqAYSSFiKorUt+j6hKSnjtu42YhFJCnHdxcUe0iq1+amVl8/MoQrbA5p1vpBtf+MxasURTHMy0AJV3IJw9bqISAUBzBtYp6G8UP24U8FiLl91viVUusLNWjY7gt8JWie5UTiVbyubGOqIXHIwleMStKsbOkNoo/MCIc2iYK7QNgNEw3ATMY2vdCSZhWZlZx7SB+2FgoFr/a091tmO4960LO01gH0qSJk0HpRP/UUG8GptGyauRYv39swSSwefUiQUoKctIfLpMIJjk2binKk3aZdTnK5x++UmzuD+x2DdbGOTmqHwqqrDDaymNMFJKCTrCUnw8moRtPzN3+9YMi6cRwc0rvU4zXtaTauUS1zqs3L+X3v10/8hfDa0nWyvXlesexEb/iohj58d0nfD7ZpQn07cihl3nhq7sntuuaT/GGEajfG5QRTu2JknD8gahUy++SpB1amF47rms5jsePLRrof+BQtayU6a/Guc3A9QXTSydt6qNhVCU/+eYN4w/k9v23b+5wqwrfaNp3iXGjsB3zXGJGiVsbrxSreuTj04qwDlid+OGNrGr85vGaH9w9M3rLzaLj1WLH4abiKh8HpRJN4Yirjhg1hQ2s/skDRV4GCFHzox984L4+8LZbc98cWJQTRsvrlfE8Ni2buueDjhid+PRxNR/zw1Ty5/dv+KvC8/7fvUQlUfrSXhHzHGp7IYn4WtHfKapnhVto2vd59Sjzs2NlcOsSuyjEt5mvaxUjuChzY1NijhPdV+286hFLpOt/UCx/KSsxdj3hh7ya2RmGF17m8bcG2wtJ5fi6pOjkPbaLoBqKGIXnHCOptJ+tXJaE2lDuYPjJgAamu8989wtPKEU5TYvAlNXSU2knXu1yL/OTWxmqpzQTIlJlMB+OMDnCq2vcuqJ6u0Mdetyrq7yfivJmICbhaScr1KB0Wv2xCbyiuO+ZDqUwkIvz6k9KoA6WtMiMdh0xZSQamQCb2uHqAu8N/VSgTCK9HBmPMo+rUYONMGlJKdTCFrbdmcSQKogOSIr+RYntI9OmoHw+TbKR0BTE4nOH8/+3LgrupS51qUtd6lKXutSlfq/qouBe6lwKUe6skU53kG73Re42bSvxxoZIzOqo0lr8mKMTVVcpwqJEj2H2gqoxEBaldMbX5aw6zk+1ucs/toV0lx+mvI3srfJpTl4xvTzB+et27uDVPs6d+GqYSHVmpJ68nKUWHmdVEhYlvsm8w7uEymrR/eKID4YXqwPfq0SMihgVbnHy8eYkmYUn2SjpVIM5BR6hJ+Ge2oNsr/kgiu7pdRWEE1rayI9ffATgl8938/tNLx3z402i2EoyU2hE9aqfZSOu1agkXFUzeOlIX1WzBw2ke9t0E+o4EDcLYmkoTt2zPhJXNSp7aqNRKJ8Iy6x0a43O6VZMnrBZUXTDTKPQ+4FUWMLdFXrfiYJ6UlJPTNvSkowS9VVzTpM7ldUzz1g9bknrjZAP3IlzW6KOPdqv0M8H+Wylzkl3hUEPUSgSKUnKXoxzqp18hgU/5JWGnK5nFOpzn29ZkPZCc6i2AZTi+CqrdFvwDYQ2YbeG0Ea0U3PHeRxEaddeVFw9wnid0KekMwPFXlT74VqzfBvEp+eyF/pjN6f96Zj3BVGQ4ktR9Yoi4EyJPhjMjSfoJFzVMSv+gxFCw9KTvEVPighzIlIsREWMlawG2GNCa5hW8vu2T9SPnv6+ZHi9xC8U9c0wK25p0sJDVaBrLx69oEjhM3pJ44mdxdSioCqVZiUVEM/wIuCuwUyGYp+JD3lIFF2kvzH0XwVeLTp2uwZ3KLGL/IYs51zf7UlJMTpL0TqMjlQLmZ+GrqFqHNNYEIMo4OVWuLIgKyt6hNgCneL4Sjyc2OzvT6KKv3r9xD//4b/hT+vfcm+ObDLst1VyXI8xsU+W3/oN/7b7hn/57o8AuK2PNNZx/cc9UzSEH2t6VzB6w3Ev0qPSQn9wV4UkU0UoFo6PD6Jg2q0lLCN3X2x5udzzB8tPNMbxOIlv82lq+MXjHVfNwOAtu67GBcMfX70DoHcF3w+3hFqx/UOZA9wVc4qU7RXJiPK37RrcsWB5f+S//fH/xp81vwZg8e3ElR5Z6Egepjx/WfIQ5Dt0qeIYK55Dy9a31NrxVfnArTnIcdAdrfYUeVJ2KFw6j4WQ/99lk/n3/pp/+9XX/PXhhYzXpPi2/cTDuKD5T75j9Jb9UHF4URPymEqdRQ8aPSiqZ1kdc63GTHJdh8pASvjWyGqi19AU83znF5b618/n/gwf6W/0TDA48Z0T0N8pUh3R5qxc6l4TFgHTa6a1cJ5PKv05ES7l+40mWo19GvH3a0nrBA4/bNE+SXpXGfCTodieucppmUg5DU6XIad3qpk3bEa5p/g2E4BSwi0LioNcM6Gx6GWN8iX6OFIgq1oKZvKQdokQNE+PS8ykMAfFdCWMZRAPt91apiTqsaoDyWlUTnQzz5bYiMorv6CJvSHl/oDDoUbbSFEEdp8WspLTJvQyk0eMpVg43LYiVQFzKCQ9tD0r1faoKIDFO8d4bSnfTgz31dxXE1c12oWLgnupS13qUpe61KUudal/WHVRcC/1WSniZildpyf/qxaCQmoq9K6DcZIEmPeP8vqiRcWI6gZSPxB/+FJUxOMwe0NjaYTV6jxaa1JOcineSyd8skswCj0F9K4TlmOMkjYDqKZEvfkE99fi19Va8smz8qvefiK9viOVFrU7omLEfXlzTlJrCsq/fU+aHLp6QdLgVhCuHeSO7+/qDc4ZtkVNCJoYNHUzEZfnJ0TzyxoXLSpKLnfz/pzUEyrpnvZXCeUUfgH1h7NPMxYQm4DWkZgUv3y4IewL1KnRNYpP0fSZnFBnJu6QZpWg/eAxY8A3Jj/N5vS1MqvAYxC/mZJkJb0fYFl+lvJlxYs4RpqHQPUoHcgzRzJG8AHzIKqt7hzxZjUnodm9EAxiY0G3mKcjdANpfUfK70lGYXcDYVVhjpmEEYVJCcK5TW0h42W1ICmFMka+G6COw8yQTPuDJJ75MKvxyjmhI5B5siFCU88pRvF6RVJg3zySxglVV8TCoI4DKifbmZ0GnRVd5yl2E/bgcG07n0vtFOko6V5h0uK7yyp+qCWtqjgK3zZpBdmDDdJJr52cU9slolFEqxhfnZSmFaHWtN93xNLw/OOa6UoShU6817EvUL3GjIrwrhUF5yxwgE3ogyYZQ/1BE0vx56nTe04BVGNmxRYwVmf1lATT2uJaBYgnfbPseHhe5uOsSa2nqD1+tCw3HeNYiCduOpFFJgadZrhEnAyqM6hNXv25DSwWE/2xZBor/EJhOjWzRA9fGewxYa5HRm+FvtCE2WeslKIpRbENUdOPBcZG9ruGxSqvSpgkynFQcux+5Ogea/EFA7612E5R7hTTSlR1t1IzT/j/fPMF46Hi65snbsyBW91REE+2TQYSLs1BVWx0x4tiN5+Gp6HFJ8111dH5khA1UzCkZFGnc6AjIVn8MpC0ltS1982cxlY+KaakuV8c+PPNd7wsdlTa8W1OW/vkV3zdPtIYx6+6W/5ies3P397zlBO+nDdUbwr8IhIrIX5Ew8zoBuG6Vs+Jw1+vKQPUrzx3dsdGy8lolWehoyjWSoZPqyZuchTjSZF9jhVDKjAkauVo84BaKE+rwCjFlBJGsu3yn1JBKULyODQbc+QP6/cccut+Hwo+TCuWxYhPGk3iqAu0iYTs5aaMRECPhlDJipc9ChEEMhs1icc9aUUs9cxaBrBH6byfy0fMCO37rDq3iuatlgS/L4V/Ox1L4f0i7FY9aTmPyqCCktTKYyLlk10cAqGWpM5kNeFuhXYBdZAehfHqRu4FRjFtK3RnICdUAgy2wg6yOuRDRXGQ+0Eczvch24mvWgVZiVGZFAIQKk0RZYXT7IfMgC+gNMS8UuQWhvCsaL63ed6QFaeTF9n2kiBYv7MMLz3pIP0uaSaTKPBgBkNoogyWBHp3+uekRXkYFgXKySpQ6i20IpHrvcFPNXrM/molPHGirJzJRyTKHRxfFSQDw32FGc6rXX5RUL4/oqqLgnupS13qUpe61KUudal/QHVRcC/1OxVLQ7iuKZ5y96zzqIdn8TWeyAk+CP8WxAfZDUJS2Cwxn3aSFGXNrOwopUiVQTlPWIlKZj/tia141PS2EyZqZp36+zWmm2bfpd4eSW5CdQPxxFN1YfZtKmtxq4ri4Sj+W62F3ZpOySqK8PoGNXhCWxKtdGyqo6V6lH06mgb7UPBxldUfBZOJxKweJK/QWlJezCg529Eys1G1F3+U+aDmJ9r6KdLnYzZthBUaguYv37yUX4unTHNo3wlTt3oSf5V24p00E+IRBcwQsNuRpGqSNpghyhP7KcUmJEJrxWe8qii/f0I78U7LlwSUwu4nTK8xh5FUGFSXCQQnj6rz6H2W2cYJruWYx0VDuKrxjcFoBSzQhxE9ecJCPsOMWW3VSogV6xq7G2Ymo35+xhhFXLakpkA/H0lNNavEOvtl9eBJX74Q9dVH9KfMvS0KSEloCUEUZz4bO0mB2faiYhtNWrXi/z70JP+ZV7nOxyQEQmOx+2n2S5sBmofA8aXBDDB9DsAF7EFRPyTMJIrDdAXrX0XxdgLuoBluFLEQpcUtFKDzn+JBMy7x9JMlm7/pSSo3yN9GYpc7jR9KioOMMT0olDOSKpY7qlOZ/ywSbind+dWTIpwuy/w+HWQ1wDei3lbPmdJQKqalxvaJaSHq8b6vMTb7vasAvcFNwgF2zmJMxK4GfCYOnGgKSiVJrfKKVEbIr5ftREqwWA0cFYSPp3F4Sq0TJStMhsdDy+LVkT+6f8+rWviwt+UBTaLWDq0itfJ0sUSryNbLPPL+izU/391xdfeJ3hdEFN2q4DDIZ3VNha0chw8L6UZPCnPUFFkxm0ILK89fv33Bfz/+M9rC0bmCmM7nfHSWlBRaR0obqEzgzYN0pLunSuYLr7A7I77zSXzYZZc9sEcRr7ST9Ltxo2QeySJ0uU+oqPir717y3fZKKAUmYLP/c/SGRTVRW89hrNjvGopfVzzk46md4uo3idBokpLtxUJoHgDth0i59YRGUz9pXKP49GrNv7D/DJ0lf6MjlfGUJuCjpjYOq+P8uo8arRI+ajpfolWi92dPe6kDJq9OdU788DEpbPZ0K5XmbY3eMgVDiIrJy7xw3NVwkO2ZQ1ZeHeig2LzL19VKUe5k1cqMCTsk2g+TpDUCqh9xL68onmVVKrYl5qmb5z89THn1T+4TcdmweDsx3mT1sxC2dTJyzUWrMuc17+OjZrqOUEbSqHFX4nEHNR9rO1iKYwSjUF4Ux6QU8atbAIpjonuhcUtQvWH5S03/Ks2ecdsrymfFtBaFNlq5F5w8un4BtjulvCUOlWHzi3DuqygV08sFegyEZSUM3MPIeN/O81u0oCfhehdHmNYJMyg48WkNhCYSyyQs7M6QbidSL2ppcZD5Ai2UkqQiyQglCJDVJp1QvQGTMAdRxVMmBiUr85hCYfcGFHJP7c4rTMnIKutwl33HwVD0GpdXVG0XGb9YUr0/8vfVRcG91KUudalLXepSl7rU71VdFNxLzaWUEr/sFOf0F1ISj+PJB2kMFHb2wigfSEUmGPRCMFCjQ/XjWZWbPBGIVwvCssTuR+KyQR9EJUxVCU/bmXeqc850WmZDTj+hVivSogGl8OuaVGiUy0zPys4cQPd6I0+z8TN24BTRvSM2Bf3LksOXiuE+oiKU2U437a2w+I7iNwplwk0anVNWbKfkiRZR8IpD9tVmEaP+mIglNA8R20f6G0vRR7rsE3arhKk9brRokwiTQY8alQ/zcCeKX/s2oZ0wLMtdIhqhSJz3xaNSotw67H4ktAWhsfN+JqVIpcI+j4TrBfbTYU6Ei6uGZLUk+uwH9K6ThLGc561CIC4rtNHEVmgXSiv0Pj8lay3Egxdr8buNgen1GtM5Uu7QTQHM9oh+2BGvlpDTc9Qoj+apriQdL0TpelYKNbmZmKG3R1JZCO3BR/kO/TR7cEmJNExQV/jb5XxcTnQN83REDVn9ryvUscdoLXSNrOCq1ZJkDWmzJFmNbw1Q4k5JY4WivxdKgfKieh6/ks5eEOatWyqCl+Qjt0r4WmU/qyh0dhBFfbgRf6xbKuGRAtNaUW0T/Z1CBUm52r2A+ts93S4zhxcBtStw1158bEm8eCeygPIQ2ghFJJZGErsacKt8TRwlOcgtRR0yXU5Jy+ppsUsM98JNjYWsQBzfL6CW1YL/6B//ilf1ni+qZwrtqZWn0g5DJGRd5BBq9qHmECrGaPk0Lvk4LFlYWVHofcHH4wJrIvevHvjb8QXqscC9OtFRFMVWo54Lhr3lJ3/6G/7L25/xRfE0n9eIZpV9ooUKlAQKFWZv53NseLe5olCBnw1f8L98+hF/tPnAXz1Ld/7d8shV2fM3+p5lPaJVYtfXc0JhXYjnNyXFm7fX2PclYRFFiQbpIB/NzNq1H0vh/eaO8nqvMcMpZVA816dEwkIAAyzfiMJWvx+ZNiWLd4rxSlM/yrH2rSZaAz9t6G5r6gfFcZNmxrYZFM9KzlEyUBYJ5RVVTs6zHRR9pH6O2C6SrCTlzQmFPmHGICrYPhBfFzS/Knm7fXGWuM52XfSkxFuZgGxxTCph94ZkksyDSdS2E9PYLyLKy8qWuw6ohYfdZ9SSIL5SokIHGcban32fVS9+WhVlbLv1qScBysOJYwvtJzkvzdsBvyrQQ5C+DCBeL4SW4qOsTIVEuG5nvjU+wG4ru7peENsC4yLj6sRMTvhW5etekVYe+2jxGzkR1a8sbq0k7TLIMdNO4ZZnBvawEb74cN8wXRnMmKieHNPm1GwhymT3tcPsLONN/tk603qcns+7duKF1YPGZQ6t8sLY7n7gaX9jsT18+tOK6unkOQd7ZWg/eEKtMVMk3pckLWMCYLxSlI/ihXcrSW5zV+cxf6pk5Dwnm+CpROVVoVCJ79etA6iE3VpimWbSA0nUae0Bp8Rrnpg958mk2U8dGyie5L22O/vGzSgq8fDKU7+19C8Vo5OVMwAdEscbi1us4d/x762LgnupS13qUpe61KUudanfq7oouJeaK6UkNIPDhNpL12daNqSUUJMDsnrb9aiyPP1SVnSVME+HkTRNqPUK/SQSRrwSSkIyFnvyR6Ykym0uVVWkGKEk/FEAACAASURBVEl1ifm4JS7bufuflEirVjy30WL6CD24jahd7rbF9JJig0aeVp/H2VtabI+kqkRNCtdq/EKeQE2n6F+e1VEVpcs6KVFsUZpid85B106enpOB8VpUm1Lsgrilotwn6apXiuFWMa0t3ev8xHk3crXq6IYKN1nSqFl+J94kOUiiVoRSvLdJSc55UlBus8fWR/y6Ro8hq0iZaziE/KdHFZpYWcy7B+L9hlTZmR4Q2hLtRTnVnSFcr4ilOVMUlHwZtT2gQ4u/WWBdgGNW2tdL1OQwO6EjpMJSPPWofsKkk1GtIK4a1PtHUmUoPh6IywrzIAcqVaUoyimhu0n82mUxpw2dmLfKR9SbT6hFI+llmcucDgdQoiSbvkR/2pJW7TnJrCzm5Ly4bEUdftqRxhG1zBKt86hhxL9Ys/+6IVSKpK147hByRfeF+N7KR81UKKJJs1o/rYRZK+8Vxat/cSZmjNeiZpheY48w/HBCHQ2rv80sZQP2vfgxh1sl18O3B765eeRnx1fyHjTjbUAvHfFYiB+vCnxuB1Y2wmBEPaki/jahhuwtLYOk6WlY/GjL5CzDx4ZkcupdI97CPqcoVY/CDx3vZdt/fvUd/8Xy/2alJwyJQklHfEDN6unn6kiXDM+x4p3fYLLZ78Ev+Z8+/BnXVUdIiv2rik92xd1tnhcSPD2sSE6YuzEp/s3+G/5V/JFs05d0vqTUAZ80m7JnYeU8f1GJJ7tQgTFZNImf7l+TkuIvn1/w8USDUPDOrpjGgr4TL2Y4WklTApxJ6Px3rUSJ01tNaOS4hKTQg0btDCp7FNu3Crc6KeHic9aTJA7aIQGaykP9LNeV7SPKJ+xzR6zEc1h9CvOqw/GbJavvPdtvLPWDEFrqDwq3zuc5QPsxiX/zpRZm9uqsGi6/D6KA9ZFi7xjuSpJm9uZPmwodovBXj45yZSh2GtCz9zMhKwJm0KKcDprpNmK3WR0tskJ9zOSYk2c8r2rVH82sxtbvLdNaC4kkK92k7C29jhAVxss1VD2euc0A9ij9D/pJVjyKg6TdAdhBrrHq0WMfj+J59mFelUi2RE9B5r/nDgpLbIp57mEYZSWoqYlW072umBaa45dZhV7mvo0kzHJ9NPjrs2//+I1HTZJSaTvF+IUjfuGp/rqZ5/HxWjFeW9r3SUg4rSJU5ZzQ5RtZPSw3I/pNwXgviYWniymtPFOpJQ1QJerlxLCrMI8ncz0MLyLFZqTTifpNQbGH4xcn5rFQI8ZrQ3evKXeiKMdCMa7lQ05Jd4evA1w5muXIop64rvt5X33ShKhxmRPfjeXsS5+cZdmMVNbL9aESRqX5vT4YSit+bpAUO2D2ZveuoBsLuqcG03q8l+tyumY+Dg7xQbcvjozrEve2ymMvjxdr6F4pfHOhKFzqUpe61KUudalLXeofUF0U3Ev9TiWtQGvCnXQJm+eD+BlDIO0GVNui2obU5ae99VL8tN0oT9NVibJWEtCq7KndHkiuRk2OuGjQ270osrnCuiYV13PSiukGwrrCdCdqQ0Qfe+mUz3xevR8o9MnfWmJ2gzytHybxBys1vze1NUyOVDX4RtQCFTJfMD9Zm0GJj64XtdaMknl+quIAzcfIpz8TL66elOTK5wdr5WHx1jFdWQ7fiDLrKggvREUpbKApPPtjTfQKQu7OPXW2lqJONJ8SoZFtj2tN+zFw+IF4ketHj+k9uvfExgoRotRU32WecGXRoxM2rBG+YqqMkCXIPtWUQJXit60LtIvorXhs47oVtm5Vog4delnJ+zNxIFVW+LEhzL5e9dyD0XKMgWS1+GbrKn+++KPD7SqPhU7Oo/PEZY2apOM5ZZ5hKJboYRIe8vUa9kdUiKT1Ih9nP1My9PYo6ufoZgJEqgswGvW8l3FXFSTvRb09+Xh9IO72HL/6lsNX0k0c8zkHUdBVSIRNxAU1d8XPaUXx1Aks59+vIs6pswqjAAV+Lb7HdtNT3Xu2NzLm07YklJJHP97I++vKCQXAZUUiKH7yp7/hdSMm8bXtqbSn0Dn9L2l2vubDuOJX2xtc0GyaYX5dq8TfvH1BAl6vd7ho+NunGn+fZb+PBcmK59D2immTx3Lex192d1zbr9n6FpcMdZYLt75hmQ/Uygy0emRjOp5Di1GRIRbsY15ZiRafNH/zfM9xLHHBkAbDoc+KzWgpvi9RQY7vz37+JT8LX6GzCh3XHnUw2E7jbr0wb21E6YQ+0STe1IRVQFWRly+f+fCwJj1UVA96PlflM7CS/Sv34ps+sU9DKd53lPCPQ+aChqzWlp8M9SfFtIaxiDTvfze5Soezxzbm+at+joRK0byVOdJtKordiOpHyg+KsK4x+1ESA4H210fGFw1mEJrCiW9qhuzpvpbPC6WMRdvL66cKlaLcebQXZvniV3vpU8h9EvW7I/ppT1o0JK1Z/NwRiiviB8X2R1nVuxMPrZ5kHiqOCt9oSWREjt10lYhWOu+LoyjVbpm794uE7SQR60SZQQt5AHJX/FWaVznKZ4VbMTORQ5S/b34uVINoEtEqql2YexDk3gTlQ0dcN6jeQWEJaxlP9rknaY0Kkm6ougFzum8gRKDU1pKCNXhUEM9qOCVoRYhfDsTRUK4mYtAQFXUzzcfamogPmqEv0cCruy1vo6L+uYz50/aSVUQj4823Cl/nDSgZPDFo/CZy9YMtr9c77mpZ1bgvD9wWR1ZmoNKOQ6j5zXjD//H4JQDvnte09UhTeKa14WO4ZrxV1O9l3hjupW8DrYhlzPc1RbFXmP48ZmMBP/kPfsN/fvtz/rj+nld2O+9jSWRI8k/DgCKgGWKBy4bskDS35kCRzcIxabSKDClTMIgUKhCTnt8DzNscUsFHv+a37gYXLV0sGaOlDyXvBrlP/GZ3TW09RkemhWHbNHT7CvVLOZCHryOpTMS3f/8/YS8K7qUudalLXepSl7rUpX6v6qLgXupcSfyjSYHODNpkDSgrZALnSH2PahpYZUWtG+QpSalZZSNGUmkl+QxIXQdVAUoR2wI1ltLlesqVLgzGR9Tg0YdOmLr6M7OhRlipTYF5/yxP5NacOblTyF6sSnydMeKvW/RnOdU6JdyqFAWkU0SbKPaKUMvT++I7xXQlOdzGiadOJen4BaieEq5VVE9CUKieJc1qVnCDHAPXCksxWnDLRLWQp/+XV3vePcvTqTIJNUo2/EnBIEpX8bQWRcktJF1mXGvseOo+lQ7h2BboKaCOE0lrUiMKrXKSlGSehC2rUhKh6USneD7gX1zN7zWjF4ZxJ+eJZUsqDLFcYN72koSWzycgqmpVijqwrM+KaE5Ag8yaTIk0jJhPW6av71AhoQdRfNNCUseU88I/nhzpcIRvv5Df72UlQDiVNWZ3EAV6yjnm9xvxdheWVFjUMJKGEXUaj4+DfL4XFYes3qAUcSVKuHYetVriGsVwnyi20jk93MpxLvZq9hP6daD5rcWtIraXMdl9ESUrftCgE6mMpOJ3WbkkUE4RqoR7bPnqD97yxVrU2J92X9F9E0m/LdAThAZ271bs0oryMY/ZPzzy55vv+Ef1WxZ6YqV76kwxAGY/7HNseXezoVCejemIWbP43l3zP6s/warIp34hDfELz3ol5/qJFeahIFaJqUzENoovNXdC/6+/+ZZ/pb5lOJTgNWbhCLsSu55QmWfqnivMlZjWUwRTBGIwskIBlI0jRoX5mfhhY5Fojwr3JAe33ipJg5sS05WieVfglkKlADCZCWw7RcJKN3dmBvv2dByg+U1BaBOfPrwgVYn2rZ49j7aH4iBpgKcEKJKamcTaZTUygm4UqhYlP9RnBVh7qB/BLxTFPjHeKMoseC3fhMwTjlRPnmlj8bWi3Me5u98eHLp3cu2ECDGhHrfwlRieU6Up9o72kyEUSuYAKzxbEAWw2kbcQqO9+PyL7pxwWD9JwiGAOYwwOUxhZg44Xq7PVNqZZqKDKLUnJXvaQLHThCahoqi0JidqgRwj3yiMk7nOtfLaia3qW6HImE4Rq0T1LP0Kp/lVeUX5JKqtmSTdis/mz8WbyLRWuJVF+wRIGqNv9JlDuw+oKTHdtqiU0FYTazMzYN39guLDAX/donzEvn2Se0n2ncf7TV7hShAjtguAZvOnnwD4D++/47Y4cpfxF5V2bEw3UzxqJcfujb/mp92X/PJ4y8FVvIkK/xNZBXPHAjUa/FL2eXglnl57OKVNKuxR4T7WpGXgv/r6p/zTxS94YcQnvNITOu9wnQ/ufmn51+03chC+kmS7f/38NX/58SV65VAmMaS8gtYGiuVECJqqdnivce9b3FVETSefrvz5x+t3/KftL3hpDqx0JN9m2MeCgGJIBbVybPTAwhz4fIqbk/6SJiqFJrHRZ6XbJVkZG5Ih5MaBk5q7UI5VOcy0FIOovy5ZPni5R/5i/ZK/ObzA6sDz1HJVDXynr9i/zFSehYejZdr8Lvnh/10XBfdSl7rUpS51qUtd6lK/V3VRcC91LqWwhwk9+pkrG69aUVqtQcUo3tsYhTV6+rX9EapS1N+6QD93oNuzgnCzEQqDD+jOkZqs4ObPKB47YmnFg5uSMFhjwl3LU6ndT6QSSEm8weZ31bLyVx9F5UsJ1Y+kpiJpNVMG3LqkjJFiP9E8GUItHZjTlfjGQNQM24tyerINlVvx5YEoDUkLp7baR1yrZyXlVL7RTKuchqPAX3viIJfYR71g0Yz0Y4kfLcaJCpQDmYil+HyjFfWmOED7EBiuDPqYE402Ft9oyp0nlhr7HNH7jnAjCplxARWy+rmoUL0jLOVYA6jSYo7iUY6LrHYbLYo84K4byt98Im6WpKuVvHbsJc0MSCGitCaeVBGrUT0yNp5zp/KJjHF3DYcOMwZiaYht9shqRfFhP/t8CRFe3s3EjbRsUJy9vPLD9JlaLJ5qNYykVYNytXjrTlxma0iHI6ooSM4RF6ICA7/Ldl4v8Y0wHWsnDM5pIy9rj6guzwZ/4+l/4NFLR5+7fWMTKa8HpmNJuZi4WfYUJlDk1Ka75kDnS6yO/PZ5w/PbNW/3KzaNKOLFesQ914Q6oZ0ilIn1X1l8Az57P6dDyc+P93w/bCi1Z2lGKu1xuV37yvYscxSWIXFlPB/9mk6ilXjv1nzZPvM0tXz/3Y1IGV6xIw84pyi3Cu2VkDucrGaoTrbvdgtIUB4UsYIwaMygiV2NOnmR64T6bUOyCQUEA3av0FmV08capWQsJwMqKsrndKZ/OLlOmodIeVB0LzUqnjvrQy3vUUHUxFDJdywOiiHTH1CJopNt6yDJaOU2EbJ/3jdQPUO5TZJQ5aDosk9Rfp3iGIlWtl8cRMFcfC+vjzeKaSUrNvWDqJLNe1Ez4cRzTZS7gO0DsdI07xwYhTnm6yarqamy6H2PORq5TvxpBUq4re3bkVAb7GgZNnqmMGivsUNCxciErOhUTw57OFNmQlNgjk6up7IQ3nW+JnQECps55QWpMBSHgDGK/jb7JgeZ71zuTQgVlLs0e32Pr4SuQJLXtAM9nhXc6kHjrpKwTEs5Lkkn6o8nTq68t9irmYZQ7JjpA7FQNA8pq6rgF2Zm+YYye7KNotpNQqKIMkckrbA78YSHtiRVhcxxVoMX0ovOFJi4avI9YoLnHeVVQyhLfnL9EYD/+uZ/58fFE/fGUqkCjcIoTRflOD/Gicdo2ceGFxmg/rPdK9r1wBcb+f/eFWwzZ9mPFl0G4mTgKPPidC18Ydsr4p2nUIF3fsNjkDm8UIF9qNmGlqUZWOmeW3tgY/IqG+Jh3U4NhQkMJhEOxcyvVp3BhQq7NQwvNWnS2FGWPU7KLRHcOvJ9v+Ff6n/Cztdoldg7ud8+TQ2Pg6yI1dZxVx/ZFD1re6YsdFGoCu+GNVolrIpUWe6PKPauEub0VBOy8fpEV9lUPY1xWBUpdGBjOwIaQ2TMS2fPrmUIlm2/4v1uxTAUhH1x7svpSsonLeSLv6cuCu6lLnWpS13qUpe61KV+r+qi4P7/vJRSPwD+B+Al4lb6Fyml/04pdQP8j8A3wK+A/yal9PTv245Uwn7cEdctYSVKkNkNxKogLi0mJZRSoqCd0q2sDKFUlajtHliRhhGWLXGZ1SKjhLQwOvT+SLxaiJc2Uxb8/Rr7eASjQWv0rkeVBTp3GftlSfHYiY+yssTSkAqN3WcFoyxQxx5VV9KZ7yPlm/P2lY+icAwePSWSlhQeX0LInMZQi/fVTKJQnBJj3PrMZz110+ug0S5Jp/SDPDlPa8PiuyPD9ZJprUglkvCykKfa/s0S/6LHmEgaDLFI9C/PST4KUXOLvWTUV0+J/trgW8UpTqj95DF9RLlIrIwkwe2P2FNK2KqFmIirCvPhGf/6GnOY0IfMiG3KrOwoVB9FVcqJXgCxkKQ6vT2C84QX11AV6MxEVqMjXC8wn3bE69X/w96b9Mq2relZzyhnEdVaa6+19z7VPbdwWkliQNBCQkJI/AB69BANJHfoINEw4hfQ8g+wRIMGHSSQANG0RAPZsoRkLGQneZ15M8+95+5ylVHNahQ0vhEzzk0y06JjK4/ik7bOPjtizZjFmCPWfMf7PS9MUbZhLFyJd4qUUfsjWSmoK8z9FrVuZz8iReVHKdLdBv1yJK4qzHRWV7PVxEWFe/9EXi1QuwP5lcir6sRkVko8wdaAE0VXxpoRFWvhhOQAQlFISf4dyH1PvloVf6aiv804r9BlE92bRFwmSFBd9bT1SOMn6pLAtfEdG9+zcR2v3Y4pG/7B/c/5evEspwCF15HKBMJa0w2OlDSPhzM5hCzJaNMyg4b+JmM7NSdDVb91/KPwC8gKNZY0IJ3hxBUN5XzahGkDdT0xDpbpeE6PMnXE+QCTxj0ZqmdFLPdE9KJUmo45HSg0+nc85emkxB2BJ0P1lOlvNbaISf1dxj+LmTLW4resH+Tvct+VNKox43eZ4UrjD5mwPa/A6DHjDgl3hFBJl/xwdSYV1I9Zkq1WCjNA6hTN54wuPt9slKSHLRR2K/fQtFLzPtbHjN8nmg8Dh69rqueIHs++vVjrOfFpXIt6Oa5kmyDKL1kYrNVzYvF+pHvty30pBIPF+xG7nwo/W2O3/ZwECKIsmscXSYp8fY069KAU5lFUv+wsqvLEhcPuRuxhgtzM9IBqGyHB4mNHtZIkNf9pT1yWOfrxgAo1eteRljVhXWO6Se5BQO87uU/Kilq2mmQVOmTqp7JSV8m1d4dy3Y1QG07UA7TMjaKwZ6alJE3V92fKQvWdMKRB5tLF97B8J/fM9luZ6FKhVEiPg5w/gFDDq//zmem2JbSGxW+O6ONIWNdChQHMrkc9bcnLllxXYBRxWaELTzguPaRChYmJfL1GDSN5K6tLyjty44XcohT2t4+gl9wXtfJX42sSmnexY1X8pMfk+BBvAPjHx2/5k8Md9/2Cx64lJsVu3xCj5rtR3qN0ZuotbB16UOjJYyf5PgGZ31WUVaLdleN/+/5fZ3uoSQXnM20r3GokZUUaDdpHmmZkXVZ/vIks3MjH3ZLd/QLVGdr3ZibATAuwnaShjX2FHoXkkDUs35VrbeHlZ5p/+E9+j3/o/oas2pgsHneAU6LZpNBt4I9GI8zt/XluyXVEuQTPTnpkXMLsC2XhKkBSqE6jgiJdT9CbmeHt7w3JQ7gbZR6bFPiE3p1/Hc0uo65G0mCov/PUo3wnnnznAIuPiWHz53of/lxdFNy//hWA/zLn/AfAvwv850qpPwD+K+Dv55x/D/j75f8vdalLXepSl7rUpX70dVFw/5pXzvk98L78faeU+kPgK+A/Av6D8rb/Dvjfgb/zV29NEW+WxKVnakunc0iEpZf0LKUIX96IB28lMo36f/4M/fpWOtatdLbrpp5TyQDM/YukmRUFQT9shclYuvtjY7HWoLpBOLkleevERs1a6AHq0BF+esu4KvxWX16/XWF2jtw48QcPhdFatqOV/Pz4qimKjahNsRF1CIQ1mW3x/2VRJI4/neYzYwZHWolnNxtFrEtH9ea0j+L1JYvSYTs4LjPplMe+iCiVsVbSqFKVCEqj+x96+eTpWmtJuEoO6vs8+w1jbWnuE+2HiHvsRU21BvbFGzVOswc2e4d5OoJWwq8FwqZCdwH3eY86dMS7K/KmmYkV/uEHft7PL6iTh/pFPKwsW0kI2yxEET+OqG4QEoI/RREZ8tUKvT+C1uTak7WeVWSM/h1PrQpRVgnKeNL7gXjdokMiV14IETnPPmC0himQN0vx/TaVqDpFQVa9+IqFA5yE6nBzBbsD+bq858WQnaHaJUiasIpUf+PAdSvn8a458GX7ws+az9yYA6sCkOyLvHliwnoVeY4tf9h9yVPfUFkZTO93a3bHCu8D42iZBsuwN1D4rv7BUBevozsIlWNawPUv4+zt3H+t0ZMjtGef+HiVsIeiyg3y89M6k7Uj7VtYZKqTNT6rWb3xJXlKfq68vlSsfp3w+8TUCLd5+S7QXxcPbqvoW1FNT77UaSnq6VhSvKonRfWQCQtZdchGFL8f8jazKulKEdqPMoe8/QeiXPZvGpIV3qrbRxafJKFPj2Uflgrb5VnxGjfir8367P10u4ztwG/FC7t4J/t3sgs2DxG/k875+n7CDAm77WdfevV9x/RaxsXxtsEfxJ9rC0Bk2CgWHxLHNxq/ldUgHTKLj4Va0GdRWkPCPu3QV0uyM0INCWcGaHr7StIe+wDeEZYV5kQpUYq48OIrNZr+dYVKefY625eJ4ZVnvCmrU9sR9bTFKCGipI3cl7mp0Pserht0H2YKiipEkXizJjUW9+GFsHiFHhOukzG5eC/Haga59vm0slTmx+oxo6fSK3BMgMbvYFyWzvyiyIaFon7MRC++2uGqsMqXMpe1nxLDWpMtLH8bmNpTUppiumvRfURbhT4MqH5ErSp0XxTaVY2JCRWiePT3Haqy88oMCfTTXnz8x15U3EWNjhIJlxqP/vBA+OkbbKG16AH+6JfCmP27H28Jk8FVgbGshLg6YF3xBQfN1Dn0k8NvNdEL1UNZqAov2O0zw7UQJm7+mbCJu1dniHY2Svo6EjS/tRy+u0M5oNAmfIb8YKm24lUerjODa3g4NQggvvFYZ3yUFMXNnyQOXxSf6yh+6sXHxPgs+6RSScU8zfPbxPo7cEfL1MJ4k2ASrz2Ae3Ci5u8UsbaEhSQl+qfi6Z4kyU9FOc7kIGzOUHnzZMVzHxTNR0U/evyLOqd2KpkjwsrgngyxyfiPllhnTPEJJ5+x7xoZdzETnaK5T4yrkx9bEtuq57/ag3v5BfdHVEqpnwL/NvCPgDfll1+AD4iF4S/6mb8N/G2AmvYvesulLnWpS13qUpe61F+ruvyC+yMppdQS+B+B/yLnvFXq7E3JOWd1glf+uco5/z3g7wFs3F0e7oQheCy8udgsiF6xeD8QlxVx4dBjIjl5wvV1dU6xqjy5MsRmg5qieERBVDxnUFMg3q5RQ0QNI7oog3bphX5QezBKnjSdJZSEGrcdyN4S19dkJSlTOmSyKyk5TqMmj34+CGvSaBgSeSF0AMaJ6fWS5DShklSZ5CG1EYrvKL9YVJKEs/EmY/cKVSX0vRzntMhkIz5JMwgL0+34HQ9QNo7hRpS1/jajBwVfiKTWNCN95wk6Y5YTOSpyUoSjnGdzFC9SNuJtVFnSfrrXalaTpqVCJU3WFe7gsMco/rRWVFu968Vf2wemN2tMH8Tr91mgnS4kprsF5kk6rfWhJzTLmSMJwscNVzXZXgtlAeBGlKK4rFBjEFZuZbCfe+HtHjriW1EY9H4kVwada0mSGyfxGp4U2KYm3K3R/YT++EhetsLiLX7C7Cy6m1BTFFU3Z9LNcu4It796T77ZnNWx0zgv/5+9g9evYHck12VcOQvWoo4iXyqtyZVhaoR/+/anD/z7b/+Er71Y1DfmwJU5sihy53Nc8Bxbftm/La93fJpWGBKfxyX/4Fe/oGkH/vC38rqxkeGhYbAZ+2gxEbQF/1RUnoMoY2aUbv/1d4FpoWk+9oSycqKDI3rYfqupH+R9WWvMCU3sTp3oMt5OSv+J+iFpV+Jt9LtcOM38Dh92WGtRRh4jh7eGxfvE4r2oYU9/02N64Z+ufh3Yf21RUZTY9XfFd77UuGOelRmVES9s6bxXSZiyJ46zO5zY2nItq4eBbDTDjcceAgELSrH6vvgfX8u913yeGK8ssdKYUTyyJyU6VuIF9dvyGceM7UX1Aajve/R+RA0jsCJVhvFVi3sqq0S1k3kkQ1W4s/VTnr3IKLknk5U/3ZuK0Ciq50KAOQTcxy25rUjLFrXviK9FMTSFH61ylu79IZFqW7yiEFbrcq0T5hgYXjnsPmKGjD0EYiXnKZYxEStF1SWhgTT1+R7QXvzz3pIXQrOZrhvsroz3Y096tSa1QhiIGyFk2Kdu9rcev6xRqXhu5zGS5/EiG4L6KdFvNOvfTIwrw3B1wiAoopWViNAqSELBGNdljrZw/EKRnMYM4m1OVs1KvPQ0GNxjIR4sKpTVZKUI14VfXVi/4zfX2IdOVO6UCTeL8npg+MVrkteQ1phJVEv7A+pO3qxkjjIaFSLt58j0p3J+uzcanWHyDlwW/+ijo68KK/3XhvBNpHrQqAxZCfPXHs9e5VjLyohKYPqI6SOLMdG/Ol1DoYqERpTX/kYTPdiykmcPMFyJEpu1/PHPalbzT6sLI4rldxkzJkKtivdZKDz1UyJUcm7NmNFTuf/LilzzsUOHWjzvSRFWalbqQVThZDPVI/S3ChGwzUwWSQ7MoJlWwpK3Rxj27sx0z3LPxKoo/oeT/1heDq3MDc33lmkj29CjfO5pG8mrOeWz/Rw5vjIc3pj5vg61wkya5uEHO/4X1MWD+yMopZRDfrn973PO/1P5549KqS/K618An/5V7d+lLnWpS13qUpe61L/Muii4f81LiVT73wJ/mHP+uz946X8B/lPgvyn//Z//hRszlmFj8FvmpB/bF6NYBQAAIABJREFUydOpGhO7ny3E9/JyzgdXbUsGUuuFfhANmUTYVLhTUpm3ZKdJ60YYmFqTlh7zUBTcjy9ko8neFqXXyc8Uz9DJS5YLS9V24mM97cNw47FG42MkFTXTjNPss0vrmtAYbB9BGVKV8U+K2GqyLftYJengvlaEtnirnt3sCRpfRczBiHe3EgWpesl0t8VX+SIdntELT1JlGG8iDMWjWyucDzTVSEqK4VikNn168gblINeptK7DeCUe4SC2WKoHRX8jPMPqJaPHSDZavL+A9Rb7sGd6syY2BtMHUWBOHrVxwhwDaSmKiIrF81h4muG6wfQBFTLmpZPzl5if/GPjiv/LYPrI9HaD+7xn+skdpjv7lVVIhCvx9vo/3ZGX7ZymhlJCcvCW+NWt/F0p7Is8uodNgzmOhKtG/MzHcSZnAOS7G1Q/oA4dOUSSt+hlez6mlFDHQVL1hhG0FuJHXcGjUA5Yr8QLWRTN1k2EpPk4iaL26+GG1ozsQ8Xz1PJ5WPKb5yuskfO4PdSSJd9blEmoJ0+8b1CFyagPilqLSuuORTVVZ5Vx2CgM4B8zUxJlrr6fmNYe08t4dPtIXhvMCK/+sOf+bzXU9zBIszb2KIxXlU6kgqL27M63s9tJt/u4Eb941ueEtuo5yyBVku6lInR3DtvJMVaFHuB3meHasPw+8Px7roxVuSeW7yb2XznMANUusfvaYIY8q1n1c2bYKJJV+IN8vop55l/HWtjX9hCFbaoUdj8xXst4rrYRFcHtJ+HD9rlwbM+kBbcXdXvxKTCsDTpkxqWm+c1Q9lVjcpbVJatxzz3TVS1jH9CfnrDGkCuDSo7m88jhi2pWod0hYwahQEiqWcYMnD2qKYOzpQehUGC6ILSPeErxSkL88I7pVlK4wsKeexKCEEzsMZGcZvHPPhBv1sTCp66/34JSjHcLSLLKEjeLmcKgYyTdrdG7o9AFtJLzWSovGlF9c6GoWI3dj8R1xbgpnv0TuEIhPQhWxtXJ/68H8Lsk16PL6CkBBj2djwHEk2m6Mz9cn+EoUIZctjLmho2Zz5EdyjZP5IdtR1pW6CkSi+KvBqG2xMpgGicCX8zoKB8Sl17GZsokr3HbkfG6wpeVqFw52WYfhNvuHe6Q8Fs9j5WwyjTvNdM6C4d5OKeQqQj1J4M7wHAjCq89KNz2fG9nI35X02e5l5ViuLYz+7n9FHDPA8evWqrniWQqDl+qmYKQnNzDq+8jh9eaaSUe/XFTzudY+jVGZE7ZJ0yfcHs5B4cvK5JTjGu5d82Qad/19G8q6nuZo8PSkRX4cu80H0UB7st3mdtK4lpy4m1XUZUVjUL1eE6Ma039Sc0Saf1wJgJNy7Jy00HzWQgn04qzp7yTfdcT6ChUiWQpXvrzeFJJvLeh0lTbRPdKz9SN9rPMISdu8l9Wl19w//rXvwf8J8D/rZT6v8q//dfIL7b/g1LqPwO+A/7jf0X7d6lLXepSl7rUpS71L7Uuv+D+Na+c8//B7Jr6/9R/+P9rY1rR3WqGKzUnbIVa4Y6J/bcN/ZXCHRTJKaqX4of69hb73MuTckjEhWNaOdw+MN3KRsx+xOx64qqmv62xx4iOiXwn3cvu/TM4S1aK4aaiQrpNT2pWrsSbZ5/PKt945Wk/iFw13HjsfqT/Yok9BFETvZPOZOSJVSV58pcub0X/OsFqomrkqXY0Dl5EocxVIg9GOKWbkjY0aPxW0d9F3NZg++JRK0JJey9P0LtvHYOVDtTpSgkrEPA2oFQmZ0XlA0OucauBaV8UlKYoFUmUv5yR7vnMrCKHVpS76BTjykDKuP00Pxlnr8m1w3QTyWlUH4hXtfBxEQ+t7ifp1m4c402N244z81iFRKwt7uOWtKpFbeuHHyhNCRXE16aHQFh50rL63dSxlFDHiVRZsJq8WYrnsigzqZFrofrAdCdeyNg6cqEq2G1P2NQyhnYT2Vvsc0dalBSxpcd8+AxNQ359jT70pHUjSXiA/bwl3shnuvc92TthN3uHqotX3BoICTNkTK/59acb7vcL9rtCBjEZsiL2BkaNigq700zXpXP+IHzHZquE31qL4nBi2CYriqrbAwqa+8TxtZ5XRUQZyYRaFC+3i0IgMBRlDLKxoupHOL6pROHoM/7lnNBlelFik5Ox4XfnVQ13hOhE6XF7UWh/6LVGyViqdhG7n6ifNDqK4gPQfDyy/XmL7TPjQpMqIRigzgQDHSTpKvqi0r5k8RgWhmwyRcXuMtVzEFas0zI2AFOS9vxjR1h6QmNI3swqjenB70bxOjpF9Aq/j/Q3BrcvqwpeocfM8c7S3Eeqx4FsKtxTMfOFWDzYGrMfUccB68w8HvN6KYl8xdc/LS065PkY/VaO2e2zUAE2htVvBnR/WvkxhKsGPUb0y5G0qNH9SKo98UoUWPvckTYL9P0LyV9Rvdtjy8oFQGod9ulI1gtshtzWZKfnBC9AaCWxBc28OqJi8Z7ue1RMhLs1qTboIaLGRCyrWfbQo6Y830dh4XAvA8noWYEzU2b5LrB/a/H7TD7KOLUHOc+mWOhFactMCzlPiw/Fi7wLbL/12INc7+5G4w555hEnL8pfrIpi5xTVNs7H6LeRWGshUEyRcLMQFTwk6SUAdD+SFzX2ENDHUVbrQhJPM/Kd4Z9EoXe7gB4jto/nJMfjIF+UKZGuFnPPQnNfVi2eFS+/EMpO9aQ4vpGkwVOy5XCTad+rQvoQlrpK8ue0atHeyzG57STzYs7YPkEZjtWnoyjoxyipe/vEuDe4cp7HtWL1myge01HU8GzAP5/vORTooAi13Hf1bsIUSs3qTwPDqxq/07ISNyTc+yd0v5oTQMPKUz30uKNFJU9/rcVXXij5p+8ZPckKUPsgZJKCBiZU0n8Sa5lHdBaucVqcqQ06ntXm4ZrfIZ9MC1F8sz35kcVXrCf5I9uQtMDqJVE/yLV0B8e0KCrzIf0L1Vu4eHAvdalLXepSl7rUpS71I6uLgnupubKSDk7U+akVBcc3ophMK+nSXr7P9KV7Vk+Z5BeERhN+shB1L2TGKzc/YaWrmuo4ir82Z45vHGbKuH1JqDm0HL9ZYcZEbDTjdUX1bs/4WhSKbBtiZeSp/2Ege/07fq2sYbypMX0kLCzmMIlSMXcfS5f0tNRMrWTWk8H6yKKRJ9/hvsEMGtMrdKcJ64gKkiIF4kUerxLZiWIWG+iv9exdG9aS1BKbzFhlcpUw6xGt5RhD0ty0HUplYtKkW4VSmTD84BactPhCbYakSL6kQZ18bAamtXCB6xfoXznplD3KG8LKkW4X2OOEjgnVD2TXzolHYeUxVuM+vDCtK3QQz19sZB+qD3sIEVW8rWY3oIaR8KakiMVMWHnsbmRaV7jdKIlkTjPdtOU9CftwQGUwLz2pcaC1+C1BvLZaSxpZBr09kvxqVlmI4lszY1GKjyOME6oovOGuhZ9/Kd3grccoJQpO8fJReVEqQyItGlEt8kL28+bEyp3Qxx7brameIP5Jw9E3+JPfeiNd0K6wZnVQRJ+xz2XMB/Gl1Y/iU01GERtOFjXcTq5VWMDit4lkoX8lfrbTdcxa7rf6JWKGxLQSduS4Ll7lfFaH3D7i9kq838XzSKto7oXZWj9mhiuFPWRcd/LJRbY/EfLB8l3E7QNhYWZfe6gVfidpdiiF30XsbmK8EjVMeKuJaanJGg6vDe19YveVmZWY7tYV9VaUvWqXOLR6VvvMkFBJxrQeE7GxVO+2xE3xSw+RtNEMt43YgVNmvLKzihxaTTZe/ME3RW3MlO774t3sMv6QOd5qzEqTbUX1cPaDp7ZC5czwusU9D2A0apg4/kT81tXjQNaK7DTLXz7z+O/csPrNQCweVpU0/ZUpXkhRwrNW85qZ/7Sn/3JF/XQkbVpZJXL6d9TybDWxccSf3JKsIq4q7Oct+vQevWD4cs20NOgxM26uSO48B4erBhUqUTJzJlXS0xALOzrVZVVkCow3Hhq5zs2HIp96R3IG1Y3i3/WafFPjn4ZZ4upu/cwbNmPGdonQaKbCZVZR1MVYadqPET3B1GpMGY/DtcUOkhoXao3tc1mxOPs23THRX9uy2lFWMYpX93hnWbyfhBiRoL/1rH75LNSJkmhJ8Ge121vUMIHW4mcGqoeeuPRErwmNKMHmGGbmdlo3mPePxNeFEJPEu3ryfe6/rKgeRXHUk5ANzMDsj1VJMS3ES6qns6dURRiLJ9xMhvoxoscovQpDmSNc+a5yhtha3G4iVYb6fiRWFc1nGbP7rzyhVjMP23biV55/XstnL55kXskahlcVvhCF3P0R/3JiDzuS16SrJal1UMab7oW+o0Jm2AgJJbRnf76KosiaUby/WZ3nLBnPcuy2E/U1+aJonxjcnjOFpCjXkhp5uq+lV+Xk/xded1GJT771KB7hbBTuqUdFOZ/1UH5nOAbQimn5V/8Ke1FwL3WpS13qUpe61KUu9aOqi4J7qR+UIraZ6kHNvM1TN+zJQ5MtHG/17D2tthplM7uvLLaXDur2c2T3laW9L0+Vu8jx51eMS8PUKsyUGWpNqE+exJUw7oIRVmaj4csloTk9f4lqdHhtmBamdItHum8KR3LKTAuNfx4Iiwo0jBsvPESgvzbUT5FxKU/Gw00mrQN5sGyzKI9XX215eWmZciXq2aDJVZo7aMcvJsnMVtB9EXEvmnFVfIkI19AdMrHKYkqqIjkqvroTBm0uUm9jp/m/Q7SkVJigbmJXlTQvnRmea6IW1erkT20/KvqFPNUe3mhsB+OVpTy8M64N0SmqrcZtAypl4QaXJ3fTBezDgbSspUP+N0+E1+uZLwmQVjVh6TFH2c/py5s50Uw/HclrT6ossdbo0RSqgp6f7v3zRGqlO9tU0iWt+zB3zqdWOJ3pqsYeJqa3G/H97uTxX40BcxghOXQfhJKQ0qzwxsqgR6E0JKdRMdHfeBa/FiJHapz4gUvndWocaV1LelXx6aohkNoaMyaaxyQEDw2nsCC7l+5gPSlMV5jGAdy+dHgHmJbiPbVdPvtfy2msHxK7nwgrM7Qy5rIRlQTEd1a9ZPwuzcpMcnKdTFEoUKJgytjStA/xd5Sg6Mt9sBePn/mkmFo1UxDGQmCAfPayKzV3WyfnSFZRPQlz2HTimzZj+fmritBq+itFbES5ipXch6HgpadWvLHt50R0iu5GGNZD6fhWwVC/JKpHSRLLWtF/s5mVJNU6YiXeX/9hz3i3IFaaw+tCzDDQPCiyF76oOwghwR7PSvW4VPOcYiZ5XU+W2KzKedJUD4OoltcVtrFMSzufW3eQ8aT6SLiWuSDUhmllyva1qFRFLbN9Zve1Z/1dke9iIiwNh19ckaycf/eSMYd+9q4TxTcZlo7qcRAO67JhvJL7vbuTbv6sJU1Kj/L+9s9E8k9Lj4qJ8bqWbvxCmzjNb3Fpqd/L+NeT8GX1JAmUAKpxuMcjad3Iyk3MJKPQ247h1bWM+T6JIjtmQvFbK59xp5DEVWG9Zhg2mqkFf5BUNxBVu/oUSFbRf6GEjNLlmYGsI4wrg+3EgymrFYboTymNYIZ47hF47YmrSs5JmeC000wrR2hWuO0oSVrbDvKpYSSVMVq84lNkuqrx96JkZ6sJX71C9xPhusW9e5KVwUIFcV0ia4iNZrgSmsDp2sNpNUUoAfYoymVWFO4yZW7INL89CHN4SNiXDtNZhldyrcPSkYzCdYG49uQa6vtp3gfbJXQUYsW40nO65WkshVYVX7xi9dtJVk5rhZ5kJ12IkLLM4X1kWmti6yDnmXSTaitjyYpPWodc5jQ5huaTqM7DtaL5KH70rOR4Ze4Q9rQZyupiFtZzc1988U7ILWbIHN/ISkT9kOfve3vgB8dViEMr4feexoNS4PZJeLhG0b9eUj0OMzFI5UzO4Lbn1Zq/qC4K7qUudalLXepSl7rUpX5UdVFwLzWXJO6ISjtz9ybF1a8iLz8z8uR6EH/NyX8UK0V/ZUgVjF7RfE4c3pj56RbkST1Z8XAlpwhJldx1ed2MevbgjEtN8xg5vraz188MmfpxgjeG/lqx+BCZWi0eQsTrNy3Fd6aCdN6GhRFfLHB8o5gWBj2JShcWicVNx6rp52PXKvOSFoRVxD0JQQGl6b6RJ0S/GBkf5Sm8ejTyZBvPT+7RU/xoCjUp1LMjNYltLxL4bXvkuj7SmIkpGd4f17x0NX1XSA+nnHObiApIilwlsJlcUm5MB/W9nLtxLYpKshozFIUjCDfQ9IUzer0gNhpUVX4+EG4WxNaii6Kqhoi7FxpFriv6uwY9JYxWqCjd/fbzQQ4yZ+xuJFuNfx5JlcHsJ3Sw+OfCHa2sKHRaFMlUGcxuwO5ERQmv16LYPolfToWEjkk4ooDaH0k3a0w3SSrbFIUteujLtRYSRDYat+uFmzuk2adnukm8uYcBfejItQVnUCGh+pKs54S/6j93kGqi85gpE2s5j/VB0b3OhFqupzsoss5zh++4yTSfxCNNx5w0N3tHa4V/YfZpZiXX7cRWBVj9WcfxK0kTsp2Zr5l7KcxnZ2jHxLCp2X9pUBEWn+L8GW6fqZ8SbhdIXggNOjJ7dE80g/opCWljiiSrZg+u6RP1p272LuthQr8coaRwJa9JBpbvI8PaFJVIvIAnJUY8dTJnmEnS98aN5NQDXP3ziD0KeWNaOSEANHq+r8nQfBIPbFjX2N2IipmpdP+rBNVTYFqZwqEtfs5toilMT7exdNeG6kUUIr8XJXL+DCWkg2wgJ4UeIroyNJ9lxSA5je4j09qBUrOiNS5lLJhRxkByojgNG1ndmlby1RkW10yNnH8zio+4mqIQEk60iUPHdNeixySsY2dwD4dZgU1WEVpN9RRk/0IiLNy8cmI/vpCbivS2lQ79XSAsrHC9EfV0vFvgXgb0lLCHhDmMM6Uhrr2sZHgjqzjbnlRbiHFeEbDHiJ4yY5kzTwlltiRk5bVi/WcT/Y2Vc2rVPBZB1HyQFRshhMD1H5/n1/7Go0Omv1LUz+Idd/uInorGtodsFbqbIGfhQCtRomNZMTC7nnxXY/qiUHYBc+ypHkti2xTRU0KPqfhGhXusytyhvCW2FvN0IL1qhTrh1Oyd12OGBSzeJ/ZfF/ZtFvURZJVOT5nNrxLuIB7l0MiY0+P5XIRNhS1scRUz09Kf+0WU9KikygqRpvB6TeGAx9+/of31QQgSXy1QUVYv5mtZyc8LySGhvHwP2oOszAxfbXC7Uebwg3ia9ZQgpHOSo1K4lBlvanQUIoF/0TPxQk/Q3yj8LnP1xz39rSTlne57PWXqg3x/x9MqrFXUT4VH7DUqS7/LaZsqQvNQvpf24sFVCTmOIKq+SufUQzNkklfUnwamdYUZEmHp5u+t0DpZPfqjvzq/6qLgXupSl7rUpS51qUtd6kdVFwX3UufKwnW1+3Ni0lTD4++LIpuKj9DtIZ0ejTJ0d+IdU0GemvUk3c0nnuXxtahdthNmqDvAeH32NHZ38qRXP2b6G4UOQlsw5anY9onnn1cMN+KJnNriAS6pTcc3jmGtsMeKWCncwosX8Ors71JJujzHTcLedWidGIOZPbBTNOSjQbWRdNDol9JVGmUb02BRSUEhMIRWfGoz89Mqjre6JBgp4s2EX57aSiFkjSYzJcOV60iNQqtMW8l7joPH+0D3yyvSFz1UEaXEenXygO1+Jiq73auSBy7qzf5t4YpO0HwOjGuLShm9sgxrgyp+QtuVpCkDdDB8e4PpAuqTPNmPX1+TrcIcE9PSobtJ1KTbYs5KmdhYUZ5yxh6CKARjon8jMot/EmU3NJppUaOnjB4rcsmTn5YW/zKKh8oosi2e2rWozNlcSYfvS8f0zbWwU0NCFYajmhKpdoSFpf5+EG+hL6xd4Pj1AjNk7P2O+Ep8mCc1ONenz9AlMS+iY6Z5DER/7v5XQXzo/W2WJKMjDK9gWhd1tBZOrttlqhfxncf67Fv3h0S/0WQrfrTV95Fhc1YVk4PjVzXDSlO/iBJkhiS+t7u27KO8V0+gTWa4VpjxvA1h5mZJSSoJZe54VqFVEu9c9IrxqqJ66DFTYlyVNDKvSF46/nUXUJ/25EUzK7wnGkp9P3J83czd28mdVT2VhY/qi9o2rrX4kO1pHxXV5yOxcdg+cnzjC1f75NuE0BjQ0p3vdoFpZWkeSsrYKCsxbhfRYyZbxfHOoEP5OeT8umOmfpgYri3+JTBcO0KZe+qnyHDtcAfhjobWYXcT/evCfo6ga0Pycg3sMRIag9+fTrSouNNSPs/thQ5wosicVPoTScJ2iuG2keMuHlhYzrxXFTOhNeipJp3GdIKp0bidItaO+pMMpP4Lue/qd5njzza4l4l0K2lkodHnVbRaOMXJ1rKSZRQ6VFSP03wtOVFM9iPJC5kgLq8Y1rJfw1q66XXxMdsuz4SE03lCidfZHhLZFi7xTq6VakXxNxE2//zA8asWcwyz9xTOqlyoRIVXMc+ebxXF420WFfpZKCzDq6oo0qIMZi9UkHSi6GhFfH2NKt7S/usV1YMw2ZNVNO9GSXArPQxZSfd9vF4I+eVNI/dZOcxxJauAfifKcnLQfMyYsnLjDqnQH4TZq5LMt2FhqB7LyotShJWTnoOU6L9aoadEfy3jpXo5+0b9y8i08nKObuU8Ra84/mQxq5v1/ZHui3b2v7YfRkn1OwRUTIDFduHcb3JjJY0yJkLx0ZM8OmZsGW/TppJzMUSiM0yvLaE5p4h1d+fv8GltZ+/1aU5yW0lQi15xWClUkLngNMajF99ttmWeCuCO6cywPQqhYWqVJAKGcxLjD+c3MyRMNxEWFv8y0b39wVjqpe+m//kd/Cl/aV0U3Etd6lKXutSlLnWpS/2o6qLgXupcSpV86IwZzvzDbEoi0iBKUXOfGdfFh3dIqGxQpZsyG7DbPPtvQTpoc5Jcb+XFb+Nf5L8gyljWivqxdKZqaD9G6nclqez1QtTgLD6e0Cjq54TtCiAWg99npqWQDbJpxNt4UpoexC/WvZG0qNWiZ1WNhKQZozyWpqzQo6Z6faTL0GmH7RT1p9J5/95iBkl50SVhpn5ONB9E/cyqJnpF9SDPjN2VIkbNFys5hpvqwNoOJBRDsmiVcToSkyhJMSsW1chuHWEyaBfRJhM6K15cIKwSqExYWsxeo8OJvVjOwpjFO6coSoR0OZ+UmPrzIPzMyjBu5LiaPpJenWgUCdNFUsmrj61H5TyrA/7Tgdgs545eM0RI4n+NlSiDeoxkp7GHSFgY/MtEaC2xLopbrajvE2qUdBr5Nys+XGC6qvCPPTiLe+yYbhrpgC/KojsGCBm7mxjerjBdEJW5bD/Ueh4XceHwf/wRtCYvm5nEcFJ7k5eO61BrklP455Li9RQZFxp3VIRalIjlr5nHvGkkgezqVxPJKZIzsMvzeFv8tiMrUaxjpZha8ZWfKAsnj6g/ZGxRNvSYyPZMJ8laiZpYy5hPXjrOm8cyFqqS9KcMeowkp/G7iVCYxm43lWSwMh7fNthDnBVHlTLmMAp1IkMOgbRuZslDpVyoKKV7Xol66w6ZxQc5v+PalDS4RKw1i3cjejqnDekfKIDj2lI9R9IrQ138s6YP6CnJeNMePYmn+MS21KOoOLHWhFbjtxF30LhjmpVq22eiR7rqU0aPifZdR2jdPKbBUn0QXq2ekvgwi9ev+nik/6IVNXGI2JeB7svFTC7RIaOHTPWcqB8jw0ZIEtVWLuawEUKLjBVFcy/nvX/bUn0u3s8pUn84EFYV2WlylusbFufPkPMdS1e//p0x3f1kIwSNyoii3EfcdmT3beEJJ1HFxrXB7UVdHVeaptxf2WlMH+hfN4zXnqzF/2q6MK+SqSTXy+8mYqUxY6K6H2fvqJ4c9fc77KsG3QXM04HpzXp+XW2M0ASmTHYGMyTGjZ+/B9xB+Lvtp4TtEtXDSFi62acs5IcsKWudl3nIKFHx+tM8L+NajxFyRh8Gws2CbP38evIG/9Axvmrk/rYa/d1HGW9KoV52xK/vhMtcadrvj+y/lS8iO2RSEMJE9SgkBrKojwDDSpdkLsvyN73MaVlU+RPn23//hH22oLUwfL2WlL/SL6JH8caqGNEvB8LiTu7n5rTyIisfekpUjwOpsthDRIWifj51qJUwkVUf0GsvSaHqlHBY5gNvsLuRZBTjRlLnbOmTcC8DYelxzz3VQogiemImk3SIett+KH0QCeqXhC29IO1HuX9jo+dkOpRcY9kH+TwzyMqpKp7204qDfwmMa4OeFFGVNMWjfPecvO1ZK/zjKMxnLb0D1dPEVJjH40rmg5NP/S+ri4J7qUtd6lKXutSlLnWpH1VdFNxLzXVSycaNIhUladxI/nZohRRge+nQPvFxH/7AEVrpKu9vQO+li1ySTwqrcgXJZerP4l2d1uK/DIvSwT0qUpXZ/kIyrpMX1t7Tv1EStJL4t8JSsfuJxh4gtIbjXWELNgrTn1h+wsVEMTNqlZH0sdBm2MjT59+6fo/VkftBfG7/7P4N5u0RYxLaJdSbgfFocQ9yi7TvRMnKC8qTqeStRy/7MCsVO5hWoA+G3/+971k6eWo2KqNVYm0GNrYjZs337povGmFdTlnzPLbknygOg+e4r4iTRtkE7tQSDkpnUhNJo5rVtVN3+eGtpn7MRAcoqJ6LMqDOT7mxMgwbMzMvp41Hj6KQbH8qKvSJXZqNYriu8M+Fn1hIBSoW/6xRpMoxXnvcVrbRfdGiQhZ6Q5YUuf7W0Xwq2/CW41cNeqxJXhGdov0wcvhSBpQ7JKZNhbUafRwZ15ZpoWfFzQwajCjF09oQK43tRC0+lR4TeCdKwje32PdP4sMrKsd45UUdqcVPPK707+TJJ6Pwh0Q2Bj1lYiXX/DSeTszb2a+qFMv3I8c3cn5CK6ooWfxoZDjemll9XX7fEysjKmRRIPzzwPCqRp083wvN7itDWCiGqLEHGd/Vc0kHvDXE2nB4a9j8ScSMgXFMfysXAAAgAElEQVTlzp7GKcLComKm+X7H9vevaN71xLokzgXQLwfi8oZpbamGm5mbCqK4miHz+K/VdHeKaZnxL+L9nukQWjzhOiSOa4fbBhk/ZR/9c2D38yXuKExP489cVBBlMYeEHgN6smSrhI08qzgyfqOXsR69xg6J452ZFbGsFN0rTTYLms8jeozobUeqCge3kQTEXDlJRlPyOarwW+PSS4d2K8rpcNvQX5t5PyXdUc/jqv2U2H/p6V6VFQkvvlszFcVqSvghsvtpg4qFuhITetczfbnEHYOoald+9uBWL1F87Dee9rdHwsoTKz37fkH4qMmKWta99jSfx3m8wmkOULTfvYhS3QgzFkQhtkE4u2NRIevPA8kbqtL5rqOo38O1bLu/cfR3Fas/epLXp7qkYUlqpF553GNP/7Yt50ERGk39GNl/I977ZjuBPq0EZtw+kI1FT5n+dYXpzxxoKJSLMTLdtkxLUQGzURy/ks+oHifMfiRcVdjtQKo95qVn/zcF+dN86DH7QVZsCkUhVRrzA+99ur1muG2EI2wVsXFUzyUJsjV018L2/qFPfGrPJ3rcSL9If+uFfKKUpM9dlTS1X2dyiKixZ/j6DSrmkvoln2G3A6mxxLpGXzeS1HUM6LKSmJWQPaaFRkWPPcgqx1ToFqkR2kesLWlTYfoonPGjrBZU3jBtKvznA3FVU306EJu1cIeLH9o/9NjdQNhUwpfeR4w7pyS6gy4MW8fy3Ug2iv1Xdu4x6G8dfisEjsX7if2XDn/IHMr8FytmRnhWWXpv2rPH9zRPVtvI8U7Y7cqLr/w0p6qcGa88ZkzoKc29AuH2xMEVNf+Hvty/qC4K7qUudalLXepSl7rUpX5UdVFwL3WunDGdqLcn0S8bCAvwz6IS6GKHCiU8RkXx2UQvyta0LikvjpmBZ0aILUwrhe1hLOqWOfFdewBRfZvPuahmZ95mrBWxkqfp5MQTOTXgdicGH6SVov2Qxder5I8/FGWz+H9UyuSouG2PfOhF4TkGeSI89p6UNPv3S0mxWk4QFLp4kZvPie3PNP5Fjnm4EZ9Wd1u8gMc8kyOiz3A7cFMd+P2F+L++9E88hiX/dP8lf3Z8xTftE0O0NMVA240tVkWsTliduLo6cOgqrE0YLWrV9qklDwYSmEGRlXSodq9K92oN0Qkvdf+FITRqVn5AUnSOd+Jpcp2wKGOt5s796OUaDGtN8z5i73cMt7fz8NCHgf5nC8wo16h7W+OfA9X9cO4mL8lUDOJxi43B9qKQAfO1TUaYyONKYSY3f4Z/EQXPPZZO+pBRhdUI4HYj43VF1MVz20pHtSo817ooyeSM7QLZaMZvXmG6aU5Ty1qRjZYx8rnDXlv8NjKUTudkJX2s2kayVpguYfs4H0N/ZfCHH6SQWVFeZq/ejcX0kvMO0jE+rvQ5pWeKmJTRoyIsrPggYyY5IS9A8Z42Vsa2ESUiOWZOaXMfRd10isOXjuo5iTdzL+NpeFVjBuFskrN0gO96TnpHbBxpsyCrwuq8acUTW/idw81G8umdIrS5pLLJvbz9pnhkAyx/MwkZY5/o7zy2zyx/+QzA+GZJcuJBPjGAbX9Wo+qPQTyT3z2RnSEsHMNXDW53TtZTURTO6DXTUtia2YhPEcBMSTrwPfQ3jsVhIq1qYlXGR6Pn1QZ7TOLv1rIyAUL1aH67I3y7xu5GDt+0uGOeE+FsFxjXjnGpMX0ktkIoOV1Ld5Tr3v7xE/03G7LTJCUe7Xl81E6UwqdB/OYhi3JWxosQMRIqZ2IrCXPT0szzn5BRDG1hHqsyVpa/Ldfq2kEuXe+vFmSri5p9VvP1047aG9zBERojtJshoidVtuGpxpH+2sysVT3l2YscWofdj5huYrxy2EMirv3M4l2+i3R3HpRcY/8iSYpuW3jDlcG9eyH5m6KkM8/TMhYGxmuPfe6Y7haynS5AEva6jIVErowkmylFXMu84fayD9PKobuAPg5Uj2ZONcu1n+eEXAkT2/QBvy0e9fJdt/jVC8P6msUHIXKYPtPfmHk1UweoniRpzO0j9jDRvalldalsY/riSuae+z3Ja+wx4j8eGF+Lzzd5K95jgAjTyqAH4YqDKJdm23P8dkO2CvfUEa7OdJPhpsIdgnDSjax4hHWNnWTeM9szV9p92pEbWaH4YSJcaqyQU6Y0c6ZDpbHFEL36TWD/hSVbWc1qvt/RvbqW1RtAdTK31u/7mfTRX6l5RaF6EVKPjlDdSx/IoTJU23S+JxWYLuG3soJWPY2MG3tOdDv1z6RMbIqn+7qifS8ysu4CaGZ++F9WFwX3Upe61KUudalLXepSP6q6KLiXmku8QnD4Js3sVXsQ351KkuQ0rRTTSniyAP5FE2phgZpePGLjRjqbZ8Uqg3uRJ7z+9tQpCsmXp7QoPLz6Xvw7hy81bnferxO5oH7IHL4S5TKXrm4QCoOelDB0o3h+UXAs6uq0FO+wiuLN/enqgUOo+Kef3/KLm3sAvn31xC9/+aXkm29G4qjRg5738eHfUoQ20nyQzllhopbEKpjTfUIrfN/1lVATXJG8r8yRO7ul0hP/64d/k5Xr+ecvd3y9FLXr17trxmjoJ0tKGgeEyaJUYBxFQjBVhCoSj1ZYrEgm+Ek9cDs5J7FSoliXfz91fPc3lmkhCTXRw7hUuCOzslltRZUcF6IsZr8RNal4IvXUkKySruCSGBUqX7Lf5Vr7bWRaGuxRlGN7THRrMye+yfXOhfQg/99vzMw+HV45TJ+FBRnBdpF4Y+dkJYoGmUsql0rQvXYzocCWPPnpbkGyGjtM9G9r7N6Isoyovio62fZCPHixPvt8p4UwPbtr6ZhfHCJk5q54qDneWuqXktSVC8u1KHY6lrz2Vgl5o1G0n8JMF+i+aLBdksSfsk+7v7FkXGraezmQ3dcG02WmpSpKoKSInZTQ0GqGlezztFCopKlinpXJ0Ioq1N01pJ9fiTL/zRXuWY5B+MWRbDXmGEiV4fhFgzuI2pUMJflM5oDsZIUAxdk7+pwZrxwq5TPnN0NcVj+41qK+i1daxle/OflXG5a/PnL8g7fERktSmlV0d24eS7IzQhnIWvz9OmSGQrQY1gZ3EB51qBXjTS3vnWkRZa6pDKaPTGtP86sHpuWrMo40cVWLGr5wolyOeU5oci+J+mPH1C6EG6zk2vp96R+YgFQSpPYTw3VF8qrQOQqLNwohY7yqyAb880S2zGM2eo01cl+h4Pja4bo0j6fqoWdaiarpt5OoceZ8X56SHOW6V6JoZmZyiXIadbchtg41JSzFm2xFTQbxGce6or9VcK/RIbN8NzC+EeVR7g9DqIVbPW4cpo8zpQPAjELByFqhim+yvy7X8iUQ7lZU7/akb1fYfWTc2HkOiI0V3/GrBd2tl3HdWrJR+JfCub2rMIN4hcebGnMMjNd+VqrdbmK8azCDx30+EKvClG6LB1drUi37TRTvuDkGpnUZ862sQFTfvxDra/SQsJ2av8eqF1Eju1emqODnazYU1ng2wh/uf3I1K/H24GdiTP+mwh4T48pQRVF+Y6MZkX2wXSTWsoron4RZfHxbzUx5lRRp0CQnCWi7n7Ysf9OTFmVedDKnhIUl/vRaWNKtqNQzh9sbUmWw2wH7dCT+/Aqj80xySDbPaaWp0mQvKzTjqpAc9gkVMtN1I2q8Br8/35NmythOPL16klQ6dWfm+c+MmWGtyeYEzIZxY3H7yFC8zPXngeMX9bxilbWak+QA7MOBtG5Ezf4r6qLgXupSl7rUpS51qUtd6kdVFwX3UucKkeQhLuMZC9AZ/EshK1hRp/q3cSYkTEkRq0zYJOzOkHXxFl5F/Msprz1jg6ibWYvaOq4z6HMylB5E8Qq1YtxkVFDzE2d/l3EvSry4PqNrhd+e09ZiJVzabCAVX1dyMEpzLclDUEJycB8d4x9Yfn/5gSvXsbBCOfgnT1/hb3rG5wrnAzlbUp1IJ9OiTzApxo0wA6PPVKOaOZJ6gmmhCItEWECrMnd+x7HgJv7x8Vtu7Z4pG143O0Iy/GT1RFUU3reLLSEZtEqEbBij4fVyj1KZT3vxOT39diPe4CRKeX+X8c9qZqc29wkU9Ncav8tMjfhhc/Grdq80zX1iXCrGjZzfZNXcATysNNNK0X5KdHeiZvldUQBKnbi640r8n6ktfsiiFrcfpVN6Wohf0r+URLtynqIX1dgdEkEp6kfhGlcPch36N5K+My5FdaiMXM+TijItdfF9Z8yQxFP8AzVYR4PtEqO32KModuNC1MFQFLHowGfxX6qcGZei7p78X8OVwh4yyUvqXncr6VonFeaU8pWsKGnNQ6S7PfvHyDL+hivF8ntRsseVwQ6F9dtqhrXwU6eVePzGlfg2j3emHKeivhf/sT1mhhs5xydVfmoUdsjkEVyXON4a/E7Nvk+VoX9dMa4Vqvj1slH0NyWVTin0KPuWvKO/tjJWyiGYSRTjrAvHOp8T3GxXxttDYthobJ8ZV4r1d4Fpaeb0KrcPQlooiVX+sef4dTuvDg0bhfqqkeSjBIungVh5oaAgxIxkITkrHthJlFphXZcBqUqinxHf36z4/ID3a/9f9t49WLKzPO/9vWutXn3d19l7ZjSjEYMAgwQISQyEW2LZiX0gcQyxqQDOceAcKOyqkGPw8fElSZXjOKecHGIT42sBAWwfbJJwCwe7bJwQ2QkIw4ABSdwRQhKSRjOz9+xL39btPX+8X180zGh6C81lb72/qqndvfrr1V+vtbrn6+d7vucdKIPl4EWMhezw0jgHevQ6WUfI2nVT8LLScr0xpTtbSBCF/mqNxnpB40w58bU3LM82m4/RqGZV7ep2TYx8wvl8St42b2syqFCBrBNR36jGn4mkm5OmEduH05BYM/WZy0rzJSt0D9ZpnsoZ7KuNPxP2+Rhl2UJt2zb3V0w9jcMqfxXLVtZIqFKxKochxSPrCMP5iKIBg2Vh/p6KrSP1cR+j3PKwyzSi+e0ttr5ngaJVGyvAGpniHg8KymZCtphSNmRcUS7pRagI+VzNZiJ6SvOhjGwhVNarWebtcLlmqTiZqfbDxXg8SxbllvWbWvAMRTsBtZkp+9yaZ72xJlQH56yaX16N1xhA8PSH5I4qS6jSmHiUm90yRTtf7RANq7FC3lgPn/uwbiDObAYqyiuiMmKwEE/WrESW75yvNGicyiibMf0DDeqjJJpYyOZjsk5kyQk6SaUByOeSsSqcLabovrplbceTtInhYkKZCnnHUm/6++t0vmYzM/lSg/qJbfL5uVDZz/zrta2SpB9yatuxzR4VymB/K1QGncxAde4dUDQbViXzUEJ/eQ5RfZhnvL+/bklCSxGt0yXxYPL/jIRqn82TOWUzHs/+jj6TWUjFGeWKt07abEXZiMaZ7UUrIRlUZJ2YWrcyZbtuCShgMybxcGpa8Dy4gus4juM4juPsKVzBdSaUJXlHkWYJW/bLeuR1LZsgOXSvKaBREW0GdXa+RDKBekV0OraV/A1TZ0e+yygzJahKTWXRCqpmRdQPnp96RW0zokxDSoLAYEUpW+EXY6pIGRMPhaQvpsi2TZEF8/6WDauw1l8RoswqpQ32hbdVV8omJFvm9f3y+n7mkgHtZEhRWR++vbFAWUZE7YIij6my2BTmcvSzMyLZME+mlOatjArGK2zBPIpSmv/19HqH3qGUa+qnAfhK7yD39JcZlglZFVOp0KkN2SpM4c2qhEqFBKuqllcxzSQnL2MWmvbrfLg/Ic9j6s2cvjaBmKIjVHGoQLMSUdvW8ar+kco4klctV1TGP2uTvlI0ZewFLJqm0if9ytSdyvyoI6Vo62jLqrVt5OQtU9fSLfslnretzcbRUTax+bnzjnkkR7/Mh/OWvxtlFTQjugdj5u8xDyiYulmmluRRNoUqCRWSiknFpaxj1az6y7GtbO5WDEJiwWj1e9EUaltK0Tb/b39fMlaZo9JU5eFCTOe+jOHhhDiTsQ94dICizI5hWRNQZbAazlUnYrgUUQyUVqFIYUp486QpQRtPrFGEKn1FQ0i75nnO43Dg1VQyjWLqW5brWt+ozNvdGXntrHpQWbe0hyizbo0UsVrPjkm6EdSnwwnD+Wic32lKlY6TNrKGfQZHKk2cqan0JVZ1rim0TpZjL3WyXdJbiUn6SrYgNE5DpkL9jOW9jo61Rqa2SwXDxZjmQ/lYHc07QY2VhPYDGVtPbCM6qbxXJYzVqmRoObBlOpkVSfoVw8WY/rxQ6ymDRaF9wrzLIxW5CvXu7bZ5X7OOUN8cZXBbDrRVVqoYLibknWTsj5XK8m01hrwVZjXqlm0Kppb1VyI695dkc9G44lY/JG7UtyqrJDasGM5H49zjeFBR2574A7sHatR6FVGubB1tEBWM+wBWaU4qu9ZqXfs7WnVetlNa92yy/ZQFsnkhGVjCQn1rUmFLKvvMJ4NqvB5gpISnm6V51ms2+zBK6BDVsTIYZ6aCpxvB47xgPtxRBnbeSYiGJbVCKeYb1NcLhksJjdO2+n+wL6VoRiTbQhn8zv196fhc5u2Y9j3bdK/p2LWZV4hCfd2eXzZiokHJYF9qx29ola10JRmrvFIqqFWsrBJTw/NWNFYWq8TWYFR1AbHPTePBIVVIW4hKJelm5HMpwwMd0oe6DA+2x8om2Gch2RqitSa1bknejmk8ZBfb5pM6tB7KLIsbSLYyS+tYhEb47tDE8nZFdZy3HOU6rjCYbpnP1P6fkzDbKOP8ao1sjUmtW5F3zLee9CuqeFIdsIotd7ms2VqIqFCGh2xmJpuPEW3TX0nG3wWo2rqDkJeetyyFIh4UbF9dDzN5jGdWBqspta75cPOWIInNStTPhOtt3hTrqLDPskaWRT1S+6VS6pumSkdZRfdgajOBzZAqUpus9ymamCd5NaFKonE1teFyjc49PaKhZYoPl+z7ezSbGOUha/0CuILrOI7jOI7j7ClcwXUmxKai1uoFWRayVRWKvmWv5guKdAo0i9AkKEWdnGqrhiQV+WJFvB1ZZidQBP9q0VKqhtK6P2K4pORzFczlVGq/rJPtiKKpJAPL3Kx1BSmgWAom3LSyDD8Nv5DFkhpGfUi6EZIHVSyB+qmQR9sIvs+mrVaNY6FoKpu9Bvd0l6mYKCgrnS73Zwl5kVCupSS9iLJTjb3G6amQ41oKSdfUgnigFEEl0djyfJO+KaL5do1v9xc5UDPD2OH6Gco04nTeHr9mJEq/TKfum//WjrtQk4pcI7Lw07pdy1gfNCmriDxLiE8kZAsVUWZ9yOanqtHNmUeyd3CiwMZD8+rlHSEe2K/yeACdUGWsdzA1ZakZ0Vwr2D6Y0HmgHNf/zkMOapVYhmcVx5SpWK5tMjrXmMrUA+lqWPXO2LOosamr2XxM3hKGi9DvxuPVsYPlCKqQddwxb/Vo1S2MvKHQ22/3298qGC7GNDYm6kJUmtLZvSo11Sysuh+pfsMFoazFVAlsHk3JFoXapo79r/HQ0gtqXSVvCfUNU5tGq4DztpDNQaqWCDB3X0ZZi+ntD3XSF20Wo3FKxjMSSV/Hq5CTvvl78zkhKk0xSwZKf1809p1HOWxfFdsK7U7ISi0mXuPGesFgOSE9PWDjujmbZUlN0bH3YNnApnzaSufBXDSuxjbt3zRvJfRWE5ohxWG4UiNvmyIW5RN1p3mqelg+tVSmgtY3lKwtLD7UZbhk5vftqy11ICoto7ZoSPDU2r6SgamYcabjRInRsbJrKRr7E0ce5cGieeBH77NK7JrprUbUN5S8aarY6Dja/ipTdkN1NplaUR6V0D1Ut6zh0q61wcKkklkt5B33l2MaG5Y9PFiKxmpUrW8+01pfg0fT3leVRgxC1aV4aNes+XMjsjlTwkcKVNI3L3M8VJrrpb3WekUaMp17B+s0E7vWiqYEJdkqewH0Vurj66OqCXk7RpqTPhJB4+SA7WtaFIsx9U1TzaSaJFpomOhprNvnJc5s9fsoOxWBspFQWx/Qv7ptSme3sjxSYLDQML92YVXYpLLrejSzYyvyk0kSRC1iuJCQhipiZRqRnupRHKqHz1oc0jNgsBy+E0MWepTZ3yzMDtVC2oYlPVT09tdIy8ryfDf7REv2H1FViygbCfm8fd8Uiw2bEQoKb9GOSM8U9A93aDzUt4qK/cr6jc0yFE2rBNk8mdM/2Ar+Un2YshhvD9HllN6BlNq2zQ4UDXs83SwtZ76ApFcy2BeuySAi95dsFmaUcV3bhqjSsce3t5KY915MXU23KvJ2NM6m1tg+I6NrOx4SMnChESop1rq2JkCyAo3NY5x3ovHnOmtHRGWY3SiguW656oOlyXlorNvrjvzJ2f6I5unw//F2TraYgsh4vxpP/g+wWRehsW7KdG9/Qt6yhIhsLiRWbCrZgn1+iobNEjXOlNRPhVzlNKa/vzZJWjkPruA6juM4juM4ewpXcJ0JYv5MEaV20n7V5gdyNIqICqFsT6qGaC34bRTiXmS5kWlFORcaRDr2yJoqWDJcFiSH6lCGVgLNUIEmVoig6kdorOSR+djqJ8Lq2KtzqqZStiuioSm5VX3Sl2gYvHOjSishC7dshZ+9wTc7UjlW57ocam2wlrW4d2sRgE4tY3Vhm81Bna1egmZKNJBJikIEcd8SI6LCFMqoZFx5KhqaT7lKdPycB3rz3B4dBiCRitV0i2acU5OSWCoilE4wNNakpBYVxMEoWo9y4lBZZhCU7vVmmweH85wetimriC1tUXUKqmLkbYqobUf0DyqNk0Lej8jbpqAD1NeFKLdzls/ZL/RRAsD0eyhrQn85sYzZfYll7QJVKpRNiPJRNSc7BlQTH2StC4Nly0gVNYW7dbIcrwqvdc2HXdaF/or1u78q4wNZtKD1gNI7KGOVs0qjyap1NS9YNh8FJcdW/45Uv+GCKX1Smr+rTKNQXcmUBwir1nu2EnqwTxguma9tVHnPPL/mL+7vm6zsHSuLdbveal2laJk/rGwKWdh/0VTigXnpyoaQ59A6VY1VGsuJNCWjaJiaN1gwpX2kskQ5dK8KCnCilHXb1joxep+2Anvjujnz5So010qmJiWoahGNtZLuwYQ4h36Tsce2tqm0TpnHMG+PPI0TlXm4EJkqHLztRVBGkUkVMdSuiaJp57OqQTnXGKujRdNmNWrdKuRe2uzB6DhqFPZRs0qFrYdKNI7Hvs3NIwnN0xXpljLYF1E/UzFYjqhtTSqJlXWhsa5sH46of6sgbyXUtnXsaZz7tgZ/bWSK05mSrBWPj1O6WdJfia2637Za/mw9JmuPDmTwtYeJlvqZgt7+dJLiEN6nqVUSfKJ2/EaKWDycqGFlavnNUanjVI3aZsHWkZSiLrROmR+6dl9Jb39YBxFbtTKN7Xg1T5V0DyRsX52O9z9cFqJNu47TrZK4X7J1JFTmqwmD1YZV8gqJJ6P80nGWeMc+E2VqqqCUStLNxxmx9bUh20eaJD1LACiaViEvKke5y5avnbfMY9/fFxPljP3aUlmaxHAuAsxr3X5gSG9/On58eKBtal/wmuetGslgcq5HvvxYlHRbx2rfKAUmyoO3X8x7LSXkB+bH12vRiiBKyFsRrRP5OCd4tDK/TIXeAfOlR0WDohUR9yvyju1fCvOZj/ozWI5pnipAJt74uZMZVT0Zr12o5xVZKxnn2OZzsfn110v6q4mp/+HzYdcztB8sx4kCyXZO0U7GlRyzgzGdBwq2rq5ZSkVTGCwJw7DeJMoA7FwUdaFe2TU72JeMq/uNZvny5abNvpweUDRb4zUKlhCh9FatAmhtsyTaN5nVKOsSUhcs07msR0A8Tkko2gnDxYj2/TlVElPV7HvC+kbIYRfyZkRzvWTrkB3fKp3MrMRZxWA5sTUCIdu8ioUoZLYPV1K7HuqPrNG6gus4juM4juPsKVzBdSYoUIUKWiPrVVIRD4S8o2gcUgUqgeB/jZOKqhIYhtSBuRztJRBUWYDGSSFbEfL5kmgYUWsUZGsNaov2876oxWglVLXKlN/tBK0p404UAiUQg+RClYac3KBcNk5bYoJUo1/ZlmQwzsIMf6uaEg+FRpKzmPRYTHok4Sfj7Q9dRZqUJFEFtYoqFdL1mPh06EJLgwpj/uCyDqiMPY39VVN3R1nByWZMPS4YlvYRWy9STgzmSKRkUNZYaWyzWOszDObGZpzTijJqUUkUfkpHMlGpAVpRxmKtT7eos9jss3Ztn0igykMahUDvKqFoK7VNYbgY1MyRHzG2f/U1pXvIkiDyTmS+VyAeKGVTxsevcboim5/UGJcShkv2QqITFS/vBJ8zlucaZ+aHbqyZSrdxNJkoZmeUpKe2WjiGzn3K1hNkXHs87gvZvCnNGlm7oiU0To88uqaA1bbMl5Zu5PRW44nfMKQGRLlStCPq67Z6t7FWjj22o+MxVoZSU7Zq2yMJA2qDSU5j0YgoGtFYRQFTqsfV/voVKhNJL87sGOTzlkla1WCwOKmk1l82v6WKHVuNbDVyvGHKMUC6XTJcSsjqoyQTBSYeNeZMEdEk+F/nhXhQjdWZkRdTJSjFLfPTjlSaxpmKwaJVkOut2qruKNfxsTGlNlz7Tbvm44GpKKPkkMbJIWvXt6hvKt2rgqKePPx6mbuvMhU8M5VQ40kltDI1pb+qhWzNlcQya0dev4Ylg0SFjvtdptAc6th7l3ciU5tjU7WTvtJ+IKd3oDZuX6Yj77jNTozUIAjV2kaqdm+UmAHN9Wp8rpKBXYdVIubv3NTxbEBRt89Y0leGC0KS2bHL2pMc7yoxH6NloJpS2rmnN1ZHRaHWt5SI2nYJJHQP1iiDulrrhzzkvnnauwfMh9kPlRqlDMkpiVVei7KKshGP1dnhXGwV1LYqmqdKBsshHaMdTRIoUtABdL6dkc8nlGlE61uDcZXDbLkRsldtVXsVm5I/8p5qBO0Hc/orCa0HM/rLsanywdOtYm2r1JJUVKB3oD7JyR1YwoXGYQZuIYBB+rsAACAASURBVFQBbAiLd9n/E92DKZmaAkj4m27bdWyfOwVqdv5OF3QP1hiupERBeRSFxoM9+vvmKVoxrXu32XrSHO17e+E9NOnvi0lPl2RzcfBOR5OqdtuWYWwr+M3Dn/RKWIrH14PkFfl8aucjgqRbjPN5IfhmtyuytnlsR37w6aqfZcPSEzS2ymTZfDyeRZISevsTywoeKo21gv5KOp4tbd8XZgoadnzyTkTzZIaUCVEeZgSXa0S5sn0ote/XtmUqjxXa1NYdxJl9djQZ9XM0k2W5yclQGayakpr0ob86GU4mAyWfj8mb4Ty2mCRqhFmcMg3Vzgq174e2kG6OZncsE18q++zGQ5tNK5qWadw8WUDHvPePhCu4juM4juM4zp7CFVznYUQl5IXAgWy8bbC/omqWSLNEIqVSGSsqC3N9to9WJGrKryrQKInScuz93HpSZYruyMNXCfF8jgYVUPPIlOFEidOKSoJSG36VUqtof7VG92hpOYeJqcmNk/bLvXeV+X2TnjD3bdi+xry+cd/2HxWQL1RUdcivMlX1TNGiHQ/HlcTSpGTta8u0jm6O+zk8ko1zcKUfUTaF9n0R+RwkXRisTDy4eaci3YggUuprYmkFVcxK3coKFWlMv6zRjHMeGnS44+RVLDQHnOmZwbVUYbnVR0TJypgDrS2yKiESpZXYuWjHGblG3Le9SFFFJGlJ1k0hJF5ILhQLFZKNqsaZbzM9M1lNbaot1NdNLcybMlnV3oPOfRXbV0fEfVPC081JlZ3BsmUKR6XtP+nar/p8frK6P5s3VS7vCPWtknRbOH19TFUfZTIKi18v2XxCElZ9m8d0++qRGmUV6NKNkEwxtNcY9TEqzNNJO2Lhaz3Kpqk+44pymYRqa5jKs12RDIP/OnhgywYMYxnnBUshNB/SsYqssZC3GFdcSvoV3QMxVRL6WAUVp2E+2+1DifnKgorTfAg2nmzXb+sB68dwScYVmPKOqYjx0BSbdMsyLzWavA+Nhfb9FcMlU+Q1VdJTEYMV20d9zdoWsb0HTWDrSI2lUNFouFSjCokVUWEKXX1NLesWUzuLhlUJG1XAqmpC87R9HraeYCv1pYL+qs18zN9dMffNLuvXdcLnrkHeFvPWlyFLM47G1dQap5TuAUuGWPxGnxPHmpZN3Jqo7f1VYfGugu6B2CrIPTg5jvEgpBrss3M1Oka1ruXj2vUQlPYIBvsiFr+WsXFtOlbEGmcqylRY+Oo2+fUdU4gXI4pQ3GogsaVNdO2cdg9YVaf6VvhM9JXhYsT8PQWDpZiyZorqdIZtM8x0JANTWqVUNImonw45tnXziff32XfI/N0l0aCgWrI3mm7l1M9EZKFyVpyZQjnKwS0aEel6RtKL6e2vUzQZZyuPrhXUrrPBQkR+Vj6oqFImEnzQpkQPFyy9YjTzUaZQ7BeqWp3mKatwly+3qK2Zuplf06ZKhKJt6SdRYaraKB0gGShxr0CjhOG+mqmUwQcLMAxJHs3TFb0Vu12EBBGA9rcHnHlKy1IWuspgRWidUPqrEd2rwmr64Lcezgtlat7+4UJE2Ri/U6sYl0KZmtIcDyyRwK6niu0n2jVQNITNp8xR61ZsPjkk24SvKCmhvlVQ2xgwXGmOPbjjSoWYpzseWs52nE2u6bKVhMzZ4DteqNn/eaNZxBQ639zm1I3ztB8qyTrRw/zc8QCaJ+xYtE8UdK9KyTqTyn2th6qQJKLM3ZszWLYKdaPZgmzRZhctpUEtg7xXUKYNtg/ZcSwbowxaS4rpHkotFWj03SOwdXVMY81mLqK8CtUiw4xW0/ojlVA0GCeTjI5f0bJrd6Tij73YMjp2EqpFQnd/QnOtsnUNZchxBtr3DRg8s4VGsN2K7fslFbqLEq6nlCgz5feRcAXXcRzHcRzH2VO4gus8DCns52bSCBVszjSIcoE5JU4qiqGt3h79GhNRmvWMbr+OiFJ2axApaaeg3wg/KxslDGPiboTuH5Jv1Il6EeW8qUWSKFpEpA8mZPsiSCtTTsMvwqReki2FX5dANDDf3GB/kAeCChflQn8VC+EVqG2H1aLzaikKNSWqVdy/toCqkFcRaRwyFCuhmi/odU1V0U4BwxhphT5uxUgl9A+GFfJDIJry+YbKLGVpqt9wEe4+sY99je742KZRwZmsyaCssbnVoqwiNteCelAK2ULCcLtOa6HP/fctk85ltBoZSWzHsTtI6W82kH4M7QIthGStNl6dGuVCtlDROBmRL1g/kz7jCjT9/aacSAnptmVULn25x0PPsT5IaWqTFLbSHzUPZPNUNTrZ4+pttS01j1fbfnlHoTpVrTuqYGbe21rXKl9V6UT92DoSUzQZpx1Ew8m5ztum6g732ftBYeGunI1rTcJtnpzkn3aPNGnf2wdJx+rKSLEra6buaSQ0Tudk8wmDpTBjEIMMgFpIJZBQASt4bk05NFXKcltNIRkpvEm/ZPNIQm3LlGoV6Nyfs3V1bfwek75Zx1unSjavMY/rSM3SCJonbRW4lLB9OEYKu54aQUHtryT0V0fZzxVUpv5EYVV681TF/De2Wb9+zs5puOYH+0ylGSxEwXdp5yjKlM4DBetPtj7O31OSt8zf2l81pa3WregeSMbHIMphsKxIZf00P2x7rKbX1wsGixFb+2HublOmt69Ox8cp3VbyOcu7JKjEUcH4XFeJqU39ZftO0Qgap3IGi+n4XI6UHWrB+7dg52M0YzAy2NfXzdO3dWSS3wumjpe1mP6h0YrxIZtH2mNPY5Sbj3M4b/m2UpnCNc76TYP6n1Ukffs8FY1oLA1VNfMOShG+EsUUrqSrzH3LgpfXrm/ZqvOmjBNHhvtb1Hp2rot2Yl7XylaFN9Yq0q3SEhmAratTBqt1at3ScsBzob88UUdH+bBFy3zkeRsa60p93R4fLkQ01i3FYzhv/sqsE1FfL9HIpMH+AUvJyDuw+A3zoIpCsWRS9yjrt7dqKQJJX2mvF3SD13nhGxnDlTT4NiMap3LKtDbOty6a5hUfLESk2+YfTh8qxxXjsoWUsm7XzGDRjvPou2pa/ew3JlUT082SratrY99mMjQ/rvlYLfd1sBSPK521vrXJ+g1LIfUgfD42y/G5XvjmkO3DKZpA0Y5BGkSF+VzBvK+1no6zakdMX9PZXDxWIe3Yx2hkqvToehocMKV663BM54ESDZXJwPZz5ntaVs1sI6O32gqJMJPrMc4h6it5JwaxtBlG6wrqYW1AbKp5Y62kd7hpWeBB/ZRy4kmvde37skpkfBzSbZshyTo2g9FfqVHWhSIffeZstk8jmLu7z2B/HRWYv8e+nNa/p05/KbZkjQ37nmuuVWSdqYgXtcpymkD7nh5nntYJ+x710SoEVql9v0aoZVWH/+t6By1nfJRTfz5cwXUcx3Ecx3H2FK7g7nFE5MXArwMx8A5V/TeP2F5B1mpUB00ekGFEPABpFszP9VjbXASF+avMpFaUEapCtt4gaudIWhHVKpr1jEEzqKFhlT8Hh1SbNZqrPWpJSTO1n4Qn1+aI5kuyJEEGMXRKVKfSDyohfso2NRXy+cRW8ccV2g01yrOgImbm9Wk9aL69kco8XFYaJyKG+yrKuEbRq1MsbFOpEIUSMfONIc3DOWe6TXrDJkmjJGrlZGtm8NJ2iWzHlG2ltmH+RY2UOJv4W6WA2tB+ZecLJWlS8ldfudb6GFdcfXCd7WGKiBJFFXFUkbbtJ2l+oknRjmkt9Ol3bel8kcVIU+kNQy31yFRVjRXZtqQKFcv4BfPElg1Tr2obluta605yJKWAtFsxjCN6ByKKBmw9sTk+95aDaopV0lP6+y1Hd/QeBytKsm2KQdmUsa+0tm37hlBhaF7Iww/yhbtLojwepy7UtpThPiEOVcXmvp0z2JdO1KjGSPFRokxontZxTiWYf6scqXqAJqZy5sEDF2e2Gr4XVphncxFZxyo9FSOrXWTKpnZNdVz6SmnVyUbe0Y1qrJ5lHcuzbK5VIcMTkm41XsEdZ5Z32tufkHdGypJSXzOP7tbhhCizxIT2iTz0KWW4ZMpE696Sh54dKiVtgx4N13TI8S1aStyLSHqCJkq6FRTLjZLB/iZxZuesSkxtGa0IzzodBotCfVNDCkXFQzfWxl7ofi+iSoX6RsnmEyKquhIPo7ESv3BXwfr3JFR1aJyCzr0V24fN+zfy2G5cm1K0hDK1ZIyiaZm4o3MzWDaVrKwJ609tMFyG/upUqkdkvvyiZakYVco4/QBMzV67LmX5KwVnnpRQNuz66O+LxokX/X0y9uFJpXZNbkw8kduHYuobyuY1Ce0HK7qHG+aDfNA6UV/LOfPkOmXD/Ob1zYqtZjxOKFj6Ssbpp6emlmGfbVG1yBLMuzpcFJa/krN9KCFv2+dj6asFWpvoR/UzJVtXR5ayoKYGJn3bZ962pIYy5BlXibB1dW3swTWPecX6U1Jbjd6f+OvBZqjKphIV0ThvtEp0/JmURBksRcRBWR8sRdQ3lc0nJBMPo0CZqinjAulWSX9/ffwdGufK9lUx+Rwsfq2kvxIR9wrAztf21SnNkwVF3dr2V2uWh7sy8kpD60TG6Wc00ASSru1vpPBqbB7P1omMrJOGdBarFjlcCB7aTNm+BloPmNc/n4uJs4nannViFu8a0j2Yms9z3jKwR/mvxTOWSLcqeqvxOL2kt7829reeubZOrR9mrpZjdNXSJkazASqCivn7NTJVtVcL/txw2Q6WIhpBOc/mJFQSs8pf9rmtTB2vbE1AmZr3tRYm+oqG5SkvfDNn89omVQKt0yXdJKw3ORBZakfP1PhRYsIo8z2fs//DO/cIw2Wo7hO6B2LzwAaxMw2fncZ6yXAhRiqlt1/G3veFb9rtqiYUbVtPEhWWMQ5htrJXMf/NAWvXtSy9Y1vpHRh5fMO6Dg3pCUOrcjg6znPfLhnORbRO2uyWxjaTpHFEHrzx8do20Bm/XpWESoZhDUK2XFI2I9r3eorC4xYRiYHfAl4CXA+8SkSuv7y9chzHcRzHubi4gru3eS7wdVW9C0BE3gu8FPjieZ9RQdWoaAcPbn9ZyBeFffM9Fpt99JCQFQnDzC6dffNdNvsNon6ENgVZq7H81NPU4tJycYH6woB20/w5a1uL1JKSg3Nb9Av72ZvWC+K4YqCC1ktanSF5Ho9TFuIvdFh40QliUYZFQqVQVRFnhvaTMFmvTSo4qVWn0oixbzfKrFoUEpFVllfbSYccaZ9hPjEpMRIl15gH5+Z5cG7eDoUKazX7+V5VQjaXoJvmM8vnQk3t8B7TM/ard/7uivXrBJ0rOLy8wephS1FYTnvklSUpfLtrHuBmmhOH55+q1xEgG1qOY7QVU+VCv5FSfMN+yZYHMySpoBubalsKVatCs1Gdc/OuZnOWKJEtWiWw0er9pG+Zg0XLVr9qzFixBVNBi5YpUEXDfrlXqZo/DygbFfPfsJXvRUvHla5q2xPPY9Ey9bVsmu957amWgRsPwmvMC9mCksTC6udzzlxbCxmI4fKrB/VJoaorG9dGpBvRWCEpG5PVwelGQbZo1Y4mvk4BNTXMqiCZSlLVJ5XMNLIc4MZpW7FfpkLzVEmZBv9pv4LI/HTtB3OGiwnte3v0brLzMNgX0zhtFcKWv9Rl48kthovReFV70bD3WSXmQZ37ljJcgPUnWwNNTPGRCta/JyHvhKqAZcQwqNWj/MdyroRYKTsR9ZPxOE9z63BCum0e4bxlislgKWIjrAhf/PI2998yx2DJHts+EpHPhyxroHtVRNKz1eAaT/ozUnc2rk3IFm2GYLiolGlE/UxoL1Pq5RIgVmlMQs7uSP2s6kBl12DehnyuompM+iClkGxNKoXFIRtz5K/VxBTNtacmlE3I22oBJxXjPODRiu5sTtDEZhKm1c1swbJnR8etv89Wv48qR+VNmz0oGgTlM6LW1bHPc+PaGlXNXqdo2L5GK8PBlC17/Zi8I2Rzo4plEcPF2rhNfyV4Wtv22SoaQutB28lwIaW/PKpaJ2MldzTzojGcubbGcNkU7PSMKWujPhTtCm1UDPeZF7i2JZYNHK73pS/nrF1Xs5mWyny23QOm1meh8mSZKsVcRboesX1VYudSJp8Zqew7L59TNp4UE/ehd6jxsOqA9U0JqR2jlAkZp1XM3VuyfbWtfB8umle4aEHSn1xLotC9qhYqZQX/bc2SAcBev5iv6MYR6RlhOBfRXCvZOjKpZNbbn9p13FWyjlDr6dj7WSVCNh+bH7wECnvdUa4zCci2+Wbzjp3npK8M50beVJulWPhaj/XrWuHcTc4DWF510rcqXEXL1P04B41HSQ5hBkPNR99fjtj/kW9w5vtspi/v2HXQX05MJW4Lgyom/DdlFTpDSkLZFAjfcUUrqP2pfe9uXx0h2AxG3hYaZ6nEZR3iQUy2GK7rRaVK7XO51hQQy5pHNOT9TmbNLHVF2Hxi05JkIqiXk1zmqgajWPCiOVGxx7najZh0U9m62rKae4cbIYNbx7Ngmzfut/SROmEWCBqnlXw+fNE3S8pocv2cD1dw9zaHgXun7t8Xto0RkdeLyHEROZ5V/UvaOcdxHMdxnIuBqD7yKjRn9yIiLwderKqvC/d/HPgbqvqG87Q/CXSBU5eul85FYgU/j3sBP497Az+Pux8/h1cmT1DV1XM94BaFvc23gSNT968O286Jqq6KyHFVPXbRe+ZcVPw87g38PO4N/Dzufvwc7j7corC3+TTwFBF5ooikwCuBD1/mPjmO4ziO41xUXMHdw6hqISJvAP4Miwl7p6reeZm75TiO4ziOc1HxAe4eR1X/BPiTHTzlbRerL84lxc/j3sDP497Az+Pux8/hLsMXmTmO4ziO4zh7CvfgOo7jOI7jOHsKH+A6juM4juM4ewof4DoAiMiLReQrIvJ1Efn5y90f5/yIyDtF5CERuWNq27KI/LmIfC38XQrbRUTeGs7rF0Tk5svXc2caETkiIv9dRL4oIneKyE+F7X4udxEi0hCRT4nI58N5/KWw/Yki8lfhfP3HkGSDiNTD/a+Hx49ezv47E0QkFpG/FpGPhPt+DncxPsB1EJEY+C3gJcD1wKtE5PrL2yvnEXg38OKztv088N9U9SnAfwv3wc7pU8K/1wO/c4n66FyYAvg/VfV64HnAPwmfOz+Xu4sh8P2q+izgRuDFIvI84N8Cb1HVJwPrwGtD+9cC62H7W0I758rgp4AvTd33c7iL8QGuA/Bc4OuqepeqZsB7gZde5j4550FV/xJYO2vzS4HfC7d/D3jZ1PbfV+OTwKKIXHVpeuo8Eqr6gKp+Ntzewv5jPYyfy11FOB/b4W4t/FPg+4H3he1nn8fR+X0f8LdFRC5Rd53zICJXA38PeEe4L/g53NX4ANcB+0/13qn794Vtzu7hgKo+EG4/CBwIt/3c7gLCFOdNwF/h53LXEaa2Pwc8BPw58A3gjKoWocn0uRqfx/D4BrDv0vbYOQf/HvhZoAr39+HncFfjA1zH2WOoZf95/t8uQUQ6wPuBN6rq5vRjfi53B6paquqNWDn05wJPu8xdcnaAiPwQ8JCqfuZy98V57PABrgPwbeDI1P2rwzZn93BiNF0d/j4Utvu5vYIRkRo2uH2Pqn4gbPZzuUtR1TPAfweej1lIRsWUps/V+DyGxxeA05e4q87DeSHwwyJyN2bR+37g1/FzuKvxAa4D8GngKWHFaAq8EvjwZe6TszM+DLw63H418F+mtv/jsAL/ecDG1PS3cxkJnr3/AHxJVX9t6iE/l7sIEVkVkcVwuwn8AOan/u/Ay0Ozs8/j6Py+HPiYesWly4qq/oKqXq2qR7H//z6mqv8IP4e7Gq9k5gAgIn8X8yDFwDtV9f++zF1yzoOI/BFwC7ACnAB+EfgQ8J+Aa4BvAf9QVdfCIOo3sdSFHvC/qerxy9Fv5+GIyIuA/wHczsT3988wH66fy12CiNyALTiKMdHoP6nqvxKRazE1cBn4a+B/VdWhiDSAP8A812vAK1X1rsvTe+dsROQW4GdU9Yf8HO5ufIDrOI7jOI7j7CncouA4juM4juPsKXyA6ziO4ziO4+wpfIDrOI7jOI7j7Cl8gOs4juM4juPsKXyA6ziO4ziO4+wpfIDrOI7jOI7j7Cl8gOs4juM4juPsKXyA6ziO4ziO4+wpfIDrOI7jOI7j7CmSy90B58phZWVFjx49erm74ewxqqq6cKMpzpw5M3PbBx98cOa2R44c2VE/drLvJJn9q3R9fX3mtvV6fea2AGmaztx2a2tr5ra7seJlo9HYUft2uz1z2yzLZm67tLQ0c9sHHnhg5rYAO/m+3sk1utNjd9dds1ep7XQ6M7ddXl6eue3JkydnbgsQRbPre6dOnZq57U6+73bSB4D9+/fP3PbAgQM72vdu5TOf+cwpVV0912M+wHXGHD16lOPHvbS989iyvb29o/Yf+chHZm77K7/yKzO3/bVf+7Ud9ePNb37zzG138h/xhz70oZnbPulJT5q5LexsEH/rrbfO3HY4HM7cdqc/aHYy8CrLcua2T3ziE3fUj+c85zkzt7333ntnbvuKV7xi5ra//Mu/PHNbgLe+9a0zt93J4Oi6667bUT9e+cpXztz2ec973sxtf+zHfmzmtm9/+9tnbgs7+/H4zne+c+a23W535rbNZnPmtgBvetObZm77xje+cUf73q2IyLfO95hbFBzHcRzHcZw9hQ9wHyUi8hYReePU/T8TkXdM3f9VEflpETmnHCUi7xCR6x9h/68RkUNT928VkWPh9p+IyOJj804cx3Ecx3H2Fj7AffR8HHgBgIhEwArw9KnHXwCc1xCnqq9T1S8+wv5fAxw61wOq+ndVdXajouM4juM4zuMIH+A+ej4BPD/cfjpwB7AlIksiUgeuAz4LdETkfSLyZRF5j4gITBRZEYlF5N0icoeI3C4ibxKRlwPHgPeIyOdE5GFGHRG5W0RWROSoiHxJRN4uIneKyEdHbUXkOSLyhfD8N4vIHZfouDiO4ziO41xWfID7KFHV+4FCRK7B1NrbgL/CBr3HgNuBDLgJeCNwPXAt8MKzdnUjcFhVn6GqzwTeparvA44D/0hVb1TV/iN05SnAb6nq04EzwI+G7e8CfkJVbwTOuzJDRF4vIsdF5PhOV6E6juM4juNcifgA97vjE9jgdjTAvW3q/sdDm0+p6n2qWgGfA46etY+7gGtF5DdE5MXA5g778E1V/Vy4/RngaPDnzqnqbWH7H57vyar6NlU9pqrHVlfPmbThOI7jOI6zq/AB7nfHyIf7TMyi8ElMwX0BNvgFmM7XKTkrmk1V14FnAbcCPwm8g53xiPt3HMdxHMd5vOED3O+OTwA/BKypaqmqa8AiNsj9xCM+MyAiK0Ckqu8H/gVwc3hoC5h7NJ0KC9C2RORvhE2zhxQ6juM4juPsclzt++64HUtP+MOztnVU9VRYT3YhDgPvCkkMAL8Q/r4b+F0R6TNZzLYTXgu8XUQq4C+AjUexD8dxHMdxnF2HD3C/C1S1BObP2vaaqdu3YtaD0f03TN2+ZeppN3MWQdF9/9SmW6YeOxpungKeMbX93021v1NVbwAQkZ/HFq05juM4juPseXyAu3f5eyLyC9g5/haWq+s4juM4jrPnEVW93H1wrhCOHTumx4+70OtcXnbynfTBD35w5rYf+tCHdtSPG264Yea2H/jAB2Zue/PN3zFhc16+8IUvzNwW4NChc9aGOSc/+7M/O3Pbm266aea2M1qzxmxtbc3cdmNjdqfV3//7f39H/Xje8543c9uXvexlM7f9zd/8zZnb7uQ6Ajh16tTMbb/+9a/P3PZ1r3vdjvpx5513ztz2JS95ycxtT5w4MXPbnVxHAAcPHpy57U6u/wcffHDmtp/61Kdmbgs7u/53ejx2KyLyGVU9dq7HfJGZ4ziO4ziOs6fwAe5FRETeIiJvnLr/ZyLyjqn7vyoiPy0iHznP898hItc/wv5fIyKHpu7fKiLn/CXjOI7jOI7zeMEHuBeXUU4uISVhBSvrO+IFQHq+J6vq61T1i4+w/9cAs89JOo7jOI7jPA7wAe7F5RNMIr6ejhWD2BKRJRGpA9cBnwU6IvI+EfmyiLxHgoltpMiKSCwi7xaRO0TkdhF5k4i8HCsJ/B4R+ZyINKdfWER+UERuE5HPish/FpHOJXvXjuM4juM4lxFPUbiIqOr9IlKIyDVMyvkexga9G1hmbgbchA2A78dU3xcC/3NqVzcCh1X1GQAisqiqZ0TkDcDPqOrxsJ3wdwUrGvF3VLUrIj8H/DTwry7yW3Ycx3Ecx7nsuIJ78fkENrgdDXBvm7r/8dDmU6p6n6pWwOeAo2ft4y7gWhH5DRF5MbB5gdd8HnA98HER+RzwauAJ52ooIq8XkeMicvzkyZM7fnOO4ziO4zhXGj7AvfiMfLjPxCwKn8QU3BcwKec7nGpfcpayrqrrwLOwohE/CbyDR0aAP1fVG8O/61X1tedqqKpvU9VjqnpsdXV1R2/McRzHcRznSsQHuBefTwA/BKypaqmqa8AiNsj9xCM+MxAsB1GobvYvmFQ+2wLmzvGUTwIvFJEnh+e3ReR7vru34TiO4ziOsztwD+7F53YsPeEPz9rWUdVTM4aiHwbeFZIYAH4h/H038Lsi0meymA1VPSkirwH+KCxmAxsYf/XRvgnHcRzHcZzdgg9wLzKqWgLzZ217zdTtWzHrwej+G6Zu3zL1tO8ofxQU3fdPbbpl6rGPAc95lN12HMdxHMfZtbhFwXEcx3Ecx9lT+ADXcRzHcRzH2VO4RcFxnCuKGX3pADz3uc+due1gMNhRP170ohfN3HZ5eXnmtjfddNPMbT/96U/P3Bag05m9nsvRo0dnbquqM7fdyfkDiKLZdZZ6vX7hRoGdHAuAJzzhnEmK52Qn19JO2p44cWLmtgC33XbbzG2/8Y1vzNx2c/NCSZQPZyf93klaz6lTp2Zuu7KyMnNbodzZ/wAAHaZJREFU2Nl5WVxcnLnt85///As3Chw+fHjmtgB/8Rd/saP2j3dcwXUcx3Ecx3H2FD7AvQIQkZeJiIrI08L9oyJyx+Xul+M4juM4zm7EB7hXBq/CSvO+6tE8WUTcauI4juM4jhPwAe5lRkQ6wIuA1wKvPMfjR0Xkf4jIZ8O/F4Ttt4TtHwa+GNp9WUTeLSJfFZH3iMjfEZGPi8jXRGR2s6LjOI7jOM4uxge4l5+XAn+qql8FTovIs896/CHgB1T1ZuAVwFunHrsZ+ClVHVUpezLwq8DTwr8fwwbPPwP8s4v3FhzHcRzHca4cfIB7+XkV8N5w+718p02hBrxdRG4H/jNw/dRjn1LVb07d/6aq3q6qFXAn8N/Ulj/fDhw914uLyOtF5LiIHD958uR3/24cx3Ecx3EuM+7dvIyIyDLw/cAzRUSBGFDgt6aavQk4ATwL+0EynW3SPWuXw6nb1dT9ivOca1V9G/A2gGPHjs2eBeQ4juM4jnOF4gru5eXlwB+o6hNU9aiqHgG+CRyZarMAPBBU2R/HBsGO4ziO4zjOefAB7uXlVcAHz9r2fuAXpu7/NvBqEfk85qs9W7V1HMdxHMdxpnCLwmVEVb/vHNveytRCMlX9GnDDVJOfC9tvBW6danc38Iyp+68532OO4ziO4zh7GVdwHcdxHMdxnD2F7KTGuLO3OXbsmB4/fvxyd+O7Io5ntyhXVXURe7K3eelLXzpz21/8xV/c0b5vuOGGCzcKRNGV8Rt9J9+jV0qfHcdxdjsi8hlVPXaux/yb1nEcx3Ecx9lT+ADXcRzHcRzH2VP4APcSICL/XETuFJEviMjnRORvXILXvFtEVi726ziO4ziO41xpeIrCRUZEng/8EHCzqg7DoDO9zN1yHMdxHMfZs7iCe/G5CjilqkMAVT2lqvcHhfX/EZHbReRTIvJkABFZFZH3i8inw78Xhu1tEXlnaPvXIvLSsD0WkX8nIncEhfifTr32PxWRz4bXeNqlfuOO4ziO4ziXAx/gXnw+ChwRka+KyG+LyPdOPbahqs8EfhP492HbrwNvUdXnAD8KvCNs/+fAx1T1ucD3AW8WkTbweuAocKOq3gC8Z2r/p1T1ZuB3gJ+5OG/PcRzHcRznysItChcZVd0WkWcDfxMbmP5HEfn58PAfTf19S7j9d4DrRWS0i3kR6QA/CPywiIwGqg3gmtD+d1W1CK+3NvXyHwh/PwP8yLn6JyKvxwbJXHPNNY/2bTqO4ziO41wx+AD3EqCqJVZ17FYRuR149eih6WbhbwQ8T1UH0/sQG/H+qKp+5aztj/TSw/C35DznWlXfBrwNLAf3Qu/FcRzHcRznSsctChcZEXmqiDxlatONwLfC7VdM/b0t3P4oMPbRisiN4eafYZ5aCdtvCtv/HPgJEUnC9uXH/E04juM4juPsInyAe/HpAL8nIl8UkS8A1wP/Mjy2FLb9FPCmsO3/AI6FBWNfBH4ybP9loAZ8QUTuDPfBPLr3hO2fB37sYr8hx3Ecx3GcKxm3KFxkVPUzwAvO3h6E2Der6s+d1f4UE2V3ensf+IlzbC+Anw7/prcfnbp9HLjl0fTfcRzHcRxnt+EKruM4juM4jrOncAX3MjGtsDqPHWVZXu4uOGexvr6+o/bvec97Ltwo8OIXv3in3ZmZ06dPz9x2ZWX2ooE7abtTLrDo1HkEdvLdEcfxReyJc6Xh18buxBVcx3Ecx3EcZ0/hA1zHcRzHcRxnT+ED3EuMiLxMRHSW0rki8g4Ruf4xeM2jInLHd7sfx3Ecx3Gc3YAPcC89rwL+Z/j7iKjq61T1ixe/S47jOI7jOHsHH+BeQkLJ3RcBrwVeGbbdIiK3isj7ROTLIvKeqWIOt4rIsXB7W0TeLCJ3ish/FZHnhsfvEpEfDm2Oisj/EJHPhn/fEU/mOI7jOI6z1/EB7qXlpcCfqupXgdMi8uyw/SbgjVgRiGuBF57juW3gY6r6dGAL+NfADwD/APhXoc1DwA+o6s1Ylu5bL9QhEXm9iBwXkeMnT5589O/McRzHcRznCsEHuJeWVwHvDbffy8Sm8ClVvU9VK+BzwNFzPDcD/jTcvh34C1XNw+1R+xrwdhG5HfjP2ID5EVHVt6nqMVU9trq6uvN35DiO4ziOc4XhObiXCBFZBr4feKaIKBADCvwxMJxqWnLu85Krqobb1eg5qlqJyKj9m4ATwLOwHy+Dx/p9OI7jOI7jXOm4gnvpeDnwB6r6BFU9qqpHgG8Cf/MxfI0F4IGgBP84Noh2HMdxHMd5XOED3EvHq4APnrXt/cyQprADfht4tYh8Hnga0H0M9+04juM4jrMrcIvCJUJVv+8c297KWQvBVPUNU7dvmbrdmbr9L896Tif8/Rpww9RDPxe23w0847vovuM4juM4zq7BFVzHcRzHcRxnT+EKruM4F5V+v7+j9uvr6zO3jePZbeYLCws76sdkTeeFSdN0R/uelRCJfVHYyfu7mP24Ung8vEfn0bGTz4pz5eAKruM4juM4jrOn8AGu4ziO4ziOs6fwAe4uQERKEfmciHx+ugSviBwSkffNuI93i8jLL25PHcdxHMdxLj/uwd0d9FX1RgAR+V+AXwG+V1Xvx/J1H4aIJKpaXOI+Oo7jOI7jXBH4AHf3MQ+sA4jIUeAjqvoMEXkN8CNAB4hF5BbgN4AfAO7FSv06juM4juPseXyAuztoisjngAZwFVby91zcDNygqmsi8iPAU4HrgQPAF4F3nv0EEXk98HqAa6655iJ03XEcx3Ec59LiHtzdQV9Vb1TVpwEvBn5fzp1p8+equhZu/y3gj1S1DFaGj51rx6r6NlU9pqrHVldXL07vHcdxHMdxLiE+wN1lqOptwApwrtGol+Z1HMdxHOdxjw9wdxki8jQgBk5foOlfAq8QkVhErgK+o1Sw4ziO4zjOXsQ9uLuDkQcXQIBXq2p5gco7H8S8ul8E7gFuu7hddBzHcRzHuTLwAe4uQFXPWY9UVe8GnhFuvxt499RjCrzh4vfOcRzHcRznysItCo7jOI7jOM6ewhVcx3EuKmma7qh9q9WauW0Uzf4b3SY1ZqeqqpnbXsAudEWyG/t8MfHj4ZyPnXzPOFcOftYcx3Ecx3GcPcXjboArIm8RkTdO3f8zEXnH1P1fFZGfFpFDIvK+sO0WEfnIefZ3t4isPAb9+mER+fnvdj9hX7eKyLHHYl+O4ziO4zi7jcfdABf4OPACABGJsEzZp089/gLgE6p6v6q+/FJ1SlU/rKr/5lK9nuM4juM4zl7l8TjA/QTw/HD76cAdwJaILIlIHbgO+KyIHBWRO85+sojsE5GPisidQfk9p3FLRH5HRI6Hdr80tf1uEfklEfmsiNwecm0RkdeIyG+G2+8Oz/+kiNwVFOR3isiXROTdF3qNqcfjsK87wmu96dEeNMdxHMdxnN3C426AG8rWFiJyDabW3gb8FTboPQbcrqrZI+ziF4H/qapPx7JmrzlPu3+uqseAG4DvFZEbph47pao3A78D/Mx5nr8U+vQm4MPAW7AB+TNF5MYZXgPgRuCwqj5DVZ8JvOsR3pfjOI7jOM6e4HE3wA18Ahvcjga4t03d//gFnvu3gP8XQFX/GFg/T7t/KCKfBf4aG5heP/XYB8LfzwBHz/P8/y9k2d4OnFDV21W1Au6ces4jvQbAXcC1IvIbIvJiYPPsFxGR1wcV+PjJkyfP0xXHcRzHcZzdw+N1gDvy4T4Tsyh8ElNLX4ANfr8rROSJmDL7t1X1BuCPgcZUk2H4W3L+qLZRm2rq9uh+MsNroKrrwLOAW4GfBN7BWajq21T1mKoeW11dnfk9Oo7jOI7jXKk8Xge4nwB+CFhT1VJV14BFbJB7oQHuXwI/BiAiL8GsBGczD3SBDRE5ALzkser4Tl4jpDtEqvp+4F8AN1+EfjiO4ziO41xRPF4LPdyOpSf84VnbOqp66gLP/SXgj0TkTmwwfM/ZDVT18yLy18CXgXu5sO1hx8z4GoeBd4W0CIBfeKz74TiO4ziOc6UhO63u4+xdjh07psePH7/c3XD2GKdOXeg348P54Ac/OHPbl7989iS/Tqezo37spN/tdnvmtnNzczO39epal46d/F/o5+XxxU6qGnrVs0uLiHwmLLb/DvxMOI7jOI7jOHsKH+A6juM4juM4e4rHqwfX2aP4VNKjZyfHbnPzOxLnHpP9Avz4j//4zG13MlW8vb29o34sLi7O3HY4HF640aNomyQ7+4q+WFPncRxflP1eTHZqvyuKYua2Ozkvbmd4OLvRCrKT77Cd9PlKeX97Ff8f3nEcx3Ecx9lTXDEDXBHZmbwyed5Pisg/Psf2c5bavZyEMr0r4fajer+O4ziO4zjOI7PrLQqq+ruXuw8XA7G5CwnVyxzHcRzHcZwZuWIU3BEicouI3Coi7xORL4vIe8JgDxH5NyLyRRH5goj8u7DtX4rIz4TbzxaRz4vI54F/MrXPWETeLCKfDs/9iQv0IRaRd4vIHSJyu4i8KWy/VUTeEkrbfklEniMiHxCRr4nIv556/odE5DMicqeIvH4H7/2oiHxFRH4fq7B2REReFfpwh4j826m259u+Hd7rnSLyX0XkuaHfd4nID8/aF8dxHMdxnN3Klarg3gQ8HbgfK2DwQhH5EvAPgKepqorIuVaAvAt4g6r+pYi8eWr7a4ENVX2OiNSBj4vIR1X1m+d5/RuBw6r6DICzXitT1WMi8lPAfwGeDawB3xCRt6jqaeB/V9U1EWkCnxaR94fts/AU4NWq+kkROQT82/Aa68BHReRlwKfOtV1VPwS0gY+p6v8lIh8E/jXwA8D1wO8BH56xH47jOI7jOLuSK07BDXxKVe8L0/OfA44CG8AA+A8i8iNAb/oJYRC6qKp/GTb9wdTDPwj8Y/n/27vzaDur+ozj34ckkACSCMQsyhRAMDIGuCIqIqJWVCpgGeoKlaiFoqaiDBpalkZbKy1VKIhCRAbRpSKDRGwZjBEpS4ZMECJBNEotUAgVQhhMIHn6x7tvOBxucs9J7s0Z7vNZ66573v3u992/c3aGffbZZ/+k+cCdwFZUA8k1WQzsLOkCSYcBtV8Z7x0gLgAW2n7U9vJyzfbl3CfLLPIdpWxtbdV7yPYd5fEbgJ/bXmL7ReC7wMFrKQdYAdxYE+Ottl8oj8fXNybppDIjPXvJkiVNhBkRERHRntp1gFu7j85KYHgZyB0AXA0czkuDuEYI+DvbE8vPTrZvXlNl208C+wA/B04GLukjtlV1ca4Chks6BHgn8Cbb+wDzgJFNxPpsE3X78oJf2odldYzlzcIrZuxtT7fdY7tn7Nix69l0REREROu16wD3FSRtDoy2/R/Ap6kGoKvZfgp4StJBpWhSzembgI9JGlHutZukzcrjRX20tTWwke1rgLOA/ZoIdTTwpO3nJE0ADmzi2np3AW+TtLWkYcAHgVvXUh4REREx5LXrGty+vAq4XtJIqhnZU/uo82HgUkkGamdoL6H6eH5u+cLaEuDIMpDta6flbYHLJPW+ATiziThvBE4ua4YfoFqmsE5sPyppKjCrxPkT29cDrKk8IiIiYqhTs9leuomkw4GdbZ/f6ljaQU9Pj2fPnt3qMNZLMpmtu8HKZLZixYqm4thiiy0arttMJqDnnnuu/0o1Ro5sfGVRM9nJmrlvMpmtu2Qya0+dmMmsmT8bzfxdaZfn18kkzbHd09e5TprBHXC2b2h1DBERERExsIb0ADe6z7Jlyxqu28yMJcDKlSsbrrvppps2XPeJJ55ouO6YMX3tjrdmzcwQNDMr1Wwc7WCTTTYZtHuPGjVq0O4d66bZ2bERI0YMUiRRqxNnLZv9JCXaQz6jjYiIiIiukgFuRERERHSV9R7gSrKkr9Qcny5p2vredw1tjZH08ZrjP5N09WC0tSGVtMBHtzqOiIiIiG4wEDO4y4EPlC23miZp+NqO64wBVg9wbT9iOwPDiIiIiFhtIAa4LwLTqZIvvIykv5B0p6R5kn4qaVwpnybpSkm3A1dKmixphqSfATMlbS5ppqS5khZIOqLc8mxgF0nzJZ0jabyk+8o9R0q6rNSfJ+ntpXyypGsl3SjpQUn/2t8TkvRzSeeWFLb3S3pDuceDkv6ppt6pku4rP58qZePLNd+UtFDSzZJGlXMTJd0h6V5J10l6dR9tv6PEv0DSpZI2KeXvlbRI0hxJ50u6oea1PL3m+vskjS+Pj5d0V3m9Li5JISIiIiK62kCtwb0QmCRpdF35fwEH2t4X+D7wmZpzuwPvtP3BcrwfcLTttwF/Ao6yvR/wduArJUHDVOC3Jd3uGXVtfQKw7b2oMntdUZJCAEwEjgP2Ao6TtH0Dz2lF2VvtIuD6cv89gcmStpK0P1ViiTdSZSs7UdK+5dpdgQtt7wE8BfxlKf828FnbewMLgM/XNljivRw4rjyP4VQZ2EYCFwPvsb0/0G9OXUmvL8/5LbYnUqU8nrT2qyIiIiI634AMcG0/TTV4+2Tdqe2AmyQtAM4A9qg5N8P28zXHt9j+Y3ks4J8l3Qv8lCqz2Lh+wjgI+E6JZxHwELBbOTfT9lLbfwJ+BezYwNOaUX4vABbaftT2cmAxsH1p7zrbz9p+BrgWeGu55ne255fHc4DxZfA/xnZvSt0rgIPr2nxdufbXdXUmAItt/66Uf6+B+N8B7A/cLWl+Od65vpKkk8pM9ewlS5Y0cNuIiIiI9jaQuyicB3wU2Kym7ALga2U28m+B2hQ+z9ZdX3s8iWqWcv8y+/hY3bXNqk0ztJLG9v/tvWZV3fWrGrh+XdpbVy/y8n7sfZ0EXFFmuyfafp3tafUX255uu8d2z9ix/U4MR0RERLS9ARvgltnXq6gGub1GAw+Xxyc0cbvRwOO2XyhraXtnXJcBr1rDNbdRPoKXtBuwA/DA2hqR9G1JBzQRV317R0raVNJmwFGlrE+2lwJPSuqd5f1r4Na6ag9Qzfa+tq7OA8DOvWtrqZYe9Po91fIOJO0H7FTKZwJHS3pNObelpEZmriMiIiI62kDvg/sVoHY3hWnADyXNARpP1wTfBXrK0oYPAYsAbP8fcHv5ItU5ddd8HdioXPMDYHJZUrA2ewOPNBHXarbnUq2XvQu4E7jE9rx+LjsBOKcsvZgIfLHunn+iWtf7w/I8VgEXlaUcHwduLK/lMmBpuewaYEtJC4EpwK/LvX4FnAXcXNq7BdhmXZ5rRERERCeR7VbH0BKStgC+ZfuYVsfSCEmb236mfNnuQuBB2+cOZBs9PT2ePXv2QN5yg1u6dGn/lYqk6n25ZtJRJjVtRES0mqQ5ZUOAVxiymcxsP90pg9vixPJlsYVUSzgubnE8EREREW1pML/8FAOozNYO6IxtNxo9un6nuva3ww47tDqEQdXMzHez9TfaqPH36MuX97di6eU23njjQYlj2LBsRx0RMdiG7AxuRERERHSnDHAjIiIioqv0O8CVtLKkel0o6R5Jp0na4APjkub2vTXH75c0dRDbG7D7l9S/fS6CHkiSxkj6+GC3ExEREdHOGhmoPl8SBewBvAt4D3UpZjeQicDqAa7tGbbPHqzGBvv+g2QM1XZiEREREUNWUzOxth8HTgKmqDJS0mWSFkiaV5IyIGmypB9JukXS7yVNkXRqqXOHpC1LvV0k3ShpjqTbJE0o5ceUvW7vkfQLSRtT7Rl7XJlNPq608bVSf5yk60r9eyS9uT52Sd8oKWkXSvpCTfnvJX1B0tzyPCbUPIfe+19err9D0mJJh0i6VNL9ki7vr42a88PKve4rbX16ba+3pGmSrpT0S0kPSjqx5twZku6WdG9NW2cDu5TX6BxJ25TXb35p8619txQRERHRPZreRcH2YknDgNcAx1dF3qsMDG9WlUUMYE9gX6rUsb8BPmt7X0nnUiVvOA+YDpxs+0FJb6RK1nAo8Dng3bYfljTG9gpJnwN6bE+BagBaE9b5wK22jyqxbd5H6P9g+4/l/ExJe9u+t5x7wvZ+5eP904G/6eP6VwNvAt4PzADeUurdLWmi7fn9tAHVLPS2tvcsz6GRTU33Bg6kSoE8T9JPymu7K3AAVUreGZIOBqYCe5b0xkg6DbjJ9pdKTK/YnFXSSVRvWrr+2/wRERExNKzvWtqDgO8A2F4EPAT0DnBn2V5mewlV1q0fl/IFVOloNwfeTJW1az7Vvq69mbZuBy4vM5aN7KlzKPCNEsfKkha33rGS5gLzgD2A3WvOXVt+zwHGr6GNH7vKirEAeMz2AturqPal7b1mbW0ALKZKuXuBpMOApxt4btfbft72E8AsqkHtn5efecBcYALVgLfe3cCHJU0D9rK9rL6C7em2e2z3jB07toFwIiIiItpb0zO4knYGVgKP91O1dtPJVTXHq0q7GwFP9c421rJ9cpnRfR8wR9L+zcZZF/NOVDOzb7D9ZFlWMLKPWFey5tekNv765za8gTYo5fsA7wZOBo4FPtJP+PWp5kw1a/tl2y9L9iBpfF17vygzu++jesPwVdvf7qe9iIiIiI7W1AyupLHARcDXymzmbcCkcm43YAfggUbuZftp4HeSjinXqwz+kLSL7Tttfw5YAmwPLANetYbbzQQ+Vq4dJql+t/8tgGeBpZLGUX1RbqD124akrYGNbF8DnAXsV8qnSJqyhvseoWqt81bAIVSzsjcBHymz4EjaVtJrqHuNJO1INdv8TeCS3vYiIiIiulkjM7ijyhKCEcCLwJXAV8u5rwPfkLSgnJtse7mkRtufVK4/q9z/+8A9wDmSdqWaqZxZyv4bmFpi+XLdfU4Bpkv6KNUs7MeAX/aetH2PpHnAIuAPVEsgBlSDbWwLXKaXtlk7s/yesJaY7qVamrA18I+2HwEekfR64JfltX4GON72byXdLuk+4D+B+4AzJL1Q6nxofZ9nRERERLtTNREbrSTpBuADtlfUlU8DnrH9bxsijp6eHs+ePXtDNBVDSFL1vlxS9UZEDAxJc2z3mWeg6TW4MfBsH97qGCIiIiK6RWZwYzVJS6h2wug2WwNPtDqIWC/pw86W/ut86cPO1q39t6PtPreAygA3up6k2Wv6CCM6Q/qws6X/Ol/6sLMNxf5b331wIyIiIiLaSga4EREREdFVMsCNoWB6qwOI9ZY+7Gzpv86XPuxsQ67/sgY3IiIiIrpKZnAjIiIioqtkgBtdTdJhkh6Q9BtJU1sdT/RP0qWSHi8Z+XrLtpR0i6QHy+9XtzLGWDNJ20uaJelXkhZKOqWUpw87QEkNf5eke0r/faGU7yTpzvJv6Q8kNZ4JJVpC0jBJ80oyqSHXhxngRteSNAy4EHgPsDvwQUm7tzaqaMDlwGF1ZVOBmbZ3pUrfnTcr7etF4DTbuwMHAp8of+/Sh51hOXCo7X2AicBhkg4E/gU41/ZrgSeBj7YwxmjMKcD9NcdDqg8zwI1udgDwG9uLSxrk7wNHtDim6IftXwB/rCs+AriiPL4COHKDBhUNs/2o7bnl8TKq/2C3JX3YEVx5phyOKD8GDgWuLuXpvzYnaTvgfcAl5VgMsT7MADe62bbAH2qO/6eURecZZ/vR8vh/gXGtDCYaI2k8sC9wJ+nDjlE+2p4PPA7cAvwWeMr2i6VK/i1tf+cBnwFWleOtGGJ9mAFuRHQUV1u/ZPuXNidpc+Aa4FO2n649lz5sb7ZX2p4IbEf1SdiEFocUTZB0OPC47TmtjqWVhrc6gIhB9DCwfc3xdqUsOs9jkrax/aikbahmlqJNSRpBNbj9ru1rS3H6sMPYfkrSLOBNwBhJw8sMYP4tbW9vAd4v6b3ASGAL4N8ZYn2YGdzoZncDu5Zvjm4M/BUwo8UxxbqZAZxQHp8AXN/CWGItylq/bwH32/5qzan0YQeQNFbSmPJ4FPAuqnXUs4CjS7X0Xxuzfabt7WyPp/p/72e2JzHE+jCJHqKrlXew5wHDgEttf6nFIUU/JH0POATYGngM+DzwI+AqYAfgIeBY2/VfRIs2IOkg4DZgAS+t//t7qnW46cM2J2lvqi8gDaOaBLvK9hcl7Uz1Rd0tgXnA8baXty7SaISkQ4DTbR8+1PowA9yIiIiI6CpZohARERERXSUD3IiIiIjoKhngRkRERERXyQA3IiIiIrpKBrgRERER0VUywI2IiIiIrpIBbkRERER0lQxwIyIiIqKr/D9dM3cJQ9siswAAAABJRU5ErkJggg==\n",
- "text/plain": [
- ""
+ "cell_type": "code",
+ "metadata": {
+ "id": "HiLKl_rVUqHJ",
+ "outputId": "1a7c0344-6dd0-4228-9e62-611ad8847f84",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ }
+ },
+ "source": [
+ "# The graph is designed for a sampling rate of 16 kHz, but higher rates should work too.\n",
+ "# We also generate scores at a 10 Hz frame rate.\n",
+ "params = yamnet_params.Params(sample_rate=sr, patch_hop_seconds=0.1)\n",
+ "print(\"Sample rate =\", params.sample_rate)"
+ ],
+ "execution_count": 6,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "text": [
+ "Sample rate = 16000\n"
+ ],
+ "name": "stdout"
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "bHNJU9JUsxQs"
+ },
+ "source": [
+ "# Set up the YAMNet model.\n",
+ "class_names = yamnet_model.class_names('yamnet_class_map.csv')\n",
+ "yamnet = yamnet_model.yamnet_frames_model(params)\n",
+ "yamnet.load_weights('yamnet.h5')"
+ ],
+ "execution_count": 7,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "XCrhG2WrsxQx"
+ },
+ "source": [
+ "# Run the model.\n",
+ "scores, embeddings, spectrogram = yamnet(waveform)\n",
+ "scores = scores.numpy()\n",
+ "spectrogram = spectrogram.numpy()"
+ ],
+ "execution_count": 8,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "XN67xLQasxQ2",
+ "outputId": "9b0744bd-bc2f-4996-c9d9-c7ad2e08725a",
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 483
+ }
+ },
+ "source": [
+ "# Visualize the results.\n",
+ "plt.figure(figsize=(10, 8))\n",
+ "\n",
+ "# Plot the waveform.\n",
+ "plt.subplot(3, 1, 1)\n",
+ "plt.plot(waveform)\n",
+ "plt.xlim([0, len(waveform)])\n",
+ "# Plot the log-mel spectrogram (returned by the model).\n",
+ "plt.subplot(3, 1, 2)\n",
+ "plt.imshow(spectrogram.T, aspect='auto', interpolation='nearest', origin='bottom')\n",
+ "\n",
+ "# Plot and label the model output scores for the top-scoring classes.\n",
+ "mean_scores = np.mean(scores, axis=0)\n",
+ "top_N = 10\n",
+ "top_class_indices = np.argsort(mean_scores)[::-1][:top_N]\n",
+ "plt.subplot(3, 1, 3)\n",
+ "plt.imshow(scores[:, top_class_indices].T, aspect='auto', interpolation='nearest', cmap='gray_r')\n",
+ "# Compensate for the patch_window_seconds (0.96s) context window to align with spectrogram.\n",
+ "patch_padding = (params.patch_window_seconds / 2) / params.patch_hop_seconds\n",
+ "plt.xlim([-patch_padding, scores.shape[0] + patch_padding])\n",
+ "# Label the top_N classes.\n",
+ "yticks = range(0, top_N, 1)\n",
+ "plt.yticks(yticks, [class_names[top_class_indices[x]] for x in yticks])\n",
+ "_ = plt.ylim(-0.5 + np.array([top_N, 0]))\n"
+ ],
+ "execution_count": 9,
+ "outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAArgAAAHSCAYAAAAHR7iOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5wcdd3A8c93ZtvVXBrpIYGEEkggEHqVDlEQC4IFUJHHR1F5eBRDFZES7GJ5BEEUVATBAoYiobcAoYWEAOmk9+Tqtpnf88fM3e1dbu9297bd3ff9yr2yO/U3v92Z/c5vfkWMMSillFJKKdVfWKVOgFJKKaWUUvmkAa5SSimllOpXNMBVSimllFL9iga4SimllFKqX9EAVymllFJK9Ssa4CqllFJKqX4lUOoEqPIxbNgwM2HChFInQymllFKqR6+//voWY8zwruZpgKvaTJgwgfnz55c6GUoppZRSPRKRVenmaRUFpZRSSinVr2iAq5RSSiml+hUNcJVSSimlVL+iAa5SWWiKJZkwaw73v7a61ElRSimlVBoa4CqVhQ31UQB+++yyEqdEKaWUUulogKuUUkoppfoVDXDLnIicJiLvi8hSEZnVxfwLRWSziLzl/12UMu8CEVni/11Q3JQrpZRSSpWG9oNbxkTEBn4NnAysAV4TkYeMMe92WvQ+Y8wlndYdAnwPmAEY4HV/3e1FSLpSSimlVMloCW55OxRYaoxZboyJA38Fzspw3VOBJ4wx2/yg9gngtAKlc8BavL6eC+96lVjSKXVSlFJKKeXTALe8jQFSm+uv8ad19kkRWSAiD4jIuGzWFZGLRWS+iMzfvHlzvtLdbxnT8f2sBxfwzPubefPDHaVJkFJKKaV2oQFu3/cwMMEYMw2vlPaP2axsjLndGDPDGDNj+PAuh3NWwEV/nM+MG55onyDef2+v2QnAbdqrglJK9QvGGO5/bTVNsWSpk6J6QQPc8rYWGJfyfqw/rY0xZqsxJua/vQM4ONN1VebmLt7IlsY4H25rAmD55qYO85dvaepqNaWUUn3Iy8u2cv7vX+XyBxdwwe9fLXVyVC9ogFveXgMmi8hEEQkB5wIPpS4gIqNS3p4JLPZfPw6cIiKDRWQwcIo/TfVCuqoIq7Y2FzklSiml8u28383j+SVbAJi/Sttk92Xai0IZM8YkReQSvMDUBn5vjFkkItcD840xDwHfFJEzgSSwDbjQX3ebiPwAL0gGuN4Ys63oB6GUUkopVWQa4JY5Y8wjwCOdpl2b8voK4Io06/4e+H1BEzjAvLZS7xGUGghWbW3ijudXcN2Z+2FbUurkqBL5x5trOHv62FInQ+VAqygolYUtjfFSJ0EpVQRf/8sb3DNvFYvW7Sx1UlQJ/c99b5c6CSpHGuAqpZRSaexsSZQ6CUWx7zWPMfvR9wBojmvvAarv0wBXqR78803tfEKpgWbh2noAvnDnwGhJ35Jw+O2zy3hs4XqmXPs4C9YMvL69//7GmlInQeWRBrhK9eDS+95qe710U2MJU6KUKobV2wZuryhf/dMbACxYk33VDNc1mM6j4fQhf37lw1InQeWRBrhKKaVUite1eyhiSTfjZR9+ex0TZs1hjysf4dYnlxYwVYWV7nN33L4btA9kGuAqpZRSKVKf2gwEXQVwtz65JOP1v3Hvm22vfzb3g7ykqZz89TUt2e2LNMBVqhtLNzWUOglKqSIaiKV1O5p37R1moDSuy8TsR94rdRJUDjTAVaobn79jYDQwUUp5XlmxdZdp+17zWAlSUjzffXBBXrd376t9r8Szuxubhpj2KtEXaYCrVDecPtxgQimVgy5O+ZaEU/x0FNHcxZvyur0r/v5OXrdXDO+uqy91ElSeaYCrlFJKqQFtiVZH63d0qF6lurG5IVbqJCilimj5lqYup6/e1sy4IZVFTk35i6fpbWHCrDkMrgyyvTnBvCtOZOSgSJFTlp3L7tcRy/obLcFVKo1XV2wrdRKUUkV29T8Xdjn9mB8+XeSUlN4ry3etj9zZXlc/mnbe9mavodpbq7XbtXL16optvLayf/7WaYCrVBr5HtThiXc3knAy71tSKaVK6TO3z8vLdqIJve6Vq3Nue5lP//blUiejIDTAVSqNfI7H/twHm/nK3fP5eT/sI1Ippbpz6X1vcV6egmVVGNEMG1LeM29Vh36Py5kGuEql8fO5mXd03pNtTV4/k8s3d12/TymlSsHtod/fD7emH7Y4m2F5X86guoMqrtTP/u3VO3pc/pdPLuGafy7k4bfXsbE+2jZ9/c6WgqSvtzTAVSqNpjyW4LZ6dOEGnl+yOe/bVUqpXOxx5SPdzo8m05fstdaxzVRLvDy7W8skUF+1tf8VTlz5j/bu3D5z+7wub3a2N8U5+AdP8NnfzeMnT7Q/gfzBv99l9bZmvnrP6xxx81O8sGQLALMffY+bHllc+MRnQANcpdLIVxe4O5sTHYb+bL0QKNXXPfT2Op54d2NWJXnlbGvjwOo1JV0PCKmSTvrP9kePZzfC177XlueAGZkMXvfFP7xW+IT4mmJJVm9LX3KeL399bXWH99c9vGiXZZ5bspmtTXFeWtaxBP7fC9ZzzA+f5rFFGwD4/J2vMGHWHH777DJuf2554RKdBQ1wy5yInCYi74vIUhGZ1cX8y0TkXRFZICJPisjuKfMcEXnL/3uouCnv/zJ9LLOyH975q4FnZ0uiQwnPzpYE37z3Tb5y93z+Nn9NCVOWPwffMLfb+cl+1ki0ux4QWj37QfonTve+ujrtvHRi3ZQIl8rrq3ru5aGY1cv2+97jHPPDp7n50eKWhN798qoON6tbG2N8669vdbNGeve++iEvLt3CvBJWTdEAt4yJiA38GjgdmAKcJyJTOi32JjDDGDMNeAD4Ycq8FmPMgf7fmUVJ9ADyzzfXlToJShXFzpYEB3z/P9zy+Hss3dTInlc+wgHf/0/b/MvzPNRruXp4wcA752957L28Dr2799XlV4qbSf1ToCi94KTeRN327HLWbC98SW6q1JuWr/35jZy3c8Xf3+Fzd7zCubfP4+n38jtSXqY0wC1vhwJLjTHLjTFx4K/AWakLGGOeNsa0ngHzgLFFTuOAdctj2T2ea2WAh99ex4RZc9jRHM9vopQqgNYA57Znl3PST5/FyeSZbj/0P/cNzMEAuhp692/zsy+9LVeZ9m6zvanw1+tzO/U2sWDNzoLvM1VqvdxX8tQXfDGrd6TSALe8jQFSryJr/GnpfBlIfeYUEZH5IjJPRD7e1QoicrG/zPzNm7XxUzEYY7jjhRUArEgzapJS5WJjfZTZj+Z2M9eX9Jd6xJnK9ng71wn9zgP9p9S+KcPGb53rrBbC/E7VJXpTitobmVTbyMaEWXN4+v3iluRqgNtPiMjngRnAj1Im726MmQF8Fvi5iOzZeT1jzO3GmBnGmBnDhw8vUmpV3lqwKVVA9732IYfd9GSpk1EUf34lf4/h+4JsS+E/e4f2Y/vTJwrTj/ni9fVMmDWHo295qiDbz1Z9NMEn/++lvG/3i3e9VtSnPxrglre1wLiU92P9aR2IyEnAVcCZxpi2ZsDGmLX+/8uBZ4DphUxsf2GMyWtr6ldWdKxk/8S7G9ta7YpI3vajVD45ruG7D+76aDqdP81bVcDUFF66IXr7q2z7pV29LX99nT713sa8bas/OP0XzwOwZnvx+pPtLtCcdt1/0s7rrX8XsR67Brjl7TVgsohMFJEQcC7QoTcEEZkO3IYX3G5KmT5YRML+62HAUcC7RUt5H3b//NU9tqbOVMJxuemRjo93V25t5p21xa1XpVS2bpiT3eWimAHih1ubB1yVgnzLpSBtZ0t2/d6m88S7GuCWWn2ePsts5dorQy40wC1jxpgkcAnwOLAYuN8Ys0hErheR1l4RfgRUA3/r1B3YvsB8EXkbeBqYbYzRADcDLy7NX7cmrv4Iqz7qrhdXZr1OMYLOf765lmN/9DT/e//AbPCVL2tzKC38i1+N440Pe1c/M5fuxQay9zbU532bv3gyfyN1lisNcMucMeYRY8xexpg9jTE3+tOuNcY85L8+yRgzonN3YMaYl4wxU40xB/j/31nK4yhHi9bt5OG3S9vtz8d//eKA61xe9U7ScZkwaw53v7yyYPtoznEUv1seez/PKdnV/z2zDIC/v7m2JOdOJoMj9AXPfpB9g5/WnmM+8Zv8189U6cUS+f/O/eGllXnfZrnRAFcNWDNvfYFv3PvmLtPzWS1W6HljSzY15m+Hql974t2NTLrK6yjl2n/tOupQvqzbkVtdwN8+uyzPKdnV+xsb2l4XoiFMT36YY/eA5aa+JbebGO3aML/2uabnwTb625PAhUWqoqcBrlKd5LPZl6HnC9OGnVGOvuWpLodmdF3DxvpoHlOkMvXysq1MmDWHCbPmsLO5NPXVUjXFknzl7vkdps289fmC7KtcBjHZ2ZLgpaVbiCddbn5kMdOue7zD/JVbi9sJPtDWxV9fl20js1YHXv9EnlNSWqWuyx3NoHT2oRI/acy3j/7yBd5dV89jC9d3GB0x3zTAVaqAMrl2/v3NtazZ3sIxP3yaDTvbg9kPtzZz2M1PcthNT+ZcoqZyd97v2rtFOuD6wrUqzsTf31jDft97fJfpi9blv24ewK+eXlqQ7Wbr4rvn89k7XuEPL63gtueWUx/NrdRRqXRSnwgUW6Yjo+VSH77cnXHr83z1T2+wx5WPFGwfGuCqAa/zHXyxu+5KbZmcOjrap297ic0NXh3DP81bVfKShoGk3OpZXtZNgypjDMYYjvnhU9z86OKClogUW+tISp17IlEqX97fULoA93sPFa6aUV9SHy3MEzINcNWAd1/K6DR/m7+aD7uoKpCJrqoYZCJ1HPR/vLm2rRHN9qb2k/43zyzj4QXrc9q+yt5tXdQlve+1jgMBzFu+tcOwlqXysyc+4IY5i1m9rYXbnl1e0BKRTPQ2wN5UH+Wfb66lKZZZaa0+3VC98VKWvea8sGRL3vb9lywGF9nzykdYsGZHzwtmoLXgpFxMu+4/GZdmZyOQ9y0q1Qeklpo+8/5m5q/azvLNjbzxYe4XkP+9/23u/+oRHaYd/6Nnst7OLY+9h+O6xDud8OV2UerPftLFiEXfffAdjp48nKNmP8VJ++7G3MVeK/QLjpjA3iNrCpaWnkb++curq9nSqTeBF5du4ahJwwqWpu5c+IfXeO6DzSy58XSCdvZlKIf6I6dNHTMoo+V//fRSdh9ayZeOmkggh/2p0tjRHKeuMlTqZGTdgGvZ5kaOnlz8c8txDWf+6kVWzp7Z6209+8HmPKQovyZf9Wheji2VXg3UgJTah+ZjizbwwOtrehXcAry6ctsu0zbk2EDsx//ZNcD640sraSjQoxzVrrsSmqNme0Nptga3AOt2FrYEsaeeAjoHtwCfu+OVQiWnR8/5P57/eqt3DWMyHQzlz698yE2PvMe9r+Y+1G6+BjBQmXtpWf76G++N5oST1fI3PrK4QCkpnkKUlpYjDXDVgLS+wEFJIXy4rVnrbBXB5+/MLji85dHC1Q9NOi5vrc7PY8li+/bfsh+IYWkvusxbsCb3rof64vWgr/van98odRIAmJNl1a9yq5+fi5/P3bUApRw8+k5+q+FpgKtUHrXEnbYf6UI0CtvaqH1Qlpv3CtRIJZpw2vq8HShO+umzOa/7t9fX5Lzuis1NWS3fnxryqdIoVMOqTGysL8/qbpfel99hfLUOrhqQMm3Akq1v3PtG2+PrTx08Nu/b728dfpebcurE/qFePuLP1UDsrePBN9Zmtfycd9bzsQNGFyQtzy/ZzD/eWMtPzjmgYD26DMTPuNxc//C7pU5C2YnluXRcS3DVgFSoDuJfTGmR+0AvSpTSeX7JFlZtza60SWXmw63NOXdiX4jSmMsfXJD3bWZiex4HtegrdcbnLt6Y1fKPLdxQkHQkHZcv3Pkqf39zLR/71QsFy78nF2c/TG8h9NVAu6eGn5nI9fehr+ZZKWiAqwacbU2FK6VrybLBQi6Oy6FnBtWzY3/0dM7rTruutANBdOWND7eXOgk8n8culcrJnDzXFWz13JL21u0L19YztUDfqxVbyuMmea3fxdtFf3ytT9U1f/aD0t0g5CO4Lmf5rP6jAa4acF7JcYhK1X898355lGjlU65VHN7bUJjR0XqyfHPuDcz6iy/9Yf4u0z4swNOmP768Mu/bzMVdL65kxZYm5i7exMd//WKpk5Oxrj6nYlm/s38P3Z7Perga4CqlBrwL73qt1EnIu3vmrcppvdl57BXikSxKOk/4Se4NzFo9trDvDoayJM2Qscf+6GliSafL7uByVS6NVe98YQUf+fEzpU5GTkrV0PCZMuzDNp8eejt/bQ80wFWqDzrzVy+UOgmqk+15rPqSj8eQuW6jN91tdfbi0uJWUfjqnzLvesoYQ3O8MI1Nc3Hyz55LO2/vqx9jxg1zacxT49hiVKXq7/a+pjQ9nPzf00tLst9i+tdba9uqr/SGBriqZJpiSbbmsVRiIFmwZqeObFZmLrjrVSbMmsOEWXO45+WVXPmPd3h1xa6Df2Qi1+GiO5swa05JG6Vsb04wYdYcftrFyHClNvmqR5ly7eM518n/yX/eZ8KsOXlOVff2/97jba9/9dQSXl9V+nrWA1XCMSUpxd04AK773/rrW22D6vSGBrgqrZVbmnbp1PrVFds61GF9fdV2bnpkMRf9cX6HH9L6aIIr//FO2lbAi9fXs9/3HufgG+bSEs+sNGHOgvXc/GjvR5HpL3WYTvjxMwNmRJq+ILXk85p/LeIvr3zIObe9DHgjZU2YNafbgPeDjQ3cP381AHe/vDJv6froL19gwqw5JDP4rizdVJg+fW99cgm/fXZZQbbd2YRZc5h63eM9Lpf0g5M/51iV45dPLW3b37peljZlG2RPmDWHH//ngx5Huetrin3DEO1lSfYeVz6Sp5RkznFNr3rx6Euj9vX25lwD3DInIqeJyPsislREZnUxPywi9/nzXxGRCSnzrvCnvy8ip2az318/vZTjf/wMe139KC8t3cIT725kwqw5nHPby3zm9nlc//C7/HvBOj75fy9x+3PLmbt4I48v8rracVzDdx9YwF9e+ZDfPbecpONyy2Pvcd7t8zjpp8/y/JLNnP6L59v2te+1j/Hmh9u5/bllxJKOfwKv56ZHFrNhZ5TF6+vZ3hTn6395g9ueXc4+1zzKl/7wGv96a21Oo8pc/+/+0f9gQyzJPS/n9uOs2hWiO7dUv356KQd832sNf85tL3P/a6t3WcYYwyk/e47LH1iAMYa7XlyZt/0vWuc1GutphLZbn1zCST9N/5i8t1Lr9v7uueVs9IexfmzhhrwHNg3RJP96a21biXp3P5Q/yUPp8pGzn2rbVy6lqrdlEfzneqMQTTh5CcYLrTUfM7kh663ejJzXyhjD44s2EEtmHiz39kbyq396Ped1v3Hvm73adzFNvOIRJsyak3PDcNE+1cqXiNjAB8DJwBrgNeA8Y8y7Kct8DZhmjPmqiJwLnG2M+YyITAHuBQ4FRgNzgb2MMWnPwhkzZpj58+fn/cdmRG24oCOnvDTrBEIBi+pwgEjQ3mV+a7216nCA9TtbOOLm3j/6KCejBkU4e/oYZk4bxR7DqqkI2RhjCtZJfDEV4ziKXWoE8My3j2fCsKq29/91z/y2G8RiWHbTGdhWx3wtRT4su+kM9ixBKdiSG09ncoFHiTvv0PFMHTOIj08fTWWo45hK76zZycf8evR/vfhwzr19Xs772WdkDY988xgsq/vzpBSfb2/97DMHcOK+IwgHLF5etpVDJw7ZJS97oxB5cszkYdzz5cNYu6OFkbWRDufZ9qY403+QW1/bXXntqpN4e/UO3l6zg031MdbsaOYbJ0zmuw8uYFBFkIcuOXqXdfri9wBg7mXHMvPWF4glXZ79zvGMH1KJiCAirxtjZnS1jga4ZUxEjgCuM8ac6r+/AsAYc3PKMo/7y7wsIgFgAzAcmJW6bOpy6fa3x77TzHW/f4hr/7WoUIekSqwmEqAhumtDld2HVrIqpTuizx8+nvc3NBC0LXY0J4gELZZsamxbd1h1CGNga6dHq2PqKjo0DhhWHebAcXW8tXoH25pi7DG8moqgzaqtTQyqDLJ6W/uy+4+pZeHarruoigQtBGG/0bWMH1LJsi1NvL+hnrOnj2GvETU0xx2qwwE2N8RwjKEiaLN+Zwtvr97JAePqWLa5kQlDK5m/ajvLNzcxbewg1u2I5rVlulLl6uzpY/jHm9mN1tZXjKyNMGm3ag6bOIQFa3e23RQPqw4xebca1u1oYfdhVcQSDhUhm0TSJZZ0Wbiunofz2GJflcaqWz6qAW5fJCKfAk4zxlzkv/8CcJgx5pKUZRb6y6zx3y8DDgOuA+YZY/7kT78TeNQY80CnfVwMXAwQGjnp4FEX/Lzgx6WUUkop1VvdBbj5K+tXfZIx5nbgdoADph9snrr6JA6+YW6JU5Wb4TVhDpkwmKRjGFIV4tUV29jaFO9TlerzYWRthHDQ4vCJQxlRGyYSsnlj1Xb2GlHDYXsM5Z6XVxJNuLywdAsXH7sHM6eOIukaogmHxevrGTu4gkEVIQBCAaG+JcmOljiL1tazenszx+21G6MGRaiPJmiOO1SGbNZsb2FnS4LxQyqpjyZojCaxLWHyiBr2GlHNC/6IViMHRRCE5niSMXUVbG6MsWZ7C+GAxfTxdSxe38CidTtZtyNKOGARd1w+ffA4hlaHcF2vZKYmEiBgC9sa4wytDmMwBG2Luoog4aDNpvoorgHXGJKOIeG4iEAkaLNuRwurtzWz98haogmH83//agk/KaUK7zefO4gDx9VxZB5apZeL4/YazoHj6ogmHY6bPBwRYeSgCEn/XG+IJqkMBQgHLKJJB1uEcMAmFLAIByzCQYvV21o49eeFq3OuSk8D3PK2FhiX8n6sP62rZdb4VRQGAVszXLeDoC0MrQ7z9LeP71Xn250fUz/3nY90GAb1oqMncscLK7pc97+P35OVW5p41G8leuGRE/jDSyu73d+Km8/Iqp5mX62DlM5dFx7CEXsO7bL+cVeO22t42nlHTRqWdt7Z07NOWpt9RtZmtNzBuw/JfSe+6uHVaeftNaKmw/uVs2eW5PuwcvbMDu831kc57KYnC77f731sCt9/+F3e+8Fpu3xfipEPx0we1jZ8b11lkLeuPaUo+50yqpZ31/uN7Q4fzw0fn1rw/X5ww+mEAl2343Zd09YCv7ffwVs+OZWpY+qYMjqzc6wvWXrj6QTswrSF33tkDUfuOZSXluVvZMv//M+x7Dm8epf67aneWbOThxes4/bnlvd6f6nXkeZ4klVbm5m8WzVJ1xCyrS7rZffV37+Vs2dSH02wqT7KpN3ar+NyS/p1tIpCGfMD1g+AE/GC09eAzxpjFqUs83Vgakojs08YY84Rkf2Av9DeyOxJYHImjczA6+OxtRsc6PoifMf5M7jobm/5577zEcYPrWyb9+TijXz5j/N3WfcfXzuSA8fV8fiijR1agt7z5UMZU1fBHn5wYoxh2eZGJu1Ww4tLt7BmezPfffCdDvvfc3gVD11yNFXh7O7TPvGbF3njw74z7nl3fnrOAXzioLGlTkaf9u66es649fmeF8zRY5cew2dum9f2JCHdj3brObJy9kyOmv1UXjo6T/WLcw/krAPHpJ1f6HwA79iSjsv6nVHGDfGuF42xZIf+XfPl7e+d0tZ7RecbCijMD/1Rk4by54sOz3q9xxauz3iQileuPLHDzVA2N/hvfrid6eMHl32Q89vPH8xp+48s+H62NMaY0csnll19t3qSeoNTzP0C/PLJJXnpOaSYujvW7hqZaQluGTPGJEXkEuBxwAZ+b4xZJCLXA/ONMQ8BdwL3iMhSYBtwrr/uIhG5H3gXSAJf7y647ex/T9mby07ei4RjMHg3QT/85DQuf3ABV8/clymjazlyz/bSvtTgFuDEfUd0eP/Mt48nErQZOSgCwKn7jWDSbtUs3dTIzKmjOGZyx1JFEWm7S2stVTxhnxEMrQr12Fq4JxOGVvWLAPeioydqcJsHhS752mdkLW9eczJPv79pl/Mi1ZIbT28bfWy/0bV5C3DPP2J37n55FWdMHdXtclNG1xa0RPvNa04GIGBbbcEteL2b5HO/VSGbaz46hUEVwYyCgB99ahrfeWBBzvs7fu/h/OycAxlcFcp5G4fvMTTjZUfURthnZA3vbfC6msrm6dX08YOzTlsxXXjkBK47c7+i7W9YdbiX6+f2mff2N+yuCw/Jed0vHzOxzwS4uQbxrTTALXPGmEeARzpNuzbldRT4dJp1bwRuzHXfIkIo0H4innPIOM4+aAzBDB8ZhWyLMw8cDdChS6TWbf/8Mwfy0V++wNGT0z8WTzW8pncXo2IZNShSlMEkrv7olILvQ+XuLxcdxqETvSoXliXdBrcAQduitdbAmQeO5j/v9r7bsPMOHc/1Z+3P9Wft3+tt5eoLh+/ODz5evP0vuv60jJa79bzp/OyJDzh7+picAtzHLj2GHz/+Pr87f0avu7Krq8wuUHrs0mPZsDPKoIpgr/ZbbooZ3ObD/KtPLvo+z5g6ko/ss1vO60cCmVVl6w80wFVZ6RzcvjjrBJrSjI/+wY2nd7ut/ccMYt4VJzKitriB65TRtfy9gF3mPPyNo3v92Ev1PStuPoOr/7mQLx09kT27qQeciY9OG80lf+l9h+w3f2Jqr7fRW5kGt6MHRVhXxFEGzzxgNGceMDrn9fcZWcsdF+RektbZy1ec0GMf3S/OOqHtdevTMFUavzj3wFInISe9LT0uht6W3LbSkcxUr4ypq9il4U42Rg6KFH1Agi8dNbFg277kI5MYnGVpjCq9iZ2eMORCRLjx7Km9Dm7zJddS0wPGDspbGrK5ef3GiZN7vb+3ri1+iVq+jBpU0eX0E/fZjVeuPJF/f+NoxtR1vYwqvu7qsxfS0ZPSNxJWHWmAqwacQt7BfvvUvbEt4U9fPoynv318wfbz1P8eV7BtD0SPXXpMqZOQd5+YntsP8HdO3SdvacjmHMjHk5xsH/WXm+ouGsz+4rzpjKiNsP+Y/N14fDMPNxP5cMzkYbxz3SmlTuem1/gAACAASURBVEbWPn1w6do+7DWiPG6gC2XuZfn7bdMAV6kCOHryMIb0otFJdz4xfUxbbxMqP8K9rJf2+KXH5ikl+ZNt7yKt8lmvM5thVT+yd+71CvuLv3/tyF2mdRX09tb4IZU9L1QEZ08fQ03EawyYr8fSxfDDT00r2b6nja0r2b6LYdJu+ftt0wBXqQIpcs0L1Uu9KUnae2Tu1XTSuaZEjQi768OzkIpdVSnVWQdmVxf37BxLx3syudOP+9zLCnPjdND48giS+movMKX8rqbrW1ntSnNKqQIp1CUwmy6FVOZqIsGyqqrwpaMmlGS/hXryUM566uGiszOzDIgzJSJ8+5S9AK8aUmqH9vk0tKpv9EjTn/30nANKnYR+TwNcpfJk7OCODUAKcZc/uDLIp2f0zVKPvmBSGVX9EBEuP23vnNYd2osgtZSt8/cpQEl4Jj42rfs+gjvrXNKaT5ecMJmVs2cWtBrSoMr+1b1YX9S573cF5+T5t00DXKXypHNL/EKU4N509tSSPh7r73IZFvRbBWyw87XjJ+W03r0XZz+aVjnobhjpnnzlmNx7R8n2nBo7uDzqsKrem5ZlryF7Du99jytQ2n7dp+axwWI+XTUzv9WyNMBVqkAKEYdqbFt4M3ZPP9rTG9fs2g3VxcfuUcjkcF8OwWpvuu7Ll9Q+WzP1v6dkX2J96ARvMI0jJ2U2YIxSqT6ZZT3gcg0Os/GFI3YvdRK6lO+BS3SgB6XyxJiO76UAZbhaclR4D/z3kV0OG/vvbxzNkKpQW2vvjfVRFq+vz7m3gkwd1kOd64cuOYob5izm1RXbAPhrGZTePvjfR+bUZ2suDWju+6/DaUk4WfXYoEqvqx4jSuH0qSP53kOLMl5+XIl6oAhYwmtXnZSXbZ06ZSSXk/vw1IVw/N75r7KhJbhqQPrledMLvo98l7b+7atH5LUvTJWdznk/ojbC8UXq2urqmfumnbf/6EH87vwZANz7lcNL3ghx+U1ncHA3peA9aS2RzZSIaHDbBx00PvfvSF6ZnhdJddExhX1i05VQwGLpTWcwOE8NQGsryu98+cMXD837NjXAVQPSx3oxRGc6FaH8jPH96YPHcveXdj3Z8zHalsrM29eWV+fzXz46ff1SyxIGVXh9iR6xZ+l72OjtQCqHTCyTwEcNCDWR7B6L5/Mx+h3+jWlPAnnuuq/c2nF8cMPpBdmuBrhKpehNB96zPzG11/vfrSbMjz59AMemNLapjZTf3XZ/V26tzEWEFTefscv0UA6N4srdt07ci598+gAWff/UUidFdeHrH9mz1EnIq3wVTOTiyEmZ3ZAeNjG7pxp9wexPTGWfkTWsuPmMgvXt2/+ujkrl6KwDR/dqCMah1bm3ij19/5EAnLBP+yPvP3zxEH74yWn8z8lev5g1GugWVWt/pMNrwrx+dX7qvvWGiFcH78H/PqJt2qLr+18QGApYfPLgsQWv26xyk8+hnAe6TKvWnDNjXIFTUlxBWzj30PE8dumxBS1N1iuIUr7p4+ryerJlMyJUdTjAS7NO6NB1TGr9zi8elXsXSCo3l5wwmUtOKFwXYLkYXhNmeE244MOa/tdxe3Dbs8sLuo9MfO34PfnNM8tKnYwO/u9zB5U6CXlRGbJpjjs5rXvKlBH8592NeU6RSqe31X7KzXOXf6Qo+9ESXKV8x+W5wVDQtvhmFn2kjq6rINgPHzmrvudTZTKE6rdP2Zs3rjmZ68/ar8v5//7G0UVOEew7qrbo+yyE3vQ5/Jt+EuSXg5cy6E7vyDKoW59PowZl38NKLvTXVClgj+FVBWnElemIRxeWaFhWpbqS65OM/z4+v/UzLUsYUhXic4d13W9nKXoVKVU3Ufl23qHjc143lwFRUp20b3F6H+kLRmfQnV62DeEycfb0MXnfZrnRALdMicgQEXlCRJb4/+/StFhEDhSRl0VkkYgsEJHPpMz7g4isEJG3/L8Di3sEKlPfPHEy+43W7r9U+ch1tKZCjepmW8JdFx7SYdq5h+S3XuKs0zOrW5pN1aNyNimH4YbTlaRnS7s7LL0zC9CTUCbu+uIhPS+UJxrglq9ZwJPGmMnAk/77zpqB840x+wGnAT8XkbqU+d8xxhzo/71V+CT3Xak/Wd88IbfhUbOxz8jSjzSlVDq5luBGgoVrkf6RlAaYNeEAFxw5Ia/b728NeXqSSclhZ+cfMSEv+95jePbBdTmYOXVUqZOQN7sPLc2TiGL2CKEBbvk6C/ij//qPwMc7L2CM+cAYs8R/vQ7YBOR/OJAB5rJT9s6oi6L9RudeF+/aj07h0pP80q7OQ6AppdIK2RbvfP/UvNeF7R/lsoVz+Wkdh1Ged8WJOW+rNwOBlNLQ6vwMtNBZvp9GZKK7KnmFLN0t5qAsGuCWrxHGmPX+6w3AiO4WFpFDgRCQ2uT4Rr/qws9EpMs+rETkYhGZLyLzN2/enJeE9xV/+cph/CDNI7dMuig6bGIvKv6L13n/pw4ey0XHFn9kHKV6cuUZ2XUHNaI2927yMrX4+tNYcF15DcIxUEwf1zEoHTkokvO2chnGuZA+kuEwsZ8pUCB609m970M9W909pbm1CCN9FoMGuCUkInNFZGEXf2elLmeMMXQzoKCIjALuAb5ojHH9yVcA+wCHAEOA73a1rjHmdmPMDGPMjOHDB1bh75F7Dmsb1jSXR7IVod6dPjWRID/+9AHUFqABgVK9dfGxe7Jy9kyuOiP9MMGpfnDW/gVOkdcpf6GqQeRzhKq+4qPTMnvkfvgeQzh8j/432ECrTHuUKFRbCcsSVs6e2WX3f7/6bGmCzXw/IZl72XEF796wMw1wS8gYc5IxZv8u/v4FbPQD19YAdlNX2xCRWmAOcJUxZl7KttcbTwy4C8j/QM9KqX7vKxk+YTh5SrcPmcpeJn2NVpVw1KtCuCLDm5e/XnxE3voIX1iGI9SNqO25NPrCPNf5Tqdz47/WQphi+ZLf5/oj38xfF3wHjqvLqVFjb2mAW74eAi7wX18A/KvzAiISAv4B3G2MeaDTvNbgWPDq7y4saGr7KK39qlTP3rzm5B6XKbfx7XPRUwlTpgFhX5FJVYHOdW9747+O24PqMhyh7tT9Rva4zJePLs5gO5277xpcWZh6v+l893Tv887n+fzPrx+Vt21lQwPc8jUbOFlElgAn+e8RkRkicoe/zDnAscCFXXQH9mcReQd4BxgG3FDc5PcN44dUMm5IBdd+dErW6x4yof8+slMq1eCq9h/Zg8bX7TJ/WC+Gqe5LPndY7n3Hlqs/X3RYt/Mn5bHHg++ckr9gOZ8yKb1PHWWykL6UMmrlFafvU9Ru6V6+4gTCgfanFI9+65hebe/y0/bmvR+c1ttk5UwD3DJljNlqjDnRGDPZr8qwzZ8+3xhzkf/6T8aYYEpXYG3dgRljTjDGTPWrPHzeGNNYyuMpV5GgzfOXn8CxXdTBOmbysG7X7U0dJdE226qPaa0jeEmnbvQiQYu5lx1bolQVV38ope7sqEndX+eO72aEx68ck12pZm8HiCilQnaBl6oiZPP61Scxc+oo/uu4/A6c0tmdF8zo8L7zCGP7jqrt9qnGtLHp6ySfPX0MXz12z6LlW1fK71mBUmXipH1H8PySLWnnZ3JnXR3RU0z1LweNH0zItrj8tL2ZtFt1twGQ6vtCgfRB6TkzxvG751f0uI3vnrYP+4zSvr8zNbQ6zK+LMBzy7kN7N3rnQ5cczc+e+IBfPLkEgDOmjuQ3nzs4H0nLC/31VSpHmTyWPX6v4dRVBtnRnChCipQqvLrKEB/ceHqpk6HKwOQRmQWtJ08ZUZJGRqp7wzLs13fl7JnEky4tCYcDvv+ftmngDc+9cO1Orpq5b9kN4NF3nxco1QeIyIAbIUmpvuzerxxe6iT0K49+6xgNbstUXUoDts7VFToLBSwGVQR54n+O7TDIRyRoc+eFh5RdcAtagqtUwfW/WntKDTyjezGwQV913cd6bnw7bewgFqzZucv0t793CknHZegAaYDY1524b2bd/GVaal8OtARXqTTqKvPU8XsXEW6N1s1Vqiyla0d295cHXlfiFx7VcyOy+//riC6nD6oI9qngdvIALWV+6n+PY+5lx5U6GQWhAa5SacycOopT8tB5feceE847dDz7jynMiDhKqd6p7GIwhzMPGM2k3fpOyVUxlbKVfD7dfn73j+j7qz2GV/fbKiQa4CqVRsC28nLR61widMp+fXvEJ6X6M6uLItybPzG1BClRxTRxWPoeBYrYFa3KIw1wlSowvTYq1Xd0VZpVVYajb+XT97qoaztz2qict/edU8tzQIdcfXCD9hrSF2mAq1SB9cO+4ZXqtyJBu6ijR5WDLxy++y7TpuVYjeqDG07n6x+Z1POCfUhfHqBiINNPTakC6+qRp1KqfPXXRjfpdBXAXXDkhIzX/+JR3rJXnrFPtwNDKFVM/fu5i1J5dtK+uzF38aas1tHwVqm+JbU+5j0DrPeE279wMKfsNzKrda796BQuPHJCr0fGKrVPHjSWB99YU+pkqDzRWy2lejCmrn187imjs39sd/KUjj8WGvAqpcrNA189gupwgCP2HJr1uiLS54NbgOvP2q/USVB5pAGuUj14cdYJvVp/6thBnHuIjmamVF80oR8EbpmYMWEIC79/KjWRPPX/3Qf198aEA41+mkplQUtflRoY/vG1IxlTV8FutQNvBDOl+gMNcJVSSqlOpo8fXOokqDIw6/R9Sp0ElSOtoqCUUkop1YWvHrdnqZOgcqQBrlJF8ImDxpY6CUoppXrw3g9OK3USVJ5ogFumRGSIiDwhIkv8/7t8XiYijoi85f89lDJ9ooi8IiJLReQ+EQkVL/Wqs0MnDuGYycMAr8WxUkqp8hMJ2m1B7vlH7DoAhuo7NMAtX7OAJ40xk4En/fddaTHGHOj/nZky/RbgZ8aYScB24MuFTa7qSWtr7EEVA7eVslJKlbtI0Gbl7Jlcf9b+pU6K6gUNcMvXWcAf/dd/BD6e6YriFRGeADyQy/qqMK6auS+/v3AGB46rK3VSlFJKqX5NA9zyNcIYs95/vQEYkWa5iIjMF5F5ItIaxA4Fdhhjkv77NcCYrlYWkYv99edv3rw5b4nvr6p70U9iJGhzwj7pPkallFJK5YsGuCUkInNFZGEXf2elLmeMMYBJs5ndjTEzgM8CPxeRrJp8GmNuN8bMMMbMGD58eG4HMoCcPMULUHerCQOw14hqoONoZ0oppZQqLe0Ht4SMMSelmyciG0VklDFmvYiMAjal2cZa///lIvIMMB14EKgTkYBfijsWWJv3AxiAWtuHRYI2AA9dcjQf//WL3P2lgTVevVJKKVXOtAS3fD0EXOC/vgD4V+cFRGSwiIT918OAo4B3/RLfp4FPdbe+6r1I0OaxS4/V0Y6UUkqpMqIBbvmaDZwsIkuAk/z3iMgMEbnDX2ZfYL6IvI0X0M42xrzrz/sucJmILMWrk3tnUVOvlFJKKVUiWkWhTBljtgIndjF9PnCR//olYGqa9ZcD+tw8T/568eFsrI+WOhlKKaWUyoAGuEpl4PA9hgKwamtTiVOilFJKqZ5ogKtUFsYNruRzh43n/CMmlDopSimllEpDA1ylsmBZwo1nd1krRCmllFJlQhuZKaWUUkqpfkUDXKWUUkop1a9ogKuUUkoppfoVDXCVUkoppVS/It6gV0qBiDQA75c6HWVqGLCl1IkoU5o36WnepKd5k57mTXqaN+kNxLzZ3RgzvKsZ2ouCSvW+MWZGqRNRjkRkvuZN1zRv0tO8SU/zJj3Nm/Q0b9LTvOlIqygopZRSSql+RQNcpZRSSinVr2iAq1LdXuoElDHNm/Q0b9LTvElP8yY9zZv0NG/S07xJoY3MlFJKKaVUv6IluEoppZRSql/RAFcBICKnicj7IrJURGaVOj2FIiK/F5FNIrIwZdoQEXlCRJb4/w/2p4uI3OrnyQIROShlnQv85ZeIyAUp0w8WkXf8dW4VESnuEeZGRMaJyNMi8q6ILBKRb/nTNW9EIiLyqoi87efN9/3pE0XkFf947hORkD897L9f6s+fkLKtK/zp74vIqSnT+/T5JyK2iLwpIv/232veACKy0v/OvyUi8/1pA/6cAhCROhF5QETeE5HFInKE5g2IyN7+96X1r15ELtW8yYExRv8G+B9gA8uAPYAQ8DYwpdTpKtCxHgscBCxMmfZDYJb/ehZwi//6DOBRQIDDgVf86UOA5f7/g/3Xg/15r/rLir/u6aU+5gzzZRRwkP+6BvgAmKJ5Y/DTW+2/DgKv+MdxP3CuP/23wH/7r78G/NZ/fS5wn/96in9uhYGJ/jln94fzD7gM+Avwb/+95o13XCuBYZ2mDfhzyk/7H4GL/NchoE7zZpc8soENwO6aN9n/aQmuAjgUWGqMWW6MiQN/Bc4qcZoKwhjzHLCt0+Sz8C62+P9/PGX63cYzD6gTkVHAqcATxphtxpjtwBPAaf68WmPMPONdRe5O2VZZM8asN8a84b9uABYDY9C8wT/GRv9t0P8zwAnAA/70znnTmmcPACf6JSRnAX81xsSMMSuApXjnXp8+/0RkLDATuMN/L2jedGfAn1MiMgivsOFOAGNM3BizA82bzk4ElhljVqF5kzUNcBV4gczqlPdr/GkDxQhjzHr/9QZghP86Xb50N31NF9P7FP+x8XS8kkrNG9oewb8FbML7oVgG7DDGJP1FUo+nLQ/8+TuBoWSfZ33Fz4HLAdd/PxTNm1YG+I+IvC4iF/vT9JzySuk3A3f5VVvuEJEqNG86Oxe413+teZMlDXCVSuHf0Q7YrkVEpBp4ELjUGFOfOm8g540xxjHGHAiMxStV3KfESSoLIvJRYJMx5vVSp6VMHW2MOQg4Hfi6iBybOnMAn1MBvKpi/2eMmQ404T12bzOA8wYAv976mcDfOs8b6HmTKQ1wFcBaYFzK+7H+tIFio//YBv//Tf70dPnS3fSxXUzvE0QkiBfc/tkY83d/suZNCv8x6tPAEXiPAluHO089nrY88OcPAraSfZ71BUcBZ4rISrzqAycAv0DzBgBjzFr//03AP/BujvSc8koN1xhjXvHfP4AX8GretDsdeMMYs9F/r3mTJQ1wFcBrwGTxWj6H8B6LPFTiNBXTQ0BrC9MLgH+lTD/fb6V6OLDTf0T0OHCKiAz2W7KeAjzuz6sXkcP9eoXnp2yrrPnpvRNYbIz5acoszRuR4SJS57+uAE7Gq6P8NPApf7HOedOaZ58CnvJLXB4CzhWvJ4GJwGS8xh599vwzxlxhjBlrjJmAl+6njDGfQ/MGEakSkZrW13jnwkL0nMIYswFYLSJ7+5NOBN5F8ybVebRXTwDNm+x11fJM/wbeH15LzA/w6hZeVer0FPA47wXWAwm8UoQv49UBfBJYAswFhvjLCvBrP0/eAWakbOdLeA1hlgJfTJk+A+9HbBnwK/zBVMr9Dzga75HXAuAt/+8MzRsDMA1408+bhcC1/vQ98IKwpXiPEcP+9Ij/fqk/f4+UbV3lH//7pLRc7g/nH3A87b0oDPi88fPgbf9vUWva9ZxqS/uBwHz/vPonXkt/zRsv7VV4TzYGpUzTvMnyT0cyU0oppZRS/YpWUVBKKaWUUv2KBrhKKaWUUqpf0QBXKaWUUkr1KxrgKqWUUkqpfkUDXKWUUkop1a9ogKuUUkoppfoVDXCVUkoppVS/ogGuUkoppZTqVzTAVUoppZRS/Uqg1AlQ5SNkVZhw7VBvwFbASrjtM10XLAuMwdgWknS9AQJbB8KzBFzjTUthrPZ7KDEGIyAGaB1Bz4CxxZsG3nzHYOz2DYnbabQ9Pw2IIK63TUQQxwWRtu20vsYY3JDtbwxc/1vvhvzlgFBlgppAlJA4BMQhLElvGYSIGKLG+z9mIGaCVEoc1z/YejdCixMiYWyMgXgy0HZ4uP4OLAOuYAVc3IQFbsox2+0HLw6IC+J4edO6TGs+i+vnh6RMF3960u147MZ4n4sIyYjghvzt4v1vJQxivPz2Pjtvndb8NpaXRi+PxPvXmsepIyCm5nm6z6w1zZblvbZ23YaxxD9m4+3bNd4yrYu1ft9a1zV+BrRO8z/rts89NZ2p01u/y4AbsklUCSaIl9mtX1fTui/T8TNM/UxTXwuQcrqI/1qSrZ+Z92clXe/73XoeAG7AwqkQnJC/D/H+F6s9fwRwXcHyp3mHYnAcK/Vr7n2vkgIuWEn/c04aJNl6UvufhS9ZaeNUQiCUxLa8RAcs189eb8MJx8Y1gm25uP6HbFsG1xUMguvngTGAI94f3v6thH/s/udlRZO44QBOxFvGqXYJB5NY0p4mW1wMguN6H4YlBscIlhhcIyRdC9e1vP0lrfZ8drzjs2P+d9o/N9rOHSBRJbgVBtt2sMRg/OMJpBw7GBKu3Xbs+Hnfti8XrHjr52m844s7mIC1y/feCGAJbkBwIoIbMmC3zcWyvL/WPGz9yrr+sbd9D13x8jIJluMdnzgp+2r9nou0HbcbskhU+fODLmIZLDHYliHpWNiWl89CxzQ7bsd5IpBI2v7n7k0PWA6O8T6DoO0ST9pt301bXBzXwnEtRLxja/+O+scZl7Y8FMdLrxV3O6QDYxDH9X5vXIOxrPbfkNZ89s/r1msV6UZmdQ0mYOGGbZIRMEEDtulw7oiAcaT9nPe/z3bMz1rTfs3ENe3Xp9bPqZUl3u9X2zXbkKixcSL++4CL+N/31s/dtg2O0/G6InHBSoIdN961xT9u41/rWl9L61e9NT2Ssh0BN2h5xxzxvgPGsbADTts55xrBbT2/XAvjirdcyqZaD9O23bZzz/K/T63nKYDxzxFcvHx0pO16Bl5+etcD79oArdcot8N1SZIOWJb3O+8nRJIOJhbf5aNtYPsWY8zwXT90DXBVigq7hmnHfqstsI2sb/RmWBbS2IKpiiDRBMkhVQS2NkIwAAkvEDShIBJPQMDucLF1q8Jt25eWBCYSQJIu0hIH24Kkg1sTabtYGxGspihubQU4BmzBavHPhKQfnTkObk0lJmhhNScwQRsTtLAbY5iAd4KZoI0bCiCOi7iGpnHeld7Y0DLUW6Zhd3BDBnGF8Qet5ejhyxgf2srwQD2Tg1u8ZUyQvYNJliZsJgUdViQsliWGc1B4Hc3Gu+g/0bQvbzeMY3O0mpgTYNWmIW0X8mSzd4pZYQc3blM1uIWmTVUE6m3vR9lAfLB3XJIUgjstgk1CaIfBSnhBgrEgEPPyJ9BiCDQ7uCHvGOyYixOyCDYlCW6P+sco3o+C4+JGgrghi61TIjSNM4S2e+kKNkDNuiSShNCOOHZTAhP01rEavSu6Wx1Gkq6Xp5aFEbB3tmCCAcRtjeD8i7llYUT8AAqseLL987ItpDmKCQYwlWGkJe59X1J+jIwITk3YCwBbErjVIazGOJJItn+f/O+bqQgh0YT3A5h0vBueSv/XIxaHcKhtm+I43o9RIokJB70kN0e95V1Dy8TBbDw4SHS0gwm6SMRPswETs7GrEzhNQTBgVyUxLrhxP0JxBKvJBgETMFjR1psAwfY+CsJbhWCTIdBiCDYbIptjBBpimKDddkMSG1bBtn3DNI43OFUOBAxWRZJwRaItf2zbJdoSIlIRxxghaDuIGOobKrEDXppdxyLZHCC4JUigSYhsMYR3GiLbHMJbvQQ5lQHspkRbnm45qJZtB7gMnbCduooWAIZGmnCNEHWCBMRhdf1g4kmbQRVRmuJeHtZGYjTFQySSNs3RkH9aWjj1IQI7vPyJbBWq17oEWgxWwmAlDZXvbaRl0nB2TPLW2Xl0lMmjNxGxk7gIAXGoDsaIuwHq495nWhmI05gIE7GTRJ0Am5uqaGyOkEzYsNm7voS3WASbIFkBdctcQjuTWEmDG7TaAicxhvWHR2jZN0pdXRPV4TgtiSAihmGVTQAMCTdjicvG5lpEDGt3DgKguTGM2eGlOdBoUf0hBJsgss0h2JgktG4nTl0lVrP/mbVehwIWbjhAdHiIrVMCNI9PIpVJ/9QxVFbHqArHaWjxjiMUcIgnbVqavPfG8QJrabGoXGcT3mao2OYSqncI1ns/9sYSrBb/hrwigJVwcYMWTWMr2Hiof5qOjhIMJYmEEgyqiLKlsYrBlS0kXAvbD3RabyAaomFqIjEcIwQtl6DtsG67lw/VFTGSjsWw6iYaYmESjsWomgZWbR9MdcS7btSGYmxrqaS+KUI4nKC5OUwg4OC6QjLuXQ+Dq8NUrYFgI4QaXYwNleui7dcbwIo7WDubcWsrsBpjuLUVSDyJCVjtvwmJJISCuJGAF/TH/OkpBSskHSSewKmrpnHParbtY9MyNoldkyAQ9PItmQhgWS6JhhB2VRInakPcIlBvM2iJfw7GoGpjAivuYkeTWNGk9zsGHQJrUxkmWR1CXIMkXay4w/pj66jfyztPQ7s1Y9sulmVo9j/n2poWGhor2gLfZEuAihUhwtth0MoEdtTF2IIdc3DC/s1GzCFZEcCOORhLCDQlkISDGw60XVvccICWESG2TrGJT24hXJEg2hSibnATFSEvr6KJAC2xEJFQgqaWMPHmIKHKBE7SxvJv/Fp/z2prmmlqCZOIBaisjhEJJqlvirRlQXJrhXcjGxOcSpdAvY0JGJwq/wZyh02gWQhEIbLZO9bIDpfI5ph3XQKwBXvjDkxNJU5tBEm6uCGbwOYGnCXL6WyueWDVLhN9GuCqdsEA2/cKENlmiGx3aJzkXdQim2IE8AIGKkLYzXFMZdgLRv1Ax60OITEbSTiI6+JWhsAxSDyJxP0gOBzEaklgAhYmGMBqbMYE/B96/1bRiie9QNlEvNctMS8wigTbgyorgFsRwG7yAiC30vvRTQ6ubLvQuGEbYwnB+jiSdAnVJ7HiLm7IYtu+ftA9sQlLDMlokJpgjKA4VFkxdrMbCPlFcLYxNLsOo20HsBkbSLLBiTE2UEFQvAvN69FGagJRlsWGU+YI/AAAIABJREFU4RihrraZ+qYIyUQAu8IPPhIWJCzvjj3kkqyB4HbbDwb9C3pUCDUIgSYvsA03eKU0TkjaSgTjNRbJCsG1hUDUxQ14AZaTtLEqvHwwAQsrlsTY3v9WXDBWBDfYfveP5d05i/H2YcWT0OzgVoYxFd4PuTgGNxzADdlYSRe7wfsBM0EbWu++XdcrbWiJIqEgbkUQKxb3Ps+AHwgag6nwf7AtC0JBTEUQaY63BcGmJoIJWDi2YAk4FQGsxjg4TtuPlgnYHbbpVkWwGlu8gNe/oEssAa7BranwfixtG2yQeAKJ+j9Gllf67wyuoHFUABMAiQlWXbK9ZMURjMH7YQ5ZXgmT5WJEMEFvX3XDm4jGg7RsryBQlcBJWNAQxG4WXO+jaAtsA1HjfwcdL12WhVPl5XNsSJB4HTiDkgRrY7iuRWWll9dB28sf2zJUheM4rkXAdtjZVEFFOE4onCDYGuAaodkRktU2krRIVnqBdmyQjbjtN5puwMIJWW0lyNbgONXhWFtp5eaWaqqCcUJWkspAnDE1O4kEEmyNVrWV8lYF41QF42xqqsZJ2lRUxmhqiGA3WUS2eN+xUINBHAjvSBDc1kJseCWJ0YPBEqzWe5+AQ3MiRHUwRtQJsjNRwbZoFSOr6ok53s9TzAmwpbGKYdXe+RpLBKmMxNnRUI2E/ODMLwWMbDOE6h1CO+PYWxpIjKlrCwgMQqgBYptCNARcmqMhYk0hrIDL5tWDvc90VD0ihopQgvUb6wiEkxjXwm0JUL3a205oh6FuWZz4oADiGpKVNrJbTfu5AZiAYDfGvVJdxwu4KzfYQAA36B1XdFSSxuYA8boAiWgA0xKgJeIgAReT8AO0hEVgp02wQaha532Pwtv871HMvzG2BUn43xPjleJZrkWw0SG00/siRofZBIIODY0VJBybeDzAuuY6amuaiSW89FRF4rTEg7iuEHdsGv2gOxBwSCS842owYeLNIRKO3fbd3NhYg4hhR2MlANvcKkQM4XACx7Fw4zaxmI00BQg0eMdVuVao2OIQbHSJbGimZWwVyeogdovjPSUCErVhgnhPWSTpIrGEVxAi0hYEE4qA62I1xzGhACbkB3cpJX3OkGqs5oBXIJIwBJugxXg3GLbt7ysOifoQUuF4BYYttnfDm1Ko7PhP/NyQhRUXkjVhrEgAuz7anh4A13tKYyW9oDQ+pILKjS4tI7xlYrYXBEp1kkhlnGhziB1bqpGojQl4O7Sabey4d8OWqLIJb44ixiuNFdvfl2OwYw5W1PFKQP1rIIb2m4SEgx31nky5DUEStqF6UAtJ12Lzjuq2JCeaQ8RCQdy4jVgGJ+k9jQwEvc84Hg+AEeobKqmpbqEhaZNM2mxtiECD9x0LDmvBhB0wghMQCLok6wzB7QHchP+UxwYnYrBjghOGcL0hvMOLCeJDvUDZDVkEIwEC/u8NjsGKJZHGZrKldXCVUkoppVS/oiW4/YCI1AF3APvjVcf5EvA+cB8wAVgJnGOM2d7ddox4j2GMQNMI7w4SwLXD2PEQVtyrHxje4j3utJoTXkkttJXcErBwxfZKblviuHVVHSryJAdFvNIHHExVBW4o0PY4D4Cki6mrxg0HwLKwaH/MJ8HWerSCCVokayPYluWXZiRI1lW2bcaKe4+GjIDxSyCNLSRq2irA4boW4UicZCzAlpYqtldXEg0FqXcjRI13V1ppxVierKTBraDS8u4oX2naE1hG3K+isCY+lA3RWjbt9O6IYw1hr85cg91eAlBhsGJCbE01RFzE8R7nV6wXEs1+tYFGiGx3sWOGqnUx3KBFoClBfHCY4A5v38nqkFfqmvRKauyYd4xOZbCt5EMc1ytJco1XlzPmUL3ewVg2ofr2R2mh+gSOXy3BBG2sphaoDmNa6wP6dXhb641iDG5lGGzBBFof98dxasPYzZZXVUTwSnGjybbqK/j1pd2KIBJzMGHbq65iC5JSn1Vc01Y/125JIo5XmtNa/cAELCSawK0OQ0XQS1MwgFsZ8vYHUFsFiSSScDChgFff1vVLf9t2JOC4uEGbZJXgBgxmSALbcglHvNLiWDSIaxuSSRuxjVdnzTK4jtVW77a+sQJnRwhxhGQihBWzsKNCsMGrHgAQajReaXuz45X0OUGsmP/UwX/aEGh2qdgkJCsCOM02bnWSqP+4uC3JQDiS8ErX4gEsMTQkvXxxHP9xuCu4TUGshBCqF4KNhsotDoFmh0CD/zjb/4ysmEOgPkrj6ME4DUE2V1QTDnrHXhVKsDVZScKxsS2XaDxIpV96PLZmBwDbY5Xt9XEDjvcIvzHg1zn10hxo8arXOCELqQljJ1zcsI0kTVsVjsT2CDsiCRKuRUNLBMtyqauIsrmlmoaYV4JoiSGeCLB+Ry3hYJJYNIhV6SIxm8BOb2ehHQbLgfBOg7EhOjxCRdIlWRHAtKXHoW5JnHhNiGS0gtjohFda6lhIhff9iSUCxGNBmgJeSVQyGkS2BwlGpa3uvh2DplFBwjtcIpujuEGr7RrXWpIqjuBUBBEDdkMUSThUr7MxEiA2tP1ztZps4lbYq+7SYmES3vcx0OB9X604hLd5T3BCja5X1SPh4kTs/2fvTX5lW9P0rt/7NauJZjenuzczK6sqKbkBCdEIpgjJfwAzpoCQmDGGP4GppxYSYsAEMWHGxBJTSyVseWCDjSlXZt6b95527x0Rq/s6Bu+3VpwsLKic2FQqXunq6J4TO2K1X+z1vM/7ezBL2u4bSYnSeiQk0rEl7j3+Ejn8Ujd6efBMi0HaxGXpMY3+7JePx80MnrNhOrWYJrHMnrYLTGNDjIZUVbrUWJpdYJo8l6c9dGqo9F0k/1rX39xmyi4xmxa56H6IATsY3LleM0th936pHRGDP0f855F4111nP4rak+zLDE7Xl9g5/KcLclI1L719QERI970q2lbVXVOu/lh7VkuQfR5pdp7jL4Xl3jEnYX7U7cmLhSRqS9oHuBgoBjvL5vftnrJaAc7aHYs7i12E1B2wdf2RmJEpYsdAPLaquOZC9yXS/7BapyzxPlMWw+IcxhbERWK8+tfdRbsvx18l7JJ1fa/7U9zqedf3NktUy1PW+9uEdPXpJrV/2AnoEykaxqElJ0Hq7EcOBrmoeixW74d08kgSqoaK84ll8IgpPH0+IDYzR0+ZLNQuZfzNTq+H+r7mxUFZ96XaKiagQP++4OZSZ07UY+zq9RxNvdYOrXqne4cdArjf/dfVm4L7+1F/G/hfSil/E/i3gH8M/NfA3y2l/DXg79b/v9WtbnWrW93qVrf6va+bgvtXvETkHvgPgP8UoJSyAIuI/EfAf1hf9t8D/yvwX/2/vlfM3P0ykhoh+68nMQV/ikgu2Kma/JekCtvLX5hqdBaJiXzXU2yvSlpVXmWJ2MtSp14N2EJp7eYVBfQJXGR76s67hmIFc1alECDuHG6IUCDet/rEv2/Ue1bfxwxL9eSCPc1IVj+pGzL77+qTsO2ZDy00heWN5cNy4B/wh3xoj3zjngHoTODBDPwyvOY59byyF0Kx/N2Xf2Pb5ZfYMSVPTgbrsk7ePjuaZ52CBbj8ImAGR/tRoBhMBH/SgZH4VBXcIdP/ODO9bVkePM2XZRuSiYc64DIEPV5LIu0c82tVPMxXwxlx5+pEMdgpIgYOfz5gQr8pynZMyJKxSYkUuQFxFnOaKF01kFbiwW9NLFOV1Or3Ssd2U69Sazclo3j7laoVKb3HDPXcY0l3bb2G6tP+84Ax6s0urdW/j4nSt1dfY6ODJPY0kbtGH89LwZ7mjbpRRJCvPXrGIOOkvt91e5Y6oJYLb//+wJe/3vP56LFPLcN9HaqwhebZkDqPrSSEdap4fevUFvrPBooOHJkFbCj4c6Y5rUN4kJ3U7ogOp+ReB/+uKkvh+OuICY7ptWF85+GjBwvpsKp0wlBazCTYUQgPGTsaii2Y6m+L+4w/GdonVZBNhOyF2Fuy03vHzhmJmbh32Nlz+D5S/tTx9K8fuRyrqtclHW7KAsFgDoFp8cRg+XJSlS4Gi0hVf5487izYWVWn3Q96TvuPEZNUcSzO6NohQu4s7Yt+Vv9rx4kj532k6QLzuedse1wbSdX3aWwhTg6ksDSeNFrOH1soQv9jVbzGgkmFYiB2htQI5Wc7mpdIalaPe2J+9GQPy08Cx1cXShEaF/HVi/n5eY91icfjwNh6chHkYeD86zvirqp9ztA86+S+id2mEEsGs25zzISdw00J2TWQC89/7Dj/EYRvdM30fSD2lrYPOiC4s5Qk+C7S/0Rf8/Jlx/Kt0PzokWzZvc8U4yuVoapiS6ap+IXwoKq+WTLh4Aj7em++WtjfT7w5XPg89HQ+cpka7D7TVg937wPpeOF56Hl7PFOKcGkC+2ZhOOqaMMwNy2Jpmkj7swmRwhwc1mbiL6oCFw37fiFmw7xz9F1gCU67D7/c123W45day3LvsFOGVz0A4ahrnR1j7dZZvddFu0mlcZQ39/X+huIcqXe42mUq3pKNUWUdtGPSeVxIZCd0nwMP/0R4fyek6lGmzchOtz8tlnJMNJ8s/nQlorghq+c1rENdpXqsvyIbiOj6kgzWCuWuBYHp0TG9rQNkh0wx6otNLw24lUgkm3JfrF7TUgp2rENt50WV2u76a1vudO7FjEEpEyFRrN3W1dw7hjeW6W3GNhnn4+Y7Xv9cFkduEr6JhGDJ2eAOiRjcNoTnXKZ5iIxjQ9NEHRiMFvYFU73YuU20PrKMekzNXSb/2DG/yjRfKrXG6jkLB2F5EJrnwunnLe1T3pR7/xIwIZF6D5WmU6yQHw/wK36nuim4f/XrF8AH4L8Tkb8vIv+tiOyBb0opv6mv+QH45l/0wyLyX4jIn4rIny7pdzdx3+pWt7rVrW51q1v9/61uCu5f/XLAvwv8l6WUvycif5u/YEcopRRZx8P/QpVS/g7wdwDu/btSjGK09u8TbqgqXbtyCFcvLZgxkLtGp+/1jSiNTtzbi5ILSIXS+yvTtnIMyXljY+pEuSgSDEj7RhUf0Ce4quhKzNiKrypGNjbipujtGmUCrniUnb6PhEQ+NBRrsJdlUz0Awr6Q7hTL9KofsFJ4Px052HlTcH+9vObJ7mkk8taduOSWXITfTPfceTUSjkmVnjB4AiBdIhWYeoMd9BnSPamqmh3svytkD/2njD9nVjCmZJjfNOy+GzYFG8AER9zXp+IpEh47ihHCzlGckJ2lOam/EcCkgh0jeVV0jy3+aaJ9CoS92z6rOEM4OLoPI+Yyk14dVOluVhVBGYT+y0juPem+JzuDiepf1Q8T3NMIS8BaUU81iirb1NngVDE15rpPc0TmtHm4y75BQt4wY7mx2KD4n9VfK8ukCsYcMTFR9p2qOV8hgcww6HU2xytxwevnl6p8rOiwIhDuPX4ovPqHQuzBTlVp8HqummfoPhWm16rGS4RX/7tuz/Ro1R/XCDYUYqdKrUlsHZDsRXFVreA/BuzzqDJwbgl3ValaMqk1dE+Z6dVX3ve+bH429lGnm8+e/CbBiyO1leUZr/dl6hSCevfny28xWDcMnxHcFKvvLZO9sNwJ/Q/CZSUSHALNLrIsDn+XcC4pAiwYUkWklckiu6gevIfA0llFAlnZeJ+5Febe0j4n/CkSDp7macbETOwrX7YB2Ud2h1kbBAXaXaDxEdPrgQjJ0rSB4eOO4jO2egntydB91m32QyY7JYuYUPDnouppvHYgwp0neSH1heawsGsCIRne7AZilWGbV4lUhGFucDYxB69s3AK4st07q6oXe6PK/Jwxc6JU1VCWTBMWilWlPu08y72QuutYfi6Cq17YMHp29yPjpSWcmu01yiMVnT5vYdkL2Rn8pWBX/FkqSooQPc/qy1T6R3Nauy/KK74s+t5TcLQ+Mi1+Y+6eR8V5WZP5cDqQs+Bc4jIdcFXlNSbjvWKjLmOD94ljPxOS2d7nsJuZg6MUoW1V9YuzU/5r5aGmRhi+aWi/RJqXq5fYjBEzVWpK63VNZ/2+cUhW9NXXXSX78QXyofKlBSkJahcQIB07/R6oimD0hsu3Rk9iU9XMThXMZXLk2YIthPuMXSxh0vtrfGPxF8P+HDaWdQbFK65fgwJYo/5/0VmCYoTTzw3zt7pf0mT2d6p+x2hJ0SiL1hu402s+ve9Y7gxgOU6FYDxN9Rhv37kimJCI973iHceIWSLxzmNXjFrReyy/WWiqertrA84qjk4v+vVQCn0XaFzi0M58vuwYBu38eJ9oXGSkIQaLb/S9rF1p8DBOlebjdGahZKE8Brg4wtoZqy8OR+UE21FJCgi0nyuecr2HYtb9WhnTw+oI/svXTcH9q1+/Bn5dSvl79f//J/QX3h9F5CcA9c/3/4q271a3utWtbnWrW93qX2rdFNy/4lVK+UFEfiUif6OU8n8Afwv4R/W//wT4b+qf//P/55tZw3xniTthORjOP1kVRLAPFjdlfKeKBaZnefC0n/Spyj6Pqgj17TbNjwPilSWY66S/QrrVv7R5JavKa19m8qEBMdi5QvxdTTOpSp0dg07lNzqRTUan7+1VIcytwz2N6hHNBawSCKZXjuGb6ks7RvUZLsLT1HPwM//2/a+5tyN3dcy7kcRT2rHg+JPmR74Lj/ykMXxYjnw/3G+H7uOw5+71hWFoiaMyA4spqrIBbhSyL/ggjO9W36B6BefHFWyuzFT5pqevkPDldY8dIs1vXuoxbGneX8i9x71URXNS4sC672nfbslkxYC7BIoI/sMFUA+c+zQiOWOn9kocgN9KYzLjgjmP6l+tvlsxqhLZ+jSdO6UYiNNJcjssyBI14GNVcC8T8fVBFYdhoexb9eOWQln5tc5glkh4s1OV8WWGecEs4crlzXVKvW1U2X0eVHHedRoIARryUIoGQtjKxCxKZ/it5CeoaVAQdrIpC+ufJui0/Bq2EXequpsA+3/6Wa+Nd0emtw2p+UpB7YELm7pmF5BY1cRUSMcO+zxi5kj7XlWWeNcRe4tZ9DPUO60s3VSZl+YQSGNVoG0mt+r1pk/EVfGYDaUpjG8Lpz9ocJMqIwDtl/U6jIS7FndeSLuG5bj6tqG0tfsRDf1xwdQ0o3nS6el2F/DVk3d+2lEqv7kEA6ZgR8HM6kUGVbu7Lwl3Wf37CQmJ+NAx39v6ubpdw6lld5x5fHsCVLVdQfTOZobZK6M01dwtl0EM4VCZu5fauZj1c5Z7t6U8NZ9rAMpelb/m2XA5NbyfHK6NzMFvnH5jCvPsEGFL4YrGYCaz+eklwe59pv8QqlJqcUNQL3hdz3LrlEXtDLnzpNZw/JXyj8Ok3ZjwrgY92YxpEuOlpUwWM1jys3qdjRT8i6F9gu5TpvuSyI3gT18FPVSlXn3r2gWROeFLQX5SPaZRCIvj48cdu7cXUjIUr2SQWNOnjCmMY6O852yIs2V/P5GzbOpsSjW9zCf1bubCeWrxNjHV/UrekJKml8VgCaOHxVCKxeaVkcwWxJEb9bGuRIpUw4HsGBC0q/c1paJYrmz16vVfWeqS9RhrAE31KD/rOonTeYbTL1pOf10DN3zlvK5BBsYW7UxkISdPagrLvf5b9wX697OuQVmVSzPH3+byipC7RgM3xkhptNvoRu3iAaR3i95TlZRSklCyYP1X6v4xkl2DJIg7gxtged0p131LScyQM3ZN9jRKsJHCxkimdfQfM8+fG4I0zHeRpVPWcanEhrJY3D5sYWynaPhs9psfHGAcWiZpKFkIwRIXS6nkiS3B0RfMxVIeAunskGiQILiT4M/rnMY1wQzAVpJC+2nGftCuqdn32sldImQlBJlwTZ/8Xer2C+7vR/2XwP8gIg3wfwH/GarO/48i8p8Dfw78x/8Kt+9Wt7rVrW51q1vd6l9a3X7B/T2oUso/AP69f8E//a3f6X2s4elvQtxlTr9gy+A2M9UjY2m/GPrPSfl+sTC/1qft8PN+88HZ3pFawc6F9sNAuK8JJV4VFdl5Td9qqwJkBH+pPsvObZP4uXVkbzSicBTiTi9XO2siWRFof7hQOkfaeyTk7clenFHPFeA+vJDv96Sdx18yzXP1Hv3S0TzB+Y8K9+3EL/af8KJPvt8FTTYKxfJgBz7EI386/GsAPMeeU2y5azTadIgNc3DEZHi4G/hS9qTFQJORqnLFvlD6xJIESdB9FJZ70Sf7y+qPVApA2BvSH+/xp6S+3c5S2qqwtpYSVNkmlWvGtzFfeV713yXmzYtVRMi7Bjvo8UnHFjsspM7hZlXE097jztfHa8n61JyOLTIrNcOgykH5SlFJe4cpmhxnTyN516lSu/p0nbIZzahpYmZl4gZ9T732NKo3eYMbaw75XmMaV+9V6VolMjRGFQpjyDu/qc+6PZqEV6wyc8mq7JRde2WUTkpz8L/5glmOzPeaQjV5aJ6/oodQo5GnQvdRk7/cUJTDC+r/GzPO6uS+iYXmpJPC6/FxY1VSMvgfn4lvjsTXe1UkVg7ueSE3lriz7L8vXH6q3NP+R6HUtLxLbrGzqoju106T1QKEo8WOVR3pC9kq1rT/lDBRmbBmzkodgc0jut4v7ZeEZLh8a5Aav7bcW748P1LajL+bSclg/3nHfMiEuao+99d46d2fOeIVQY1dVkoJyFJY7j0mKrM5Vc91+6w/78+eMrYsD5lBYGks4dQibeL8Wd/U9VGT5WarXOfRsvvekhv9DIDYCcc/n8iNIRwdJhb8KWz7C+AvkWI9zVNhuDgwhZgb4uOCqV7nUqDfLTQucr50hFODBINfNHYZdF1sXhImVT60aHJi6vYbq9RegsbKNh32suAFciM8/tNC2Ok2XZ4bJMLw00zeZWQ2m58x9zXa9MXgz2CnQvuSMUtGssEOEftJ1e686yjVg7vur8mB1Dqas75P+4NjeW3AZYbPO8gw2wJFed0A5dXqEa2KZjRc3u/BZfKhqsXZEEfHYouqnOHKYJaXqky/qd7XyWJODj8o8zs11/WqeQY3ZtylJrB9xZG1z7quls6rEj4FXQ+sIKGyXb+650vra7eobLzcYuTqv0y6zhSjcx+l+vzFFI17BozLhGAVZvC+U5X+k8HOqtiDxjKHo8deNDWNVOoaFpE6r1IaT+k0LhlnNAUyZO7/LOAvus3j+5bpbSH2GROEXOkl6dlvMxvdpCQUOxfMomu5f1mwXy7bfudDT963OqNg68wEenxype6YMeLGwuGXlulVIS8eM3vim7RFRgOkjy1mVqXZzEL7BE//ZlwDKymDw4yGvKtybaje6MnQf9Crdjkq0SUuja53J2H3g/JuY10yx3ca09t+0X0DvYdza4nvtCMqlaYQ7zQt1U5RO8Hhur1/2bp5cG91q1vd6la3utWtbvV7VTcF91ZbFWeI+0J5CJQkJLem6RhVF53+9/zHju5zYfc+bNPixQjjK0dzyar8iZA6YX7Tq2cXQAzZQnOJLPf6tJ/9ytyt6uyUSJ2msqSqikmsk/1mVaos2QqpN9jQq3KbC8UbwqtdfZ+oKtWSSa80YcwETR7aJqB36jdd7h3Pc8cltYyp4aftE7a+6GM4MHvP++XInD29Xfhn5zfsXGBKug8/XO44XTr6fuEyNeQomNkgg2aW6/YIqVVPrKrR0H65qgPA5pn056wK1DkiKf82S3idpJ0i6diqX3VRr9/qeV1JEmbSJKC0azRZZ98Q+6+UD2cId5oPL3PATKk+KVdV1VvS6/01KclbVYaXuJEL0r7VHPRUMCGRD50SDOw13YmYkLq9K8PRnCbyUZVegNLodrcfBuJdd1VerXJtQdXrdf9UzYmURhPBViUYIxRf97GCa1fuLfarNLOcyTUx7/CrSjag21KdxleG4sAPBTdlHv6ZEgdMKKSqjgzvGvxFt7O5ZOykil6pyWWgXQp3ieTWsvz0gdxamg8X3a71ODuDOy8gDbuPBT9aYif4IRO7qpRnSzhCdoX2s6oeJhTcINsEd24EOxX8oEqVKswFNwTiTq9VO0aMU5VnpThQ4PyHGfmm+qqjUBaLzIYwelW33kT8cd5OqZNCmB05Oy5/VHAvlu6zcnC7z1WNS9V3m2xlAev5DDtH7Cu14BnGd1C6DKMlDE69xEE2/zAfevyiVAspes/YCfwJ/LkqplUpzo1OZzdPesyLFVz1lMuSaV4C7dFw+KVh+LaAFNLFUQ5VvTQwnFuG3FGy4J4c7lJ9ozUFsH3ONM8LZkmY54H8swe9/lMm1fvLGCHvW03s6/1Gb1kOV9/w/KDpaxiQRVQxf7JIBH++ckPtrPtdrPpW7XnWOYZFt9mUQikd8a6rXNRMue8JR4+vhILmybE8oobdCBKE0mvq1KrKyYvDLEJ8UMWVJOAzmKI+WqgJjY68T4jP8Owxi5AOmXKoDHKT9fVJNrKHqeez+bx27Ur9N4sb9VzZS0BiuXK4UY/rmmgF2hkxU1RfLlWt7BrMtKiaeZm1yxTS5ost1mImVV1lyRx+iPC/OeYHR21aMP4s6fUWlefsLoI/673RPl3XVVuZzpRSkx4h3u00aQuQy4TkTN41pP16zErt7tTjnA3+AtMrS+oK7SePG/W6Xr8PJBfsoteaP0fcJeg+7a7c8dJavcad+m/tEJVY9DW73Ap2yfiXwvRK/fHZowmS57omTILU89N91PUkt7D7c0cx1TfcV/pFsuQuQ5Z6TRbCoZ4sgfldwp0NdlSlNhyFuBPa56rcv0D3OdM+J7r3I6lSliTmLZkvN/aqws/KHA6PnXYn/ozfqW4K7q1udatb3epWt7rVrX6v6qbg3uq3qnkyTK1FFkP7SZ9/fLX9uItmyD/9jUxuDcV4/HjlUPoB3CWRveCG6k8rEA7rhD74k/p39em9smwzW4pJcap4FSs0TwvFG9zTpLzTyiNcHlpMKvgPC5Iy9mXeGKcrq3fLaZ+TTpd2SnAookoqQPOsikM4Wk3xKcKYPaFYzqny/ySxMwu91Sf0j8uBg5/W50FeAAAgAElEQVQ5h3ZTcE9Tqx40qczKYPBn9VGu/shwLDpJWmC51yf/YgV3ls2/aBeh/Vxq/rlA0SQkWnslEsRMahskKvtSSlHV2oA9L9s+r5683HpN3qnK2fpkbxdlDBYjLK963LlmmHujCjzqwZVKc5BxAVEyQmmcTmujKrHkrDnxh069tjVVKddzYscFM0zqjxPBDDOUgjlNqtQCsuj0tFIaNG2rOIMU9c9CpR7kXNUToXQNMs1Iaa4Ttquqa6tCGmLlg+Ztqro4C07VPTuETekxqXD+aSVRdJAaCHsDGNwEsVM1R/9OqQ5xp15wEwpmyaTeYkL5yg9dSDtHEUitx86Z5e2eIl/5c4t2H6ZXTlX8opPFy94wfHudPpYEpYPxG1X07CS4AcZv6zkdBUl67E0oLEcln6SuvW7PIpg5Eh5a/It6sIupHZPqFxejXsnSqFKTT16Vqslvk9emSeSoaYQS1Ds6vSrsv5dNYfL1esQKzY/jlUOcC7nRc+rGQnHKBjVNIr00yOsZ79Pmjyz3C/PkMM+O/BAwT564K7iL2ZjB8gyXb51SFJbC8uBJjdA+xa8m3FXlMqHBLJDuIu3jBMFtyr2RQr8LnJ96ZLTEo65X3XuDq2tdc1LVPty3yHFVEgu5s0p+gcrdniAmcEbXgrmw3MmmeMW7hHux5D4hs9m2AYHx3erpLBRn1bO/N1tSmsyJctSFI+9bzBiwo17LdgxbJ2F5rJxpW1XbVlOziin440xc3NXHvyukYBCf9Tzbgvvkicd0ZQD7SvAw6sGly5THCMFiviIS2C4qHaB6dPPqe63dMxNrd2NIuOdrEqGUgnnRwKH0+kj2mtRlz3NVYKOud/X1Zq5M6+Kv8xeXCbxTDz5A32w+3fCmUVX2pTC+E6Y363Gu1JPBKUtawA2Cv5TNV40R5SrHvHWx8s5XekpVQ2vnJHuLLJm8W9nbV8/p5VtheqtJiJJgfp1ZguBPQqsgAYpVCo9k/XwJaevkrdez/TIgx24jxZTGbWlnZlo7Y47UXgkg4SFT9lHV9fVenh2p1+/h6Y2qu6nR+3IlJMTHiERD8VlV/Sy4D55wlwhSlXJf4BBJwVNA1yLAn2FZaScven++/KFjenWgOWV23w16TOeqgouml7n3Sg6i8ZWqkDZgw1+2bgrurW51q1vd6la3utWtfq/qpuDe6loCy0NW5uMsV4pCAH8q2FDITshdIXbqr3H1qTTsDN2noBOqIiQvuDFtSU4AJauCggC9xY25+m8LuebFuyGx3HvcmDBLUqbioaU4Ia0T4K2oT3WO5NYR3uz0CTCXLc1r5e2a00i+212fqlNmOej7LAfD0183hLf65HgKHe+6E+fU8t34AMCrZuBT2PPgR5zJGPTp/msP7qv9wDg25KyMSNsn4s5hguBfrod3epsxs5D2WX1rrT69lzXdKAjhqNP0dio0nbD7UVmR/lzViZCwNUXH5qL+11I23xIAWf2zEjNmivpEf5npf1M4/YkSA0z1lc13qqZDgxvU57V5vKSmyMHGmtx8basnOKv/VkgYI5ghqAdt315ZjNYQ74+aLGQMpW+QYSbvWsz61B4SqXeYRXmx6/ZLzDBXmc47SJnSN6rmhASNoziDeR7q9hRNMAt5U4cpRZW0XlXD0jdKORAh7Rxx54gHy/MvLFExwcyv1JNXfMEMpqrvSr5YOxrtcyLsDLETpBjmO0NqBTcVapMAu+RNSZWEJgutvvXVY+oMoSo9YW9YjsqWDMdrKlg4FEyA7Iv6JLtCOBaaJ0Ouq3h6LJioiWzzvb0Kglnz3UGnlbMV7JyQXEid0dz7QUiH6rk/BopNiClYm8m9IUWDSMGstsI1PW0xIBD3GXsxpFaVVFDPYvYGu+hU96oqFnNdE7IT3AWSKeTF6sT+bMmmkCv3F1uwbSLdq7qc2wyuIOnKL06N4Eb1tPrL2oEQzHJN+MvWUGxmeGuZ3hb8/UxOBt9cp7ONycRoOT4ODE2HSCGllmJgfGO2bW4OBjdUqkGB1DtV2ioDPO48udWLyQSlvmSv2zi9qeelCMUV7MnqtZYsBQivEvZcz8VFiFVd82NNk6z7vKYArhzW3FiyN7jtvpMrZWIP7SdD+tmwsV9zFowJxHj1pqcklKhrguki6dukanpNsDM+U1zGre+R6nVhNb1qfV8pQnecOWehjE7XmHLlHqcO/G+0Uzd9s9N5g5CQELaZCYkZWmWKy6jJjunQIgXM2h3IqFq5Ul2mZVvvV+W11G6OGRfkdcfyyjPfC+FYyIfVu18gGoorFJ+JLQzOKD1lxbpfdCYiN1bXXcCeZ+JDzwqRXbtbZomE+27zzob7htjXDuQdzK8TxRZY134pxDvLUpno3UdTu42Ow/eiTN19o/tRlen4eq8e5Z3XZMgxICkh87J1tGSYkXjg8jMh3GU4Bhgd2EL7o95f2St3t+4lElSxRcD0+vfW6v0pgLiMMYVwb7D3C8ZcddXw3KravwjhTr+n4g71YAPnP4L+R12bYgK7GM5/vFeP8pN+ln+akDHqeh8TxIT78KKdud+xbgrurW51q1vd6la3utWtfq/qpuDeaqsigv/phflLh1kcNZ6d1Kq3tf+gDEF30mQVEwrToz6Ztc+qzuRGk87C3pK9TnCuilV2gp0cqVOFIDv1NOFk8wgud47UCFIK/qX6Dzv1S65qjRtzfY+GuNeJc5PAn9P2WdAiIWPr5L9ZEtnDy5/0nP9Ad2x+VUid+t+m6IjFMCZPNsJDZdz+vPvMr6ZXXGLLgx9wJvEPP/2Un+xfSGu6T9Yc9mnyfPvqhR8+321JVOM3q4pQKK8W8pdGn9wRTICyS5h2pRYYFgETHM2zkGZYHhxuzJsylDuPGQJ2DuTWEx9azKITqPFQub9VXciNVU9rQTm5lwk3VlVpUQW8fc466V4K9vOFfOw25TPd9+pfNgL7jnjfYqd49fgCyTfK3c1Fk5taC1HUo1bV3nRoVVkthfjQIalgnPmt15TG4Z8nJT8EfXovVsidww41iao1mgR2nkm7Rp/Oo/p/aVXNylaVWQFK61S9XpXAylGUUqBxxH275ZzH1hD3sKyZ6U3BXgwyXnPm/UnV0FBVXruoYmsixFbP98qAze3qKTTsvxuJO49/mcne4r9EzHkkP9Q3KoVs1Y9erHD5xmOXQux1An2teJ8pXSLtDHQZuVjlK9fdi3eJeHZ67TmhfU7YWYkTq8cRAfd5pLTKIS6mw43amVkl31LAN5Fl8nRdYFzspuZu5YX00qhEsgvI50aVrsKmzsa9pX0KZGcoVpViciEcPdPjV0QLUc4mbWL/amSZHV2/0N2rVP756YB1iRwMYgsEQ0G9pHNdf0zU1DjzVSKW5ILMSX2SgF2UoWxiIRwKkiz3dxeO7cKcarJasjyfepwTjocRYzJfFu2O5CqYhqPgJvXYm1QwU8LErL7qj8qm5d0dxQip+o7jTkk0YSdbumGxhXQo6o31er2lY0b2kVRTrWS2NJ8My71ga5qaxEJ57DaFUkKG1mLHgAmZ+NhjzwvZmY25S4bx55E3+4lX/cAYPUuyWJPJdWFtra5DH897GhdpqiLrTCbWtW5Y9Fi2LjEFR0qG+/3ItHhcvT5yga4m3pUinMMOFrv5WgGS1+uj+7ioclvAXGZll6+i5hKx1d+f7/fqsW3dNl8BlSRgGnLjsKdZ6S2lUFr//+hoYTpMKsROGN9UP+1YX3MfoInk7MCAe7aaPun0ewpgufd0HybmVy3m0CiftVTiwctUL45IftiT6vpYRPnecWc2csjyUNQLnQXfB1IyiEDxmbS2SD7W+ZehaBfosaVY9enG6vs2c9L1cj0WnSO7ZiPT6PUTsUvW73JbKKNDukSZDctjXeu8zoQQhbTL2j0oot2UlRMsCddGwlnTzOxuwRyUthRO2hkz+4hMVn26aLeJAslCrtd8OiTG4pACYa9UIbMI7SlvvwMUEfKx3XjIedfCrsU+nfld66bg3upWt7rVrW51q1vd6veqbgrurbaSXIjBKeMuw1Qnef2Lof9xfQ08/iPwgybDmK+4fSu3NjWG5kXpBXbKWxoTDvXStkZVydWHZ9kMiXbOtE9powQA6m/KRYkCVEXXGMySaELG7CxuSMS9w1TmbrZCBR9sT/Kpd5x+bhj+WP/B3y2YIqTBcRo7fjB35CI0Jm0K7nfzA9+Pd/zh/gu/HB+xUlii5fO052Wqns4ipIvD7iNGCvvdzOnQqrc2r8pZoekieWqhqG+yGFRBqApud5gZl57slFbh5lL3Hexlue7LqlJmpSBQlT9Tj4+EpP7Hxio/2Ip6Ic9C80Xfx0yR3DmapwV3monHlvRqXxmzerzcadan6c5hlog7C6ZOapeq6NgxKFnBmzrF7JBxofS/rSQQE/nQX+kW61SwXb2GM7lrdEq4cUpBWNac95V+YLCniWIt9jLrtnoLyWzJYWvCkUxB2Zd9e+X3rsvd6terioGdE3Yx7L4vuMofLc7gT+qXa15QP9qiPlgb9OeGd6ru2qVcfXqTKkSh/ypVqK3T9SLYYcGcRkqvzE6A3HrsGIn7FndJVSVWQoKpk8jT64LMgpn0/mQ0kEXV2qqO+C8Wf9G0teakym32lfKwTl6POoUuMSPeYEImdZb2s4BUH7CB4hNlMWvYGjkY8uC261l2EVym+cETD4b+N0o0cJcr79MsBQkZfwmqZIpgUqb7OCFFzcXToyE75dEal0nJEBfLYh0hVDJKgeWpBa8KlClgLhazXKfyi1FKgT8n5lctdlF/rEkL9rwqVZYs+jPNk2E6OD6f7/kMSFtfU6fLw8VjXhz5IW5K+XYvA8udTrybJbM8NJhUsEMkPaoqH+4a/Mui5IbzTGd03TsIYPR6HsSSXaG01U9s9V7nSwN1qt+MV7926sCc0VSw1tB+mrZrDGPInccOC7l1G2PZTV+lys2Gj7+558tuTxotMlrkcdn8o1LT3NJLw/xsyT+dKF8a5HGhvFT5+hjg5Dn5jBksZhZ+3O3hPmBc7fyM1+uEAvhM8YX2g926gnYuxN6w3Hv8RVSF7j3Tux39b1S5T3ed+vhF5zRYAma2ui5UGkw+dtucgZRCfqhKb1V+AVVXK9lFYmG+F4Y/jNi7gFm7FnV7u3cD40tHvAcTrLKX39ddsarIZ2+UOGM0ZUuWuHWHcBZznjCD0c7Qsa0+84Sp64Y/OcziMEmYZ0PZR+TicINQRXNSqx2j2Gl3UwCZteuW1lmCmvK5koKKtRsPfCNIAO4SaD+3LA+iXYgOvS7HmrK5C+qlzygXuSlK9ZjNVY1dDLlRbjKLISx1OCCzpa8l0XONaCJkbq73yzqLUqzBTnq/uqnQnArdp6h867Xz2NW5itZXrm/92V3H71o3BfdWt7rVrW51q1vd6la/V3VTcG91rQL+n/TkV5nsoP+hqj6zPm2l+kTWDJn+x5l48PhnVeni3mkqTc0Wl1KIndVp/6VOnC7qZ+o+LvgfX1SFHCby/YFSOYIUnRQuzlC8wYwRf5oIbw6Emn6WWoMdVaG0c8KfArm1G1MXdHJZVWWHnSLx0JBanfiWqPvV9wvWZF7oGS8Nv5k8H097/vj1Z16CPi3+4f4LRgr//PyaH4cDf3L/iXFWYsJU/Wje68SpSOGHz3f4JlLaTHM/09QJ7RgtYXHkbxdKFOTiiIeqHjypOpLeJjCF5kVITeWuPlfV+qBP7fZclcs6Ob1O6abe456qD8yKTvoOQfPJq1KZjp0yiKmUiSUSjy3xrsN/PCOngfTt4+btSzuP/80TZtdt5yV3rqqvqyqm2e8rZ1ZVk0YJBfe9flZVk8wYKMHoRH1IyLRQ7lXxkmGCQ1cVtgKpkHeNJjZ9peCaUyA/Nuo9iwkRwZxOW3qZhKierZSUfbkEzFg5kVUBKFUdtqjiq2k5HsnX1KLUCcs9uAH1iAos99rZWO+DeCi4i+AGJRc0L0Ax+KHQ1ong7A1h72ieA8t9g50d+bXyS83qq7aGuFu7DAZ/LrgB4l6u09lO+cip1+nw5kkV5uwg1e7Hmuzlx8rOjNd7YU1ayv16nxZSZ8mNwV0ybrTM6zqQYTk3+MPCPHvEgHGZvCj3FqpntkC4z0gUlnvdPlsTmYDqKTZkr10XEwslClg2Xmzce+Je1VHXJEQKrkm/5fc9HCYu0pHOOlGf9wn75Cr3d91HuPs/T1z+6IBkTQG055nS+q0TJLlw+YOe8a2wPFQSw2gpu7j5j1+9OxGrH9e81W7N0nm6P1g4f9Lz1v/zhvGN0H8yTI9dJTckjDfM+3rNJ117ViJIscLwjUeKdgVAPdMPP3lhmBpSMuRoKIND+oCt3NmcGxYPciiYYGheMqkx+FPUextVxXKr9JVuUTpGPjS408Lp57q2LI8J83ohnTz5qVHF7HHB+URKdXK/CwznFukjMav6VY4RY1B2KmBdpjwsqugXsN8seFMwprBMq9fdYA6Bpo0ss/5dc1iYSo+9rOdCyF+AYgl7g10K3Ueh/XJl4mZvEasdCP/ji3rWU6kJiJW+MkVknDFWdOJeKpd4Tpvyx0qDySvVQ5P3yl3YwvJyVmbvODqVItsEYmlehPalkjF6gyRH9oI/LTrjsESlWdT0tVK5vaAsdhMzCWWSL8f6fTrpNqW2KA9YAFc2jy1owl/zUpSiUjSZ0z1PFGfofqXr/ErRSMe2Mp4Dcpkou1ZnKdCZDUph9yER7qzylZ88/izatQHOnaX0GXty5DYjWdcb/2LIde1NO+0ImYul1GuzNBms/pseIE3wtJNs6XXzq0zzxXD4Xl9z+caQOu2imEU52G5QBdpUjrEZFsKbg86COIfktSP5u/+6elNwb3WrW93qVre61a1u9XtVNwX3Vlup4qIcwFIM86P+vYkQe536R1QxkFLTpWJNq5o08SccHMWq9xYRUme25LDUCnYppLYhHF+przEU3OmaJpVaQ24Nkio/8tAguRAO1/z65IXmSRXaYmoalRHcnDYfj5nj9jQtIcGxITdKYjA1Ec1IYdcEnUoHvj2euISGOTlCVXH+WXqDt4lchMdu5Gnpud+PfDntWGo++5RaCEK8eJq7meGpB1/Y95smRggWpCA2Y5tClOr3q/YyUB+YTJblWHBWeatmzoSjw/1SPcFp32DDrLHpvcfETG5VpV7TfdJeFdTSFdynM/nYkQ4N9ryQ2quPyQwLLl8V1nJ/wJwmSluXhQLFO3Lv9TifajJT4yltZYK2TtPLpqBP4cOi6WmXCRuuSV153yqrc9dgn0c9N7B5Z+ObI/PrDjclTSeSytSNGdI6Vi2k+z0yJ1X+972qzSnp9DRQ2gZzHsFa9SXeKX/VvAw6oV23R0KiNE49yiiBY3owxF31vL4tyof0yiH1p6sqkZvVmKr3Rve5kKpF0Q+Fwy9H/J/9oOfiJ280be6hq9ntQtxZ2s/zdt5No0pxMTC99vRfEt2HhQ//zu4rBqdOdEtQagMZYq+Z8dv9m9Z7TNUmOxWaJ2WE5r76WUWwU8JOETsWwnFH7up91a8bhLJtJ1+9scp+RoDKs5R9pASDOXtym4nHgj9Z4k4Y31QP92gIR8vu+0k/cwx6XRrZzrt/URJCrt0MazPeJS5Du/nycxMRqZPeueAPC0EKIXua764UheHne02MmrKuQ05Yjld2pqRC8rA8FtzPBpbBYx4W+t28qZgxWcbJ433i2M9chpY4Os6jwz6tHm5wI0z3huZScGPGhLKlKOr2FExQj7zeI1UZs2wcW3wmFSFMTtfVi1Mv7mTXkKlKxNA/2y8FNxeal0jzcSDX+zTtPRKrElYKxRhlSsesTGXAvF4QKTpBX33GTROxVpmmANPY4GrHKWShFFHucBRcW9sBUjAC1iWa+7h5WMex2Sbu3d2CmEwMVtPMkiGYoql3dUnwJ1Uyi9V7xp+ieltLwb6M9bMq37d3m7e2eIuZ/yI5YKcdP2tV9RPR7t+ldrSc1XmF1mOnSP/RIdHwkruNgU40SBL8k8GOwvgHeo8Vc03hap/1HEsuTG+7qnTKbyexVVqMzIl012qHyxm9J+rt5SaYX2uHJTdFyQNtImWINfmtGL2P1+9BM0fiQ6/K9VdKZu78dj0VZxDvKhN5Zdwa7BTrdisdhiTEfWF5W49j9XvnPiNBaD8bzIJ2VupH+RdDuA+UUVnBSEHapN7lNT30WdnvEvXWDcdKXRqvDGl/KZRROy7tSf262Rv8adnS4Iq3uC+afJh6j63XNO5312NvCu6tbnWrW93qVre61a1+r+qm4N5qq+KqN80W8i5dPV6LYKJOdacO4k4Io6qxy71eQmav/ltZYH5wSKPcy+Tl6iNdrklA84PVfPbKDTXVp7vcO1Ir6jXrLO0X9Zg2LwE3XlVeNwTsy4SMM3nfY6xBCuSVq+oM5ulCOfQUY/BfJuJuj525ZmS7xF97+MA//vwNzxf1zj10I09TvxESnuk4djPeJkKyzNGxRIu1mTJVykNlDJKE5VOHvVjSmwVnM/vmShKwptC6yBgcea/bOc6euXIE4+cOd1YPk0kQDsL5Zw3tKTP+XI17Jijzllw9yFMEZyhGiI+6D837M7n15N4hjSetiUfGbL5PqWSKNd9cUlJ1+DRd6QJj0DSZ+vrSOPJ9Xz3AbPtuXyZImbxrdeI3JUrjiQ+qFvsvI6lz+MuM++4z+dVRc9z3/aYsrnQFO8YtwazI6qtbFYJB1eD7Hak9KNv3eaCcL9tkutwfkPozK30BIB+6Td2Xy0R63KuHLhfm15pUtfuUuLg1CU8IbyMxCubVwlR90rLIpqr6k/KcUwPN+TqtPn3TMr/+Iz2nO4MJqrqlxuCmRP/jqGlHNS8+vN4jUoh75UNe3lmG1z1mKYSqHmnyXcEu+hnFQXKqKK9ebrJ6UiWDnYXSg2SvanG+8nTNECidw4yB5llVnB//fYO8U8Vr1wbGS4tIoe1DTfMySBtJ9d7pusCyOEKbKJP6cefXSnWwy6qqFpajoXmpCuN5ITy02CGQajcmN9XDV1XElAzD6LWbUf2+06ceScpzLrtEODUQtfsRNPQKd4GXnzt2HzLNc9RruBS69yPxoOeuOCF2Quoy8dIgpuAbVSHH2W+f730iLI73p44yWexFqSf2K6KBJtIp9UV53oLIlQFsloT7dCG8PdTJd48fMuPr6zXZfO8Znu7hzUK5uMrBTUg0SFXKu/eVOR4Lh+8jzfOC/+GZ8JOHqw81F0xISPWamzkqQcUZ5tf8VonLYIQSHMYUShHCn6sPPh0y3buFefKIsPlzSzTEqs6KaKJVc1iI0eJcom8C8+IwTe0EAYd+4fnLHvPRkw6ZfOqUflEV3OUBui8Ff8k0TwFXuzq584Sf3gNgXxbs0wk57kh3Da7SYWRctlSrjaSyMoHnhOSMRCW5ANvri6zdvo7sNX3PdCtTN5HPntQVTVszRZO4LtfkweWg3mEbCiYW7QyNSlFYl8NiDBiQJWCCZzl4shfCTrZ7Oe4gPNbPzdQunvrtw8PVl78MQv+xrgtJ11z5ig5RKuM87BzuEtXzHZWoYS9r97DV73HRe8S9GMLrSLEFu6sKbiUJlTYDhultwp8MqWFLMw0PWa/R14se6oImD4JiVwD5dqJ83yFFiLuCPxmKKcyvC0sVyk0Uug/KgJ6swc5gZ4NZrsmLxRql6NSUToz+LmFPE1cC8l+ubgrurW51q1vd6la3utWtfq/qpuDe6loZ3GAI+6RP+lNVQ0X9UqmrnqSjKkR5umbKI1LzpAPNs3oNTSxwtJvap/4kKI3Q1OQSqfzI1QvphoydVZ0xc50k7j32POPqlGXuGk3wGgzsOpZ3X2W+V0qAvcyUXbt5PeOxZTla4h7SO1XOvE18mvecp5bpqeM7e0/rA0bY8tnD4rhcOh7vL8RkCMkyXhpck7aJcpkt7QfL9C7SfLJ18lU4DR2l7vwcHI1LzLHFSqH1kcvcEGaHDFUdiYIdBTtD/7EQ9tV3+ZX6piekUBpH83lUWsIYKN7iPw56fGpKlUE9anYMZK/nQapSLktEQlRCwcpODF+TEFAl1hlM9cSVxmOGBZkCpU4NSy6qlraaAJT2De7jifT6iKt58TLOuBejSqyzymjsG6SqTaBc3v67iDlPlNZfCQopbSoyrqowXy6UviUdW8quhe7dlr1uni+q3LaNXoPnWfcTKF1VdGLCXhYk131IhdgZpkfD6Rf6UeEhQqNe9DQ43MXgBsEs13QxSdU7a6n8ZyF5Id9Z3LjCGwv+kur0dlbFt7WwcyxvVXHX6Wz9M3l933Cnn5WrSJJa9S8WgXjIFDGkPtN+suqNR4kG6tUVii20T4n208Ty0OKGepznqHn1Q1X7QmZ+bNQLVxWZYdCs+u64kJIhzKrQir2mmQ1DS46GpgvMswVXqndeiN16utRHGPaVUdrs8S/LNWEK5X2aWUh7CLMjuazvuwsbg5epVd9fQe+VAraej1VhMomq3iaa756I7+42JXOlgpTqmZYklCjYQyRFw4Ijh6vWY62me7k2EhZDbgtmli2Fy5+VKuMv6k2WXPBDJPZuU/v8UyQ97rZrwISEZK8c2LpfdhH8Rbgc9fhRp+lZ9JgAzI8Ff9ZrYUujmxdl39ZulX9/3uYNSqtT85IKcW83kkD+3MB9oERDd5wJLrPMnhwF3tYu0+gYhwYRsE0iXDxuF4lZNoXStokc9ZpYubdPs6PvwnZtxGg5XzpMk0j3ylcth0gpENZOS9JUNjvpmp0OLcUIJmZi9WK6nKFV4oodIhKSekv7RpVSwD6dKbsOebnovreNzgDEtKm6OIvMC1j9PhhfV699kY1/W4JRT/QecBmiASnY5XpdNJeCH/QeLgJujNjPZ8quReo6JilvMwzK49U1x01sJz7uBTMa8i4hxVCkbB2M67WhPGs3Ztwlqoe7Um/WdbU0tnpsExhRbnkldrDyxWMm3DWk/5u9N/e5LFvTvH7vGjMrQzgAACAASURBVPZwhm+MyMi8N2/dGhpR0Ng4OICwoIWFBUIYSLhIqAVqDwcDC7CQWsLAQ8LiP2gTo9oFhKq7uqry3syMiG8435n23mvCeNfeJxK1WnUxqlTJeaW8ETfifOfsYa11Yj/reX9PI5y/0rVDD+5iBc9VkfXbiZB0NzE7SDeRcrrMVX8/ECeHsYU01vdv0jJ3BIj3Eb9v6D8KqQV3NMR1xtYdCTtAuNFeAJOg/1gY7i0mlIWLb1ImrRvMlLBjQkb9vpHxwvb9q9ZVwb3Wta51rWtd61rXutbPqq4K7rWWUr6mEG4MZTK4quD6neCO6u2TXEit4M6w+jEs6SzTnatd3Fb5rIcAVmhei/JnQZ9OhcX3JwXsKWoHbH2JP0XltlrBnjSRhqz+z7ht62c1uLN2wZdSMFPSrtkxYmbrkTFQPTxzotP+W8PwTcTXjuAfn2/4Pt4hnxtsgXPXcKZBbNE0nnrMpks8v6y1S7gAwTB9kdYjk/I/7dkwvU9IH2Gy5C+Sj0KwnHY9rouUIuxRT7J99viantW8qedJoqZB9Z/V62XHdOGkiuH0qy3+GDUDvV4vSRFzmvmIHaW1yg5sPeYwaWP8FImVTVucWVK+ZIqQ8sWPO/vZYtaEnpoQZM4BhqjEgjkNbP6Z86T3aWX08wXM/MQ96uen2zX57l6fzH94IVdqA6g6LEMgPW6Rc1jSicz+RGkvvEdzmEi3a+zrAff0Cn2nim315JXGq8dQRI/rNFK8Q07DolgQIvK6R+SG8es1uRFSI2QvyqgE5MkuDEgToP9R/W7+UJbO9NTXMXsoZKu56to1fEnzsmPCDJoZb4bKd0TnyUwFmTuzVfU12FDUy/soS168ifUzm6Lqjy+4k6HYQv+D/t35a3Anod0VVZIbfX93jOqnBr2Xq0a9i8aQOsvxG1WB/W+qwp1UlR5XntxnzDqQzw45Xrr7803E7BxT6+k+aiKXHVWx3f6lHnP3EmleRsbHjvHW4vfKZY4rS/Oqx7P9S2XxhuhJ34zIdz3GFaakRBE9d6CI7hpZlu5yf1CCBUD/lLCj+hSHP3ggNYb2WVm089qSvcGfC6vvDcdfq/80Do4QZCE2mDYSJsdqNRKjJa8jqTjM3uGPdWxkveftWyKsLdmCCco6nneQUu+JG4+ZMnZM7H+9UsLFCOvf6DEffqXd7M0nR+wLaZtUBjPgD9WvuS6kVn2gR2OR3DA+fEuzD8tuDJtO1zqZt8oEExLD3QWx0bwYQmgQYAD1H/eBnJwSKlCFOw8WadQYKj4TR4vYjO90zZzeWk25ugnkYBgni/GZmBLT7GMOBtdGcrCsH08cX3uQgmsT+W2mecBwL9jR0n2eiBtP+/FEvG0XL2a86ShO57I9R+h93XlKF/Vx3VN6jwkRGSdNOXzdk+9vFs89Sb8HZr+uFFUSm50Qo4753BVk0jlsd5bclkufQbn8qgzchNtPFGdIDxuKNZh516uq6JIKMkX824SJqtwPDzUpcFOQqGNObiZKUKXdDmb5/rJnqTumQuotYevofjwp47judsZtQ/aG5vWnHnsZ07L2xr4ntUbZz4+J0iduHo/zV/FSU+uU5w7EYIlA/3gmbCuNIQtx8IjLiBRsHylZFXCzrp91mHfEYLxTnrs7gWSzXMOwKRSvVJhSVClud3qt41bPKzuHP2bap0FZ+3Xn9sueir9qXRXca13rWte61rWuda1r/azqquBea6nsjHaQNhk5X7h9kmF8oGa/C91zwU5FeZ4v+vTW/zCQqgctdUKx6pd057R0TEuE5nUibDXRJwsU62mezkiqT/bWqK90Sky36qMsVvDHiDnrk1yzDwsrtRjBHkNld8riFw3f3GhClTXL0607gzkZYlKToBkEF7Tzu5jCGI0elPDTLPUCJRnt3M6CBEGCWbrF3UHVvXQfsV3E+8S480yx46mycs2Thz4ThwaJQlknzN5hT0L7oh/lj0WVtGNWP/Mu4J+OxPsVxl8IFKkVinWYWMjO4OfksXZdb5h2oKZVg3s9UazFvJ0pq5bU16f/9Qr/NjHdNnQfT5i3SOk9cg5LqlwRwZSiSUFvZ/K6g94rb3GcI6RqJ/MXnlyZAsaKcikBhgFpG1XZp4Q5jpRVp59X/bGSM3hNScsrr8zdJErBmCkKpwmZgnYS9y1S77/sj+Dq+LndwBSgrX7h83jx8OYLTxeoOwwjdnBIbHBjoXu+MJunrY4FO5a6E6CeWH/U6736WHBjJnkdo91zIjdSfeQ1zWhMjI8t0GDPHamt+fGpLKl/uZ0TjjLT1hBboVjltc4pQfJW1Y1VpvlskWS047uH0zf1/veF5kUIK+ifMsXA+esVJhbc573+/Kavn+mY7tX3qKqiJrUBnD9Un6Gb1WK7KKjhsd6vk8WOgnsx+IP6WlMD2+8y/edZLWbhhbavSpEwoyqtofJp7ajqTbMT8rkjbGuEVjCUVb0+opxQO3j8XipnG9qXQv+kr3HHhDsFyOrrXljQhkVVRSC2wvBYr9dodU4nof9ex8h01DVgf+fVv5oEezJIhFCnV/tSyF6UGpNUpW/+8oWy6UkPen1NKpryKBC2DW7ISDFMX6TTta8w3ilfPFQShnlVjvjsnW0/q5dxZh+HtVGKxx7crrKxb2Ymq0POgXSryqc/Fda/rZzar1QxjOt6bydD8lY5t4vyJso2LZrqZV8duSmUPjGFSmKRon7oyeh3hEBaJc5PG5qXurb8IpBsIR89p7dG2dHbSBwcTSVRtK/13jiUcb4yxNUGe874Q01NO07ETYM7B9KmITkDa485x4W6YnZHylT5yt5pT4F3P6UN9B5zDuRVo98pRr/PirAwrWUScqdc1mREz9Fn0tGTuurTNTCtDf4tLezx1NrKt76kytmXo64XtQck9pZihOO3elPTbYIoSB8xtuDbURXSs1083O6k47xYmLb6GXHbYoe4kGfaHw7Eu57cWtyTjgUTJv3s9sJ/bl4mJHnMoKrw/q1XFvJ46f0oTSZtp4V9TJcYTg2lvgZbICqjWW4mcjQYW/TXp/odZwt5k5TEVHTcxpXuQCzrWBTyKpGdxZ20x8SdM3bM+H2lgkQlBBEz9hhI6wb/25ergnuta13rWte61rWuda1rXRXca11KYHyXEJ9htPQf9WmyeS1Mt4I7ayezOxWat7QoUaC+2GJFKQrVd+fGhD3Hn3hwc2OJK0OzT/jjJZFmSacxBok1iScVZEjk1uI/HZfs7WwNuIKh+kZbq0/QpSCrmrAWMuZUu/jHiPWOdylTbM/5q0pnaFSRM6IsU5LAZKBPF4NSNOSTw5wsbhDiTfVEBlm8fdrhLpDhZnOmcYlPcY07GeRJnzrtIIymeqtWBY5WM7tXBZ4vSTnJVwV39tfFhP/tC25+eq1qZHhc4/YjaeU1n33lsbuhnq+ety2F6cMW/3JmzmOflcX5fjSvU0386rVL1ZlLt2pVOot3yHlEOk8xFkmF8FAZt7tRqQhZ+YtyGrVr+TxdlNP7W32PVMkZL2+U+xvlFK+rmv6yp3Qt5jToU3cpsO5Vja4qLykhgyZzYQ35ZkWxFll36tsG5ScGUU5w12BC1I5iZ5FjTUgqBRolTcgUMeegaXk3zaKG58bQvglmKrTPI29/0CtpYO5kRz3k3aeJ0zctEgphbchOFTr/VjudRcdFXF8U+EW1fKqGX6OKarhp6D9HwtrWsWWIq9qRHTW1LJ4dpja9u5OOwTmBTA7K6HXzaVrtSi8C0y/v9PKcwsL/9W+BcONpd4nkLamKPvYkhAdViqeHqjYFIfVF/ZdA+8nWznAwozJhH/+PwPmdW9jYkgvZCd1T0HWgc+o1fj5R3qscWoyh2es8nCkteZWQyWA/1p0EqyxuOwj+AHENzb6qmrXsmLQLPyTM24l4o+Oq/XzGvKh6Pf3+OwCaV89ohWSNeiG3gfPv1/k2GdqPDhMc/iAM7zObvzCq+C19AtS1zODOSk+IH25JrV2UxdIYisiy9vlDwsTCeOMuaXnvilJSitC8GtKg9Itil+Z7JZN0RT2LdTp1zwH//duS3rekNZLA1rTGOs/n3Ya413Mpbt46g/zaIDcT+FzvhVAOXq99UNbyTLiQucsfKKIqMNtIGQ2MSs+YZrZrQekEGexBk6/koD0aMx3CnmHzQ8KdMnFlcEPGnpXx7X+rW1rFO0xT/alH5aHPiZczUWJe9y/IDSBlfV1dE0xQhZeYGd553v4Q+OWJrgvYulOXs2BtxtrMOHqmXau7dfnytsOjYfPbxHjvyE7Y/rl2+RMz5jTjPAQ5nJTdOq9tIZN6w+bPK/v8K0GycrrtUPnWQaD9wkeb9RolL7QvCZOUeALatzJfA7sfSJu2rnuVhjOGRcGd53v3VBjeC/LZE+8S7s2S5s8zBf/sCEmQyWji3Pz95mcMB4tPPb+0FFvIU2VeV9N082qIkxI3zKiJZhKF7Artp0s/Q/bKdnbnmmxmK9+97tBOjx12SKSbZvnez3cbzO7I71pXBfda17rWta51rWtd61o/q7oquNdaSlKh+9Eh2UGG9feVa9hqDro/q/fWjpn2eVR1quZQt8/j4sssraV5C6rYhUT3qSoEVrBPB+x4A7moh0xUbZv9mvFO88JtZbsSsyZhbdqFB2umtPietBtcsLtBO2QPKl/ZmJXjaivntXXKUkzawQkg+5lhqr8SzZLLvXhwrapWpSnkWfw0gGgaC6CdvkZ9iV9tDhymlrxJtH/hFwWgiP6MFPX++oOhuNpNWh8zhzvBHwupU59mcQZ702NOE+FxfblRpWgq07YlbB2rzweKN6RKmXBZ6QelcepRrZnsZpguj7TGIFMkt4649rhjAFMoIsgXhARiIm96pG2WruRiNXEMNF1Mor5uObYQkPVKCQxA6dtlfMl5UuJBqp7bc1U6NyvkNCw+q7JqlXVpZVHiZ+WVUtSDOwRYGU0q+jK1qKvqbCoLe/dLhaesOnBK+8id+sHNOVJuG3J3UVpjKzgB93rGnzpVEqXQvuh5pc7y9Hd7mn1heDBkr/5MxNB9oa65Iak3ORXap4Ak9U5Lmj24DXHlGB4sydfMeKuK/kwJAIi9UgTiWlVgO0LsYKhz0A3Q7C+vT412MWuWfZ2b3mKPQceCCKl3pF65rrMqFrfqtzMBii9INJQ2a7pY5VliNBkpbFRRtYOmtrlzXsgqZsq4VDCT7qZIzJUNCk1lNp/f3eo5vM+UtlB8hiyUNhOrB1dOFrmbyKeO2F/4wMdfivqDAcme5mViuu/wlQXa/OaVdL+Grc4de4rIlDHJk24Tbh0owIeHNxqrnxWS5fiLhpAspYAMDaehoxjo6rpx/EZY/VA0sTFZVSF79Uk23z0DMH37gH87YzuvlBcr2M7TbC3TzcXTicB0o97P3GYkCf6t9kKg15Wi42H2Z5ox6a7Guc65dVvnpdUdmFSw+xF31y682Fn9XX+raraRQuMS1mRC0teMwTNNlhQsaVSaQk5Cu76kMQJ4rz7dvtGdnrEyw3Ouc6eqoWUjhEdLSgaeWyQI41d6zN2T5Xxv4NHgjwV3LDSflWVbVqp8pm27pC3KVMidQ1IhrZtFrYzvb/R7JETKutM/n1nb865XysTHNeYclSt7toxnR7B5WRZyNoTJUZJQJos52spZlkUFl6xe72JVAZeQyJ3HvZ3JdefQjIFyf6NrryjJxB0jkqG5qV77CUB93Iri1gRCMwnt66w6FyRDt1PPuj1H3O5M/sJbm1eNfo9+2iNToLQNed0hzizfg+lhQ1p7bv5iYHjsiT34oyOuysJ1VkpHof+NY/Vj4eVf0WMprujOJkAR3KulWKUkSNHek7gt+Le6bvTaQ2KirmFpVWifhfZFlsTS1SfdyYi97nbZqdB+GjDDRXV2x4gZ5l27jDmOpG2nbOPfsa4K7rWuda1rXeta17rWtX5WdVVwr7WUGRN+r6pB9he/YbdTH58di9IMpowZKo9wZq9ue/Kqwb4NlJyR06hPlY1XhQ8opzPkgu2bpUuYmCjtRcnznyKkRFm1mFNNrikFe5wWFU6mykg9BwgRY4x28Tuz8FA1xssgNQQ7d8pHbV8zqap0432hOPUcmknTZSjqGSt99TutIrkxyKvXnPguIU8NqS2LBzB1+rTtP5wpRRijw392tcO6nmYPucuUkyo9qa08wL0w3lffp6vKaYZ2rB23nVM6xDizAA3u0574bqtPuQbi+y1UHjBAXrekbo1EffJPDxvsYax+1tmnNep1zwX/fIIMMoxI21xoAwDGYI6DKgOnqDnnuzekqmJl1cJpVKJB1yjBYN3D4aTdzQBdo17ZqiTrQWZkuPAb8/0GvKuJNcr2NdNEXreX+34eKV2jKU5QfbXDxfuLppTN+eWECax6b0vbXJTk0wDRKmkha5f18M2K6Qt1TT2h1YO+23J6b7ATjLfCtK2paUFVOJOge86kRuifUk2Rqj7UpGqi3wdSP7OVi3KIv8iUd8dI+2rIjXrXml1gfPDILIw7iJ2+3h80RUuKUk0WvyY1XW0ouHNBcsYdVDGed1rscVLCgFXiiJkSJhhOH3Tcg6q2xRbGB5DRYAchPiiPevamT3eZ8V2hebK4k6pAuz+wbL7LS2pa9kY9wE4IDyvcMVCsJd34ZayOd8qCzbXLmur/lzZh3ZyMJbi/7MgWwjeJ4gruzWIHGN5VNVQcfO1Yf0xMt+rfL+Ze2car2uUdlN7QvBbkbOjfj3ibyEU4TnoRQ7SkIsr6NAWzGjk8OtyzI86bKEXnqImFcav+0fbT6SdeULcfIau3Oq08YauEmfODIWzrvYraxZ/XCbOKtG0kBsfk/OJ9zCdL82IxEzRvhf5zwL6N5HUPdX02b2fi40b9x7sT5W5FESWEpE7V0OFdIa9UeW19ICZL4yICi3qds0HaAm2Ejaq81mRWTcBUn+XLqceZzBQtKQvWFJyp6XaTju/77YkxOMbgVMktUG4nGFtmkPLpm0L3JPg3HavulDD7s2LPV5cBbXdn8raDqvxTCvYw/mQtKetOPa/OqKq67VXBneVZI9iT7giu/uLI+usbJDaklSdu514H3TUgs9AgmldVVmdVdaaDmFPSNajudMWH9aI4zkxwKYX4bkP2Bsm64+bOdR1LdccvF6YbIa5KVYdlme/Nvqh3fUjYY6D4y/ktcz2o5xrvdOdLRKk2Y74kmY2RuGk4fdPgToXVj8rxfvsjlvluR9F0RKtUi2annO3Qg6mee3tWVTYbfb0J8M3/PvLdv9UQbi4pbCZoumNca5+KxMoFP+h1ntfOeQ7ZMVOsUXZwZXVbK5jPO02jrP0fbgyL5/x3qauCe61rXeta17rWta51rZ9VXRXca11KYPObTPbqu22OcxpTpv/uQG6demUapyrfFBa1T84TRoRizKLG5ruNKq3HcX578rrHHAbyqiPe9bhPe2UZdjP9QJOzGKMqjCFp97sxyjmE+nsL50C+W2M/v5G36r+aPWdyVmZq6VpV9Yyql/1TImxqzvsDxJUqb8UXyiph2kTZe6SmuCVnICi1Yfbn5m2qf6/v49+qolez2PenFskQNwWpnafFAbaQm0LzahgftKM5W0P7fPHkxV6q/7JyF+8a2peLB05JBHlRDBa6RCo/eVyVlEm9W7yy8z0zLwd9QSmKjjBGFdBx0u7bxiupAPW3SVAftZkiedWqx6xtlFsLFAx50+ufOwOm1b/LeVF5OZwRIyBC3qz0HosoC7eqmOYwqF84ZvWzWUHOswJTT8xZVXFXnXp450S1lBZVtzirefD1vEhJx01KyE59n8wqdYiIMZw+9ORG/XCzHzpsVLEY3hW++3ca7FC7wkeIc666FNq3QljJ4pfzx0i2X6SIhawKjq2Z8bmop3imPsDFn/s8kp1hfNdgxsh42y2UktQIYVO9bAlsKEwbVYfm7mY7FrIHO2nHtTupmpzbL/iR64s6lm9X2CFy/rpVf/EsrhhNdWqfVLXODtLmp1qIO8jC0M0eSl99uJ0sXvnsBAvYQ/X7rj1uN5I6S9yoqjo8COE+qTpnQGzB3iiPMx7rbkwWwruoNIdKK5GoPsFUw7pSr8QLTZJTjrQyuQ227oaYU8C0luZQsGfD/uNGO+Un7WoHfc94G8EVxGXKVFV2q8oUgN8X/Fnv6/qHC3EkrZuF3qD+b1Wxc6NJjXbI2NGw+r4ec6ds2uwNw1eO8T5ijpbuzRArGSNtMiZAs9NUw+HR0/xYKQGzV34KmCkqQ3zbaX9Cp2tnmsfhQcjWcTQrxhdHdoX9u4lyckqSAO2cr30FZjSkOz2el21c2L1MRpe9AudgKJuof+YzctJr9TlaTbXKQv+XjvhVxkR9/3ZX18yDfte0z6HOWyGve00j63SdN+eoJIWq7JVGv1/MaVrWqNJV730pUHcUy2ZFvl1h5vme9V5JUC++P6p6PAVBol6fuC4QBImCCfNYAH8q2Dmt89bRPmtanQHS2lROq0eOc/+JUm2kfqY7RopTmoapc7lEmLayJIKOj5psJkk/DyCshHZnsMeB0lrIhfCwovl0XEgS87jLrde5k8qyXlD9/WmzIa4sYSUM74TshPPXVUmu9z3cJbCF8StlYbuj8msRJRyAUj2kems1RVH4/K+1mAipEjbCQ6L5bHW9aIru/FTiSWqr39cL6x8ixQrTjaEYIXdW1/rlvofFh63XtNG1fqbp/A51VXCvda1rXeta17rWta71s6qrgvszKRGxwJ8Avyml/D0R+QPgfwEegX8M/MellOlf9B6kTPcSOT867eSehbOTPg3al6N2vo+DqmLeXRTTKcBpVA9sZaLm1lG8Jbc6zNyzepRK35JXHrcb9D1gUYIXnu1poHhVgMuqo1hLnr10Q/UVJlUK81rTg4j58pr5KdcaSttgTpOqySFfOs2L+q7KuoAp2Caz3Zwxt0cOJ32CvN+qCvDjD3dKVjg4aDKly6R1Vaq8pftk2fQjx9BgTCFaCF8FpvvKwXyzyGTIX01MpSX3WZNyVonTek56E/rvLcUJ+28dzaHQviSGd83SQd0+R/Kv7tUr9t2zqhPeKhFh5gS3DnLBv43aZW5EFdsM5UZVVTmcVDUvRf3SIS5kgbn72ExJCRe9V1XSzd7TuNx3nKF4Q0bJBRiQ/Zmy7hclr/Qt7PZwu9XPiQkxZkn/mV9TRCid03vVOBgn5djOZAVnkVKQENVzlpJ6zbyjmNnD6eBwgtYD1Z9mBfP0RrnZLK+RtyMiSqIYbw373zNMt2XxmLqzKpJ20o5pO9S327N0BGPUp658U0P/VAhrp77u85zco8laUnR8zkq3lEK4v6gU2Rv860jceF7/0PH8x1tV7aqalH0lfgT9/XBnKAKpM5g4K0xC/0kV5eyses0FUmNY/6aSNE7q58utw+4HSusZbwxxpTsOANInzNFz/jZC9SVKoxn0c+rVzEidbtU/qOxVYboTTh90HNqxkDqLHSwyJpI3WKt+49mPPL7L3P3qla82B1Ix3Dba/X3rB/ZR5dlchCF5Xoee/dByHjxBOtzZXbz8RderYoXzjcUN6pHsngKl+o/DgyriqVG1zG0CKRhKdrgH/dycrCY8TUbn+iph95bcVNY1Ohb6HyeyV2XWHwJp5cmtReq9KBb88wm56Zh8gzsrTSK1fiEaFGFR07qPgh00PU0yrH5bvaq/NFWVZvFnTh82pMbQfq7HvFldaCZAut9ixsD4zZbTVxe/OAIkIW6SKtavHnM/UV7rurGtaqyB1CQIqpYvqi3qT40rVcqxBbEFVlH//jzTFAQZDGWTCNtCaTJM2ntQrbx0zwUT1IvefX+AVEjbFmtl8c6apzfKutfdtzFA4xYyjnyRamVeD7reDJHSVDVzDLrmANzd6Lg+j4QPt7SvmfHWYGJVbmH5vpOk90WKXq/ZK6tzGeLaYkIhGYM7J1Vrp7wo96mzeCuQlXUd10qyGW8NsXr6JSkJJfZUioKqxQiMd/N3gTKkw70SPGxQmlBuHKUm85lTwBxP+v3nzBffu34hDhRR9bSIpuWZKNizEDaZZlfV67uCTAaZ6k6mFyUhjLIwttMqV9+s7nhMgGxQ/229qRK0HyBK0esqyuZOzYUAkp2w+0NPsy/LsWmimVnoPQBiRYk43oHTdFP7clVw//9c/znwf37x//9b4L8rpfwd4AX4T/9Gjupa17rWta51rWtd66+5rgruz6BE5Fvg3wP+G+C/EH0U+reB/7C+5H8G/mvgf/wXvlFKuEMgf1B+a+j1+acD9T4dTqRfPGL2g3ql2uan/sjTAFNYPJJmSsTW4T8f5gNV9cg1y1MxU1B+4VTZottOVaZOc7xz56r/019+prJz86b9gqagiWb2WP2/jaue31a7M2ui1OkrT2qqxyoK0ifaPjAcGtLeMzQRkUKqbMcfv9cEKGL1axVwO6eqXiUkFLXS4W1ijE47sEch7dyifFCgNJnNduDsMl4KYdfibiakqghxdOTnmnzWCLGD9JVbvFsALTDdeEwo2IfN4sPTDtrZRHm5TnY/qj8NyJ2DuQu3bVQN3Z2UZBEC0jSLfxrUx4yzVWEv2rE7K7zzo3HM2LEyi53yezGGvF1ha4IUWGWRiiws2+KsemDne1oKpfqoZX+CTX9JLave63SvZAgZgnbXTgF5O4C1Fy/e0yt0LcVaSu+xv32CGCkpKSEC4DyCd+Rtz/jQKtt5Ar8XTL08doSbP0/sft8qm7i58GkvdAVVskxQckHzPBFuPAQuueq5YF91FyBvO9JNhzlMFC+Lspi9IfaGabti2qqyND6ogjOPHzPp5/hTYbhXn7YeU1mUvexheBSa10pmOCey19eM91VVXbslEUlWDalzZAfjY6L9Ro/zcXtkem/pnM7J3gViMezHlnGr9+Kw6UgC+eSQs63HDDwL+1+Z5Zi338F4t6LZJYoTJjrcKTLdzTImpGz4eNiw263oVhONS3RNYAz6mr4JhGSxe+M/9AAAIABJREFUJvPr+xc+nda8+cRJ1rQfZyVPr+fpvaF7zstYs4N+LkB2hunOVeVQKEA5O2QdLylNUVSxbDXBzW8mQmhxB7vMd5Pg/FVDs1NlPnuLGSM25oV2Eu578rrFnAPOW1JrGB69eoPNhZaSfVXlm6ocijKBz7/Qe+TerK7Fa8ENeq9PHxr8MZP6mlZldEcgr+btB0N2LXFlsaPOr+FByDW5yr1ZclvIfaYcPXJT6QCmUEaLDDW5bp3I66S9B1WdDTe6e2WOltzWPy+6QxVu6kDcNZggpAzxwwSjJa8yZm+W9Kxs69wYE8PXa7ofT9pXkAp5Tu972KpamzN53RI3De4wkTqHqTt19kX7N/K6RUbd2ZOx7vrUHoA52ZEQcc9H4toh2Wry3VRJCLcJuw3k3DA1ifbJcv5Kvaaunvv2L0adO6WompmykkrehoVBbl8C5hwIDysl3gyasFfELRSFsKlpc7H6uovO+ewh1LYFfxCOHyzt3uBOmdwa3FNBSsLWRK90u1Ye+FRj56IygNNdVXQB/3HP+K5hvBeyLwyPSl/JXWacp46pflzR7yh5NbhJGL7+gjtri/KpnZrlwypjj2ZZE0GpDKmta2KdUsWrv3qs5JDcQP+j+sn9KeMO2icw3Tc0tdcktg0mOEzrsR9fVZVv1pdEy9+hrgruz6P+e+C/5LJh9wi8llLmEfEd8Mt/3g+KyH8mIn8iIn8yleGf95JrXeta17rWta51rb9VdVVw/5aXiPw94GMp5R+LyL/5u/58KeUfAv8Q4Na+K6l3rD4G/D4sHkn/fELOIziH/bhTT+cUVDH8kpkKC9dUQkR2ET/7aABzPGOnQN6uNMVpCpV1Z5eucvfjqyq9fUveNGA00csM4dI1HPNCTCjWUBplTdrjtPhQJShLV6YIzhA3jSrK3UUVa14gft8yNQ2sE+ZkGHKPrBLlPHMEDbnLNE+qVORq7Wx2mqgEyiUdHmEIjlKE86HFO/U7ZX/x5JGE464HKeQi6sGNBqkepjIZpjtlDufG4BtZuILda+Xi9obmLZCtId60+E9VHVz5RSk0WRWsdL9GrJD6RpWNkBaVN9+uVPEYRuLvf8DuzqSuwYxB7zWoyiiCOY6E92vc05nSOdK2w33e18/tFk8tMVJ6rySGlMg3Kz2epzfKZkVpHcVa7Ouhqg5lSbADlC6QNaWMlCnblSYZVZ6uhKTe3FKUc2kNOKc7BvM47L/4fcx1jBa4u6HsDstnCWB+fGZ1nhi/vWPz5yfS2l/8mlurDMyXQv+cCCtD+5o4v3eLN705ZvxBO9X9Ud/UjppKtTAy1w4ztBSrfl8pBUmeuPZLx7PNidSJelSnojSJjHIl6yOqHZVmMNwLq4+Z3R/WVKEs6g8GxsfM+jtDuBHYF1IrrL4fSN0XNA1XPZbeIN6CQPeSKX1m0+t9dybjbaK1kVQMGcFI4d3qSFMl7ud+RS7C29ByPrXEg8fuLdPdxWfp9sJwZzAJ7GiUBLE2pKaZBVNu/hSG13vevo34VwtvPbsPmddV9YACu9HQ7AzjY+JjfI87CWFTEClLJ7gdqn+6vm/7rAzR7C8ajj1p9/bb79VErJcWMwnZWhZt6M3r7gtKkwiHRrmlsoi8eh/uDCbpGKG3tC8F9/mgzFaUuVvqjofbnTGdJzeG7lV3ZkCZyiaqJzNsy+L97D4bTNDjDuuCPyo1w4aCFE2ws+dEqkpnbi2mc8pa3U/IpIzy5s0tYyw7R9wIZrCYBCUq91uiYCr3Na60810q79i+WGJfkL0lrWuPRJOh+jjN4EjrjBmF7C67THZv6J6Fk3UUU/QadwUzsaRnxZVyhONKfcvjV6uqahbcvip5972mtp0S5jThRDDn8JN+g7nnwxwG7bK3VnedrMC6XughqIorQl61HL9paN4K5/cXaoR7s6RJsCez+HHtKPgDy/1CIPaW7A3NblKq0BgpItjduQ4OZU671wGM9pxgDM3RM1R/7bQVVj8WhkfBDoUG/U6atl+STCCudKfm7ilhz7q2mtOk6XzU9dKYZb2VoDtVdn/hnedtR7aCnfQ7a/gqYc+meuvr9+nJkjtVZM3Zkn3tQdibxYMr0ergFNH7Xa9J7jPmNB+0+ml1d1M02WyE8Z5FenNnVXTtmJEEx1809E86+4b3qoJ3z5N+30NN3AyYMZLvNvyudf0H7t/++jeAf19E/l3UTXAD/A/AnYi4quJ+C/zmb/AYr3Wta13rWte61rX+2ur6D9y/5VVK+QfAPwCoCu7fL6X8RyLyvwL/AUpS+E+A/+2v8n5ur7QB/8Nu8TVSiiaNlKI+nymAcz9JFineUXouHs4Qye9uMW+nRYFD1A9oDmdK3+h/1iJDxOzVV4S1FO80wQolJvy/Ga/z+5sQwa6RkHBBU8+KnU2vFvu0J686Su9wh4npsSd1X6Yfwc0/geHRML6D5sVgB8P5a0PaVMXCX1Qi0G7S0mb8zjPVBDJ9ItUkIGMyxqlHy++F6ab+XJ+RKNiPmuKWVlkZuaIJQwDu1S2MQX9Q1U67eC+qWDHCdKN+5GIE/1TPdj+oVwlUOWgcuVFvoDlMYOWnSThRVA3tO8xpInea767Ui0299pMybUVwT2dV5UtRskUdG8VbStsjo6qrqq5HVd4bXV7KqlPVP4NJUX1jb8dL6hyo6lrK0gmeb1akTYt7PZNn/nH1VRZroQOzPyslwbsLl7fxCyfSPr9pB3arme3L5zWe8vSC3N4gxzN22Gii29OJtJkz5ROnb1r650TzFol9Q26E9i2pWgiMt5b4ThXdYgRJVaGNZVFMU+NVnTIXPmxqLe3H04VnKUK2QugNw4NhvL/kw8+eRdBxEA+G8d4yPmbsrIb9oirBp5ptXxW47IXzh0792vV4MPr37pyIvfo7Vx8nVn/W87x/AOBzl/EPA/Gpp/1oGX49qc/27uLPTnuP21lSV1W/TpU6Owrts77GHQvrH5UNLLnobsCYmO6axVcee0PqHNPBkO2lq9y+OjCXDu7RgDsa2s/Kvm1e9dfZF+tPRTnBA7S7pMlQbwNp0144wAJmykw3ELeZYgvlNmFeHbmvc9wW0n2CyShp4GTAKCkiVdXKDap626EQNgY76lwM7zfLPDWnoHPOW+UnHyd439E+B3xVlfe/50l9oZgCVuekZL0/pe4AtC+CO6m6a6ai4/QYIENc2fm01Me9smQruFPEhEy2ZlkPz+8FE0pN6FM/pR00UWveZWpfDNON7kA0rwaJcPd/wcvfBVt7A8JtpnSJcI+SZ/aWtMlIl7CfdE2Ydx1u/tQwPFRl+KjK3+wXT72eoz9EXTNyQeKFVwy65pmT7srJ6aA7DjEhEey8jqWs6YenUdXbnDXVLoM56o5WaXRXKX19R1x55cF6ZdAOX8+0E2Us+8OFlJF6vbizx/T4daNjKBSQRpO4hrqjdKrrT6/HIrnosfieYkU5tI/63uNDYXgP2WfyKuNftc/B72VJBQs3eu9TL7z9umH9o1GOsjPYmQtcxxYoNaJY7Y8wLwfSB+0dyd5y/May+2NNvWwfzmxXOmnm9oeQ9ARTMpx2PbkzxMeC30zYmbHtNAXPmMJ49jifcL7SlSr/PQSLtRljCse9TpZz/0XMIpqI1lghrixmpO5YqT/Z12sY1g57DJj9QPxwq99fpSx9Or9LXT24P9/6r9CGsz9FPbn/09/w8VzrWte61rWuda1r/bXUVcH9GVUp5R8B/6j+/p8C//rv+PPYw6ge2NYj1bNYhgHpOlXAQlCfYynI/qTJLUBZtZipJlTNnMJSyJvu4gdat8oNrCkrc0c89ouUssarEhhSZXWOMKrXaPY1ls4jeHLbLMSA3DnMEElVHcneYPqWdNNqIo6t3eR3QmqqerQuuJNZVKOwKUz3RRUNf0nq8q92SfLxb0bTxhwXRqmFGufOy8sG+0NLs9dEmLlLN7WW4fcmOBjsKIB6E4sROH6RNtSosmBHWH1Sj6cdM2bKy3ml3tD9cNLrIarMklk6Z0tNjdIMc4tIJm5b7GlaPK9mihCUGyljBM/FF3ujXOE5Vx0RxFnySlPKZAyLl9cMk46DvkWKVwV402E+vlDe3et9rsliZWWVaBGiKq9dsyQSUYqq/o83qtiXgpkSZVbBqJ6zYpfzJqgaDCDPO32b7RqkUeKDtcrOjZXnG6sC0LfwzVekTYvZD5dx1Xpi9TUOj6qS7791mFj9185Uxmy9p72SFLrXzHArTFtDs8+cPzhOH3RQrX8IjLfqe7TnjDtrVzU5Yz9VH/N2TXnscGNhvIPxVxN+NWFMoW+qElx3B86nlrYL2GBJySBA16qiM46ew1tD82SJa2E6CPYM/XMGqUzZSmbIFhCh/zhRROieCrPecfpVJpw9kuDmzwpx7Yk3CaSQdlWRMTpPzKRKIEU9fvOOA0BzLLQvU03fK7qjMET67wPHX+suwXijXmIThbjK2CerqYHh4nnV+xrh6ClW/aqxqtRfpgC6U1G1bGNxx6QUkcOImWZiQyH1TlXxDNJHymirR7TO750l/SqQJ4MZNdUqOyUdzEqeCbD6fsRMCf9mVHk0cuE3g3oif/sMt1uwRn3rx8T5vWeqqXD+CP5gmG5h+CZiRtFb8MV5pxZu/ywT+7r7FYumwa0b+n+i2zfh61vcYcKOulPhdmfi/UrTwfxMWdH1RQr4N2G60/M5fxswtd+gmEJeZdzOMt1luo+Gl39Vx4qZNxIEzMEhCfXlZoEmI69+uRepg+QhvK+Kr4A76rnleivsqJ8/PHr8IeGGhP+nP4D3S4qVTAEJETNO2qsx++9jXtYECUmpKutO0yuHSakr06WHRMZJObkhcfh2zXivCV1mFPrv9YDCphAfImEU4rug92E0hDdLU9PXwkZwpzrOjODGTNg2pNbgzjov3DFA3yCnkXS/Jtw0uGNk90eG8C+rotz3k/rHAWsy4YMlZyElA6kmaP7Qa/pXgeMvhGmr7Fg7eXxltLcvQXswrOi5ekfuPSblZXfI/7gjrtbYuwkjBRGIyTBFuwB4QrBsVgPOZLg9Y21mqnMmzwl/pqABlcJqM9K4SEh63PNrRPS9vE90q4kYLPE2gBTd1QRSEk4PFpJgj4bmxdA9V+iw6DH3n5REkbcd4aah250p9v/bP1WvCu61rnWta13rWte61rV+VnVVcK91qVIuyWQipK9VgTO7kyqsKVPW98g4KfO28ep7QhULTb1qkEmVNfO81+SsOdHKiCo5q4a49jSfT6ruDBdvH6cBeR5J376/+EVhSagBSOtGVdnqrbOf98uT2vzU7l5OqhiLqFpZCk6EZt8RK2swrzLTfe2QNpA32hFcthHjqmI6Kfu2OFVAhq8SpSnksyzKb/ukRIX9ocf/RasdxkE7hX21FhcDRKNqgMyKkxBXBX+sT8BJ/Wvr36pSY8dMMcLw4Og/q0pnx4Qd1W9sYlbVImdVvutVMGOAA6RNgz0GVbdDQsaEOdR8dmMonaoiuXOLz6msL+laquQ3ykkMSfPunVIrZq+1TIGyXV3UlVnF366RodIYnKV86cVqvKaNNR7OigDIj3fKNB6mmsbTYF6PSMoLjYFctFsaYJxUkb1ZY47nyziMiXzryN7qDsHuQL7bqho0p/vsDvD+HhkTEpMmSn2zUgJCrbAWUiOkDsZe8HuIvXa9x9XlVMY7YXyw6gcdhP6TcPil0LzVsfGmKWKpV3aymZImxIlcjjlE7JA4vXeYAHKw5JcVsSnMoWlYKG2CaCiAMZm2DYRgiZXZbG0mtQmJbkll0pQ1WXYbZoXVDQV/CNrt74T+KSvnGU1jmu4Eezac3wvZZZrPlsm7ZWcDUe6lPwjFKfdZMrTPRVOzAAqcP7TYITM8WFYfI6YxS5f8fDwmqDcve009KlJI66IeWMDfjYRdS1oVhrbQfjaErXp/zUIJEJpDJvb6ucUKadVQGrMQWgAOv2yYHmviXhFN4yqa0AUwvY94lyih+kknPVnJcPPP8uW8vmpZ/eZEMULaNEoDSC3NTs/NDhFxTpnLMRJ/cUPqDGFtCJvZF1sWRdWcDcVq93luwM5N+S0cfmHpP2fiSlPr4u9vaHYRPqvZ2a07ijHYfZ1LK6VUuEMgzizzT7qGTbdw+kavXeoK/tktu0zFaVpb6jPcBgYamp2hfYHxrnbTT0JeJ+RskdHoGHvVf0bMqWAIlCLkRv3FpSmUweoaONvpzzXprlWfsMRCGSd4uP1ibBj9PjlPSko4Tbr7Uwp59soD6bbHfXpTikDfqk93prEAhBOyOyD+lv2vBP54T9tEvFVfKUAvhbt+IP3a0LnAp+Oa89gwrBvOtd+g2anybc9C9xlS5+g/G/rPdU4D9vMbedtTVi25pvWZqMSAdNQvjGMlZJSTU0KHK5h1YLMdWLU6fl5s5tz2Su9BExUnI3RP4N90gvlPJ11j20bTGYcJeq/r8KjHEz7c0uwKx08dRLBvhtffm7C13wOgrBMvP6zAFfyr4XyXIYHfG01yBMb7TPNs8Cc4/FHkCNiTIbcFmdcWqzs6p01m86eOvAVb6Rnz2Ch9xu+sfnZRz/HmN+rF9UedX/YcNcUScHuLDNoXJKffHWN6VXCvda1rXeta17rWta71s6qrgnutn5QU7f6U84g9qoxQGq8qbO1Ul5Qpm1XNvv7CMDYnaeWiiSrvbtWzWf/avhzVe1kKvnbsp1WDCUnTyEC76yt7NXeNdv5HTbMpToerezrq5x6SPrnmQkkZnMWevkjiGiI2JFV8B82k7z9l9QwCL2ujPM2sT5/2oIkypg+LP2kSrwqvKwxfZUozd8KXRRUzQbu/095Dr+8X86UjWq+r/s90X2hehO5JCQvhNuOP9Sl9rMlGqSyd4MVA+5IuXfCgSVi9xz7tie+2mCkp07Yq3jPNwD2d1YuaCrlzpG1LqelHctYubzkOiPR6352tnbmzUlWWTvB421clV9OaZPGtNshpIHc3NREna8rcTb+oCJIS5vWgtIC1+nt5uFWfWFE5PW8aiii/UUJc1FtyxtREtPTuBrNT1ZW2QYwhrRrs25F8X3EVNcvevZ7geQdNowxMY5ZENHFWlTU3d4Z7wtpw/HBh8p6/qoSCvlA8zCqeP1w6/ae7ot5UnzFtIo+W89eO5lUWH+rpnSU1PZRC/zkQN57mOel1mhPniqaWZa+e3tRduv6znbuqCxiDPVhW//eKw68zHFWFa16r8toXZJvxR2heLylrt/9sXDzks9IEYM6R4esVuRHOD2bpfncnoRhDauH4babcB+TQIIPFVr94fBeWuVAEsIVsldvZPtUUrnPGnRKps7SvGXuqfuK1V48sYDYGE9Sr3n6++OFB1UKAcGgwJ4OJQvusajoirL4vlNm6f8yEtaF7STRvgeQNcV19hNXzmp0Qe2geB1bdxOsPW2Q0hMe4KNNycuSkfODcFXJQz34xslxDRL2kp1+uaN4iRVj812lW7cZI+PV73E5VJxMzDOpnDRXnWSxIhPOHvPBPc6OKWJ4BNkbHRmoEGdXbKkXnIb/4oK9pHORM3PS43Vl5ziIkJxcF9yUz3QgShXivjG4zCeNDhkqMcXtD/53l9IcBJku5jYTkkGiW62xHwZ51hyC+C+RqqnVHWfy1JunuQbPTnoKS1Js93eXLmjkJ043FnTKxN/S/GSm//LDsFoESCSiF4nvdFWwc4owqlOGyRpm6ZpSureqlpmzOPG+moLslpWASpJkMYAre6pi0Ung9d+Rs+HHaMjx3uFeHsQVXx/zMm25fCv6k5I7uKWGHi4JbumYhd0guuFMke4M7w/pPdXCHjSO3eo0kamrZ+Gg5/tiy7y5KuS3qTbeD7oy0u4ydCrmt43nTwEYZ52nlMOdWaTp9gznod3e8vWP1KTN9ZytrVwkaZKVa6PXReZtNIWwK5mRqQh8L6715UT5w3IAkHUfpNiov+aDn5QaDGSFEYXwo9J9052h4hPVv6lxe6zozs76bV13j2nNZehuKN4y/fsAO6fLvjqy9Cr9rXRXca13rWte61rWuda1r/azqquBe66f18RmpKlvpqocpJeR41qSqriWvOvKmUU/nSZ+SJSSKterh7fzi2cyrhjI3Xmft7kwrj30bSTct9m1U9bCqWeVmTbGCeXrDAPFhjRuCqn2Vk1ha9YCWtlH/pQjpfqXEhfo+6bbXrmZjFsaiPU64oSdUfqR/06SWItpRK6gyE1+6Ra1Z/aXTmO9ela3sVCmzoyzqbLjRp3F7NOoLnJRjO3sg9aAFt3OYqtIGowpO82qWbvHT1wW/1y79aWtp9jXH3JmFETndtbSniewNsuqQlJkee9zRYV/UXxsfVpVcIOoFDKkmwgllVg29XXjEs39aziMmJuI7DQ63NS3O7E5IUD8zOVfKRU3Gar1yizunfses3l/3Nui9AfXCGbPkpktMpPs19vmw8HRlTNgpqoJcLERINz3mC76vjDWJrUQlMKSk4+7mklNeXKM+7JSR5uL7LZ2/KNOzt28MhHdbYqcJYiYI081PvapxWzCDUjNiVxjeF3J38aGas9G33Tn8WFW+2hQMdVyMWdWJotzX+O2K5jWCqCIhuRA3moqVG+XIbn6bOH5tF8LDCTBJfbxxpb5NTdQzy2f5vZq73RHsBP6YVYmpbFQAewqk3pNbq/zZG4uJhfatENvqeX2D4RFyW+dbUi+yhC8Sq0aDTGZR7dxJNHFrZPHFSipIKvjdpJSBMTK873/S3V+szpVZXQtrQYpjur34oe2rozj1c473euH9XpXCaSX18zPti3qZJSTIjtwYshPsWd8r3teO+cHxemyQyagKnQSZFdMukU4OuozZK5/UnQQThdd/qbJpn8GdC24oIHpP5vSvVNU1bjvsORLv+pqgaPV4mgvT2kRlF5tJ72XqCrEv2MjlRfV1sddr076BP2UOv+roq9KfOos/RFJnMWNV0/Yjb39ny+4Pqi8/KJFBiipwcV2wA5RVQk4zRUEZrWSQ8f9h702eJduy9K7f7k7n3W3iRsTrMvNVZUrVqEoygcAYIDPMmDFhAnP+JIZoyIwxDDUEGWYyIQqpVE1WVma9fO9FfxtvTrc7Bmv7uS/BDFMyI/FllhYRN/26Hz9nn+3Pv/Wt31dmMZIirPOiXqooxzp+7sUPbjLuoBfFG8A8aUL3nODo9gqyInSiWILss9FB2mqa+8j8osOMEXMYFwZ6bEpSmNEorQibCvc4oo8TeioJjutO9vxzy81o+dw6p20C6eW1UA1WFe27zPzLFWOG3uWFd67i2YecsA+WKlJ8yrD5u0Ld6RTdO1Fqq0NC+0T1aZC9czq3PwzZafQYSbWRzpsWJnTontnG81Wmvhfl1K/APSl0VMv5yabMb2T5DFm9jZhZuiCL3/exJ151MpeQKvRTL52tmEnr8hleOqzVPpOcEE+qJ4ffQPPxvMIUfg1hJWtRe0iVrJkzt1iYv+A30H5n8FtRkk2vqfbl3C+kIcW8K2thkH3V9iUxroyA+LXQgqpjxswZO8j5BOlOKK1kXT+OkmbmwzNp6beoi4J7qUtd6lKXutSlLnWp36m6KLiXei6lIKcl+UWd1b4YyZsOYkKfBrLRmNn/RpIZMRVFsCZedeR1jTnNhJXDlGQaQkRPXpReJylbcVOTjVrSWQDMu0fS3RVZKbKW/PDU2uUbXuwsFsjG4D+7Qhd/aupElYPi8eknlLPy8/2JeHdF837CF4W6+aSIjWIqiWSpypKZPmjSlTxn/6OA6fXzZPZasthDm2nfn5m7or7EVSJZjTtIYo+ZxTcFMLwyVA+ihsT2eYpZJagfyunPokyZKVM/BlSG6Uq8XGefZbbC1CRnsJpYPLXJGVTh18posag1sbFYH7EPvahJZ69zUcDRWryvMQsrcn8SJRzEoxsjVA51GsldLZxbZzBnNbR437JRi1qgYiKuqsVLF65azF6Ta4POmVx+N+1Wy2NUzvD+nvj3vhDl+TgTtjVuigvfV/WTpN9VDjV7+f1hJjcOXfx2aV1jP+xJ2460FRWfmMTbWygK8aoVT/Jty3RtOXxhFl/kdFfe1+2ENhmjE9EbpsdKcus1CxTUfbRoL8q49jJZPe2ge5cX5aM6JrJRdG/l+KqngJ4i5jTJdQTmK0d/Z/CdIrbQfMwMLzR+/cwNRRU1JUP1BPNWkeqMmZ4n7kMnKq47ZfxK0X46+3eft/nYiLKpfRKmcqUYbxSxVgvndXgpKnXuovj1QlGWPptQxRNsv2mXeyZbFpZt1uqZtewN2lvckIXnrCB0GneMDLdlovxzVZTDhD0pSQ70kF7O5EJ10E9W/OhWFEC/lrS/WLMQUaadwY4adhbbJ5q3gi+Jq4pU2NixlveqHirh2zaSwMXBkufis1wF4mRQJpF2Mv1v38vUtxvPE+WiaI23GnfM4kOcIDSG7qPsG35jiLUm1Qp3iMxbI+f6hVrUzdW3Cr8BPYh66leiFptJLd5rd1ScvsxUj6KQhwZCLX7K/vWzYTk2DndMTC8a3CEwXzcMt5rphazn1a81sYVoIa7LfbkFvIKSUBfWGoIotNoLBcGeFH6TSCURLbWRbAsnPGi018y3Ed1rSWQDptdeiBwJSDIpbw+yH/mbcg82mpPXhAaaT7qwZcGODc1jSXY8RcKmwh5m9BypPvalw/Pc8VPjRNxu0ZVD7Y/EL++WTp86U0pGL3tYyjRPifwLzbxVDHdQfSiPSQq/S6jZsPquJFBqIeecpcnmU8IOkiaXDeSk0E8n2Wu6wuP2EfexJzVWVO1OupmxeU5xy0au63j3zOPt3mVCo7j9c9kn7v+wJrRyz6/fJJqPI/o4kzq30DKwZlG8w6aivj+SpxmVMvFa7s/DjyrZV5qyP5QZAtvDeHdWWRXTC0najE2iehI6RmgzOj7PIthe/MAoCFt5vJ4Uw6uz517uZZlPgeNXmfpe404w74rveYLbfzNx+FHFzb89Mr5sqR8rsz4xAAAgAElEQVRm6T4U1vv5c/5Mm0mrVhjt/y/k2IuCe6lLXepSl7rUpS51qd+puii4l/qNUtaS7x9Rr14sP0u7Tr6NG4V5c4+qKzIJtT9CIRvkriFdr4v3NpK0YnrRFs9XUeyuVsTGLmpfdT+gfSQau/BrzXESf+XkwRUqglHoIci3YpCs9kUlSsJnTUkmbUvCjX0cSLsOPQZSZTFKoceZuHKEpvitlHyTPf04iUJb/Gjq1cSqFVXjnDATv+3gq0EmSDWkh4q5sCGrB/mGLGOvmfkqYwbFvFPESo4ntJmwkZz3bDKhFeWoeWuWSVUVJec8VorjF47QKtwxM+802ova5w6RbEVRy1ZjxoD7cCTcPE+YqiCJTdWHk6jcNy3V2wOm9/gb+WZvj/ZZPY0Rf9NhnwaUs8LVhYWJTIhghKRgjrOc77qYFo2W/PUM+uRROWN6Xzy59vma1kbU4yCKsB48uZK1ACWRaN1h70+kVU1qLPZxEs5judakRGod9u2jvH4CrKStLevn/ZNMUo9efMs+LP68s5eXkuf+9NOO+z+BcDtj20AGunLdrU7MwdA/taijoXkv6U46qIUfqUNRyW1ht1aK9mNeuMnnx5hJWLPEjHscxfv7ck0y5w6Awq/UotLO27MSIglXcn4Uc5WpPhjMBCCTzGZ8nsr3O1GO5zLRPtwa3ClR7SPzrhAkEpg5MVw5pq3CbxSHnwXMUS8KbnZZ2LZtII4WVSX8HTj3TGDwV9IiOHs1RSFSqJqF82pm6N6ASsJwJRlUgnlnOX5Zpqq3mWzk9ebPA8waXKJunkkm02DIaw97R3aSZFY9GcYXz56+5BSzE59z921PVgrz4ZFU3WBH6VqMNw39z2ZsE0hR0bSeEDRBZ3ldYLMeGCtH8IZ2N3A6NKTHhvm1J3105R6Ue1tF6epkI35jf63Y/UJea3gtXQ3fKXyrSU7x4T/IYCN6KKpYp5iupGu0+0vF8fcjUUGIannM/DpQbSfGbzvSvYYse805TQtEFTNTwu1n+s8aqifxdO9/ljCvRd6P71f4VekebSVdarseeHrqSPNZ6TSw9nSbiZNpUTYxugxVwn4o906G+mZgOtZs7o74K0trI/2xJk/yPPV2YjrWKJ1FGZ80YQVpHbl+LYDoh2930oVwMGol8w0G2o9wLPv86r3MI2QF9duTdJS8kHPO93uunSSbdbVwUkNCjzPhqsMWkgA5kzYNKiSik26FCkJ5sEUM7V9l0ipCVEw3evEcp0q8z1CoBzFjh0hYGdx+FnXRR+J14fL2osbH1oFS3P9RU7jJmeZTuaZbRBVfR5TXGIQ3rQO8/U+KEhxEbZ23xRt9W2Mbiz1MS5cybTvUMKP7GRcSua3xtyvs07Dcp8evFMPvTZiPFalJjC9FndezdBBB9pm4Lfd2gun8MdskzLHsUU0mrqH+YIhNJjvpfvjGsH4h3ZLjfSfe/NL1SV1itJn0YBb1GuDdP6lpPmUOX6/wnQJVyZxCVfbwPgHCr9b9JDMeq4bU2h+G/P171UXBvdSlLnWpS13qUpe61O9UXRTcSz1XzpKKkoWbqs9M2VTYhEkJOzRnMBacW9iiuatRcyA1FebpROx2JKdxB78QAGIniVp4iG1Rcq3GHqbntKGYSJsO3U8oHxfVUMWIKezMXDvx8jqDfvMo3tH/y1sJV614eVISBdka0JrY2YVROt1mps+e+bLqaCArjI1UVn5+OLZ03cR+lVg1ntOHTpQOnUlF8fIbmTxGZdgEQraAkanwTXlbTYadJ/eW7u6ENYl9WBEbvUzup1rUYFQWlSGVyecIZiy+spTRQ2B60VA9ZFGou5rUGBLPHFd3PxJXNSpn3MOIGmfSqma4K9SCG8fm509lQjViBi+TwDGK6gnovqQGbVaoGHH3vaQO9WFJJsvOiBL+7onwcksqaXV6DOin4oO83YiqehpkwtkHeHhC3VwtqmpWavF0p0p8x/ooj19oDM5i3z3JG9QK/bAXHrNSi78Wo5ep21Rb8rrFvH8QksNBjkc5K/7lWtR0dTKEklh3fCrnxyvx4PXPlIv6Xkv605NerhcJmo/i404VzFax+0VkvC682FbhV4ZUKWyf5NxYzfCiWtinoREaQegADfNOeMv2SaOLQpq6iG4DaW/Y/xTMmJmvE/OVJCGBsJrrj0bUqeE8nXyeZlfL+gGwY2K8MkzXmfZFz6DahcZgVoHkNVUVGE4OLKxue+bJ4p8KWUWJskcbUSqTjo7xdUJ5JZ51QD8o/Ap0FPbl/KpMzG8sw9clCW80cg2ayPaqJ2XF8bFlHhxVW3z5G0+7mhgODnvUTBvP8aelG/Je9p9Yi5KpciauHLE28KrDPc3MN9L9GO7g9WcPHMea06HBmMTL7ZHTXPF0lMe82hz49vGKq3VPzoq5tkyvPPVmIq3kmIf7hua9xfaingvlQFHtM/vfkw6JmTK+KPF+oxhvM91XB4ZThTo0ZU+AsItgMvufGfG1moz7vsIXioRyCa0z/tYzZ0fsFNWDTKCfVbF5o4i1Ydq1aA/TTcVwbVh9/YjV8jz7ly351URdB5TKhKCpbKRdzezu5P76/vsbrq56du1IW3ke9x2ba7nfju9vAKhe9WxXIx8HtySBtZVnNBVmXVLcTELZxHorCniuFSlqXOu5W8k9aH6U+RSucXtNMkLHcHvFdPXM0wWDmSFWjrDaUj969BAw80gq9JXUOeGADzN51Up3yRnsfly6TMpLAuO8qxju9EIiCatMLGMLsRWaBsD4had+44itMMvP92l1SpgpCZkmZnQ/o/uR8HKLLnMmevL4mw4zBsKmYrwRD7K/Dajyxvw2gQZzMJIct05or2U9rJ99sXZQ6Aj9C832mySd0drCdaGvxExu3NIpy0pIJbl2+J3sq8PngbuXez7kHYxahNE6kU+GtCvkmcGgT4bURrmnrjw5KtCZWJUPyyDs3PHHE6q3VLuJ+aGhvhlYlfS1o+owg8aeSkcTyOvI5PKyJ3gguYztDckq7JAZrwzbbzyhMKTjukKFjJ68zFGcJlJdyVwNv11dFNxLXepSl7rUpS51qUv9TtVFwb3Ub1ZMsBPZ8cwjJOYlDeqcFoPRwvwred/ZKFTO6HEmt5Lk4vozm7QoOnNhGsaE62dh5BpJpjl/1Yq7Vr6lWlHOslGkzpF1tXh57cfjkkIVX1+L0jzNoj6XL5z241Em7o0RJqyzksjVPacl2ZMibSY2q5GHTxua64HhY8fcO2IobMh3NftrA1Xi9NRIqk8jXi19npQvHMCwsqQuYWa1sHOf/bUKUwV04/n8as/T2IjfNynqx+KFasSHGTrxnVWHLPxAlRd/kt8Y6gclrGAfyU5LcpFWS9pZ1koSvbQio4itQw2O4bOOaXP2Ryr6H21p3/aoCUkMaoRpe2bKMk7QNuTWkZTDvnuEVUvq6oWwkZwWv/NVURWCEAtiV0ESeUSNwoJMV2tRdXOGzZr86QG+eFnWj0FNCv9ijTme43OKutsVOeDMQkxJ1qnWqHESFbgwL3MIqKYm9yOqkfWQVy3xqsMW75p62JOut9T7zOaXGhUzpy9Kck956/NtxBy1eJ3rLLzW0zNX8lzuKLxIO8h0vDsqfKcXeoaOwmjVc6L+7mlZ3829ZywkgVjD8CoTt0GUlDqh1oGAW5LDUq1IgyWvJCeepNGTIjV5eUzWMN8kzGBo30N0inmjcYN4NGUdil919asj/e2O+S6wtZGxTmRf/LSFIjA8tDRXI362rJuJd5868ZmD/JkUmEzOCuokPlb1vOa1Fz+ujvJ3M2VSpTh+qbi+k3S6p6cO6yIpGobRUddyAVwTsFaOefIarTNsPJPNomQXwsLZ15h1ZrwR3up43aCS+NmbR7Oo1tOrwNfbe/7t9FoUKiBlxWebPdtGTJdrN/HT24+EIts1NnDqJo5Dzdd393LM24b3+ZYJqN8ZspEOTaoUx6+KQn7SrL+TKflkYf7Mc9OOnD50S/jj8COPbgI5KcxuwiCM3vnVM1GG0TBR8Uc/+Z7pK8vfvnmBX9fEVtO9ldearyhUCYWZMypr5p1i00x8eJC9PG4iVmd+dPPArx+u+OrFI1Ow3Kx6Xq/EF3u8qbnqRM1dVTNpAzkrKhvZ/PEnuS4mEZPmZ1++57Y58abfctce6ZzHGdkQnY48dC27euSDiWgFnz5uyFmxn+Xz4k9ffM9f2cC7P3slnao2Y49CxrBn66wRlXt8oake5b7q3mn0LPue3DuGeeuwK4c9+SX1SvkZVWYIcuPQxxn/RUuy4qsdXifMoNj8qtzv1xm3m/CDQw2G6WVARYX93jx3QRScPnO43mL7CLrFxQwpP88tVBYdUpn1MFQHGP9oRAPzi+e9Q688cbTokyGvAj7Z35AbtZfUPXeUPcSvDdWj7K+pKLb2wx41eeLra/y2on5zgGPAv94Ry+dFdTMQk0I/CUtaD6KQZ5dFlQWqu57pWAsP2p0T4kAdLXlVPuR0BpsxVSLbmab2zLUjBMMwF065SeSXE9PJSQqhK3tCZlHBzaCwpzP3+XmOYbirsIVXPV056kcPMRE7K8z3lJdr/tvURcG91KUudalLXepSl7rU71RdFNxLPZdS4qm1ZeL9TD/oRLXNbYU6jaiYSG2FGtXi7dNjWJiqaV2hpygpLHNYWK3uvie1bkk9U1Mkrco38aIW65AwR6EdmCnKhGyRPM68W3ImXEuCjQolWUspmGZU8V3lygkFoNIkq1FtRVhXhEYzFSbfeJdQSXO3OpGS5sX6xPcqE6MmpTLlvY6okzANcxfITrx/atSLWqw9zDthG4ashY9b0o3OKm+qMs5kfvbyAwC/fLrFniSj/OzBna4z1ZMohbEV9at+TPhOLykx4seNkn61qdBjISGkvPCGVT+Rdh2p0ti9eJnTplmULDhzSzNhXWG0Qvd+8bP5KxnLr/qRtJWknOwM8cUWvR/QRRE/vy4xkSoj6rwvPrzT9BtLKxtDaqwovI8H0vUKPU5wzp03BtWPqLDBFLZkVgqV0kLGUFNRbVOGJOlruS5dhjPxIRVFxRqS1mCUKLxakeuiNBx7VEpUT4GsLKfXmmovintsiwL3aIhdoSbUmeSEU6n9sxqhJ5ivMrpkubuDQs8wXSvsm/KYOZfEo57cOEnpSZlsf8AwrRTp1YirAt5U6KPFVIG4y4SSVsWsZc1lRV4FUlCYWZEQRqqsMZga8QaHTni4WcO0MbiijtSHwHDnGD5b4deK5mbE6KLehnP8WsY0iWQyKSlSVCiVMV2QlC/ANKKiKpXJsXCKJ03uIl7smpjZyCT4Rs7bqs+M14bhi8jrohQ+7Tv8scKtnlXL69sjOSsmL6/lVjNWJ+rOM/aWuvXMkyNFRf+6kEyeJJ3L7ZUwaTtRNU+vFNqXdW89CUVKms8/e+C/+up/40+aX3NnTlwV6b5TigScUuaQLb8OV/zr/if887d/wG0j/tHaBnZ/MBKyJv5M03vH5C3HQ7NwgofBMl856gchO7jO8/7TFrO3S3rW3eePvFwd+enmA7UO3M8rnnzDzz/dsWtFUR6DZd83hKT5w91bpmD59XhLPDqefv98uTJ+K2lYdhQaQWjhqW/xvayz9d2J/+Zn/yv/uP0VzdeenZ5Y6YR/3hJ4/KLiU1zR55oxOT7FNcfY4FTky0oU3Ftz5Fb3dDrgyqbkUfj8rJXF8m9fsBzfhWv+9Zc/5m9Od4Qkj/u6+8inacXqP/w1Q3Acxprjy4YUFUNZY3ow6ElRPYJphKFsZkusV0vQW+hkP0hRE7OFmAkrS/vNTNhJ50f7BCEx3mj85jlNDQVjUVVzk9CmdF0GRVwJD33ePqecxUot51vFTHQaW+ZI1IPsd+Fug9lPnL7coYN04GwVCd7gHuV8JJfJKyAqshFVNGlQQYkPFtCTQeXzOi7vde2wB08sXlWzaqCu0KcJB6TaoaF090qHLWoe7teYWaGPhUl8UoQVmCd5nim1oMUHn4NGmYR5dKQf+JIBmBVpMOQ2cjo1aJtwLrL/VAg+XkMbMBtP7C2u8+LZrxL6eH7vEDtJD3Unee7V28B4bejeFNrJXYWak3xm+STJcH0gOf2DKZN/v7oouJe61KUudalLXepSl/qdqouCe6mlFBCv1uKVVWrxWSof5Zvivhevq7Po9w9CLzjnzvcjDCPxRy8hZcxxlgn72mDG8pXZB/FFaUitxb7fg12D1qIAA/rQF5UuCcu0rdBvP5FvrxYvL1oLW1VrzJuPxM9ekBuL2Z8W1dl/cY05eWJnqX7xHkLAuDuS+QHZ4DrA0fHt4xXeGw4uiHobNXVTWIPrwub7ZYtPVji4Adr3elHyYi2qQNjK9H3ooPnA4r8DyG3EmETKil9+uiEeZDpdFX8igB1VSR8r/NRR2Krd+7B4KGOrGe+q5Ru6qjRmkun8rJ4VBn2aYO3Q/SxTtjmjp0T3SRSC+kEmkkVRTKgYsR9FsT2nyqWbjXBnD4MkmDVOrtXDUdRXIG5r8e4ajd2PxFWFOc1Czihkg9zVsoY6h+pH8qqVr9bGLBPA+iQ/1z6SDyd0Kr9vtKTYADQ1GRYf99n/Ha9XoEV1tt/dg/dQV0LZ6CfUaUAf9HNevbXgA27vsaeA79qSva5Ip2c1NHiNjgrTC/sRJf7AHypAHESh82u5jjpK/vq5s5Ft5vTaEes1sdF03w8kZ3j4abOk+4QWUSVHixqMpNy97UhN+s0b1IA5KaIxtO81sQI9/0CjKG9Pz8LhTU7Su/QsKXkAeiuJaWBIDq7WPZ+eVqhZk7tn/2uYDevdwDxbyDDOjqadGYt0plUmq0zyBtUb9NUMN5HVeqI/ynWZx5qwUmUiHI5fSGKVvRmZglz3tHeoLpKSBpVpK/Fy+mgYJlcuV2K/b+nWE5gs1IaoqFsPvydro39owGRCZ7G9wu1LStj83CFh1vzv333BdKz50fUDN0WNdDyf5zFnfF6aM1zpnpdOPKqfRlGqQtZc1z1jdPhoGIMlZYUyGV3OT0yKuI5MSmMGRXjfkl2ie1DMRcV80Z34R1ff8srtqbXn61pxH9Z81T6wtnJ//bK/5c/mL/ibNy+5Hzp8MNTfO8Iqk+oycX/UwhG+Bh6EjVw9wfGvt5wtlc2rwAu7Z6NHOhVY6SRqtXpWuTo1c6PnRZF9TDVjdhgynZbjaVRkpQKdAqMUc84YMpAWRTcBUSliDng0V+bET5t3DNExJOmwvZ83rN1EQroDg3cYG4neCZ0D8UfryZBquYbuVNTJ4nuVtS7dk+QUJI0yYE9hSQlcbo0YMRO07zKhU3RvNMnA+HlpsQXFdKqwH5xQDSZNXEVQGpWeUyQluU5jj5HYSCJndprwQj5UVJC0z3F3TXJyvPO+Rp/M0oWrHjWjrbGTfJaEVOOOChKk8fk/yexJ0jB1RPypSdTpVJcrljP+uhU27hxIrdBDslH40hkNTxXtd/Z5vuAon0lmkKQ6gPqdZXoVyEfxAeekl3OiC9s4tUl8uFmhDpZ0sOgA48oJcaZUxpJXHn00hLlBz4rUsJBospOb69wpc3vF6bWVa1EIP2aU/XNe1dTvT8JLt5pkf1uGwkXBvdSlLnWpS13qUpe61O9YXRTcS/1G5dowXTe4pxF1LKrq46OoXuYHyVbWiheyF69Y7hrS1Rr78VCm+ItbRrNMfRofSNuSpPXxSO5q9FMvnMKixgGEF2vxk+aM3vdk71H9SHqxXR6jfETlANYStxX200BuqsWLqbIomclo4mc3Mm26csLeK56m+WipHzQn02I/Od6tG3BCLTj7sZLXZK/RWjLD9Sx8wmSfv5WK2gpmEmaiStA8JAajl7QzXCJGzV9+/0pUnqhwB8kgPyvB9SP4Tiah/UqUt/opYceI3RdPq2rI2mDGRKw1bi/pYbG1xFWZyt9WuO+fUD49kzCUwh09dhDFwhwnUTjHefG44gPqOCysQTV5uF6T1g1h2xA7i9EKrlaiECP+triqhL+oNVkrcmUJ20b4xiCpZO+ewCjSpiU1DvPYC22j+LNV8cuqMZA/f0FyRtSQT/uFpkDO8vecIUTUVNaNVphzeo9WoBR53RFXDn0chO0bAjTPa0yFSGwt9jhLutcEzX2kf1XOxQhivD5fY0XzKaNnYZ4CTFew/VWS63XUTDeKWEH9mJlX57Oo8StFNhbtM/d/uOLq54OorWdh8SaReof6VOFOQuDQk0J5I4oHQgnIRXHJLuHXmmyFm3ymgiQjKWoqQmgQz3CQ4zn7B+e1xg2yvmILx7HGmIyvI4zy3v2swWS8t2id6TYTPhrWzbOvWuvE6diAV+JLj4qq9aSkWG1kTzgpiB+L6q7Fh5usIsyGh5PsA+vPjvz9F+953Ry4rY44FalLS6MpUnmfKrRKHGPD2y+2/PXTS67v3tOHilhunmEjPs6hq3B14PR+JZtAVujC4KweDHPoYBP46zcv+e+mf0rnPIN3xPysDk3ekrNC60TjApWJfH+/xT8UVdDIBLo9GLKWxLZswPWiMgK0oSj6ITNfiV/ajJpqnyXpCfirb1/x3dOOmDTORJyNjN6yaSaqQiQ4zRWHfYv7pubTukbPit03EJtnPrM7ZlJJ6OreJ+qnSGg1zYNaUhs/vtryz+w/RauM0YnWeqyWjlJjCsNViQKd8jNh4uhrrE4MwS0/q3TElN8dgyNlRcpqYe4qlZfnnoIlJI2PmjlYToUBzEGezxxlvkAH8bpevc34QnqpnopCPWXsmGnfe6r3J9Q4EV5uy2Nm1ByIXYV97EmrWrpWWqML9zo3NWlds3rrGa+tnKsG0KCmc1cHOHu+7zX+KkGVyJNm3j0nfoGcUzta7KnMBPi0dGxQmvmLa6pTpn+p8WtQg2b9K81Q/OJhJUmX1ZN4YlWEJGj1RVUNK1lPwYinOtaG3S+iECcKIWF6ucJMibiuhUV+nJledqiYl86hnoRsYnvhtZtRkVv5zDl3iOZKEjj1aOB2Ig0Wd9CELi97VMosKZw6KFKTiG2WuYCyZZqDdBLyUMufJqOQ++RcapTUOu2ly+Q34oM2E+hYUkiHjF9rbJ8YP1vTvOuZbxpSdVFwL3WpS13qUpe61KUu9f/zuii4l3ourUjOoOcontjiwaVtRAXTkhJ1zgJXIf5GEpUeZ1FRZ4+aZpkan+LyLSrtVsR1hdnPpE2DPozkxsHDHkoyDRSW6lAe08+o9Yq0ahfFIm4bktNon7CVFR9pzvjPd0simlo8WhE1eHLr6F9WnD5XjC+L0pDB7WE+2MLnE1pCrDNz8R7pXkgHxTqGOyjcUf59Vs7aD5lYQfspSTLLjbATkwVfkmlsHZlHi7GJMFn0rNEehhfPCtzqTcb4vKQinUkHAGoqXrEM1d5j9jOmc8TWomf5/1Ilx+yeRtKuw306CTt23YqH9pwaB+h9T3yxFXpBEKVXa03qKvHLgvht9+KJdqNH3W1QMaHmwPxaPGdmKGzjlNGHHnMfSLsVNHZZP3rw5KZCDbOQLyor6WOzX96XeTySK7dkxisv6URotfjt8DO0tRA06MpzB8z9STjNIB7spkadBqxRqHEmh4had+TCTs5Xa7LVhM4AlSisFoY78wPFFqpH6L/ImF6uuV8Xz1wZGg5rISuEThKYzCge3PFG/LEgv2OmzLwV3vH4QrGPLWaC4U5erP29Pad9Q1pF1MExXwfxtWUWtY+sUBFil8DmZc3ERnLtQagBttf4dSasMrYXf2LWalHDhjvhpyYnXuLjuzXUkX/yB7/kZX0E4PP6EacDjQrU2mNIRDR9qngKct77VDFEx/284sOwprWeMVreH9fY0v24e/2Jv51forxjeu1RXmP3Gh4rhqLg/YM/+Tv+89u/4HP3IGsYzZXuiSgaJcqiIeNUxJB5TC3vrzYYMn8xfs7//FFQAn//6j0/f7rj5ebIxo38XN2xbia0yuwHUQ2n0dG4SEqKnBVv3l5j3lXEVRIVGlBtJE9GWLteYz5WIgTrTHMofvyp0EwmsL1c21gLs3T9/ZlqAu27kfmqIr1TTFtDex/wnSaVjkn+dy2nm4b6XnHcZekkjIp79ezzzgacy2ivqD9K98kOieYxYfvSZXIKd4wkKymCQlmxuEPm9JksxPZXFW/2L6XDlFnYpOKR/IHXW8t7FXW6+Hyz+NOh8Fm7JFzvUeGvI3rlyfvn/ZsoRACVpPORlZyvbKEa5HncSX6mAvit/GkHOYfnuY72kxxX92bEb5zMclhNuloJPx1QvSRaqpgIV52w0kNEH/bL/pM3K2Lr0HNi3sh9FdrzfV588OuIfbCEXaT7O4vfKvRRUi/PxAbt1eK1H680nc+Mdw3TzmCn4md98ExXjqxg3sDwY4/ZG6abfN5aCLuImjX+TOGZxQ9rJk0u7OdshIPbfxXovrHYUfHpT2rqh1w89OB2mu59IDYaPWfGu5qswZ0S0+7ctQAzQFiLcuyvErn6wfUGchamdTYZHiqUFq6zyuB3zxxc82RJVZb1koUkpIKi3KakStZwrkS9xSUSkBpwD+XztFzneSf0mdjC+CrQvLUML8sMSemWqQjDjSasNtg+XTy4l7rUpS51qUtd6lKXutRFwb3UcyWZXrRHjzr2Mu2OiFpq9uKrdRb6QXizKcu/QfyTvYdpJk8TbDfoM8/07N3VGvs4yjR7hNxWoujVFbnQD6gr7Ic9adNKQlnK5HWH8s9Z3mbwmAHmq5p024mK6IxQBMq3PPs4ovqJat8L/3SWJJywzotKZ3vF8DI/M2tnQMmEKUqO2T2Jwmq9MAmzhelG1AcnYUz4tcIdskypKhhvFfPW0H+W0XfiR9xueoapYp4tedasvtVkI89ji1oTK/HgoqA6JvHNPs2SjFOYjmaKJUmuJJKNQZJ9gibV8i3ZvH0gvdiRaydZ6Z0TVmRtyEPhDt+sSZXB+EhShTO7F3pBuBGJ0voIp4G8XaEmjzmMwjB2Bvcg70sNM+acgrZuUW8/kc8GzwMAACAASURBVKsd7uORtC7H/OkgCq6XroDuRZnNlUPFohAoBSmJevvmI6prJcFsmsnHYmzUCjVM6KbCfHwibzrUUJLMziqv0aRNi5oC+v5AnmbUqhV/cSE/hJdbDj9uCLUia0O24vPrPxdGJUD1IIl3yYB24hVLlXg6zVgOZ1YML0uC2HUi1Rk9atxJMX4lirLqDZtfSNqVmWSNjDfi1dZfi2L69c09f376TFSf24hee9LJooImrcriOFv8bILRCJu3ToSbjJrkms6VqCJKKba//8jkLcP7jmwMsT0rwc/sz+Y+EzrDfAd/uv2O/2z97wDY6LmopglDJqIw5P+bGtJnw2OqeRuuMCrxKaz5H9//Q65rMbnHrDi9rnhvt9zdHohJ8fBpI8zdHySJ/avDT/gX6fcZo6MPFVYlEoqtkxO9shNGZV5VexrtGZPDqMT/sf9iOZa/fHzJh8c1WmesTcyTZegLZaNwVdWk8UaukVzA4pN90ouvOmWFmjRqb1CNrIXue4XfKApMQZTWOYuaOiZAowM0jxFTeMM6ZOzTKPfcqGg+TOghEH+yZv2drPn9Twzqkyjz7QehPsjfM+4kr92/kg6S38ixrr9L6JAxQ8IdRDqbXlSStLf3zFcVOgh72Zw81ab4Sg8G8YOLrzJ2CT1KF8mU8zHfRtyTIbmMGcH2xeerYHxRKC5Vpv4o6xmVad5a5p3G+KIAljVWPWrma0kV0xGyg/rTc7cKRP3OqnBup0x1zEQnfkyQ+6p+CNjHAbOfZK9QimwrdOkyxdZh70+SVNk69MMRNUxCaDl3Ba2m/6zGrzSnz5V8BpTjPN/vujeE6+L5/rF0UNSosb1i+kLu5fR5pPrrlmSEdT1dW7p3mdCwqKqhrkCBbxXTXaLaTejv10x3cUkcREHeBHythRihMs16ZtzXmIfn/yQbXiXc1USvM80bhzvA6TOFLftPSAhX+k683WaW8zdtNfNVeesnOP04wtbTbSbWzcRVM5T7s5AWkmYu/td+knM2zo51O9E6WWM+afSPMk4npmgW3/jZKw7Q2kI08RW9d5zGiv6xxbaBEOvlnM830iUAMIOme3li2lb4t3U5ZkXWimQ1/WtFbA3Vk+zHK367uii4l7rUpS51qUtd6lKX+p2qi4J7qaXO085ZK+KLLeahKGc5Q4zkw4zqGuhaGEbydk1alwSyfpYc8MqhrJHHVw79dEJ5UfLU5EnrBv14JK/Fyxe2jUzx6+fvWqYfiZsG3XtUjJKk1dZLopnKoI8jTivCpsLuR+G1HueFlZuVQhktiVuzJ20aQkvhAZbniYAT+oGK4g1KFuwo/FB5kPjC2g+ZT/9QvLZmYS/KQ3SA9ZvAtDPsf2Jk6rWC9HLCFT/iqvIcTg0pSGKUEnQkqX5Oxmo/iv82W5i2mu5j5PRVS33/TD/Qoye2jrhyRKdpvttL/vngUeF8QFqS5ypLrivh3eZMXjvh08KSKKefTuKZnSO5dqhDjz6TF3KW8372zMYsSiigjsdysTS5a8jGoOdCKiivffbyxps1+kmoCfhAXNfijfZx4eDG6xV68ELNuNrCaUAde9JmhQ4/UHlzxjydxJs7e0msa9xC+FBPR3kOZyFG1GZF1krS9/YiuR+//AnHL0WdSla8lKEVr29cy2vNScl0vM2L/zVZQOfFjx3WCRVEcc8K0BA3ER0M3ZWoJPVdYH/TkR4rYm2o9jDeyLrqGlGGTr4izQai4o/+wTd81u7RKrG1I07J8TgdiVmzDw3vpw2/erohJs22GXElLk+rzN+8vSNnxWfbPVO0/O19S7ib4YMcdLLCAbWDYroSZY0Mv+xfcG1/DMBT6PDZ0BSawSE2dHpmZ3tWhYe60SOPscOoxJgch9TgkyWh+PnjnbyvqcJHQx4Mh75hniz2+0r8mGWw+s//5gv+PH6JHgxp51EHi+k14YVf5BdlE7rwb/P3DXETUU3k1csnPtzLNH36VFF/lPQn+wh5LT55d3j2TLfvMql6VhFjLf8TMkZR7j8Zmo+KeQuTi7TvnpVvVZpM6+9lmv38HprHRKgV7duReSfn2e1nVD9SvZc9zhwmcutYfXNkupO9z4waklrYyWZUTNeyv52pF2aWfckOZbq+VnR7SXjU5V5e/epI2NVko2je9piHA3nVko1i/YsnOT/uimTh6fc14wtZt2ameDZLUldjaN8rklXMO5khcCewp8z5Yvi1pPrZXuF3kuZnRvHZ1o+6rDHxWJ5nJupHUafN+OxnNQNc/UKoBsILV1R74XmfuxVZK+qPg+wXUyBrQ9w02MdhoRaomCEldD9Kt6mtYfJC9SksXDUGdBSVNXbSsUufj6TZUG8KzzsYSIq2m8kZrEnEpBn66tws4/XtE99HRf2LhtAJyzXZc4ph2RO6QmhQQJaEtbBL7L584rOttABetQdeVEeubc/GjNTac4wN30w3/Nm9dCXePW3Y1jOryjNuLR/SNdOtonlnFu9+1uLVzS6hR4VKimqvMEP5bEPugT/6k2/4T2//hj9svuO1lfVQkZjLNU1ZE1FENGNyeAwxa27Ncdl/zo/TKhU2csKpSMp6eYwprdFTdozZ8SFs+bW/wSfLlGWf72PFlCzfDzu+2V/TOY9SGb8yPLbSMe4PNeqXDacfJfHzOst0LVSh37YuCu6lLnWpS13qUpe61KV+p+qi4F7qN0shXsc5L1PnnFPN/Incj6iuEV9sP6LPpAWlSEWVVSmJgnccyKd+4ZhmU6b0xzL56iO50hC0MFcBfRhEEdRnyKwmdQ25sdh3j/I8TQ3WyOT+nCAlstXo4ywJWEC47siuTD3nTNhUoob0imRLispREdrM+lvFtPtBGpUv0D/km3D9kAkd1PcaexKuKOpZwVVRFDzfKUJT+H6rTN15Xu3kq/27pw1KgdLimVwyxgVBCMC0FXUprArdYaMxk/iiVTwnmTmhXMwRYxSpqVAxSsLPY+E+doU1mTO5cpjHI+HVTo61qKrGR9SpXJ9NSzaGvOswpwF9OJtM5XXVYRAfM5BXjTAmz+ulTC2bUQgJjBPmY2b+0YvlmPUcyatG/NY+YJ4G1OzJx5789Ws5npOHEMltTVxV2GNPrivU7Ekv5Nj141GUWWdR+xnGCTYr9P34PDEdgnj12vr5Z+sW5QN6K+SH0CrGFxm3l4nq6Vb+ni0L3DhuItW3Fr9J2AH6z5MwIEe9XLBcJXL1/DsgE8CxzvgHUSO+/PotX+6e+Df9V/Q/TuRvHWaWKe7HN6I+PrKl+mRQPzvyj66+5e81b1jpmY0eFhXV8OyHfUwdb2+ucCpwZfrltb/31/xP6k+xKvFxENnSrAK77Yn7LK9lPjlSlZldJnUJNSuoE//LN1/zL9TXsg5PFTkoTBeI+wq3m0Bl/EOD2T3LKDkqjJMUshQ0deHg6r9YL49JLtOdFP7B0jwJBcDMeUlx695U+HVhdA6iftpeAU7Uc4pSpSC2CaOg/bUjdJYP718u3NDujSjyMokvXFg7iLfznOIWVsLftCeZ0FadWmgIZ4+ySkLKqO8hdJrqkBlvFNX+mZDgVxo7ZupH6dqERlMfItloXGGHC7mlFl95zuiHPaG7JVd28c62HzXJKaatrL3qIOp6/fTMUVYho0PG9cIyrh+eUw31sfi8pxnjjNxfMUJMpMqiy54KonQqrejeZuYrcE9aOKZJfJoAq1G4yfYknSTtC1O5nCco6nIrFA47yFqvHxR+JZP353ugvlf4bel2lQ5HcrD6vrBXNwq/NuiQSShMktdUWeEOpSMxJ+bbFjIYq4mNsNfnuxXVe+kghesOFWvsG0nWzFaTXu6EAnQm6uRciBOGmz/9wD+++5Yr2/PCHZfzU2vPrTnS6Wmhd3wfrvnz/gv+rr8BYD835KSIf3TEHyuYNWGjICjG18++XnvUmEl4r/5jTd4E/osf/zn/0eoXALw0BzZ6RhfFsymtgcPa8i+7n8jzfAkfw4Z/+fhj/vLDK/Tao0xmzDW5K12dlScnTV17YtT4tx1hK/e0LbSKrOCPd2/4j7tf8Moc2ejElOGQHL60IMbsaJTnVg805og704rO2AdgzJqkFJrMrZ6IJe0OhfwJiyLsVGSlPJtqXOgopmQDjtnhs+X9esMvtq/4+fElVkf2c8u2ls+db/WOw2uL6gL0lnkn1Ib64bfXYy8K7qUudalLXepSl7rUpX6n6qLgXmoppRT25OXbb0ikXVFkfRTFNETyMEBMqKLYqZIWQ13JZHztUE8D2QobNd9cCYEBICZJmelq8d2GJAlk9Q88uDmLypsy/rrBHWYo6kQ8J5lpUWtUhuqbj2UaP6PGSdRdKGlqGr911CljDzPNgyU0huFVSXXaFVaoFuUndCWvG8lyB/HIJicqUvc2Ux0SvtNFVXk+d7HV+M3Zf5UJN4E0WT4cREnrmolxdvjRYryiepLXi5V4fAGyFSXHHaG9F5ahO2WmK0tsy7fjfSRVBref0PuBeLNCHSMqJbkGIMlioyesK7Qz5NqgjzNWqyXZTA0eZQyqbfHXLdXffSJdrclXG3JJodOnAcaJnBLolVw3q+XaFZEXa1BPx0WlTy+u0ccePQVSXfy1jQKjcO+P4gk+9qIuv7rFPMpJTKsGDUXZF1WenBc/MYgyrYaJtG6hrSXp68xlLutQVY48e1JXYfaFDjEHoXFs5FqERljHtZfkqflKJr3JYJ/kd8JNYPgyYDaePtSkJlHfDEyninolytmL9YDRicpEbpuTEAB04tunHQ/fi+r89rDhuhuothPzQ0NshGmaXGb7V3J+fPHzTYeavznd8d14RaXDb3hwfTbs7MC6IBwMmZ0J3Mc1hyhq8Tu/5YvukSff8t23NyJfRMWT6qD4zqu9QoXSKShqsxoMfv88n+wOso7TaDCjIg6iotEk1LfyWskKJzMasEeFskDfoNUP1rMGsvB/s1ELZaT9lJfH9K/Ej14/yGueVUMzqsUHH5uMOyrGWwVa6AUqCmf13Olxe2ECh1bu3YUj7WVaH4pnNkN1En9pbBTuKCrm6js5nvFGOjnVE9T3itAhk/KteiZZxEy1j5g+YCtN93YkG4U5zQuLWuUsSYGHAX00MpsQk3QzCge3ezMRG4OZLOOVpn6K6KAxU6Iu/OMJ6eLUDx5z8qiUia3D9PMzgaRyJKvlGLKQbfQ4k2u3vJY7SpdnuHWYAao9+FTYysVyX+2zdCtey7VSuXiUPQvZwCfpZPmtXIdUFYVdZ9oPcsx+JX5ldxDiSNbyetlAdM8EjzPH16807iSc01SpxV/r9p5Um3IutfiF97MkUpaOkjmVuYsQ5TOnn6B0i/RYFO6HPW7bkCrHT68+8l9e/yt+5h64M5ZayfNoFEZp+jRzn2buk+WQWj47fxAAfxVf0W1HPr/aM11bHoeGcagIs0EXjnKaDZwqpuuEymB6TX4x4VTkQ5DPr/u4plGex9jxFDvWZmSjB27tcenImOJ1fZpbKhuYjCOcHNQJ1ZfEwaCxe8P4SpFng5lFATeDWvzifpv4brjin6s/5lgu9CnUfJo67ke555XKtNbzojmytRNbK/MDU7KLyvt23KJVptKBWkdSVhyCPN/R12VtmOXYb+qe1ni0ytQ6LM8JMiswJcv9vGJOhnfDhnf7DdMo1yIcHbrXqFNF9SBcb3d87iL8NnVRcC91qUtd6lKXutSlLvU7VRcF9//jpZT6CvjvgVeIzvLPcs7/rVLqBvgfgJ8AvwL+65zzw//Tc+Wcse/3pG1H3NTYvahFqXWkxmJyRiklStqxB2tksh3EL/l4gKsNDCOsWuGgak3aiupznpJPuxXm/SO5coS7DfahXxTcbA16P6Bqi/KOsHa4+0E8vo0s1+QMyWmhJjiLGiZUV5NXzUIJcG8eoXLCkG0degqYOYE2i0obasnWjo0iOlEqQiPq0Dnpxu+E1XjOm1dRY7z44ZpP8kTz1tB92/+f7L1Jr21Zup71jGoWq9zlOSdORGRGZt5rXxsuErSQkBASP4AePUQDyR06SDRA/AJa/ABLNGjQQQIJJDctIRkjuUFhWdyU73VmOjMiTrmrVc5iVDS+seaKBCTfbNjWDdYnhU5EnLnXmmuOMcfa8x3v97x01wv0UpFqsC+WPA9078WPGF8dMSaRB+FMdq+FsQoQRSjH7oRxWD9n+mtNmElrfvsQzxQFn4i1Ru961O6AHb3winOe1Fn7aYP/4hq7H9HHUdQOq4VH3BWPYEriiV6vRf2xBr2R5LP0WiCKsV6gtwblA/FqjnnYEq+Xkj5WuLO5NrBeFDX/KElIbY152qMXhaNstdA5akdWini/Rm+OMseeyvkUxSvOHO79C3k5R+0OpNsrin1LfNpKCSXBmknRzdZAKo/3rhVGM8WP6yzERO4H8rWMhe3Ff93fSVe9GYQ5GedpYiTX6555O9C4QPM6sK46rquOpet5VVqmE4q/9/mP+Gr+QkLRmECtA2Gp6W6Lmp40j4eZiNBZ1E6/kCz4/rak3B0V2iuqd45/EH8BSaFGRa5+YNA2GXzRI1zCtIG2HRkGiz8WrEMG00RcFSBo3LOhelHE2mGKV5VUuvIfS1d+K3xJHZnS107Z9Sgjc/FOYQ/Q3yuqjZoOirWoq82jeM91ECJBsQ1T7TLDlcIdit8Z8d+6Qyw+W4i1EY7wWvzQ9ZOwgsflmYeaOkX7OaOCqHv2KF7Vaie7ICCcWHsUtbbaJ2YfRvZf1dQvUe77UrHRhEbjxsywEqXLL9Wk8lYbmYumh+YlM38/cnxVEWaKUBTl+XuP3QvL2R4jdtsTZxWERJ6JCqVedigg3V+JqqiU8KCdnXZa8txhDwF7CKjcoH0WkkDKtB9l7a0WTnaqPh2Iyxr9fECFRvob5uK1D6sG03mSMpIOac66VTpRZaxChUzzLOtHckJIiNWZBpGcwutcEs5EhW0eM355Xg+bB1H/579T2F56FZKFxXeweCf38vanZeemKurvKOSCWKtJLb77317wtzPCzDD/9og+euKqLrsoovap5y15MSPPavmeWVToTnamVOm10JuD7AZdLVG9h5ctyllSUwnVBYQn/u4J1JynYcavx1ckNO9ix/IHrfl9NrwLN/wfx5/yq8M9T8OMp26Gj3INd/uWlDS//XyNUuB7CzuHHtSU9GZH+f6wO6FjqAD7q4q/892/wvYg45WzYtzWuOVATpo4GkwVadqRVVvG3UTmbuTjbsHucY46GmbvjRA/Tt8Xvex2jH2NHpWQHLT4nAsynu3PNH//H/41/r7NqE6DQdLSfphm5jW6DfzSv0XZRN6XH24S2sm6ml+E75tdQh8M6SpAEs63OjGtrzyUBNDq0QhJ6H4U7nW5PlQJvZPE0Gwz+nokDobmtxVVGYrls6zLALOPiWEt27U/aHX4S9dFwf2rXwH4T3POfxP4N4H/WCn1N4H/HPi7Oec/Bv5u+e9LXepSl7rUpS51qR99XRTcv+KVc34PvC//vlNK/RL4Evj3gH+nHPbfAP8z8J/9s14v3iwIy4rQmomz6JeV5IArRfjyBlJGLRvUP/4t+v5WfjAlsFY8lG2DSolUO8zDVjircoKiwD1upeO1tsTWYvZGVA4gz2rUcUANoCorDNOQUIcO/81dOR+L9kXxu1titj2plg7bEyUAEMqDUhAS421DaLT4L5uTeiVq0XAtKkb1LIpOcnD82bkDWQ8OvxC/bjJKvHs7GNclU16DX0lmvfFge8VhkUhbB6XjVSkwhYmbmkRQGt0r0ExZ3tlSlD1RSZqHTH+rCI1h9iCPr+2HgeplIF7PMEajjr3wYOtzFnyuHPZFVPHsDGFdY/qAe9jL8UC8W5HWLWhF9diRirppPm8mxRSA7R7mM+lOXs9JrRWubkkFU4VFizWkq6VQMIyGupq8dPownL20WnqHVRDl68Sq1MeRsG5RIZGbShSYnFHDePZn+0BaL9AvO/HgvuzI6wWqH8m6jIWVXHrdeSg83XRbYzaaVPyI1S5B1sKs/cWB61nH69mOt+2Wn7YPANyYA0sjSlKfHHM9ElFUKvJSJPdfdm957lta6/l+t2bf1VQuMHrL2JfdhqChN1QPhiaK39QeFG5/5rNe/3nEt4rDlxrtHXGWMUc5drySwTBHNfGa/TqTtSPsZ+RZpirzR2XQY4XbQaXPcAczSNoewPJ3onCGVuby4l2gvzaE2TmNyQxQ7TM6ipLafBYVr3qRjHiQznrTQ96fPewqIupvYbbqmJl/FOXli/91R/e6JVtRjd1ePtfsszBlu9HgF0I+0DGjPQxXxTf8IqqiSqIKn7yfKmTm7+S9xqUQGtqniNsFVEy0jwEzRMy2SENWU3/X4V8JTeN4Z3AH8eqe0unGNcw/ZA6vNWzPFIPZx1gUS9AxoWLCfN6jbhbite2DJPVR0tNeXUu6Yx/AWcLNHLPtC22mKuMlNJb+VY1KonJWG89w6xiu6+ka2pdB+M5Gka7mZAW6qc9r5nWL6gN216NChJSIq5bUOOxH8ZCG+Q3GJ2xnmL8XVcwMMo4nBRdEhW+e5PrHClwn5/jD1MaTIuvnivpJuNDJKfqr4g1dyNo1+yQqebKw/D4SZppYyDPj3VxSGJ3GHEYYRljWmONIXJUExJiEp641et+hG3umUmzOxBh17CEl0rJBpxWpqTAfHvHfvAbAdQPEiB7hl3/+Jf/k4x1htFSNZzye101bB1wVCMHgO4d6cridntLOUICF+rPC7uV7Qwe4/bOACnJMfytG72RErVYJwjvL4bf3qPJWqcm4DDxa4db2MFxnBtvy6NfT+aikCE2mipKauP514vBGY8p0Nr0onOOLAsRLXu2T8IO3cn8tf6uwR0uYSdoiHlKlcQ922rGptorQWvI8k+tE9WzQXuEXCVVQQSrI91NYJ0nyexZ+sQrQfpT1uR8qqq3Mp6yEddwtDO7ZCH8YqD5ZYp0xnSJVGfO+pXnKqCjkE4DZQ2JYlqTPlGleoNpGhqsfTNS/ZF1+wf0RlVLqG+BfB/4B8Lr88gvwAbEw/H/9zN8C/hZAw+yf/0le6lKXutSlLnWpS/1zrssvuD+SUkotgP8e+E9yzlulzoaVnHNW6pQh8/uVc/7bwN8GWLv73N+36Jg5vLaEVn7hjZVi/j4TlzVh5konsKaqK9KJuarFh5srS7hfoXwSP1hdTZ28ykfi7RI1BtQQ0JsDZi5KxIl+gNaSN+4sflXjtiOpsqTV9TnJLGZUSCSnwWmUd5iXoyRaqdMxibSYoYaR8GpOcnrygJ2eXHMbiTaTt7ZkpiuGmyxd4cWjpB8cYZ7IWjq7zZAZr0XBHdbna5xMxXAtClt/l0VtezMym4nK0ncVShnswpOiIs8UobOST178Sdlk+nsgQbVRdK/k6d4vFF0qXjrdFA9jRIUKZpUopEZNLGH/eoXpg3BBP2+pQmK8n2OekngAAb0fSHdu8tIqH/FXDcleY44/iIy5WRPnNboPZGdItcF+3k3jpQ4d4c0V5jiSnSHnRlLkfMA8ls5ZH6CuCK9W6M5jPgmzUh178rJQL6xG9wE9BvFu50y8XZC1wv3moxxzvZo6x7NS0tQeJIGN9mY6n1w58eY6C9ag95JNn0pqWmjF7/nmp4/8229+xU/qR5a648qck7oAXuKclzjjV/0rFmbg0Yvk+lT+/Hu/+iPa2cCfvXuDsYn+oaW3GftkMSffsBVCgD0UP6LPhEax+l3Az2RM248DdmEx3hJqxe6nopTGWk1+StMXmocBtxF1t3kQf+zJ12g7UU1iA24nCuxJxT01MY8rRXKa9imyf2OYv4/M30ee/1o9Zdy7fWb5bWD3lUVF8eeufpsYFwr3QyIBQBa+rO1EQTKDsGdP5Q4yXslq6qeBbDTDjXhPy5mRFSy+G+leiVrUPghfNvbls/uzJzfW4gettoVlXc7H9sK2rR8HzH5ADQHUglRp/I2sUW4zkBtHchqVM9VOSBDNc566+1Hijc9W/uleV/hW07xEbPks7sOONKtJyxa97QivpDv+5AuFos6OkdxY/LJQYd4u0SFjjnKf9jcV7hDRQ8YdA7E2hHlR/ovf174I8zo3FcpHUiNkhFwn0mIxnXO4brG7AXXsSbcrIS0c/JmEk8E+d7Qhc3hbo5Io71MKH6LWnlIVs4LmWUguy289fikDPlwZ+VkrtIQTIze2MrdkrIXEkJzMh3qTyVbUOONP3mtN9STXIc4rtC1M1esWPZx34cavrnFPx8L4zYTrGXqIDD+7K69jUGmFHkW5dKeekKsl5lAUbqNRMTL7nPC/cXSvJfFurBy50A9U0KRHR9dkZr8z+K8j9ZMWT365Pqliou6kWnZDVMqYIWFKj4QZE/2tI9eyC+JbxfpXif5GTwmIphd6y3AlXufTOFQbNfUAZH0iVygWv8uYMREahenPSmf9LGuESuJzNl7u1ZwzzcfiYw4NsbaopPALNfWfnEguIMznfC9/l5Vh/r2sNabXjKtyn+3KOR+crCW5UDHqMw/eHuX83U5oJipD+70VWtGu7OaNoAfpNUnu/HlnnxPdrUy+wyth3MdGob2mfYyYPmKGCwf3/5ellHLIL7f/bc75fyj/+6NS6ovy918An/5lnd+lLnWpS13qUpe61L/Iuii4f8VLiVT7XwO/zDn/Vz/4q/8J+A+B/7L8+T/+M1/MWsa1odpGwlzhTmkoWtKodt/MxOezUeiQUfPZ6QGM1DjMridHgwLCusb5SK7Pylk20smvtCIvauxDxn7eFsW2eCidxnhHbqy87754RudOVBHAdlEUvJAYbsXrWYUkncwnxWqUtLS4XBJag+kTWRliI8k7AN1Mk10gVQm0eP7CLJHRsCmcxU4x3kbMURMbeWLWA9SbRHcnz4f1RpJ9Yl34uRn8TUSNhlwSkqyLzJuRXVKETrqF0ZLrrgqjNDdZOui1+CxVLH62R8VwU16ng2ojCW7ZavyqwlYG+3jAvylJXY0Rz7RWf/PlGgAAIABJREFU4o0ePeboJ78rILSJnDGHAX8zwxw9KiTsppvGgpyhcDfJWZidfcS/WeMeBGTqf3KH7oIoLCERrlqyVlS/+UReFstLJcp6NppUW/LbW+nuVmA3Mr5h3WCOnnAlRAhz9KTaEmuDuReqg+pGUX1DINeGPG9Jy0YU/eJHPKn22WhhNDc1PG3I64XwnRH1FwUz50lZ8X684nf5lkZ79oXtuA0tH/sl375c4Wxke2iIwRAGgzJl1j9XxIeWPM9wVDRaPGlCIDjfVm53Tu5qnjMk2RVpJwqHw/YRd4gkIzzLm18OPP6rDY1Ygulv5HVjK74+HRCyxw/8kSAe1cNC4VfiE0SLola/FOW1dCPrQRTe7t7hukS9SYxFynP7TH9tWH4XeP7jwje20iW/+1LuZTOIl3n3lcH0ovzVL5lhrUi2vM4hi4c+ZtlRqS3ZKtw+EuuyI6HA7QPDjROCQMzYnSfWmlAUZePFdzteCbc2NIrZp8i40qgStxQazeJbUYhJkMt72ZeBcFV2G3xEf3rGWU2qxLPffvYcvqhwXUlAPGjskHE7GScdMnbIJKvEJwtCbkmJbAxKa3TvxbceM9qXnZ/tkVw7/K3cA35mURnZcSj3lz1GklXMf/lJeh+alvbbLVkp/P1JeZW+grSeYR53mBAJ90v0tiO31TSfU13Wz0UrO2S5AqsnioI9eOKyYVyfd7kQ66Yk+CFzqrsTpbzdJXQUP7UkO5ZjvFyTSEmF6wpHN58Z4mRR78gyb+yQ6dcGHfOk7mufJo++2fWkeS3/z+kz/SBnYq0xs0qUuJTQQyIuKtm9Q1Th5BR2F/HrSqgxrSPNa+Ffg1ynyuEOEbeVa+IXmea9ZlzJ66Ra1EV7EEW6+WhweyGdTIQEL7zlaie+7WqfMH1Rb4tUOFxbkoHZJ0+1GTm+baleAtlUHN6ePe7JyX27/C5yfCUMdbeH8YR6H2V8lJd7RN4rY/ee41tZx6MT1bzayRydvevpXjc0D0KaOI1xtRc1tP0kO4L9nSTz6aEo7pV8LhVVUWMz9UtmXMnPAFOaYPMoOxx+IeuP7aD5XO7BuWJcyrjbjjONIwpRAuRnbVd2mAJTz0usFXXZlelu5b/bz2m6T7JWvzcP/7J1+QX3r379W8B/APwjpdT/Wf7ff4H8YvvfKaX+I+C3wL//L+n8LnWpS13qUpe61KX+hdblF9y/4pVz/l/4Pb3o9+rf/YNeSyu6O01/JQk+oSlKzDGz+2lbmJYQnaXeJsZv7rDP4vUZr2vqKCqqX1rcIeDvZtj9OHEN46phuKsxXUSHLAzcjxtwdnoCHm8aKsSTafpIrixZa9xzJ8obEFc141VF/WHHcFthD57+iwX2EDDdCcLpSDOHX1hUEs9X1pLR3b8qXrmlp248o3GTYpurDL08yQP4q4QeNG6rGe4i1dZgBvGrlUAZ2gd5PN19YxiMwu0U/gq0Szgjf9e4QEyKugoMqcEtB/yhIjeJePIzJulEz4mJE2o78RyePJTJgV+Kh8zuPKRMqgy5cZijKBbJatTgieuWvBCmse68sHIbV8arwe09cVGjx0hqrXgLl80UQq6Pw6TM6pDAKMwQ8ctKfLKI4q72URThcUA1lqw0eb04K8FAnIkKrIfAeDfDPfWkmTurONuBuG7wS4fbeVJlsC8dal4RT3zf948wa8j3V+IhXrWk2mI/74g34ovNRuPe9+LL1Vr8uE0talvx79ohY3rNt5+vedjP2e8btM5kIHZlSRx14Vlq/HVEHzU6KNqNmsYi1uJ50146xfUgbNGsoH2QOXZ8pQlzUUaqraiPOoj6efKUZ6NQPpGsFaUwwvF1RbJg+hOfVRFb8eLWL+IZ9TNJDdPhxNMt3rlRVFgVmRKNJp/lUd7b7QPNsxaP7ZBZferY/kxgp7bP+Lkm1pp6UzzaSVR/VwgJ0YlSW28ysZb3zkZUvBNFoX4JuH0gOU2qDKYwTN1TTywKU2wNqdLESriqbufFB+vUuct7n+luNG4v/GkzZo73htlDpH4Sv3jWNXbTobwopNkYzH5E96P43EF2EVYyL1VM6AB+YWU3qlynepuLYibXuF8blt+OmD5MKmm4atBjxGw60kL86amxhKtmYofH9RzzsCFWa5r3e6wxoCG2TrjfQNZzYbi2kuyYqqJK9gMqFYa0VtOuiApz1KFHJVk7Y3NS08WnG2cOe+ghZ+xLR5xXk6fXbUdyrSYlbPEucXhtcHtRx2XdENX9hIZNVuG6TJibaY4tPkTcPrL9icMdZKz7G409nP3QsRKGcKgVOsp1rLeJWCmqrdyDsTFko9E+EW7kOmif0F2QtQpJN7SHgN6PpJkkwcVFJTtET3Kdx6sat4/oPmBqI0SL/QmgXKg1V3PUvidZxewh0bzA5hea2HDezXstSq2KMF5n2g/iczfD2beqkijTWcv9LZ8nke3ZS2r6jO0S9acOrMYe5e+rfWLcF+X+IOro6tso60FRQpM5J2jqIPNQBaH2JKtodiPmMLL8jVzD4bah2st4miHh3r+gxyUohV/KzVM/DdijRaWK/loRWkX9LCzd07hrD2FRmMcLYb7HCqqd+PlP64fJoMZMmonvV0VZa06/fQzXcm1igjDPEzPZ7WXnSdaWUy+CqO+hEeW2efTowqt2B4ufm6nXRE6AaXfkD6mLB/dSl7rUpS51qUtd6lI/qroouJf6QSmGNaDlSfak+hxei3riF9KxPX+fGdYaPRriK1HOYqs5fj2fPHfDlcMeIuNVTVO68pOVrtTulUP7jNsnzKHl+PVienoLrYbrmubDgfFuhrctsTGiFj0Wn2VVFIXipRyua2wfCXM7EQBO6kVsNCrBuNCEtvjEytO2dZF5OzA8tphBYXpF7DVhFSdfrPKS7T1eJXIlGfaphe5aT0+u40qTlXSOpqtIrhN26VE6EQr9YNV0KJXJWZHv5efCYMFBPr2XUmSTRempVfG7yZP9yZuVtaLeQH9jaWPGHgN+6ch3s0nBVTGjupF0Nycua/zCYY3GftyQT9zZkMlGE1pD/eEg6maMorQVBUQNI+HNFSom/LLC7Ubx/B7OjOBkNeG6RYWMfTpAytiXntS6SaFMrcVuBvFN9x6VMnp3JNfLM2EjJPGGlm5o03sYPdooxruSmf6LL9D7kThzaKUk0ShmUWnj5AYnLRqyMdPTe7qRlKOTT9d2ifpZE38141jNcJ3CrxMqghtPFI7iSasydiPpdypC/czkBc9GTd3Cbld4yHNYfJcmysBwm6k2hQ1ZuqWbTcQMkXFlpzH1KyeqSMqopKj2kXGviKWbXntRXdoH8f+1T4m+pISdFdPI7idCPli8i7h9wM9t6XaWq+F2UfyRWv7d7TzjVYX2aWJn+rkma1Gf24fE/q2BDP2dI1bnTvlUlOjDzGDGhB0SKukzf3dMhNbQvNsJ43gMZFMz3rdnekfKDFcWM2b8TJNNhYrQ3eipo19l6cJPSRRDd8h0d5pxqUlW1P3maYQEaSbK1XDXUm2EoawHuS+OP1lSP41kLSSJ5Z+/8Phv3LD8dpiuj0qa/sowLsXvq+Np3BTVJ2Gv9l8ucc8dcS07CKIIl/lXmM2ptaSf3JGtIi5q7Odd4YDP6d8Wr/zCoH3Gr9ckJ2tuuGpRsRZf+2n+VpLcGJY1upW5rnxkuCnM3VZ2p9oPHVRCWVDdKLsCRRUer2uq5wGnFP2dIyfZfTBjnvzHvtUExC85rGTuzT8mOcdC/NC+jNcgvtBYS2d/mJ83EZuXjD0m+muDCgrjhWMaa0N3L3N+9sETZ9JX0d9WLP58Q27spNTK53LYl15U2SGQtcbPLPVjP3lMU6WJraEeE+boyVqTVy3m/ROxJDLq4wgx4XaRahM4vK2oH2G4OafuCSFH/LEqKmFU59//vjitC+O6dPc/RfSYiK3BFPKDyuIJzk5LKuPOE2tD8zgSahmv9rNn/2VFaNS0S2o62Tk53V9Zy7k1z5JKlrWiv6uonBaqBOA2Ggf4lSNWmni1IDbi9Tal30CFJL0qa+FEh5l4rlUZexCf7LiW79Jk5Xs/Ofnc5rRb1ZZkulG+l6ZjOB8Dogq7fSa2he6ihKpyYvdWu8y4lN8RdJQdg2TAvfRTCmls59RPHnsMsoOxcLS/fmL4+oo/tC4K7qUudalLXepSl7rUpX5UdVFwL3UuVbwzj9JteeJrQlExSldsdycpI9VOT96s3ZdW/IJa0ol2X1raB021ixx+Lk9efqHxM3kClKdXTaoWHF8ZVCgpU4dMbDX9mzmxPUEZJXXIz9vpXKpdovt6hfaZMNdUmxE/t1OHsL+qp3Sd5iXiF4owE85tWpXu2tGw27dcvd2y3bb43IgPdlTixQXsXuPfeCgqa/eFdOL6paIq/kTfKqpDJlXy+akSKSm+efVCKnJWzoq5G0lZUS8DYzLkrKhsYF//gG6gMsNLQ9IQ55lsDO1HRT8/q4aH10a6m68szirGlSE5RbU9JXX5ohaJmmu7iH06SHZ9UR+r754J9yvMUHxqi4awcJgfKEf+7bVcg02HX1XE2hAbUe6xxW/XaLJXVMeBNKvwVzWmtuJxLMqZ8klIDArCVY05eMLrNbEx2N2Zw2sORd0dhLmrgqhjJ3VNe0O6bsmlo324ccx/dyC37pxgl4WjnBpLXFWY7ShkjjGQm6Jej4nmKQnVQ0t6lS2JXFPHdAf9XUIHhdsrVJB7I1VnP3RoxadGgvYxsftaztPPxesm45XF73YQ2ka1SxPH8kQbEH5sUbCLUhYaTfuYJrU4OiVeay3+UNsn5p8UfqYmBW5cSVa9JU/jKilUcXqvbBXueUSHRF3Yq3pMjNf1pNKdvHphfuKMymcNMzWde/tZPkd/I2l8w1rhg6HeJJqnUzSfqNvd1ytUhjizxFrIB81nUaGG+xmq1hzvjXh4HxW5Et+hPZSUsoXGHkWp9gtV1hDp7h8Xp7nhCI34WOvHgVRpxqsa01j8Qr7m/ExjD0b8s33AX7dlDhvGpZ7eK7RqShXUfWb/VcXqt8Pk6fRzQ/ijNdkoTJeoNgl9HDE5T8egFH5hqZ5GzGEgLVrCVU13X01s4KxV8RQnslLMf7clzipUSIyF3Zu1mkgTqTKEhaN5vy+fOU1jqn0mLBzMHe7pSFq1kq5YlOVsFWbXM9w2orjONGYUKszJZ60rUcjHpdwLKpdxnRncodAhAmSdmX0KZKvYf+Gm9LoTIUHFzLjU2E52NLKRuRmd7HgAwjXthb6S7yviqpb71mpUoe74pSPOLG47QlDoXY/KM/HiToo7oDLay25h/dARrSZ9fTft5vnrlurdi/RhWGE21zoRZpqxBIdVWwgNpJm8ptsLKcAez9x0lHhsVZQdsPbdQXjDQ8S+iCfYHC39XUNYVHLN+0CsHVHpiZqSrcZ1wvWtdolxoWWnziJzCCESmD7j54rF957Yiic+tIbKn9XiMHfYY8SvivqdQR89qfizx5uGZGSnRwfx9PoFtJ+EAQ0wXCtmH/OURBhr8c7Ghkl5VQF0IWM0D0INGtdCFOpeqemY+kmIDfYoCnRyZ6IGlGTE50ysznSIWAlhZyjkkOqpx6+qaf1w25H+m2uSVX/wL6wXBfdSl7rUpS51qUtd6lI/qroouJeaSiV5qksW4vqsZq1/Hdn8zBAWokSlMmtipTi+EtUw1pAqYdcdXpsptWVcmen4YaVJTrxfZDAmE0Y9+XIAxoWifYLjK4dKwvernwK8MvTXcszig+Sau11ED4m0MKAKm7f4eMJMM6wM3WuFn1uMz4xXEBaJ+bVIcMu2R5fH2E2ck5YR92ywT2d/bfe1p5qPjE+NMAWfDe1HyYg/ddfGWrxpwi1UqI0jNYlt33DTilJ13RxpjSdmxfvjmk3X0B1rvDOMRzEzaZdEgE6KXCWwmdSLEtI8nNU+vwK0EtLEIOchyTPnBKB0NSe0hqwqTB8J1zPCzKInrypoH9EPO3Jb0d836JDRvZquYTYa9yi+Wrf3ZKOpXsRTZvfyaK+CpXoeitIn/kuVMqk+e3ntri9d3wb3Ih30ygvz9MQW1buOeLMQ76FR0g2vNXo/YNYlm77zkni29YSrBjOIMmw68fMBmKNH7zpyZYmVJBjpXlLWTup+/dChckty4gWPjaZ+UnSvMrE+XR+FPWiyzsWjlmk/K5nXRcG1feG49tINXG35f/FMms9qIiEsf9tzeFuLatcZbH/uCnabgeQMs5AZ1jX7Lw0qwuzTKSEp4/aZ9jnh9pHkFH5eKAiFD6kq2QExzwm782gfSU5U6tN7NZ/6SWXUfUBvArxaijpcfMOL95FhZdg3ogwmp0iNeFJPOwAql/fN4ttPS8X1X0TsMaHKro5fWvHQtWrqQG8+DWSrCauSLrb36JjxswqVoH4O+KXB7dRENgChGzQPnnFt6a+F7hAaTbU/3+8qFbWzls+SE+ghoova136W62b6jF9VoiAdxKM4FpX35Escnfimh7WkLo0ri5/LjkZoFX6mxSOZzuxbcp643ePdHDMm/MqRK4197IiVJlkmpbx+DphB3j/MLcSM+7ghtzWxEjUrVcINDnOLGSJZi+pdbcZp3PUxYvYjaAjLmly7Qq0IVNuS5tU6CJFUKewhon1iXJ3TyU5le+nwX//TQH9jxGtp1XRd4HT+wrWtt4lQK65+dU4AHG4cOmSGK0X9cmLGCvu4OpRLZTWqD6iUcGUMlY/k1kzUneG2Rg2JMJedJXPsqZ564RmfPvsYhI5hNdXLIJSJyghR4rH0ftzOCK9WpOqU+pXwc8P8fWL/1VnjU7kokK3M+/WvM+4QpS8E8ShX+4Qp/SJ+3eB2ReI88ZgXxZiqRGFPtVyn5JQo0YDZdMS/fs3s24Pwk7+co6KsQ6f1I+zK91mW802VfN+5Y2B4Kw0Zdic7MeY4EmYG5aPsnB2HSbp028xwXUt/wSFSb4SEpAOy+4Iwaa9+1dPfVuLlbWS83YFpTYi13FvZQvMs84isCbMzVQbku716TIS9qMCne/j0XennJdFukHstVor200hYVpOPOSwqYS3PLMlpZv/4E+mbW9IPqDx/2boouJe61KUudalLXepSl/pR1UXBvdS5siQSub0w7XyJOn/6E1FkxU+ocInJo9XdFTZtc+r415JwchRVq3t15lm6o/ic7FF8j26v6O7Fr9M8yRNwf6PQQbKozZgxfWbz84r+5vyk6GdafEU7OL520vF8rIiVws1Luo9WDGtFaApb1irGdcLedRhdVJ9oiEkxBks+GtQ8kA4avVGT6kxU+MGKHyiIAhBmUG/OvM/khB8seekQbwLVvKgH5ZFck4lZsbIDqZXoqbbyHIeKqipd3n9xRf6iRzURVCYnRbaZ3c/U1HXu9tLZbwZhWh7eGLSH2eeAL135IWX0wkpKz1LjOoMesvBWS2b68JMbTB8w3cD41ZWkAR0jYemmxC8VEv5mJrngbfFPJrCHcFZMx0T3uqV6GUm1IbaaMK/FIzkWH+F1i59bqo0kPmUtHlk9RvxKjN7ZauHt7o8MX19j96N0Uxs1KWSxsYS5pd0Nooi5Odkqjl/N0MX/N3vYE+6WonwcRd1J85r8g1QnHURlbJ5EidCjeOqaR0V/J69jepmn/Z2ot7ERtdLt8uRZzBpROwZwh8SwFoUuOcXy+3KdV0IOSQ4Ob2vGpabeJGKrzz7ZDP19OzGBtZd7bbhSmLFoEKlwZksXey4JZe4oSubp/lUJYqWFE/3Yo8eEX9oz/aAyolwPEfPxhTxvwRT/cVFZ6oeRw6t26uROrsz1fPbkuX0SgslKo6N4VpNRNJ878VsDpk8cXztCo3BHWVvizJKVeLcB3E7jl4bmMYqytjCyMzPmye97vBNqSmy1KGFH4WYO15ZqIyc9XFvxkz5HhmshuGSjxMO5E09w/0qUrNhYGbcuYY/SBX9SgkUxy/iFmVKg/EwxrM/q0amDPDmInaa/a2g+53N6FNLdn5Xs9ITWopa13H+JSRGsdpqxMTSfimL5dkHzfebws7XcK0B/V5GskuuVReUTL6nGF/9xMgodKponL77eEx9378mF3RsbS/jyinEp9Al3zLKrtdAT23jaCSgEHe0z7iBe69NYuF1Et4VwEWH9T44c37aYo2e4PfcSnNS5WEO9FUKJHvPkCY61xiwqzNMBcma4lTXDHgK5eHBVUT61l0S8+GqN7j3dl0vqx6KUX9dkq2jejaLmag26eF8LGzs5Rfe6EaqLyoxLQ1aK6pAmikK24kvVXjr7jc9CfmjOOwmzzwE/11RPIyjxCBMSKiX6LxfTNRyuDfUG3FbmpnsZCMtqol5wWxMroQ6RReVsHnqOb9vpOs8/jITG4I5h2hGxXSQ6zXAj99e8j6iQ8VeNKLi5RoWM05pxXeaiVpghkZzh8MoKB3crKZyn+912MC7d9J2ejfC8q12aKC77pZqIEqkS77iOhbpQbg0VZD0a5+V+H8HP1MTHBiE2ZC09FFkrISx0njBvcRtZXLo3ZceuT4S5pv/5HbHR07X5Q+qi4F7qUpe61KUudalLXepHVRcF91LnUtItrmLGjOcn12xKTncvvprmQXxa9pBQ5fHt9HSXjag9E8PyB2lKegRViwJabeRPv5QnueZJjklVSYr5HGnf7Rlez4m1sP2ax6K8Nop6k7BdBAzuIN3P41KRbMnpruXpsHmUP4+vhTG7XvQsavGL+WgYoyGmhPKaZj7SJUWnHKYvfs1PFhWF+xhmTPnr9Uti9uHUYiq+yvpRPnS8ghQ1Xyy3XFXiwV3ZgYRiKNJwbQK7oSZmxbyW19mtAniNtgljE75z5CoTluM0RHFpMXuNOj0Re1FLhisz8Uez1tS7iA6iKNQPI9mK8jOuz7d8+yGQblcon7Cd+LySUaS2HJNF8as+HYitPY/jECbvbGxdudaRUJJ7xB8dphSlWGsZs4eIHgN6FAUiNmd+6HhViyrjLNVTx3jTigqmmdJsVJCkq+HNAtMFUfUaQ2g09Q/oD3HmqP/JRzCavJhJSpQ9P8snJzsEsVGSsvSSaZ4Tfq6mBLnYiDqx/K10C4dW6B/rX3uSK+PsDNUuYfvM7PselLSIh/rss2yekyT2tKJ+u4MwQsNcT4pZMopsROmwB+GFto9FASuqRfNcmKNDImthSyan5Dq3cg+6nS++tRPxo8Xtxbd5GjtzGKEtrNEQSKtW+MSF5CAHCROTolTaQ2bxITCsDHY4KdyR2BgW70aMrxjnCvP/yIr3K0P9Esm3hvbBY3rxCGatyEYUJj1Gqm0W0sEoqk1sNH6mp9Sr6qCxXSI0uqhR4mdUSX4eYPYuEGdWEveUpflwoHu7QIdEmNkybzPNpyP9m9mU/mQ3A/3bGbnwa1Uovv+XTPsUGNaSGlhvI31RcZuXSDwKBaZ5kA737s2M5qGfaB7NhwNhWQuHNmXQ4pn+YWqaGQSym60uc1lz/MkKlfOZEpCFglJtPLufNrKD0ol/1u1jmc9KOvHHRHIa0wf6Vw3DrZvGwu4jtouYUdZ37bPMnVpP/trmQXy82jvad3vsdSu7PE8H/OsC4tYwJuEWa59JVsbEX9WTf9QeIqnSzD4lXJeoH31JlMyS+gUoD7G16MZh+4gvKYmmD1Nipdt59CjcZn0Y8Dcz8lJU0FTJWNSPQobItdzj+ncfMVqhXnbErwQ4bkaZU/Pvjux/OpO+BSPfVaedw6xKx/4xCVFDKfprw+LbntgUVncGHRWpMVTfvWA3jqyU8HsLbzhbISNonzHHUXalNgfi/G4aC1kLMskobJ+onkZiLeM58aqfe9SyxvQBNXjUyknfg1LEcr/rUbjlbjuSjdB0dMi4TRIGNAi//Lmnnhv8TGN84WYrMEW5n38U5T+l8r3aK2YffTnXMudrUWyzkvH1rSQAmgFSKjtPITMuJNWt3kTGlZFUtFrUYJC13PYRvxB/fv00CEtaA4WbXj97wtzgl0bWgCy7UtP69AfURcG91KUudalLXepSl7rUj6ouCu6lpjolT41r8eSNq+Jr/CD8wlgz8XFjDU9/w+ELn7X9pBiuJQkmtKfkpoxfQnLlmM/C0hyvxQ8aFtKhHuvM5hfy3tVWuIOpUjz/6booS8K/3BfOqD0Ki/J4Xzo+h4wOSli3hT+KhmqTyUaYfXGWYe1JGf70+p0cojJP44z/6+EN7s0RYxKmSvCmxx/l1rAPjtl7YX2GuXhfVZL89VgVT1V5y2oH4xL03vLX/+h7ZnbEFEqDVokr27M04h37tr/hi3bLkAwvo3RM8xM4DBWHXUMYDcomlIvkrNCmKKY5kor/FiXK3PGNpn7KU7IMCtSL8HltUSZCUW/9rCTnjJlxXdGMkd03zZQX3zwJLQGkG7p+8dKBDZNvKhtNqs/HVLvI8W0rLM5WC4mj0nS3JbXokydWluOXM8zYSDe5U8w+Dhy+kGvoDolxXWGtxhxH/MowzjV2ODNdsynM1pXF1BrbRfxcl89TjqkcYWbQX9/h3j+LwqIVfu3EOwekRhMrzbDUkyohvjNRVUCU71iL57TaZNyueBPND3xgCmbvPcfXwuskS9dzrNTEmOzuNM1TYv5dLx7lRk9/Vz3LTsJw20DM+Llh/9YIr7mk/Ey+2JdIdyc+6MNrw9WvRvQopIJTCqAOiaDkPJvvduz+ZI3bDMSmnegHenMgLK7xqxo93Exd335lJh/z0580dK8UfiEpbJJupCYWtrxXplsJASU6aF4S9Ytn9/MF9ijnMywNtjrzUZPT6JTRY5zOOVuNOXjGlRXPbCtjkzWTKmaGTHdrqPaZpKC71SQzp/3sJwXX7HpSvRQWrlOk1hFmGtRZ0dY+ExcVekiEuaQjjvct/bXBFtIFCvpCFdA+037ypLcV3Y2ZegliJ+q7HYSgoMco6mqsaQqBRO96/Ns5totkpxmvHNko3CZOc3W4dszedfiFI9UaP5ddGNsnVCwKboTjq4r2s/+9XTGA2W+3gCSsK5GxAAAgAElEQVSrhUbjl0KeYSMq+Tg/d+U3n0dhBL+Ip1OPkfFaXne4kfu0v5dEMTUmUispY8N1jV44qmdZt/rXM6ID3xrap8jhK6GvtNuRPCtKZ8y4fSAbi/aZ7lWF7dPveShzEh61v52Juhvk3jt+2VI/i3po9qOQCrbC2Lbbnt0fr5l9GCZCS9aitmathNzS1GStSfdX9HcnUkeY/NjVSyDMDcdrLZ7643nXIevzzssp+Wu4qyYKSlLiIx6uHNV3gA/o0XP86vW0o0USr6zbDqTGyS7VdStq7WknKmbpDXAKPzeoWGEPAR3SlG4YWyeJlo0lritsH1F9QB976qJej+uK+vORsKypPx4I7VLG/balehRft9sOhHUtlJdCX5EUUT3txhxfWebvPMYIz9720N05qm2cxmzxPrB/a3GHLOtdLeucqMEnj7B4butt4vC67FRuE8f27J9NlWYoyXtkJEVxTOiQp50x0wf6OwdZduyOb2qaJ6H3/KF1UXAvdalLXepSl7rUpS71o6qLgnupc+WMPYr6evLTgrDr6hdR8E6KUpiJ71YV/01yRcFcicKarfypR8mxBvBLISGMV5Sca0lMQzGpE82DdLDGSnK6YyOduFn/IPt6kGQlV1h7ogZm1JzpkS0rUcD8QrrYSZkcNXezIx968ZOlrDiGikNXE6Nm/2EBOqPnAcJJGYTZ58TmZ5pqI3nx/Y2iecpCTkA6R20nqUCpyqj7gftmz5/MP/DabQB4iTP+0e4rfh3v+Hr2TBcdrfF00VGVi+p0wqjM1dWBQ1fjXESrzPZ5RhzKYGQw48nzBP2tkCKclQQ5gMMbQ2ilOzpZ8AtLdy+epxP5IRvpZO/v25IkIx3xs/cR+yBJSf2d+Mb0YaD/mXA9tc90b5qpe735PIgvrJLUJDPIdQitnlSxMCtd8CWxJjrFuFRof+46r7bi09PPJ4KDMBXtIWELa3K8rsnGlNcU36KOmWp35v+qnLHHSLaa8etb8bD5k++zpHkpRf3QSRf+NtJfi5qXGlEcQOab7ZJ08s4Mw5UWhfcHKlSygBYVqL+x2D4zrAvjdHtSMbV0HAfx4upRE+aibJ14tMkpSekZEqG1MqdLXvxpzo8rQ/sQJpLA/m1F8xLFR1ooAcNNLWzgSsl1OCT0rqeBiWyQ1nPx2mfwNy26j5jDQH+zJs3UdD6hzRMxxAyw+9qgAyy/K/71ucUdEt1dhR0yi7/YML6aEx1QVLBTwprpM+PK0n7sRbF994R1NzKOc8vw5Qy3K+MeodoESSIrlAA/kx2UZMD683UZbiz2KJ89LRpireX6GEVWCntM1M/D5Okcryv83NJ+vyfMl7i95/BVK0SBcq62i+i1xc81povSnZ5yUatkLOwxMf/VM/3Xa1GdVCHIVGeCQWyXVM8jqTFon4i1xXZJSAonFnUWH2q2mnFhiLWifQiMK0NVmKmxthNHdfH9yHDtii834e8KK9eoSc3WIWGedzS1wR7clAaZrUIPEeUT43VFNcq8d7Wa0ii1lyTGOLPYg0d3gXTlcMdEKN5X0yfm7xPdvZu84NVGEsnctoxFbajebYjVNWEmNIqsZEzajyf6QYV96Rjv50Lu6aKk9h3VRA1IVVHflSLMHagKt4/4pUP38l76OFI9aVItHOFcV0IDqtzE/DZ9wG010cnuyfw3W8blFbMPQuIA4f92N2bq+G+ehfxg9xF7kLnZv67RQ0bljH8t6Xz26UCshS0MUH8+MtzPSJWZEvRMzPilnXaiqpcRMthtz/GnK7n3nzvCVTN9D443FfYQUVp2O/UxEFc1KkbMri9rVEu4anAfhGVuelnnZC0rbPXeo30qfGkhjVgSi+8Chy/stI5lq2i/29PdXhELNcV2kfa9gIu7L4RIMVzJTk69ySQj92L7cE65i7Wm2olfXhLuEvVWMRTOff2SChtfTWxulc49MyD9GLP3g4yxUtjeCXv9B30Wf9m6KLiXutSlLnWpS13qUpf6UdVFwb3UVCpLOtPhq0SyTB3l2ZXO2yAqrF+AXyV5Km7KU5cCPYjC6Ve5+HTFZ+O25XU0dPcI09MiamcSj1/zIOcQazi81VMqVJiJCtw8Zg5fnl/npOiOq4z2iv6mvHdh92Yt/ke/EN+wjvKz3ywfOQRRI/7s4TU/v37kp7fP/PlfvEVFhV55kteooTAmHTz8a4o4jzQfjKhay0yPot6U9yppP2GWcTvF+mrP0vbiuzVCUbi3W5yK/J2Pf8rS9fzF5p6fLJ/5p9sbQpL36kZHypK3HYJB68wYNaaOqLqkvHRWrrlShBmQRTlPjol1SmbyXFZbURf9XIlfshzjF9IxfEojik7h5wq/dKRKAtqTFT6iGlvhu1qNOyT8TBHrky9X/nS7iF8Y7DGRCk2hvz35CIvifPK7lsfqYa1pH+Vz9TeiQBy/nqMimC7S35iSsnTma57GXiU43lt0BNOls3/8fl7U0Ej3psEdhNTgZxoV5VxtF4mzivpFUorskBnn4vuM1+fEOHdIqJxpHnqgobsz0mVcvHTkkvTk1MTsDK1wdX1RzuafAn5u6N40mEKqOCl4u18s5ZwXivYxiv+ty3QL8edp8kRRsF0mzIRhaoZMmMGQNM1zZrwWJTzMNNXLSHffkn++Jhvov15TbSTpCEB7S7YKu/ek2tB90WAP4g89sZ+zknv/lEcPohLWG/EfypjCsDLn9MPCgNUlMUnuZV0+s2JYa2LVsvhdx/FvvJlYsNKJD929oy7UBJVlt+jkH40lUW1cKcaVwh1y4f8qhuu6jJfcEyePcKqNdGuvKprfCKIlLG9IRhGWNaHRhLkkbukxT1zeajPSfgyEb2bS0a6EZOEOTJQIlTLD2xV27xmvqpKQJeSD09hmqxivKrJRwoi2ckyq1PS5Tt7m4ytRd2MlXeXjcjapztUmyO6DVeI9HxLjQsbSz+SzV3tRP2NjyFGj7tdCKPDnrvPQGHLxuvZXhtDUwjB/0JOCO383MryaExuDHoSSYYbMuHKT+njawTBjptrKuWmfyFbT37vpnMP9kubDgePXS8xBVLusFbE9k1X83Yz+zpX5bORabT39fZlLQ8aMieGmxh4jw40c63ae4U62Bc1QU30+iIIL5Jkwr2NjMYX5TRKPpzl6/KoiNQ7bZ5rvt8TmunyehOs00RVub8j0t4bYmmnnJ1lRocelJhnxhfZfr0s/iYyJPVSYIdG9brDHiF8KhUPlPM15qLBdZGzmQhJ4Hsm1oXtTE+rT+qMlpdMZ3M6z+2nL4tueNKsklbHMw9BawjdXmDEVFm4u9JtCeqnFw9w8H4k/vxKfcasx9gc9G4VSkCtLtU8MS43bS8Kev26mtSVrqHaZYS33o+2EnHK6L+w+wn0ltIYxS3KpORMogEL/SAxXhubzyPGLekrUO63hodVon0QdXzbYg4y9HvUfrMheFNxLXepSl7rUpS51qUv9qOqi4F7qXDESK4jLKC2jnTxZVRsYr9TkC+y+CKWDXxGbwqZdZ8zOiId1VIR1pNpYwlwIByBqLEoUV78UwkFohUwQ2xN/VOFX8jMqQncvndyxURONIdaKagPDLaS6eH4Nv+flzU7S0lJ1Sh7KuE+OMVn+ZPEBgJvqwMIM/O/PX1PfdAzPDVXtGXHEuuSNNxlchqDwa8mfj3WmflTo8eRdAz9XhFkizDIz4NYdGJLjHx5/Iu9lD/hseN3uSFnzk+UztQ68XWwYY/GBLSMhGfpoeb2UtLNP+wXP36/RixK5E4VV3N9lqhdJkGofEllBf31KSCo0gyjUg/5W0z4kxoViXJ8VymQloUnSjRSzT4n+1k4+a7eL0okOE1N3XIgHNBTww+l6t58kOSrMtaTlbM6KsvZCxnDHjDsmstK0TwnfaupH8XR2rxtRKBfik6uNjFus1JTYpOMpxzxzvBdvX9CgojmnylXivfQrh5+r/5u9d4+25KrvOz+/ep3nffbth7ol0RIGCwFCyA0BTBLZE2dw4hgnZo3BHgeyzMJeK/bYeDx+TLLGDk7GniE2CbETL2CAxIPjJGCIx8mK7cSR7RhsaAggiTdCIKFWq7vv+zzr8Zs/fvvUOWq3pHMl9eNe/T5r3XXqVO1TZ1ftqrr7fPdvf39EeRQyMVGre3nXMtPlXTtHVSKMlk0ZnNQ57SmDtZjmBZvtbbF5GjL3WJnWujkb2HuhTC1Grfv1iir4fY4XTAXL2xGjxZjmhindpo6HtigsW1fRtZEMKS0T4GhVLFc9VveiFTL/jE3RHaxFVLtSfxdq2bpGi4JUEVVsyutoNa09kuPglVpmEaOVpJ7RHeVTtVhDbF2kUDaVGCEZQutCxSioLcmwYrwgLH21YNyNGa02SHuFqetBOItypbExonfCRgBGS0J0omkxwuG4Ghs5ZSMlbwvJKLL4wYmv60QZ6iv5gtRzAuIRVLHNup+oP3Yiqf18Te2zdsiP24jEpGw8im20opORDJXGuKxVzKKdmFe0wnAtpbFR0Ny0Ea1Jxri8EwVF0uJQi4bQvlBSxZAvmixWdGKKhlgcutjxNLZLykyIe3YvZ1nE7omMvG0KGoDkJdlOiEcF+scymudzhquWjU7FnmlUEIdHAhYyb2pomOWPWKzw5P6sMrG2Uxu9GS/GlA2bT7D4NWuM3esz83Adq2W8yyJaX99l+7mL5BMv4cI8zrPtkmhYUrYSxsuWrW5y7yR9AYnJF7ohQ6LSOjdmvJhOZ8qPKkYrKXkrIspLohxGyzZyN2l3i0UmnE/z4Z6M6kxGG5rrJXqsayMJuTI60g7XXlXH8sqoJGqYZ6yN3phyn691a0eLvDv1OK4SQTMhHhOy+U0zvI2WbE6HRhAFP9fmBcs6Bhanm20WaGz1H3cj84JXrWOCzbPWnhsqMF5OqZKMMp3OEwBlvJzUz78yE4ZHGnS/NCRfNlU1O9sjX1hguBLT2LJrK9upgrtMUrd7VFYMj7Qo2uGZkArdB4bkLVPBNYLd4zHD1a6NuPYtVnx4pFG7QwxXIloXbE5CFdvImSi0zhe1Mi2VPesRu0eSkTJatONpn5uOAExGZIpOQjxUqq6Q9qp6lKDM7Jk2Or5INLKHSTSejujsBVdwHcdxHMdxnAOFK7jOlLIk7ypRq0C3s+mv/9RcC6Ic+jcW0CyR7ZRiqUDGoVCjJL5gqkDZVIjUVKCR1PspMzUlTKFqVURDoWoo8XZE2Zh8l5oauaaU7QrNKooyIRpJnV2sbGBxRqmaL2/TYnSHa/arGyDZguGqqa3ahHRH0BQ+t3GEhcRmobbinFxjHtpepCxi4m5BkceU4wiC7yylwFhIti3+VkqIR5aXe6IoVVgdpDJl6sJGl9HxhBONDb44OArA/YNDjMqEQiMqFRbSEb0yY1ha3C1ApBWFRpQa0YgL8jJmpT1gfDRhPA6Ze1o5A1pAHFQtU/LSXa3rY7O6TZms0BB3NW0HsCw2eSsiKtV8ixOb9T6JqwQoOqYc7Jw0p4XOVkHeTqhiU4kBRotC0RG2T1oDFq3gh9iNao/JKLf87/E4/BJvmtPDwgMFVWMSTyi1m0PREqrEvEnNqcPqPO5GpP2K0arlUU97FaOlqFZvwY4z3bV4VSlhcCjEwJVKFeLbhksxCw+O2T0RE42F1nrJ5Lf+JKuT+TKa28DwcMa4GzFcieo61WUSoX2uYOumlMImGlM0TQ22a15strGaWqaRZT8bHo7JtoMypEreMbW2sVWF2M9J3nbq/aQDJSq0zvC1e9zU98l5nmSomtStaAY/2VTq4xocNreLtFdRNKVW/9NeyWBtEqOsjJeE5gUYV0Jjw/yq40FFOpnl3bVsTMPlmPY5m/mfdxuWe16sTTtncnZOtkHt2WFx3KZaTTw4R6upqZph/8VyzGjRjmm0bO3VOWuxy8kg7GNy3yXTayPvCtm2Ml4QMgkOG8GVYqJmFU2ps59ViXly2wzuqFbyxssJ/bWI7pmScTcy94FcGa7EdTYmqUwJHweHDFFzimjsTjMO9o+a+0CUK9snzSt2kqVqcF0rtJc9W9OevTa2K8pWSuvBHXafvVTfX8kwNh/TnZBlq7J7fOJSoiJUaUTeFrIdUyCrRILLQRhhayW1M040Bo2VbNscFIZL01GabKsw15NxSVRWFItNGhtF7TbQvJAzPGTuDOmu2LkbVQwOpfU1VnRiOg/06d3QDtdkBapkG+M6K1g8VIaHzBEiHlYkwxKNYsaLaR3HLGGUxWbcK3nbHEuqBOIwSjC5V1SE1sN98vD5uJdTLFgs7/hoh+xczzIg9u3mTfoVyc6IKrW2SHolRSem+ciQ7Wd3aD2SE4dZ/snuuP4uXY5MrU+C2qpQxdM45qqw7F/pTslo2RxRJvUrZ+KXNQpt2KvIu/YMSwcV43gam17Gdu1VqZDtVkipDK/rTkcttMNgLZ7e/2qxsuZ+EtwzBKJBwehEox61qxIYHm6Q9YJimgpF29aXDWhuKuNFU6rrZ11uz6LhoZjGdomUlt1Po6n7Qf9YSmPHRubKlJB1L/xPCN7Yg8OJzdEYKMOVhO7XBsTjFCqtrzE7jtieZa1JxrrRoz3I58QVXMdxHMdxHOdA4QquMyWKLfd8VjJulEzcRYu+qZfjJSXq5lTjGBIl7haUO8FvL6nIlyvinQgpoOqaqlm0tXZa6Hw9YrSq5AsVLOZUpCQ7EUXLZmTC5NeeEOVQrFSQVhZ7q9RxVwD5kqKJEu/ar+fRqv0CbZ2bxumWzel368C8PXcGTb7WW33UYR/q9DkzThgPU6r1JkkvolyY+KEq2YVJhh6bTV3FpkAU7WnMYjw0X18pIe8lfG2wylq6y3WZWS1cl23xyHiBSKZq46DMIOsRBRmq0ogqBBolUlJobH65Sc7G0JSGsorIxwnR2YR8sSIaC+OF6QxUsGxqrQsVvWMRVSYWs5iZyhUHL88yxAc2HynoHcuQwmbXNtdLesfseLtnSsadpHay0MQUFI2n8XZlw8573hbyBYuHlspmuE9+/VeJKcRFC/KFmLwtjJYg6cV1HPNwNQoKg8UIJgPqmbhRPpnFDL0jVrelr+YMl83VYLwY1XGfZUPoHQvqWWEz75MBjBeFKsw+rhLYOpkxWhayLaW/FtfZ8ibKa9E21wBUGXdCzGbXVPqJ1233wTFVEtM/kjBatpGL5vngRhCaIx2EWckDm7U8XhCktGt24gM5OBQFVRB615kzQ97lUfGsGkFjo2C4mpBdGLB5ywJVZsputjvxmzWVJxnAcNnU23whItuaxhaDIiJ2fzSgfzimfb6kt2ZOG3adW3tP7rfWhSp4Uke1YipVUHq6QuNsj9HyEjvXm5o2qfNoNaFo2vmoUrtHiqapyXV8dmmxdemgMm/j0u6n8cL0+hkuW+x7tmsxtcMVoX/Y2n6SuVBD/C11JjDIFxNTl3RaZymhf7xZxxnHY/Mujkch9rpvGbcsrrGiaJlyXzSlfkaVDfOTrmKpj8ueN1mdDa5MIYnsHI8XhOam2ujCQKdxzCOluVExWI1obVSk2wWDYw2aSVRfY0VLGC3EVDE013MGaw2q2JTXYAqChrj3omWz/JuPjNi9scVoOSbbnsTFVkgVYkhDdsPmRkVUTLMAJrsWR4+YC0W6OaR/fYd0tyTtTa6xgtFSw853kVE2giqea50l0Wbx23Xc2CqoUsui2NjM69n9jfM9escziy3vhFGWCssY15he8/HYlOjJvZltF5RNcxgAGBxJbS5IJkTbA6LVpqmUraTOChblSrTUMjeVxZS8E9HYKhmc6NB8JPjyLmXEg4qyYc+CshUxWjLv6eHRoLhX5hiQtyPiXEl2x4wOZQyOpPX50djiebPtsvbUTfoVg0NxPWoxGflIB1r7Wsuutc/kWh2EZxJCUOYr8k7EzvVJPVI3Wp5mu4tHdr7KVGisV6RmX8t4wdT4KhZaGwV5N2LUjMg7Ut+nUk29f3evixmuWBs0NyrLhofFJtv+IpoXLFPdeDkNz6pwrTbF7onEXJLyWGhuVGgU0T8SRlFaIGo+6Nm2ki+Zyl62p77Xzc2K5nnLvNc/Yu4qo0MNsq1J0Pn8uILrOI7jOI7jHChcwXWmCBCBiJKeS8mP2i8mjUxFKztl/YtIkwpViPrBL7YRQVZRLCqiApFSJRavWDTt199oVZAc9LoxVII2S/LY4nWrkF9bEyWPTRXIziaMT+RUTaXsVLU3rShUWQViqpdUMvX0m8RrLkDR0Vq10ViJcji8sMvxtqmq6+M2D+4s00nHrC322MkabPcTNFWiEO9bNe28xEOhbChSCFk/xM+Fu2eSWU1jDfHBwpn+IvdGx0kiO/bD2S7dZESEEktFhNKNR6RSkgapKkaJpCKVkjgcyFBTNlodHh5Z9rULow6VCtvaolooqIqIKo1IexH9Y/bzv3VOyAcRRccU9MaGebOikC+Epq6guWEuAPHI1PYqFYarca3SDQ4llE1TCYsWDPIkZHWaqnRSmnKQ9kxFN9XTVIfJzNkyE5KexQgXDWG4ZirRcE2QcBKLNnTOKP2jFiudd4Uyi9GEWtVobCrV0iQ21WbbyyD4MhbT+hStMCtdJuraTN70cJ0PDwnjFXOZSPqWkSkqprHFg0NR7SOahix1omFGf1CqRisJRdtUurKlFiMuwQkk1Kd1oSIqTUWKh6ZslE3LBDRR6apMaK5XFuN+TMz5I1bKZpiVDLTPwngpQRQ2b1moPaZb62XtmQpQpRGt9YLdY4mNgjTtgNMQ79vaNiUob1tsYzxUc9FYnPHBbVhMexHitlXMjUG0osgmymJQLVPzlZXK6huPIAsq+GghMiWnK8SDEHeopjJpUOk6j5RUscXb7Vyf0LpQke0og0MRjc2Juj/1cC4aQmNT6R0Xmpslu2F2f7arxGOl+/Xg5tGOGHeE5mZFOROP2NgOzheRxWonA9vvRL0Gi2cvQ5K9bLOgd6Rh12GgaNk5NTUeErVzaC4X0/trkr2PoMRno4psp2R8vT2sRg2hfb6ibAnJgyWDI6md55W0jjOezFTvH03Yub5hz5oVe+5NYhKT3ZJkULFzQ2z38JEGolDGUrsxTHxMq9QcXybHne2Udcxr0ivIF1Oy9TG9G1rE/YK8E1E0o9rHVMqMoi2kOxbnHo8r+odi4px6/oNUMF5MLV4Yi7HunBkzOJJNZ+UfbVM07foqG2IuGkOt4/DtPJvUnO6aCp6ijJYTG/3oTeKPLR48KiE/uohGEo41re/T9tmCMsRyVqmpvf0jibkbBE/0oh2TDCqKrjksDJfjepRhsGqfbZ8vLJa2Aa1zOVUzIW9bHPbkWh21zT1Dupatq7lRMVgLMdRB5R0vRJRN6JwtTY3OlbRfULSSuszusZjOmYLdEyllU8JIgjA8ND3PYO1QNkL8/qZ5h5eN6NEjeqstpFIaF4aULYuJVxGS8Hzrh4yc2XZBfCiqs1FWM97Yjc2cshEDwb2mkzBcjuk+NGaUTM6t/UU50/u1FdHcrNg9PnU7KZNpHPtwNbZ7aDh1/LDRlYrxYmYZNxsR405EuuuZzBzHcRzHcZxnOK7gOo+mgiJPkBiiNHjQjTKKrkKsaBmZR26qxHFFFbJUMTJ7BOkWaD+ByBS71jlhfChkz1osicYRabNgvN4kXR5RjM3wtEqCRBAp9BKqRNEogkKgBOKpmlVlipSCFELjgjA6pFBZ3O54eWampUyyG4VYuZHQiAtWQ4DSctInkYp7zx0jiSuSuISsosqFbD3M9j1vSjAVRLkphWUGqMVvAgwPm7pbpSEeaiemleQMypTh2OK3zgyWSKRkWKasNXdZzfqMq4RWNKYR5Mc0KolQFuJhHZcL0I7GHAp1HpQpS80hF24eEgFaKJUo/euEohMU0+2Y0ZL55UppMcOaQHPdlC8wFSBvRwxXzRmgaJkXLGqqI9gMbo2EqIThiinlE0PViaKTd0y9zBdsJvxwVWium1q3fTKo8mLqa9I3xVRj6D6o7DxL6ri0ZGBZqqLCPE41FsoWNNaDHyxBKd4xhTjbzs07tmlxhxPPyzhX8k5Ec8PiM1vrJf21qSoNpiCKmqtHEk/jbqPh1LcRLKasaCakYQZw0uNRSt5kFjtANLbjHy9Cuks9ojBaskxqw9WgjIjU2dgm13O2FdTj3ZLRSkLVCJn6Eq3P93jBFMN4bLPIG1sWXxwPFY3CDO40Ig8xc5qYo8TkuJtboU2XI5oXSpvNnNr5kipkiHuUSmltkQxNKdQYmufGrN9q13O2rfSui0xJTE0xkhK6D1YMwv0ejU29rWKIYqHI7Doo06kDSX8tIRkGxawZ4pFLrbM/ASGO3NwjpBszCmrzaGnqf9w5k9M/klJmcXBlUMuWmMqMMmQZwer46L7WmbmaG2HEZDUypT2yEZrBkdTcDTKLW4YQVzk0l4d4bK4v+YLUMeST/WkcXBD6SvdrA/PIrbSu82hZ6tnlvWOpzX8YKFIpaShTNoX+UYslHqxFwd2DcA2EuNiROTRYjLd5nTZ2KlrnTdEDG4UYd2xb2TClvftQznghrj14218dEhWWOazMoFhIa+eFMqjFUTtGI+g8nDNYS2g9UtRxs63zkwBtU4zLDEaLNpLSP5qFuP9wXywndRz0eEmIRkLRVJbuG9M/NgkuhkYYech2Jq4pcXBrmM64b14o6B1LGa5ZFjEUWg/3GRyy4aqiE9N+sMfuTV06Dw5AmgxWY1rrlmlscg6rVMwBZTeMDubmJz65VpN+iayYQhuVFXlQGDUy9duOK3hcdyw2Pu+E+QpKPRI1eV82LdNfFAtVbN7KtQtKCf0jae193dgsGBxKqVKl+yD1dVg0TS3NuxGtczlSxUR5RX/VzmGcQ+94alnkOsGxZGT/wybnNsptjkaVWL2ksjj+cXfqSjRYs6x8yQAbAQkuHuPFpI6Dj0c2cpfumud6MrDviXKtXSaiMRQdi7/VyM63xcJP/xcMDkXkrba5s3QT+ofNeWUy0hy5CtEAACAASURBVLAXXMF1HMdxHMdxDhSu4DpTwkzWshQ4OqrVo9HhiqpdIs2SKFbKytYvLQzo3WQqr6qQjxNUgVZBlFaUbWXn5mrqKRuB5qCVEC/mqApaBJU2ZCmLs4qqD1IImiqSVXS+mNI7WSIhI5q2K4ig+UjM4JhSZUrch87XYffGoDTkEA/MLzVfrKiA/FjBqExYz82wtBOPaEQFSVyx/qVVOie3guoLoxvCT9cKpB9TtoTOgxHjBfOtHK5N1ZqiW5FtRmikNNZNLR6VCUcWdhiHQqMyoREXPDLscu/5Yyy3hmz0WxRVxGrbrA3iqGJYJBxt71AE2aCdjFlIRoxCUN7XdlYpqogkLRn3MsgFySPyJVPHweJuVST4WppiFY/sF3ljY+KTGvxmU6APCw9W7Fwf1TPwwXw5qxhGKyH+uDRVNelTxzrnixqy9kC2CUUXsp2KbBcu3BpiszJFKmH5yyXbz7LYUFOh1GbeY4rFeMmy5hFZfammMV0AWc9m9S5/sU/ZStAEohB7OevXKGoz7uORKY9SQdmCUTJ1Y9AIokJonVOSoc2IL9rUil7SM1/K3tGYKonrmeJlY6oI7l4Xk+5aZrH2I7D1bKHKlM4ZarVmuCJk26Zk6uS41M7RZNawzRa3OnQfqriwItaGDSU7b2VGh6CxTp1dCbHj2Lk+YflLFvg5WkltFnfL1HtSU+3b50qGS9YWZRPKNKmzDpWp0LpQUjwrqq9nKWF42DIMLtyvLNzfZ+OWDv3rGnWs6mhF6jh0jSyLW/OC0j86dVpY/vKQs6eallmvbSrM4LCwdF9J72g4rmUhedhi1+OhnYfhoeDMMTk/Ym0/Wo6JCnPEQEzpWfmiXRxbN2eUmcU8lhksfWnAxi1t4lFFsTTxWgaVmKhU0p4pdL2jpj5nljiQZKCMliMWHigYLse1qjpRqcBGOMYLpp5LaU4QVWKZnoqg8lapMFg1f9Gl+0uiYYGuZKQ7OY1NO67xQgKVEo1NsW5sm/tIYz0n6VmZ/pEGRcti3KsMy/iodo1PnRZmdCo1JX60aMc1yXI3XApeyC2L8R4cEao0o3V+mnUvX22TrvfJb2xTJRbLmreDkh/UtaJpx530SzRKGa2kZLvmpjGJrx0tmsrbulAxWIuCG4g5P3S+bhLu5nNaoV3Nv7x9tqJ/OKJ3XVor5c0NG6UoM4vnHy3GFlOuwrgzebZAmZnSHA8rylZEMqzYualTj7CUjYidZy+Q9kq2nt2eiSeGxk4YPdsaMVprUQXv4wlFK6r9ffNOEka+oGjGFO24/p85XkrraxWxei18pcf52xdonyvJO4+Oi00G0Do7YvM5LdpnC3rXpWG0w9qy80hp7iGFsvBAYYp3ZKMEo2XCcdlzPO0rcV+J+zll1qB3PAux96ADu+eTvrJ7PLO4+7GNbO6csBu+tV4xjsxz17KBSh1/X4V7uWgGhVWnbkeNbWWwGtXtFeeWZXLy+fGS/a/oH4lprofRxYZQlqb+dh4ccOGFbbQBeSeuM72VmTBahtGyeStPRoJm22VeXMF1HMdxHMdxDhSu4DqPIioAhaRRkG81putiJUlLipFlxEFMcWw1TOnsDRpEUUW+m0GkZJ0xg0aFtEoY2q/tqB/BkRH5VoNoEFEuFEhaoXlCdsYuxfEhc2OgtJmhSVYyXn507E00tLiu4ZHSfqIVQpoLw8PTzEYIpLvCeNHiczWtiNKKh9aX0PArNK8isrikrIRqoaC3G372dou6ztIukJ0EKYX+UVO1qpHtfxLLJ5WpT1JZPGfvBHz1kVUONXt1nbOoYDBuMa4StnbaFGXM9kYbioh82Y59uJvRWRry0IOrZAtjOq0RSVzRG2b0t0PdBjHSLtFSSC6kRLnFf46XK5rn7PdqvqjEA1MJGpsV/SOmokSlku5OfV5XPtfn7Es6RKW5G0SFqZQThWOwFlmmq8gUQY0h2Q0xXyFrl5SmBCa9oHR1he2TMUlP6xnlVZiRvn1DUmfEm2SEm43lrVKLt5XCVKrl+3I2b06nsX2YYrh7Q4vuAwPAFInmZlXPeK4SkMJUhOaFgnwxZrQiwbs47CSF1lm7iKUyr91kGLJ1TbJDxSFWt7IZvslA2bnBFNu8O2l46J4paiUkGdhgROt8xfaN05nFUXCaaJ3Teub9zom4Vp1N7SoZrMUMDkeIKpoqlNSeoPHQFLHFL/dYv7Vrqlg1yS4UZuUvmmtBtqOMO6bIdB4u2Xx2wuLXbKSl17ZRjcHhoPD3TKWOh1OlfLQaYtxLc7rYLtqWaWujqNX97Zth4X5TVnauz0iGSrZrsdiTUQJRcy6R8EzRkClpsDqdlY9A63zOaDmjsVmZW0UJJGGkAFOsynTq6ACQbVrM5M4NSbh+7HNpr6LMYgbHmuYUcn7MzvVtO4dj8zFNe5YRbLBianPZNIUKppnn4lGIgx1Qz/afxFWPw3fFhdW/aNl9v/DVYR2jrNE0U1oVw+hIi6Rf2iz9SaauytxnWusVaXAzyK/PGB7OSHplfT1HuZhPdLgeVGzfk3s575ja2diweNbGhjl3jBaljrMcd4XmRolGCYOj1F7Ly18uGXcnmbGUfKVV+/z2D1s8swyUxoZdrP2jKctfHlu8a/Azbp3PKbO09rSeqHujJXOkGKxFtB8picdaK51lFpxXls2b20ZU7HxOHCyKZlRnB0x3SkaLCY1tUwkn1yFqcaitdWW0EpP2K1pf3WF023J9rUxGXbJta8ulr4zZPWHPjrw7mSfQQIqK5rrSP2I+2sOVaZbACVFhLkH5QhyeFfIoNV0juxbKTMw/VyzjYPeMedHCZJRJ2Xiubc+2xgwOt80/vkV9HUY5RAOLr9WgClvmvXDoCZRxiKtfL+mfaAcHiKi+xixm3xwcysxGaYqmxQePluP6ek4GynDN1PMit+sjCe0CsHj/kMGRBggsfG3ExnMb5g898Qsn+D2vV/Z8nDwiS5szYM9r2PjGTjjfFusrVYhJFmoPeI1tpGpw1HzF4yhk4+wkTL0Y5sMVXMdxHMdxHOdA4QruAUdEXgX8UyAG3qWqv/jYhU2FlPUMPVohw0necEFaBQvdAetbFgC0eGyHvIyoqhBLt94i6uZIWhFlJa3GmGGrgebT31BydES5ndI+0iNLShppwfmNBVgcM07DrNhhDM0STUAipaqE5Lm7ZArjhensWokV3U2RXOpMLvEY2g8HZ4NDFvOnq0rrbMToEJRxQtFrUCztAlCpEImy1BrSOpGz2WvRH7eIGwVRy+Ss8UYTbZfQiy0mcttme1exEo2nSl5UEGanCvlSSRZX/Nnnb0YSUx+vP7pBb5wSCUSiJHFJozNmfLZNntt57iwNGfQyUKHIY2hBb5gRRdPYJyKgF1wqIlPakp7F5k28adMt83ZNgpIQFaZsVXFEP8Q+lg3YvinkYR+asqixxXMNDgcluAvROGJ0SEl2LXavaFn8ZbZt35X0gvISWWxk3rW6Lt5fEY8nKov5NQ7XbGYtCt2HCstFP8lAFjLOFZ2KaCS0Lug0NjD4Wdrs5bDLNKq3VWOpnQ4GwdNxtBAx7mZUsSlcTBwYQp3zrrDyhZJxx/KmN7aqWvUA89Ysgz/teCEi7RVkO2qxvZNY56bQP5yQdwXZVhrrFqu5ezyu4/ZAaJ/NGS9Y5jSNYOHBkkfuSGoVM+nB1snElPSQ/S/uRyR9QUP8erorZNslwyNNU8JCeyUD6DxgUuf4eV2Ls96ymNfWhYpzL0qoMqUf/KrLTOhuF+zcmFA1lGhsTgjL9xVsPDepr43meVh4oGL3hGVtGiXC1s1ZHYtapUra15BBy1Tq4arN0p8onRvPbTJaJYysaK0GFW2p/YbLzFRBMBV7/ZaUlS8UbN2cBL9ZqBrmi5vtavAnDqdW7VoEc6Io2sLudTGNLWX7xpjOwxW9E816BKHzcEVzPWfzGxqmYm0r6XbFbiuur5vVz+dceH5ax/+aY0pQW8N+xsvCyucLdo/HFB2hzGDlCyVVGtWx19lWid5g51bUZsSnA4shn8aG2khFlQg716d1zHs0Vjafk4VzNVXSohzyBaVoK9GZqI4Xj3Jqz/EqgdFKZOqiWAw4mOvF9o0W/25qoAZFXMh27aYaHrEMZVGu9K6LGS/Ayhcr+msRSX+S1zJl5/qM9rmCsmGz/geHU1Pc1sIzvIT22THnX9CkSkyp7V1nZScqJmJltroZomHG/ihklQv3zs4NSuthy5CYd+1cFg1h3BWW7rORw/4x87sdhbjsMospXrBMtlPRPxzU2chUwf4Rc27YuslGHESV4UoYaVmzzGKT5xFi51Uj6mu+DJkQy1QYrkS188Z4cep/rBE0NyDdKhkvxDaHozRf7knGuKxnmQHzrrD0lYLtm1uWhXO9smsILJa9sFGMibtM0YSipejCZARA6DwgjFbMaaR3NCIemTo8UfcBWhsWgy+VxV6XGcT3S60EV6mQF/Z8jwobBUAtq9/iV+zA1m9pUaXmN90/mlmmytSu7dwGSCwbZMdGQLpfLxkvRLTPlWzfmNTxx1GhaByRtyG+sAt06nt54uISFTbHZbxaUrYiOg9GRGPqWN+94AruAUZEYuBXgW8HbgVeJyK3Xt1aOY7jOI7jXF5cwT3YvBT4kqreByAivwm8GvjMY35CoWpWtJo5g+A/mi8Lhxb7LLcG6HFhXCSMxgndxRHbA4sNjQYR2hZkPWX1G7dJ4xIipbk8pNuyX4GVwvrOMllScqS7y6BIyRo5cVwxCAqlNko63SHjccgj/ukFVl75MAL1zFlVoayEjdECyWaQi8TqPjw09UwdrZjKmm2CSkReCmWzoptZfW7obLKc9olEGVUJjyws8NDCUqir7WcjKyjLiPFCQrWdmY9f135tVsF/NNuyX8WLX61Yf56g3YITq1scPbHDambqWq4RvaLBmf4iZSU004JI4HzWnIQrMRqmaCVEOzFVLgwaGcWXu5THxrUSTC+2GNUYqpaiI1PDRmum7gAkfWG8Yt6c6Y6pslFhKm85mV0bh5mpYdZ93jZHhaIpdQxclSqDo0LZVBbuM7/foq1oSr2fdDeoCy1TF8q2qSAb3xjXonMyNFUgX7TY2MOfztm6KbFzGX79mw/jxPtT2bo5Itsyb88yxEcmfZv93tgqbYbt0NSKKpnG8pYNm+Vts6tNpagaplb1r7P9NNa1zm7WOl9QpQnJwGKNLfsStB/OGS8ndB7oM3hxl8GhhNaFgt6xhJXPWZtuPbvNeMmUj7Jhx6ixxaEufC3MKF+CzW/IqBJTNqWCjeck5llcx6lFlpVKg7NB15xHyk5E4/wkwx/sHk/IdjXMcLd2Gq0I299gKsjK53t8/S93GaxaW+7cEAWXC6V/zI4rGdjMcE2mMX1pT9m8OWE0E+s+Xla2sohs0/ySNQrftxoKRObtG5Wm4gzWIspGmFG+EBT3jjmMlK0qZPMSkp1HZwqbZNRSoVbsN55r6m3R1vrWnuy3bIVrNyjYMhPHHOWT60zqTGCDQ0IU1LXRYkTRykxRa0I5hiqJSHtau0Ns3pyag0Mrqtu0DMpW/V2x+c3mXVMXpbBrabyc1HHVgzWbq5B3wj0VMlcNlyKGqxMV3NSzojHxZZagMCb1eS5bim4KRUeCh2iFtiqGh6bZ+9JtYbRk9V39XM7681LS3XDOQrx4/6h9Pl+w+7roVqSbEb3r4lq1VLH7Ryq1TJBduw/jIfSON8L9Fe6r7Yi8JTQKreOXJyMtiw8UlnVtrIyXhWTX7vN4ILUKLhX0rjPnC00gLu18jJahCNkW80XLcpdt2X3ZWi/ZviEx1fhIWu8n7VlcfNK31yqVWj21+4vav1xjQkyqMlyK63j6shG8ghfEYslbEctfGrB+S2t6TyxJPXpQpkIyqBisJhRtcwMBG0WsYiEZWvy1ZaKrGKxGHP0PXwNg/c6TjLt23MPVuI7XHlXmYgPWTppN3W7iYXCMaFdoFu7TCnrXm5PDznGrRzy2Z2QdU94QkoEEJxwYryhVqmy0BZWpo46KPWOSfvBPVxjnwvZNzXAdSnDDsJjqasbLWmd8cFVs/W4zJttWdq43J5LeCdtPnFu9866wdfvh+hmtMyM/zQtqc2daJWVk103an8YD7wVXcA82J4AHZt4/GNbViMibROS0iJwel4MrWjnHcRzHcZzLgajuPTuEsz8QkdcAr1LVN4b33w/8BVX94ccofw7oAeevXC2dp5k1vP32O96G+xtvv/2Pt+H+4VmqevhSGzxE4WDzdeCGmffXh3WXRFUPi8hpVT112WvmXBa8/fY/3ob7G2+//Y+34cHAQxQONh8DniMiN4lIBrwW+O2rXCfHcRzHcZzLiiu4BxhVLUTkh4HfxWzC3q2q917lajmO4ziO41xWvIN7wFHV/wj8xz185B2Xqy7OFcHbb//jbbi/8fbb/3gbHgB8kpnjOI7jOI5zoPAYXMdxHMdxHOdA4R1cx3Ecx3Ec50DhHVwHABF5lYh8XkS+JCI/fbXr41waEXm3iDwiIvfMrFsVkd8XkS+G15WwXkTk7aFNPy0id1y9mjsAInKDiPxXEfmMiNwrIj8a1nsb7hNEpCkiHxWRT4U2/Adh/U0i8mehrf5NcK5BRBrh/ZfC9pNXs/6OISKxiPx3Efmd8N7b74DhHVwHEYmBXwW+HbgVeJ2I3Hp1a+U8Bu8FXnXRup8G/ouqPgf4L+E9WHs+J/y9CfgXV6iOzmNTAP+rqt4KvAz4u+Fe8zbcP4yAb1XVFwG3A68SkZcB/xfwNlX9BmAD+IFQ/geAjbD+baGcc/X5UeCzM++9/Q4Y3sF1AF4KfElV71PVMfCbwKuvcp2cS6CqfwSsX7T61cC/DMv/EviumfX/So0/BZZF5LorU1PnUqjqGVX9RFjewf7BnsDbcN8Q2mI3vE3DnwLfCrw/rL+4DSdt+37gfxARuULVdS6BiFwP/HXgXeG94O134PAOrgP2D/aBmfcPhnXO/uCoqp4Jyw8DR8Oyt+s1TBjqfDHwZ3gb7ivC8PYngUeA3we+DGyqahGKzLZT3YZh+xZw6MrW2LmIfwL8JFCF94fw9jtweAfXcQ4Qar5/7v13jSMiXeADwI+p6vbsNm/Dax9VLVX1diz9+UuBW65ylZw5EZHvAB5R1Y9f7bo4lxfv4DoAXwdumHl/fVjn7A/OToatw+sjYb236zWIiKRY5/Z9qvpbYbW34T5EVTeB/wq8HAsfmSRPmm2nug3D9iXgwhWuqjPlm4HvFJH7sXC8bwX+Kd5+Bw7v4DoAHwOeE2aRZsBrgd++ynVy5ue3gdeH5dcD/35m/d8OM/FfBmzNDIM7V4EQu/f/AJ9V1V+e2eRtuE8QkcMishyWW8C3YbHU/xV4TSh2cRtO2vY1wB+oZ1i6aqjqz6jq9ap6Evtf9weq+n14+x04PJOZA4CI/DUsLikG3q2q/+gqV8m5BCLyr4E7gTXgLPCzwIeAfwvcCHwV+J9UdT10pn4Fc13oA39HVU9fjXo7hoi8Evhj4G6m8X//OxaH6224DxCR27BJRzEmEv1bVX2LiNyMKYKrwH8H/mdVHYlIE/h1LN56HXitqt53dWrvzCIidwI/oarf4e138PAOruM4juM4jnOg8BAFx3Ecx3Ec50DhHVzHcRzHcRznQOEdXMdxHMdxHOdA4R1cx3Ecx3Ec50DhHVzHcRzHcRznQOEdXMdxHMdxHOdA4R1cx3Ecx3Ec50DhHVzHcRzHcRznQOEdXMdxHMdxHOdAkVztCjjXDmtra3ry5MmrXQ3nAFBV1RMXCmxubu5p3w8//PDcZW+44YbLst8kmf/RubGxMXfZRqMxd9ksy+Yuu7OzM3fZ/Zbdstlszl220+nMXXY8Hs9ddmVlZe6yZ86cmbvsXp7He7km93LO7rtvb1lpu93u3GVXV1fnLnvu3Lm5y0bR/Nrd+fPn5y67l+faXupw5MiRucuWZTl32ePHj89ddr/y8Y9//LyqHr7UNu/gOjUnT57k9GlPc+88dXZ3d+cu+zu/8zt72vcv/MIvzF32l3/5l+cu+9a3vnXusnv5x/yhD31o7rLPfvaz5y67l877XXfdNXfZ0Wg0d9m9/MPfSwdsL//Eb7rpprnLvuQlL5m77AMPPDB32e/5nu+Zu+zP//zPz1327W9/+9xl99JJet7znjd32de+9rVzlwV42cteNnfZ7/3e75277Dvf+c65y+7lh+K73/3uucv2er25y7ZarbnLvvnNb5677Pr6+txl3/KWt8xddr8iIl99rG0eouA4juM4juMcKLyD+yQRkbeJyI/NvP9dEXnXzPtfEpEfF5FLylMi8i4RufVx9v8GETk+8/4uETkVlv+jiCw/PUfiOI7jOI5zsPAO7pPnT4BXAIhIBKwBz5/Z/grgMYPkVPWNqvqZx9n/G4BLBtCo6l9T1b0FLjqO4ziO4zxD8A7uk+fDwMvD8vOBe4AdEVkRkQbwPOATQFdE3i8inxOR94mIwFSRFZFYRN4rIveIyN0i8mYReQ1wCnifiHxSRB4VzCMi94vImoicFJHPisg7ReReEfm9SVkReYmIfDp8/q0ics8VOi+O4ziO4zhXFe/gPklU9SGgEJEbMbX2I8CfYZ3eU8DdwBh4MfBjwK3AzcA3X7Sr24ETqvoCVX0h8B5VfT9wGvg+Vb1dVQePU5XnAL+qqs8HNoHvDuvfA/ygqt4OPOaMDRF5k4icFpHTe5ml6jiO4ziOc63iHdynxoexzu2kg/uRmfd/Esp8VFUfVNUK+CRw8qJ93AfcLCL/TEReBWzvsQ5fUdVPhuWPAydDfO6Cqn4krP+Nx/qwqr5DVU+p6qnDhy/ptOE4juM4jrOv8A7uU2MSh/tCLEThTzEF9xVY5xdg1nOn5CJrNlXdAF4E3AX8EPAu9sbj7t9xHMdxHOeZhndwnxofBr4DWFfVUlXXgWWsk/vhx/1kQETWgEhVPwD8feCOsGkHWHgylQoT0HZE5C+EVXszMnQcx3Ecx9nHuNr31Lgbc0/4jYvWdVX1fJhP9kScAN4TnBgAfia8vhf4NREZMJ3Mthd+AHiniFTAHwJbT2IfjuM4juM4+w7v4D4FVLUEFi9a94aZ5buw0IPJ+x+eWb5z5mN3cBFB0f3AzKo7Z7adDIvngRfMrP/HM+XvVdXbAETkp7FJa47jOI7jOAce7+AeXP66iPwM1sZfxXx1HcdxHMdxDjyiqle7Ds41wqlTp/T0aRd6nSvLXp9BH/zgB+cu+6EPfWjusrfddtvcZX/rt35r7rJ33PHnBmgek09/+tNzlz1+/JJ5YC7JT/7kT85d9sUvfvHcZecMwwJgZ2dn7rJbW/NHVP2Nv/E35i77spe9bO6y3/Vd3zV32V/5lV+Zu+xerp3z58/PXfZLX/rS3GXf+MY3zl323nvvnbsswLd/+7fPXfbs2bNzl93L9XPs2LG5y+7len/44YfnLvvRj3507rJ7ud673e7cZc+cOTN32f2KiHxcVU9daptPMnMcx3Ecx3EOFN7BvYyIyNtE5Mdm3v+uiLxr5v0viciPi8jvPMbn3yUitz7O/t8gIsdn3t8lIpf8JeM4juM4jvNMwTu4l5eJTy7BJWENS+s74RVA9lgfVtU3qupnHmf/bwDmH6d0HMdxHMd5BuAd3MvLh5lafD0fSwaxIyIrItIAngd8AuiKyPtF5HMi8j4JgW0TRVZEYhF5r4jcIyJ3i8ibReQ1WErg94nIJ0WkNfvFIvJXReQjIvIJEfl3IjJ/4I7jOI7jOM4+xl0ULiOq+pCIFCJyI9N0viewTu8W5pk7Bl6MdYAfwlTfbwb+28yubgdOqOoLAERkWVU3ReSHgZ9Q1dNhPeF1DUsa8VdUtSciPwX8OPCWy3zIjuM4juM4Vx1XcC8/H8Y6t5MO7kdm3v9JKPNRVX1QVSvgk8DJi/ZxH3CziPwzEXkVsP0E3/ky4FbgT0Tkk8DrgWddqqCIvElETovI6XPnzu354BzHcRzHca41vIN7+ZnE4b4QC1H4U0zBfQXTdL6jmfIlFynrqroBvAhLGvFDwLt4fAT4fVW9Pfzdqqo/cKmCqvoOVT2lqqcOHz68pwNzHMdxHMe5FvEO7uXnw8B3AOuqWqrqOrCMdXI//LifDISQgyhkN/v7TDOf7QALl/jInwLfLCLfED7fEZHnPrXDcBzHcRzH2R94DO7l527MPeE3LlrXVdXzcxqlnwDeE5wYAH4mvL4X+DURGTCdzIaqnhORNwD/OkxmA+sYf+HJHoTjOI7jOM5+wTu4lxlVLYHFi9a9YWb5Liz0YPL+h2eW75z52J9LhxQU3Q/MrLpzZtsfAC95ktV2HMdxHMfZt3iIguM4juM4jnOg8A6u4ziO4ziOc6DwEAXHca4qc8ah17z0pS+du+xwOJy77Ctf+cq5y66urs5d9sUvfvHcZT/2sY/NXbbbnT93y8mTJ+cuq6pzl91L20XR/HpKo9F44kKBvZyHZz3rkm6Jl2Qv185eyp49e3bush/5yEfmLvvlL3957rLb20/kNDllL/UF2Isbz/nz5+cuu7a2NnfZvbTH8vLy3GVf/vKXP3GhwIkTJ+Yu+4d/+Idzlz127NjcZZ/puILrOI7jOI7jHCi8g3sNICLfJSIqIreE9ydF5J6rXS/HcRzHcZz9iHdwrw1eh6Xmfd2T+bCIeKiJ4ziO4zhOwDu4VxkR6QKvBH4AeO0ltp8UkT8WkU+Ev1eE9XeG9b8NfCa8/0MR+fcicp+I/KKIfJ+IfFRE7haRZ1/ZI3Mcx3Ecx7k6eAf36vNq4D+p6heACyLyTRdtfwT4NlW9A/ge4O0z2+4AflRVJ1nKXoSl8n0e8P3Ac1X1pVhq3x+5jMfgOI7jOI5zzeAd3KvP64DfDMu/yZ8PU0iBd4rI3cC/A26d2fZRVf3KzPuPqeoZVR0BXwZ+L6y/Gzh5qS8XkTeJyGkROX3u3LmndiSO4ziO4zjX6MAKeQAAHFtJREFUAB67eRURkVXgW4EXiogCMaDAr84UezNwFlNnI2DW/6R30S5HM8vVzPuKx2hrVX0H8A6AU6dOze8P5DiO4ziOc43iCu7V5TXAr6vqs1T1pKreAHwFuGGmzBJwRlUrLOwgvgr1dBzHcRzH2Td4B/fq8jrggxet+wDwMzPv/znwehH5FHALf161dRzHcRzHcWbwEIWriKp+yyXWvZ2ZiWSq+kXgtpkiPxXW3wXcNVPu4vd3PtY2x3Ecx3Gcg4wruI7jOI7jOM6BQvaSd9w52Jw6dUpPnz59tavxpInj+cOTq6q6jDU5mLz61a+eu+zP/uzPzl32tttue+JCM0TR1f9dvpfn5rVQX8dxnIOIiHxcVU9daps/eR3HcRzHcZwDhXdwHcdxHMdxnAOFd3CvACLy90TkXhH5tIh8UkT+whX4zvtFZO1yf4/jOI7jOM61hrsoXGZE5OXAdwB3qOoodDqzq1wtx3Ecx3GcA4sruJef64DzIX0uqnpeVR8KCuv/LSJ3i8hHReQbAETksIh8QEQ+Fv6+OazviMi7Q9n/LiKvDutjEfnHInJPUIh/ZOa7f0REPhG+45YrfeCO4ziO4zhXA+/gXn5+D7hBRL4gIv9cRP7yzLYtVX0h8CvAPwnr/inwNlV9CfDdwLvC+r8H/IGqvhT4FuCtItIB3gScBG5X1duA983s/7yq3gH8C+AnLs/hOY7jOI7jXFt4iMJlRlV3ReSbgL+IdUz/jYj8dNj8r2de3xaW/wpwq4hMdrEoIl3grwLfKSKTjmoTuDGU/zVVLcL3rc98/W+F148Df+tS9RORN2GdZG688cYne5iO4ziO4zjXDN7BvQKoaollErtLRO4GXj/ZNFssvEbAy1R1OLsPsR7vd6vq5y9a/3hfPQqvJY/R1qr6DuAdYD64T3QsjuM4juM41zoeonCZEZFvFJHnzKy6HfhqWP6emdePhOXfA+o4WhG5PSz+LhZTK2H9i8P63wd+UESSsH71aT8Ix3Ecx3GcfYR3cC8/XeBfishnROTTwK3Az4VtK2HdjwJvDuv+F+BUmDD2GeCHwvqfB1Lg0yJyb3gPFqP7tbD+U8D3Xu4DchzHcRzHuZbxEIXLjKp+HHjFxeuDEPtWVf2pi8qfZ6rszq4fAD94ifUF8OPhb3b9yZnl08CdT6b+juM4juM4+w1XcB3HcRzHcZwDhSu4V4lZhdV5eijL8mpXwQlsbGzMXfZ973vfExea4VWvetVeqzMXFy5cmLvs2tr8SQL3UnYvPMEEU+cS7OUZEcfxZayJc63g18TBxRVcx3Ecx3Ec50DhHVzHcRzHcRznQOEd3CuMiHyXiOg8qXNF5F0icuvT8J0nReSep7ofx3Ecx3Gc/YB3cK88rwP+W3h9XFT1jar6mctfJcdxHMdxnIODd3CvICHl7iuBHwBeG9bdKSJ3icj7ReRzIvK+mWQOd4nIqbC8KyJvFZF7ReQ/i8hLw/b7ROQ7Q5mTIvLHIvKJ8Pfn7Mkcx3Ecx3EOOt7BvbK8GvhPqvoF4IKIfFNY/2Lgx7AkEDcD33yJz3aAP1DV5wM7wD8Evg34m8BbQplHgG9T1TswL923P1GFRORNInJaRE6fO3fuyR+Z4ziO4zjONYJ3cK8srwN+Myz/JtMwhY+q6oOqWgGfBE5e4rNj4D+F5buBP1TVPCxPyqfAO0XkbuDfYR3mx0VV36Gqp1T11OHDh/d+RI7jOI7jONcY7oN7hRCRVeBbgReKiAIxoMB/AEYzRUsu3S65qmpYriafUdVKRCbl3wycBV6E/XgZPt3H4TiO4ziOc63jCu6V4zXAr6vqs1T1pKreAHwF+ItP43csAWeCEvz9WCfacRzHcRznGYV3cK8crwM+eNG6DzCHm8Ie+OfA60XkU8AtQO9p3LfjOI7jOM6+wEMUrhCq+i2XWPd2LpoIpqo/PLN858xyd2b55y76TDe8fhG4bWbTT4X19wMveArVdxzHcRzH2Te4gus4juM4juMcKFzBdRznaWcwGMxddmNjY0/7juP5Q8uXlpbmLjudw/nEZFk2d9m9ECywn3b2cmyXqw7XAgf52Jwnx17uDWd/4Qqu4ziO4ziOc6DwDq7jOI7jOI5zoPAO7j5AREoR+aSIfGo2Ba+IHBeR98+5jzrtr+M4juM4zkHGY3D3BwNVvR1ARP5H4BeAv6yqD2H+uo9CRBJVLa5wHR3HcRzHca4JvIO7/1gENgBE5CTwO6r6AhF5A/C3gC4Qi8irgPdgWc0+B7SuRmUdx3Ecx3GuNN7B3R+0ROSTQBO4Dkv5eynuAG5T1XUR+XGgr6rPE5HbgE9c6gMi8ibgTQA33njj019zx3Ecx3GcK4zH4O4PBqp6u6reArwK+Fdyab+b31fV9bD8l4D/F0BVPw18+lI7VtV3qOopVT11+PDhy1F3x3Ecx3GcK4p3cPcZqvoRYA24VG/UU/M6juM4jvOMxzu4+wwRuQWIgQtPUPSPgO8Nn3kBj07h6ziO4ziOc2DxGNz9wSQGF0CA16tq+QRZef4F8B4R+SzwWeDjl7mOjuM4juM41wTewd0HqOolc5Oq6v3AC8Lye4H3zmwbAK+9/LVzHMdxHMe5tvAQBcdxHMdxHOdA4Qqu4zhPO1mWzV223W7vad9RNP/vclWdu2xVVXOXfYLwoGuO/Vbfy4WfB+di9vI8cfYX3rKO4ziO4zjOgeIZ18EVkbeJyI/NvP9dEXnXzPtfEpEfF5HvFJGfDut+TkR+4hL7Oiki9zxN9XqLiPyVp2lfu0/HfhzHcRzHcfYjz7gOLvAnwCsARCTCPGWfP7P9FcCHVfW3VfUXr1SlVPX/UNX/fKW+z3Ecx3Ec56DyTOzgfhh4eVh+PnAPsCMiKyLSAJ4HfEJE3iAiv3Lxh0Xkm0TkUyLyKeDvXuoLRKQrIv9FRD4hIneLyKvD+pMi8lkReaeI3CsivycirbDtvSLymrB8v4j8goh8UkROi8gdQWn+soj80ON9x0X1uE5E/ijs5x4R+YtP+ew5juM4juNc4zzjOriq+hBQiMiNmFr7EeDPsE7vKeBuVR0/zi7eA/yIqr7occoMgb+pqncA3wL80kxq3ecAv6qqzwc2ge9+jH18TVVvB/4Ys/96DfAy4B/M8R0Tvhf43bCfFwGfxHEcx3Ec54DzTHVR+DDWuX0F8MvAibC8hYUwXBIRWQaWVfWPwqpfB779UkWB/1NE/hJQhf0fDdu+oqqTjubHgZOP8XW/HV7vBrqquoMpzaNQj95jfMfDM/v4GPBuEUmBD8187+wxvQl4E8CNN974WIfuOI7jOI6zb3jGKbiBSRzuC7EQhT/FFNxXYJ3fp8r3AYeBbwrq6VmgGbaNZsqVPPaPjEm56qLPVOEzj/cdAISO+F8Cvg68V0T+9sVfoqrvUNVTqnrq8OHD8x+h4ziO4zjONcoztYP7YeA7gHVVLVV1HVjGOrmP2cFV1U1gU0ReGVZ932MUXQIeUdVcRL4FeNbTV/X5v0NEngWcVdV3Au8C7rgM9XAcx3Ecx7mmeKaGKNyNuSf8xkXruqp6/gk++3ewYX8Ffu8xyrwP+P9E5G7gNPC5p1jfJ/sddwL/m4jkwC7w5xRcx3Ecx3Gcg4bsJdOPc7A5deqUnj59+mpXwzkAnD//RL8Tp3zwgx/c075f85rXzF222+3OXXYvde50OnOXXVhYmLusZ9q6vOzl/523xTODvWQw9Kxn1x4i8nFVPXWpbd5ajuM4juM4zoHCO7iO4ziO4zjOgeKZGoPrHEB8qGnv7OWcbW9vX5b9fv/3f//cZWFvQ8e7u/NnrV5eXp677Gg0euJCT6Jsksz/SL5cQ+hxHF+W/V4u9hJ2UBTF3GWvhbbYj+y3MJC9PKv2Ut/LdWz+f25+ntlH7ziO4ziO4xw4rpkOrojML7U8+nM/dCl/15AW956nXrOnj5CCdy0sP6njdRzHcRzHcR6ffR+ioKq/drXrcDkIaXdFVecfj3Acx3Ecx3GuHQV3gojcKSJ3icj7ReRzIvK+0NlDRH5RRD4jIp8WkX8c1v2ciPxEWP4mEfmUiHwK+Lsz+4xF5K0i8rHw2R98gjrEIvJeEblHRO4WkTeH9XeJyNtE5LSIfFZEXiIivyUiXxSRfzjz+Q+JyMdF5N6QCnfeYz8pIp8XkX/1/7d391F2VfUZx78PIRA0kgiMLBrACIQVEWGAgYKgjVQRlaLiS0qDotBSqCi+IEVrNdhl0UUVqvgWEaLU+gYCUVvBIkLKApIJSUgooBJQQSRJgRAEQ5k8/ePsgevlTubeZCb3zs3zWWvW3LPPPmf/7tmTyb57ztk/qgxru5W4B+OYWeppiPIZkq6XdJWkFeV6zZK0oNTbs9lYIiIiIsaqTp3BPQB4CfBbqrS6h0u6A3gTMN22JTV6IuQS4HTbN0g6r6b8ZGCN7YMlbQvcKOka2/cM0X4vMMX2vgB1bT1pu0/SGcBVwEHAQ8Ddks63/b/ASbYfkrQdsFDS5aW8GdOAE23fLOnNJZb9qRJTLJR0A1VK4UbllLIXl5hWABfZPqTE+x7gfU3GERERETEmddwMbrHA9n3lz/NLgKnAGuAPwNckHQc8XntAGYROtj040Lu0ZvdRwDskLQFuAXakGkgOZQWwh6TPSzoaqH18fF75vgy43fYDtteVY3Yr+95bZpFvLmUbaqver2zfXF4fAXyrpBN+ELgeOHgD5QALa2K6m2eyrS2juo5/RNIpZUa6f9WqVS2EGREREdGZOnWAW7uuzgCwte2ngEOAy4BjgB+3cD4B77HdW75eZHuoNLvYfphqJvRnwKnARQ1iW18X53pga0kzgFcBh9neH1gMTGgh1t+3ULeR+phq433WjL3tObb7bPf19PRsYtMRERER7depA9xnkTQRmGT7P4D3Uw1An2b7EeARSUeUolk1u68GTpM0vpxrb0nPLa/vbNDWTsBWti8HPgoc2EKok4CHbT8uaTpwaAvH1psPzCz3BPcArwAWbKA8IiIiYovXqffgNvI84CpJE6hmZD/QoM67gIslmWf+NA/VDOxU4NbywNoq4I1lINtoNeYpwCWSBj8AfLiFOH8MnFruGb6L6jaFjXUFcBiwFDBwlu3fSRqqfPomtBURERHRFdRK1pFuI+kYYA/bn2t3LJ2gr6/P/f397Q5joyXDS+tGK5PZk08+2XTd7bffvum60FqGoMcff3z4SsWECc3fSdRKdrJWztsJ2bOSyazSCX0xFo21TGat/Ey08m8jmcw2D0mLbPc12jeWZnBHnO0ftjuGiIiIiBhZW/QAN7rL2rVrm67byqfggYGBpus+5znPabru6tWrm647eXKjVfEaa2XmoJVZqlZi6BTbbrvtqJx3u+22G5XzRuta+XkfP378KEYS0Bmzsq1o5XdgJ9gSZmVHSq5URERERHSVDHAjIiIioqts8gBXkiV9pmb7TEmzN/W8Q7Q1VdJf1Wz3SRrzD4iVFMANb5KOiIiIiNaMxAzuOuC4suRWyyRtvaHtOlOBpwe4tvttv3dj2o2IiIiI7jQSA9yngDlUyRf+iKS/kHSLpMWS/kvSzqV8tqRLJd0IXNpge6qk+ZJuLV8vK6f8FPBySUskvV/SDEk/LOfcQdKVkm6TdLOk/WraurjMkq6QNOyAWNK9ks4t7fRLOlDS1ZLulnRqqSNJ50laLmmZpJmlfEZp6zJJd0r6Zll7F0l/Xq7FshLTs56AkXR82b9c0qdryk+W9HNJCyR9VdKFpXyupLfU1Hus5vWHJC0s1+Sc4d53RERERDcYqXtwvwDMkjSprvy/gUNtHwB8GzirZt8+wKtsH99geyXwatsHAjOBwdsQzgbml3S759e1dQ6w2PZ+wEeAb9Tsmw68hirV78cHM5oN49e2e6myhs0F3kKVlWxwoHgc0EuVUe1VwHmSdin7DgDeV97THsDhJUHFXGCm7ZdSrWBxWm2Dkv4E+DRwZDn3wZLeWMr/sbR/eHk/GyTpKGBaec+9wEGSXtHE+46IiIgY00ZkfQzbj0r6BvBe4ImaXbsC3ykDv22Ae2r2zbP9xBDb44ELJfUCA8DeTYRxBPDmEs9PJe0oaXAF+R/ZXgesk7QS2Bm4b5jzzSvflwETba8F1kpaJ2lyae9btgeAByVdDxwMPAossH0fgKQlVLdWrAXusf3zct6vA+8GLqhp82DgZ7ZXlWO/SZWGF+B62w+V8u81cU2OKl+Ly/ZEqgHvDbWVJJ0CnAKw++67D3PKiIiIiM43kqsoXACcDDy3puzzwIVlxvJvgdqUPr+vO752+/3Ag1Szo31Ug+NNUZt2aIDmBvaDx6yvO359E8dvTHsb6ylKP5bUwoPXSsC5Zba71/Zetr9Wf7DtObb7bPf19PSMYpgRERERm8eIDXDL7OJ3qQa5gyYB95fXJ7ZwuknAA7bXA28HBvPjrQWeN8Qx84FZUN0HC6y2vcHcopKulTSlhbjq25spaZykHqqZ1gUbqH8XMFXSXmX77cD1dXUWAH8maSdJ44DjS52Fpfz55SG8N9cccy9wUHl9LNXsN8DVwEmSJgJImiLpBRvxPiMiIiLGlJFeB/czQO1qCrOB70laBDSftgm+CJwoaSnV/aaDs7u3AQOSlkqqf6htNtV9prdRPYy2wQF1me3cC3iohbhqXVHiWQr8FDjL9u+Gqmz7D8C7qK7HMqqZ4C/X1XmA6j7j68p5F9m+yvb9wD9TDYBvpBrUrimHfZVq8LsUOIxyrWxfA/w7cFNp7zKG/nAQERER0TVku90xtIWkfYGTbH+g3bE0Q9JE24+VGdwrgIttXzGSbfT19bm/v38kT7lZrVmzZvhKRVL1VlpJU5n0tBER0UkkLbLdMI/AFpvJzPbysTK4LWaXB9aWUz2sd2Wb44mIiIjoSKP58FOMINtntjuGTjdpUv0qdZ2tm1etaGXWu5W6AFtt1fzn8nXr1g1fqdhmm+afZW0lhnHjxg1fKSIiRtQWO4MbEREREd0pA9yIiIiI6CrDDnAlDZSUtbeX1Qs+WFYg2Kwk9Up6Xc32sZLOHsX2Ruz8JXVvw5ugR5KkyZL+brTbiYiIiOhkzQxUnyiJAl4CvBp4LfDx0Q2roV7g6QGu7Xm2PzVajY32+UfJZCAD3IiIiNiitTQTa3slVVrX01WZIOkSScskLZb0SgBJ75R0paSfSLpX0umSPlDq3Cxph1JvT0k/lrRI0nxJ00v5WyUtLzPGN0jaBvgEVWKFJZJmljYuLPV3lnRFqb9U0svqY5f0JUn9ZSb6nJryeyWdI+nW8j6m17yHwfPPLcffLGmFpBmSLpZ0h6S5w7VRs39cOdfy0lb9Wr719WdLulTSTZJ+IelvavZ9SNJCSbfVtPUpYM9yjc6TtEu5fktKmy9vopsjIiIixrSWV1GwvaJk2XoBcEJV5JeWgeE1kvYuVfcFDqBKz/tL4O9tHyDpfOAdVKl95wCn2v6FpD+lSvBwJPAx4DW275c02faTkj4G9Nk+HaoBaE1YnwOut/2mEtvEBqH/g+2Hyv5rJe1n+7ayb7XtA8uf988E/rrB8c+nSqRwLDAPOLzUWyip1/aSYdqAahZ6iu19y3toZnHT/YBDqVIgL5b0o3JtpwGHUKXknSfpFVRJIva13VvO/0HgatufLDE9a5FWSadQfWjp6qf6IyIiYsuxqffSHgH8G4DtO4FfAYMD3Otsr7W9iirr1g9K+TKqlLUTgZdRZfZaAnwF2KXUuRGYW2Ysm1lj50jgSyWOAduNVvx/m6RbgcXAS4B9avZ9v3xfBEwdoo0fuMqKsQx40Paykkr49ppjNtQGwApgD0mfl3Q0sMFUwsVVtp+wvZoqw9khwFHlazFwK1W2t2kNjl0IvEvSbOClttfWV7A9x3af7b6enp4mwomIiIjobC3P4EraAxgAVg5TtXYByvU12+tLu1sBjwzONtayfWqZ0X09sEjSQa3GWRfzi6hmZg+2/XC5rWBCg1gHGPqa1MZf/962bqINSvn+wGuAU4G3AScNE359qjlTzdqea/srde9zal17N5SZ3ddTfWD4rO1vDNNeRERExJjW0gyupB7gy8CFZTZzPjCr7Nsb2B24q5lz2X4UuEfSW8vxKoM/JO1p+xbbHwNWAbsBa4HnDXG6a4HTyrHjJNWv+L898HtgjaSdqR6UG2nDtiFpJ2Ar25cDHwUOLOWnSzp9iPO+QdW9zjsCM6hmZa8GTiqz4EiaIukF1F0jSS+kmm3+KnDRYHsRERER3ayZGdztyi0E44GngEuBz5Z9XwS+JGlZ2fdO2+skNdv+rHL8R8v5vw0sBc6TNI1qpvLaUvZr4OwSy7l15zkDmCPpZKpZ2NOAmwZ32l4qaTFwJ/AbqlsgRlSTbUwBLtEzy6x9uHyfvoGYbqO6NWEn4J9s/xb4raQXAzeVa/0YcILtuyXdKGk58J9UaX0/JOn/Sp13bOr7jIiIiOh0qiZio50k/RA4zvaTdeWzgcds/8vmiKOvr8/9/f2bo6nocknV+4yk6o2IGB2SFtlumGeg5XtwY+TZPqbdMURERER0i8zgxtMkraJaCaOb7ASsbncQsVHSd2NT+m1sSr+NTVt6v73QdsMloDLAja4mqX+oP19EZ0vfjU3pt7Ep/TY2pd+Gtqnr4EZEREREdJQMcCMiIiKiq2SAG91uTrsDiI2Wvhub0m9jU/ptbEq/DSH34EZEREREV8kMbkRERER0lQxwo2tJOlrSXZJ+KensdscTjUm6WNLKkoFvsGwHST+R9Ivy/fntjDGeTdJukq6T9D+Sbpd0RilP33W4kv59gaSlpe/OKeUvknRL+Z35HUnNZz+JzUbSOEmLS5Ko9NsQMsCNriRpHPAF4LXAPsDxkvZpb1QxhLnA0XVlZwPX2p5Gla47H1A6z1PAB23vAxwKvLv8G0vfdb51wJG29wd6gaMlHQp8Gjjf9l7Aw8DJbYwxhnYGcEfNdvqtgQxwo1sdAvzS9oqSAvnbwBvaHFM0YPsG4KG64jcAXy+vvw68cbMGFcOy/YDtW8vrtVT/4U4hfdfxXHmsbI4vXwaOBC4r5em7DiRpV+D1wEVlW6TfGsoAN7rVFOA3Ndv3lbIYG3a2/UB5/Ttg53YGExsmaSpwAHAL6bsxofyZewmwEvgJcDfwiO2nSpX8zuxMFwBnAevL9o6k3xrKADciOpqrpV6y3EuHkjQRuBx4n+1Ha/el7zqX7QHbvcCuVH/xmt7mkGIYko4BVtpe1O5YxoKt2x1AxCi5H9itZnvXUhZjw4OSdrH9gKRdqGaZosNIGk81uP2m7e+X4vTdGGL7EUnXAYcBkyVtXWYD8zuz8xwOHCvpdcAEYHvgX0m/NZQZ3OhWC4Fp5enSbYC/BOa1OaZo3jzgxPL6ROCqNsYSDZR7/74G3GH7szW70ncdTlKPpMnl9XbAq6nuob4OeEuplr7rMLY/bHtX21Op/k/7qe1ZpN8aSqKH6FrlU+4FwDjgYtufbHNI0YCkbwEzgJ2AB4GPA1cC3wV2B34FvM12/YNo0UaSjgDmA8t45n7Aj1Ddh5u+62CS9qN6GGkc1UTXd21/QtIeVA/k7gAsBk6wva59kcZQJM0AzrR9TPqtsQxwIyIiIqKr5BaFiIiIiOgqGeBGRERERFfJADciIiIiukoGuBERERHRVTLAjYiIiIiukgFuRERERHSVDHAjIiIioqtkgBsRERERXeX/AT5DbvTc/nXuAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {
+ "tags": [],
+ "needs_background": "light"
+ }
+ }
]
- },
- "metadata": {
- "needs_background": "light"
- },
- "output_type": "display_data"
}
- ],
- "source": [
- "# Visualize the results.\n",
- "plt.figure(figsize=(10, 8))\n",
- "\n",
- "# Plot the waveform.\n",
- "plt.subplot(3, 1, 1)\n",
- "plt.plot(waveform)\n",
- "plt.xlim([0, len(waveform)])\n",
- "# Plot the log-mel spectrogram (returned by the model).\n",
- "plt.subplot(3, 1, 2)\n",
- "plt.imshow(spectrogram.T, aspect='auto', interpolation='nearest', origin='bottom')\n",
- "\n",
- "# Plot and label the model output scores for the top-scoring classes.\n",
- "mean_scores = np.mean(scores, axis=0)\n",
- "top_N = 10\n",
- "top_class_indices = np.argsort(mean_scores)[::-1][:top_N]\n",
- "plt.subplot(3, 1, 3)\n",
- "plt.imshow(scores[:, top_class_indices].T, aspect='auto', interpolation='nearest', cmap='gray_r')\n",
- "# Compensate for the PATCH_WINDOW_SECONDS (0.96 s) context window to align with spectrogram.\n",
- "patch_padding = (params.PATCH_WINDOW_SECONDS / 2) / params.PATCH_HOP_SECONDS\n",
- "plt.xlim([-patch_padding, scores.shape[0] + patch_padding])\n",
- "# Label the top_N classes.\n",
- "yticks = range(0, top_N, 1)\n",
- "plt.yticks(yticks, [class_names[top_class_indices[x]] for x in yticks])\n",
- "_ = plt.ylim(-0.5 + np.array([top_N, 0]))\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.7.6"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
+ ]
+}
\ No newline at end of file
diff --git a/research/autoencoder/AdditiveGaussianNoiseAutoencoderRunner.py b/research/autoencoder/AdditiveGaussianNoiseAutoencoderRunner.py
deleted file mode 100644
index 8d8ee08654985250ac61415df96889b4a4cf5f1b..0000000000000000000000000000000000000000
--- a/research/autoencoder/AdditiveGaussianNoiseAutoencoderRunner.py
+++ /dev/null
@@ -1,58 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import sklearn.preprocessing as prep
-import tensorflow as tf
-from tensorflow.examples.tutorials.mnist import input_data
-
-from autoencoder_models.DenoisingAutoencoder import AdditiveGaussianNoiseAutoencoder
-
-mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
-
-
-def standard_scale(X_train, X_test):
- preprocessor = prep.StandardScaler().fit(X_train)
- X_train = preprocessor.transform(X_train)
- X_test = preprocessor.transform(X_test)
- return X_train, X_test
-
-
-def get_random_block_from_data(data, batch_size):
- start_index = np.random.randint(0, len(data) - batch_size)
- return data[start_index:(start_index + batch_size)]
-
-
-X_train, X_test = standard_scale(mnist.train.images, mnist.test.images)
-
-n_samples = int(mnist.train.num_examples)
-training_epochs = 20
-batch_size = 128
-display_step = 1
-
-autoencoder = AdditiveGaussianNoiseAutoencoder(
- n_input=784,
- n_hidden=200,
- transfer_function=tf.nn.softplus,
- optimizer=tf.train.AdamOptimizer(learning_rate = 0.001),
- scale=0.01)
-
-for epoch in range(training_epochs):
- avg_cost = 0.
- total_batch = int(n_samples / batch_size)
- # Loop over all batches
- for i in range(total_batch):
- batch_xs = get_random_block_from_data(X_train, batch_size)
-
- # Fit training using batch data
- cost = autoencoder.partial_fit(batch_xs)
- # Compute average loss
- avg_cost += cost / n_samples * batch_size
-
- # Display logs per epoch step
- if epoch % display_step == 0:
- print("Epoch:", '%d,' % (epoch + 1),
- "Cost:", "{:.9f}".format(avg_cost))
-
-print("Total cost: " + str(autoencoder.calc_total_cost(X_test)))
diff --git a/research/autoencoder/AutoencoderRunner.py b/research/autoencoder/AutoencoderRunner.py
deleted file mode 100644
index 7f1ab2ecd5a91c12960714ea79a864631e634f8c..0000000000000000000000000000000000000000
--- a/research/autoencoder/AutoencoderRunner.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import sklearn.preprocessing as prep
-import tensorflow as tf
-from tensorflow.examples.tutorials.mnist import input_data
-
-from autoencoder_models.Autoencoder import Autoencoder
-
-mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
-
-
-def standard_scale(X_train, X_test):
- preprocessor = prep.StandardScaler().fit(X_train)
- X_train = preprocessor.transform(X_train)
- X_test = preprocessor.transform(X_test)
- return X_train, X_test
-
-
-def get_random_block_from_data(data, batch_size):
- start_index = np.random.randint(0, len(data) - batch_size)
- return data[start_index:(start_index + batch_size)]
-
-
-X_train, X_test = standard_scale(mnist.train.images, mnist.test.images)
-
-n_samples = int(mnist.train.num_examples)
-training_epochs = 20
-batch_size = 128
-display_step = 1
-
-autoencoder = Autoencoder(n_layers=[784, 200],
- transfer_function = tf.nn.softplus,
- optimizer = tf.train.AdamOptimizer(learning_rate = 0.001))
-
-for epoch in range(training_epochs):
- avg_cost = 0.
- total_batch = int(n_samples / batch_size)
- # Loop over all batches
- for i in range(total_batch):
- batch_xs = get_random_block_from_data(X_train, batch_size)
-
- # Fit training using batch data
- cost = autoencoder.partial_fit(batch_xs)
- # Compute average loss
- avg_cost += cost / n_samples * batch_size
-
- # Display logs per epoch step
- if epoch % display_step == 0:
- print("Epoch:", '%d,' % (epoch + 1),
- "Cost:", "{:.9f}".format(avg_cost))
-
-print("Total cost: " + str(autoencoder.calc_total_cost(X_test)))
diff --git a/research/autoencoder/MaskingNoiseAutoencoderRunner.py b/research/autoencoder/MaskingNoiseAutoencoderRunner.py
deleted file mode 100644
index b776302e286ff740ba7b8e6f679a54b23944df12..0000000000000000000000000000000000000000
--- a/research/autoencoder/MaskingNoiseAutoencoderRunner.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import sklearn.preprocessing as prep
-import tensorflow as tf
-from tensorflow.examples.tutorials.mnist import input_data
-
-from autoencoder_models.DenoisingAutoencoder import MaskingNoiseAutoencoder
-
-mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
-
-
-def standard_scale(X_train, X_test):
- preprocessor = prep.StandardScaler().fit(X_train)
- X_train = preprocessor.transform(X_train)
- X_test = preprocessor.transform(X_test)
- return X_train, X_test
-
-
-def get_random_block_from_data(data, batch_size):
- start_index = np.random.randint(0, len(data) - batch_size)
- return data[start_index:(start_index + batch_size)]
-
-
-X_train, X_test = standard_scale(mnist.train.images, mnist.test.images)
-
-n_samples = int(mnist.train.num_examples)
-training_epochs = 100
-batch_size = 128
-display_step = 1
-
-autoencoder = MaskingNoiseAutoencoder(
- n_input=784,
- n_hidden=200,
- transfer_function=tf.nn.softplus,
- optimizer=tf.train.AdamOptimizer(learning_rate=0.001),
- dropout_probability=0.95)
-
-for epoch in range(training_epochs):
- avg_cost = 0.
- total_batch = int(n_samples / batch_size)
- for i in range(total_batch):
- batch_xs = get_random_block_from_data(X_train, batch_size)
-
- cost = autoencoder.partial_fit(batch_xs)
-
- avg_cost += cost / n_samples * batch_size
-
- if epoch % display_step == 0:
- print("Epoch:", '%d,' % (epoch + 1),
- "Cost:", "{:.9f}".format(avg_cost))
-
-print("Total cost: " + str(autoencoder.calc_total_cost(X_test)))
diff --git a/research/autoencoder/README.md b/research/autoencoder/README.md
deleted file mode 100644
index cba7b3b66f59ac9e3810ee1b98d67133296aea25..0000000000000000000000000000000000000000
--- a/research/autoencoder/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-
diff --git a/research/autoencoder/VariationalAutoencoderRunner.py b/research/autoencoder/VariationalAutoencoderRunner.py
deleted file mode 100644
index f5ce0045f3c6dfdd357cd874f8ee24df0d8cb3d9..0000000000000000000000000000000000000000
--- a/research/autoencoder/VariationalAutoencoderRunner.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import sklearn.preprocessing as prep
-import tensorflow as tf
-from tensorflow.examples.tutorials.mnist import input_data
-
-from autoencoder_models.VariationalAutoencoder import VariationalAutoencoder
-
-mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
-
-
-def min_max_scale(X_train, X_test):
- preprocessor = prep.MinMaxScaler().fit(X_train)
- X_train = preprocessor.transform(X_train)
- X_test = preprocessor.transform(X_test)
- return X_train, X_test
-
-
-def get_random_block_from_data(data, batch_size):
- start_index = np.random.randint(0, len(data) - batch_size)
- return data[start_index:(start_index + batch_size)]
-
-
-X_train, X_test = min_max_scale(mnist.train.images, mnist.test.images)
-
-n_samples = int(mnist.train.num_examples)
-training_epochs = 20
-batch_size = 128
-display_step = 1
-
-autoencoder = VariationalAutoencoder(
- n_input=784,
- n_hidden=200,
- optimizer=tf.train.AdamOptimizer(learning_rate = 0.001))
-
-for epoch in range(training_epochs):
- avg_cost = 0.
- total_batch = int(n_samples / batch_size)
- # Loop over all batches
- for i in range(total_batch):
- batch_xs = get_random_block_from_data(X_train, batch_size)
-
- # Fit training using batch data
- cost = autoencoder.partial_fit(batch_xs)
- # Compute average loss
- avg_cost += cost / n_samples * batch_size
-
- # Display logs per epoch step
- if epoch % display_step == 0:
- print("Epoch:", '%d,' % (epoch + 1),
- "Cost:", "{:.9f}".format(avg_cost))
-
-print("Total cost: " + str(autoencoder.calc_total_cost(X_test)))
diff --git a/research/autoencoder/__init__.py b/research/autoencoder/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/autoencoder/autoencoder_models/Autoencoder.py b/research/autoencoder/autoencoder_models/Autoencoder.py
deleted file mode 100644
index 788a14642306ece056fc53a85ba8c60d87d31826..0000000000000000000000000000000000000000
--- a/research/autoencoder/autoencoder_models/Autoencoder.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import numpy as np
-import tensorflow as tf
-
-
-class Autoencoder(object):
-
- def __init__(self, n_layers, transfer_function=tf.nn.softplus, optimizer=tf.train.AdamOptimizer()):
- self.n_layers = n_layers
- self.transfer = transfer_function
-
- network_weights = self._initialize_weights()
- self.weights = network_weights
-
- # model
- self.x = tf.placeholder(tf.float32, [None, self.n_layers[0]])
- self.hidden_encode = []
- h = self.x
- for layer in range(len(self.n_layers)-1):
- h = self.transfer(
- tf.add(tf.matmul(h, self.weights['encode'][layer]['w']),
- self.weights['encode'][layer]['b']))
- self.hidden_encode.append(h)
-
- self.hidden_recon = []
- for layer in range(len(self.n_layers)-1):
- h = self.transfer(
- tf.add(tf.matmul(h, self.weights['recon'][layer]['w']),
- self.weights['recon'][layer]['b']))
- self.hidden_recon.append(h)
- self.reconstruction = self.hidden_recon[-1]
-
- # cost
- self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0))
- self.optimizer = optimizer.minimize(self.cost)
-
- init = tf.global_variables_initializer()
- self.sess = tf.Session()
- self.sess.run(init)
-
-
- def _initialize_weights(self):
- all_weights = dict()
- initializer = tf.contrib.layers.xavier_initializer()
- # Encoding network weights
- encoder_weights = []
- for layer in range(len(self.n_layers)-1):
- w = tf.Variable(
- initializer((self.n_layers[layer], self.n_layers[layer + 1]),
- dtype=tf.float32))
- b = tf.Variable(
- tf.zeros([self.n_layers[layer + 1]], dtype=tf.float32))
- encoder_weights.append({'w': w, 'b': b})
- # Recon network weights
- recon_weights = []
- for layer in range(len(self.n_layers)-1, 0, -1):
- w = tf.Variable(
- initializer((self.n_layers[layer], self.n_layers[layer - 1]),
- dtype=tf.float32))
- b = tf.Variable(
- tf.zeros([self.n_layers[layer - 1]], dtype=tf.float32))
- recon_weights.append({'w': w, 'b': b})
- all_weights['encode'] = encoder_weights
- all_weights['recon'] = recon_weights
- return all_weights
-
- def partial_fit(self, X):
- cost, opt = self.sess.run((self.cost, self.optimizer), feed_dict={self.x: X})
- return cost
-
- def calc_total_cost(self, X):
- return self.sess.run(self.cost, feed_dict={self.x: X})
-
- def transform(self, X):
- return self.sess.run(self.hidden_encode[-1], feed_dict={self.x: X})
-
- def generate(self, hidden=None):
- if hidden is None:
- hidden = np.random.normal(size=self.weights['encode'][-1]['b'])
- return self.sess.run(self.reconstruction, feed_dict={self.hidden_encode[-1]: hidden})
-
- def reconstruct(self, X):
- return self.sess.run(self.reconstruction, feed_dict={self.x: X})
-
- def getWeights(self):
- raise NotImplementedError
- return self.sess.run(self.weights)
-
- def getBiases(self):
- raise NotImplementedError
- return self.sess.run(self.weights)
-
diff --git a/research/autoencoder/autoencoder_models/DenoisingAutoencoder.py b/research/autoencoder/autoencoder_models/DenoisingAutoencoder.py
deleted file mode 100644
index 22b5dcb44a4079b80bfcfc16e3dcda5b21ca8c1b..0000000000000000000000000000000000000000
--- a/research/autoencoder/autoencoder_models/DenoisingAutoencoder.py
+++ /dev/null
@@ -1,129 +0,0 @@
-import tensorflow as tf
-
-class AdditiveGaussianNoiseAutoencoder(object):
- def __init__(self, n_input, n_hidden, transfer_function = tf.nn.softplus, optimizer = tf.train.AdamOptimizer(),
- scale = 0.1):
- self.n_input = n_input
- self.n_hidden = n_hidden
- self.transfer = transfer_function
- self.scale = tf.placeholder(tf.float32)
- self.training_scale = scale
- network_weights = self._initialize_weights()
- self.weights = network_weights
-
- # model
- self.x = tf.placeholder(tf.float32, [None, self.n_input])
- self.hidden = self.transfer(tf.add(tf.matmul(self.x + scale * tf.random_normal((n_input,)),
- self.weights['w1']),
- self.weights['b1']))
- self.reconstruction = tf.add(tf.matmul(self.hidden, self.weights['w2']), self.weights['b2'])
-
- # cost
- self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0))
- self.optimizer = optimizer.minimize(self.cost)
-
- init = tf.global_variables_initializer()
- self.sess = tf.Session()
- self.sess.run(init)
-
- def _initialize_weights(self):
- all_weights = dict()
- all_weights['w1'] = tf.get_variable("w1", shape=[self.n_input, self.n_hidden],
- initializer=tf.contrib.layers.xavier_initializer())
- all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype = tf.float32))
- all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype = tf.float32))
- all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype = tf.float32))
- return all_weights
-
- def partial_fit(self, X):
- cost, opt = self.sess.run((self.cost, self.optimizer), feed_dict = {self.x: X,
- self.scale: self.training_scale
- })
- return cost
-
- def calc_total_cost(self, X):
- return self.sess.run(self.cost, feed_dict = {self.x: X,
- self.scale: self.training_scale
- })
-
- def transform(self, X):
- return self.sess.run(self.hidden, feed_dict = {self.x: X,
- self.scale: self.training_scale
- })
-
- def generate(self, hidden=None):
- if hidden is None:
- hidden = self.sess.run(tf.random_normal([1, self.n_hidden]))
- return self.sess.run(self.reconstruction, feed_dict = {self.hidden: hidden})
-
- def reconstruct(self, X):
- return self.sess.run(self.reconstruction, feed_dict = {self.x: X,
- self.scale: self.training_scale
- })
-
- def getWeights(self):
- return self.sess.run(self.weights['w1'])
-
- def getBiases(self):
- return self.sess.run(self.weights['b1'])
-
-
-class MaskingNoiseAutoencoder(object):
- def __init__(self, n_input, n_hidden, transfer_function = tf.nn.softplus, optimizer = tf.train.AdamOptimizer(),
- dropout_probability = 0.95):
- self.n_input = n_input
- self.n_hidden = n_hidden
- self.transfer = transfer_function
- self.dropout_probability = dropout_probability
- self.keep_prob = tf.placeholder(tf.float32)
-
- network_weights = self._initialize_weights()
- self.weights = network_weights
-
- # model
- self.x = tf.placeholder(tf.float32, [None, self.n_input])
- self.hidden = self.transfer(tf.add(tf.matmul(tf.nn.dropout(self.x, self.keep_prob), self.weights['w1']),
- self.weights['b1']))
- self.reconstruction = tf.add(tf.matmul(self.hidden, self.weights['w2']), self.weights['b2'])
-
- # cost
- self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0))
- self.optimizer = optimizer.minimize(self.cost)
-
- init = tf.global_variables_initializer()
- self.sess = tf.Session()
- self.sess.run(init)
-
- def _initialize_weights(self):
- all_weights = dict()
- all_weights['w1'] = tf.get_variable("w1", shape=[self.n_input, self.n_hidden],
- initializer=tf.contrib.layers.xavier_initializer())
- all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype = tf.float32))
- all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype = tf.float32))
- all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype = tf.float32))
- return all_weights
-
- def partial_fit(self, X):
- cost, opt = self.sess.run((self.cost, self.optimizer),
- feed_dict = {self.x: X, self.keep_prob: self.dropout_probability})
- return cost
-
- def calc_total_cost(self, X):
- return self.sess.run(self.cost, feed_dict = {self.x: X, self.keep_prob: 1.0})
-
- def transform(self, X):
- return self.sess.run(self.hidden, feed_dict = {self.x: X, self.keep_prob: 1.0})
-
- def generate(self, hidden=None):
- if hidden is None:
- hidden = self.sess.run(tf.random_normal([1, self.n_hidden]))
- return self.sess.run(self.reconstruction, feed_dict = {self.hidden: hidden})
-
- def reconstruct(self, X):
- return self.sess.run(self.reconstruction, feed_dict = {self.x: X, self.keep_prob: 1.0})
-
- def getWeights(self):
- return self.sess.run(self.weights['w1'])
-
- def getBiases(self):
- return self.sess.run(self.weights['b1'])
diff --git a/research/autoencoder/autoencoder_models/VariationalAutoencoder.py b/research/autoencoder/autoencoder_models/VariationalAutoencoder.py
deleted file mode 100644
index 3c2556ab89c2d32be0af5e61099aa12f91c1f176..0000000000000000000000000000000000000000
--- a/research/autoencoder/autoencoder_models/VariationalAutoencoder.py
+++ /dev/null
@@ -1,70 +0,0 @@
-import tensorflow as tf
-
-class VariationalAutoencoder(object):
-
- def __init__(self, n_input, n_hidden, optimizer = tf.train.AdamOptimizer()):
- self.n_input = n_input
- self.n_hidden = n_hidden
-
- network_weights = self._initialize_weights()
- self.weights = network_weights
-
- # model
- self.x = tf.placeholder(tf.float32, [None, self.n_input])
- self.z_mean = tf.add(tf.matmul(self.x, self.weights['w1']), self.weights['b1'])
- self.z_log_sigma_sq = tf.add(tf.matmul(self.x, self.weights['log_sigma_w1']), self.weights['log_sigma_b1'])
-
- # sample from gaussian distribution
- eps = tf.random_normal(tf.stack([tf.shape(self.x)[0], self.n_hidden]), 0, 1, dtype = tf.float32)
- self.z = tf.add(self.z_mean, tf.multiply(tf.sqrt(tf.exp(self.z_log_sigma_sq)), eps))
-
- self.reconstruction = tf.add(tf.matmul(self.z, self.weights['w2']), self.weights['b2'])
-
- # cost
- reconstr_loss = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0), 1)
- latent_loss = -0.5 * tf.reduce_sum(1 + self.z_log_sigma_sq
- - tf.square(self.z_mean)
- - tf.exp(self.z_log_sigma_sq), 1)
- self.cost = tf.reduce_mean(reconstr_loss + latent_loss)
- self.optimizer = optimizer.minimize(self.cost)
-
- init = tf.global_variables_initializer()
- self.sess = tf.Session()
- self.sess.run(init)
-
- def _initialize_weights(self):
- all_weights = dict()
- all_weights['w1'] = tf.get_variable("w1", shape=[self.n_input, self.n_hidden],
- initializer=tf.contrib.layers.xavier_initializer())
- all_weights['log_sigma_w1'] = tf.get_variable("log_sigma_w1", shape=[self.n_input, self.n_hidden],
- initializer=tf.contrib.layers.xavier_initializer())
- all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype=tf.float32))
- all_weights['log_sigma_b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype=tf.float32))
- all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype=tf.float32))
- all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype=tf.float32))
- return all_weights
-
- def partial_fit(self, X):
- cost, opt = self.sess.run((self.cost, self.optimizer), feed_dict={self.x: X})
- return cost
-
- def calc_total_cost(self, X):
- return self.sess.run(self.cost, feed_dict = {self.x: X})
-
- def transform(self, X):
- return self.sess.run(self.z_mean, feed_dict={self.x: X})
-
- def generate(self, hidden = None):
- if hidden is None:
- hidden = self.sess.run(tf.random_normal([1, self.n_hidden]))
- return self.sess.run(self.reconstruction, feed_dict={self.z: hidden})
-
- def reconstruct(self, X):
- return self.sess.run(self.reconstruction, feed_dict={self.x: X})
-
- def getWeights(self):
- return self.sess.run(self.weights['w1'])
-
- def getBiases(self):
- return self.sess.run(self.weights['b1'])
-
diff --git a/research/autoencoder/autoencoder_models/__init__.py b/research/autoencoder/autoencoder_models/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/brain_coder/README.md b/research/brain_coder/README.md
deleted file mode 100644
index 3e2a1656d8f145569266c19c64b41779ccbf308c..0000000000000000000000000000000000000000
--- a/research/brain_coder/README.md
+++ /dev/null
@@ -1,34 +0,0 @@
-
-
-
-
-# Brain Coder
-
-*Authors: Daniel Abolafia, Mohammad Norouzi, Quoc Le*
-
-Brain coder is a code synthesis experimental environment. We provide code that reproduces the results from our recent paper [Neural Program Synthesis with Priority Queue Training](https://arxiv.org/abs/1801.03526). See single_task/README.md for details on how to build and reproduce those experiments.
-
-## Installation
-
-First install dependencies seperately:
-
-* [bazel](https://docs.bazel.build/versions/master/install.html)
-* [TensorFlow](https://www.tensorflow.org/install/)
-* [scipy](https://www.scipy.org/install.html)
-* [absl-py](https://github.com/abseil/abseil-py)
-
-Note: even if you already have these dependencies installed, make sure they are
-up-to-date to avoid unnecessary debugging.
-
-
-## Building
-
-Use bazel from the top-level repo directory.
-
-For example:
-
-```bash
-bazel build single_task:run
-```
-
-View README.md files in subdirectories for more details.
diff --git a/research/brain_coder/WORKSPACE b/research/brain_coder/WORKSPACE
deleted file mode 100644
index 7c07b5325e71a1684fb38089adeaaa9f4f00a775..0000000000000000000000000000000000000000
--- a/research/brain_coder/WORKSPACE
+++ /dev/null
@@ -1,5 +0,0 @@
-git_repository(
- name = "subpar",
- remote = "https://github.com/google/subpar",
- tag = "1.0.0",
-)
diff --git a/research/brain_coder/common/BUILD b/research/brain_coder/common/BUILD
deleted file mode 100644
index b5f79c25096ca574d4a133f871343eedd985b25e..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/BUILD
+++ /dev/null
@@ -1,106 +0,0 @@
-licenses(["notice"])
-
-package(default_visibility = [
- "//:__subpackages__",
-])
-
-py_library(
- name = "bf",
- srcs = ["bf.py"],
-)
-
-py_test(
- name = "bf_test",
- srcs = ["bf_test.py"],
- deps = [
- ":bf",
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "config_lib",
- srcs = ["config_lib.py"],
-)
-
-py_test(
- name = "config_lib_test",
- srcs = ["config_lib_test.py"],
- deps = [
- ":config_lib",
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "reward",
- srcs = ["reward.py"],
-)
-
-py_test(
- name = "reward_test",
- srcs = ["reward_test.py"],
- deps = [
- ":reward",
- # numpy dep
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "rollout",
- srcs = ["rollout.py"],
- deps = [
- ":utils",
- # numpy dep
- # scipy dep
- ],
-)
-
-py_test(
- name = "rollout_test",
- srcs = ["rollout_test.py"],
- deps = [
- ":rollout",
- # numpy dep
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "schedules",
- srcs = ["schedules.py"],
- deps = [":config_lib"],
-)
-
-py_test(
- name = "schedules_test",
- srcs = ["schedules_test.py"],
- deps = [
- ":config_lib",
- ":schedules",
- # numpy dep
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "utils",
- srcs = ["utils.py"],
- deps = [
- # file dep
- # absl dep /logging
- # numpy dep
- # tensorflow dep
- ],
-)
-
-py_test(
- name = "utils_test",
- srcs = ["utils_test.py"],
- deps = [
- ":utils",
- # numpy dep
- # tensorflow dep
- ],
-)
diff --git a/research/brain_coder/common/bf.py b/research/brain_coder/common/bf.py
deleted file mode 100644
index f049c45258f7b78a25b5492108b2f8b37c8a55cd..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/bf.py
+++ /dev/null
@@ -1,234 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""BrainF**k interpreter.
-
-Language info: https://en.wikipedia.org/wiki/Brainfuck
-
-Based on public implementation:
-https://github.com/pocmo/Python-Brainfuck/blob/master/brainfuck.py
-"""
-
-from collections import namedtuple
-import time
-
-
-EvalResult = namedtuple(
- 'EvalResult', ['output', 'success', 'failure_reason', 'steps', 'time',
- 'memory', 'program_trace'])
-
-
-ExecutionSnapshot = namedtuple(
- 'ExecutionSnapshot',
- ['codeptr', 'codechar', 'memptr', 'memval', 'memory', 'next_input',
- 'output_buffer'])
-
-
-class Status(object):
- SUCCESS = 'success'
- TIMEOUT = 'timeout'
- STEP_LIMIT = 'step-limit'
- SYNTAX_ERROR = 'syntax-error'
-
-
-CHARS = INT_TO_CHAR = ['>', '<', '+', '-', '[', ']', '.', ',']
-CHAR_TO_INT = dict([(c, i) for i, c in enumerate(INT_TO_CHAR)])
-
-
-class LookAheadIterator(object):
- """Same API as Python iterator, with additional peek method."""
-
- def __init__(self, iterable):
- self._it = iter(iterable)
- self._current_element = None
- self._done = False
- self._preload_next()
-
- def _preload_next(self):
- try:
- self._current_element = self._it.next()
- except StopIteration:
- self._done = True
-
- def next(self):
- if self._done:
- raise StopIteration
- element = self._current_element
- self._preload_next()
- return element
-
- def peek(self, default_value=None):
- if self._done:
- if default_value is None:
- raise StopIteration
- return default_value
- return self._current_element
-
-
-def buildbracemap(code):
- """Build jump map.
-
- Args:
- code: List or string or BF chars.
-
- Returns:
- bracemap: dict mapping open and close brace positions in the code to their
- destination jumps. Specifically, positions of matching open/close braces
- if they exist.
- correct_syntax: True if all braces match. False if there are unmatched
- braces in the code. Even if there are unmatched braces, a bracemap will
- be built, and unmatched braces will map to themselves.
- """
- bracestack, bracemap = [], {}
-
- correct_syntax = True
- for position, command in enumerate(code):
- if command == '[':
- bracestack.append(position)
- if command == ']':
- if not bracestack: # Unmatched closing brace.
- bracemap[position] = position # Don't jump to any position.
- correct_syntax = False
- continue
- start = bracestack.pop()
- bracemap[start] = position
- bracemap[position] = start
- if bracestack: # Unmatched opening braces.
- for pos in bracestack:
- bracemap[pos] = pos # Don't jump to any position.
- correct_syntax = False
- return bracemap, correct_syntax
-
-
-def evaluate(code, input_buffer=None, init_memory=None, base=256, timeout=1.0,
- max_steps=None, require_correct_syntax=True, output_memory=False,
- debug=False):
- """Execute BF code.
-
- Args:
- code: String or list of BF characters. Any character not in CHARS will be
- ignored.
- input_buffer: A list of ints which will be used as the program's input
- stream. Each read op "," will read an int from this list. 0's will be
- read once the end of the list is reached, or if no input buffer is
- given.
- init_memory: A list of ints. Memory for first k positions will be
- initialized to this list (where k = len(init_memory)). Memory positions
- are initialized to 0 by default.
- base: Integer base for the memory. When a memory value is incremented to
- `base` it will overflow to 0. When a memory value is decremented to -1
- it will underflow to `base` - 1.
- timeout: Time limit for program execution in seconds. Set to None to
- disable.
- max_steps: Execution step limit. An execution step is the execution of one
- operation (code character), even if that op has been executed before.
- Execution exits when this many steps are reached. Set to None to
- disable. Disabled by default.
- require_correct_syntax: If True, unmatched braces will cause `evaluate` to
- return without executing the code. The failure reason will be
- `Status.SYNTAX_ERROR`. If False, unmatched braces are ignored
- and execution will continue.
- output_memory: If True, the state of the memory at the end of execution is
- returned.
- debug: If True, then a full program trace will be returned.
-
- Returns:
- EvalResult namedtuple containing
- output: List of ints which were written out by the program with the "."
- operation.
- success: Boolean. Whether execution completed successfully.
- failure_reason: One of the attributes of `Status`. Gives extra info
- about why execution was not successful.
- steps: Number of execution steps the program ran for.
- time: Amount of time in seconds the program ran for.
- memory: If `output_memory` is True, a list of memory cells up to the last
- one written to. otherwise, None.
- """
- input_iter = (
- LookAheadIterator(input_buffer) if input_buffer is not None
- else LookAheadIterator([]))
-
- # Null memory value. This is the value of an empty memory. Also the value
- # returned by the read operation when the input buffer is empty, or the
- # end of the buffer is reached.
- null_value = 0
-
- code = list(code)
- bracemap, correct_syntax = buildbracemap(code) # will modify code list
- if require_correct_syntax and not correct_syntax:
- return EvalResult([], False, Status.SYNTAX_ERROR, 0, 0.0,
- [] if output_memory else None, [] if debug else None)
-
- output_buffer = []
-
- codeptr, cellptr = 0, 0
-
- cells = list(init_memory) if init_memory else [0]
-
- program_trace = [] if debug else None
- success = True
- reason = Status.SUCCESS
- start_time = time.time()
- steps = 0
- while codeptr < len(code):
- command = code[codeptr]
-
- if debug:
- # Add step to program trace.
- program_trace.append(ExecutionSnapshot(
- codeptr=codeptr, codechar=command, memptr=cellptr,
- memval=cells[cellptr], memory=list(cells),
- next_input=input_iter.peek(null_value),
- output_buffer=list(output_buffer)))
-
- if command == '>':
- cellptr += 1
- if cellptr == len(cells): cells.append(null_value)
-
- if command == '<':
- cellptr = 0 if cellptr <= 0 else cellptr - 1
-
- if command == '+':
- cells[cellptr] = cells[cellptr] + 1 if cells[cellptr] < (base - 1) else 0
-
- if command == '-':
- cells[cellptr] = cells[cellptr] - 1 if cells[cellptr] > 0 else (base - 1)
-
- if command == '[' and cells[cellptr] == 0: codeptr = bracemap[codeptr]
- if command == ']' and cells[cellptr] != 0: codeptr = bracemap[codeptr]
-
- if command == '.': output_buffer.append(cells[cellptr])
- if command == ',': cells[cellptr] = next(input_iter, null_value)
-
- codeptr += 1
- steps += 1
-
- if timeout is not None and time.time() - start_time > timeout:
- success = False
- reason = Status.TIMEOUT
- break
- if max_steps is not None and steps >= max_steps:
- success = False
- reason = Status.STEP_LIMIT
- break
-
- if debug:
- # Add step to program trace.
- command = code[codeptr] if codeptr < len(code) else ''
- program_trace.append(ExecutionSnapshot(
- codeptr=codeptr, codechar=command, memptr=cellptr,
- memval=cells[cellptr], memory=list(cells),
- next_input=input_iter.peek(null_value),
- output_buffer=list(output_buffer)))
-
- return EvalResult(
- output=output_buffer,
- success=success,
- failure_reason=reason,
- steps=steps,
- time=time.time() - start_time,
- memory=cells if output_memory else None,
- program_trace=program_trace)
-
-
diff --git a/research/brain_coder/common/bf_test.py b/research/brain_coder/common/bf_test.py
deleted file mode 100644
index 2cbf505601a96ec1fc819f1d01fe551f2fae4a5d..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/bf_test.py
+++ /dev/null
@@ -1,137 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for common.bf."""
-
-import tensorflow as tf
-
-from common import bf # brain coder
-
-
-class BfTest(tf.test.TestCase):
-
- def assertCorrectOutput(self, target_output, eval_result):
- self.assertEqual(target_output, eval_result.output)
- self.assertTrue(eval_result.success)
- self.assertEqual(bf.Status.SUCCESS, eval_result.failure_reason)
-
- def testBasicOps(self):
- self.assertCorrectOutput(
- [3, 1, 2],
- bf.evaluate('+++.--.+.'))
- self.assertCorrectOutput(
- [1, 1, 2],
- bf.evaluate('+.<.>++.'))
- self.assertCorrectOutput(
- [0],
- bf.evaluate('+,.'))
- self.assertCorrectOutput(
- [ord(char) for char in 'Hello World!\n'],
- bf.evaluate(
- '>++++++++[-<+++++++++>]<.>>+>-[+]++>++>+++[>[->+++<<+++>]<<]>-----'
- '.>->+++..+++.>-.<<+[>[+>+]>>]<--------------.>>.+++.------.-------'
- '-.>+.>+.'))
-
- def testBase(self):
- self.assertCorrectOutput(
- [1, 4],
- bf.evaluate('+.--.', base=5, input_buffer=[]))
-
- def testInputBuffer(self):
- self.assertCorrectOutput(
- [2, 3, 4],
- bf.evaluate('>,[>,]<[.<]', input_buffer=[4, 3, 2]))
-
- def testBadChars(self):
- self.assertCorrectOutput(
- [2, 3, 4],
- bf.evaluate('>,[>,]hello----.[[[[[>+.',
- input_buffer=[],
- base=10,
- require_correct_syntax=False))
-
- eval_result = bf.evaluate(
- '+++.]]]]>----.[[[[[>+.',
- input_buffer=[],
- base=10,
- require_correct_syntax=True)
- self.assertEqual([], eval_result.output)
- self.assertFalse(eval_result.success)
- self.assertEqual(bf.Status.SYNTAX_ERROR,
- eval_result.failure_reason)
-
- def testTimeout(self):
- er = bf.evaluate('+.[].', base=5, input_buffer=[], timeout=0.1)
- self.assertEqual(
- ([1], False, bf.Status.TIMEOUT),
- (er.output, er.success, er.failure_reason))
- self.assertTrue(0.07 < er.time < 0.21)
-
- er = bf.evaluate('+.[-].', base=5, input_buffer=[], timeout=0.1)
- self.assertEqual(
- ([1, 0], True, bf.Status.SUCCESS),
- (er.output, er.success, er.failure_reason))
- self.assertTrue(er.time < 0.15)
-
- def testMaxSteps(self):
- er = bf.evaluate('+.[].', base=5, input_buffer=[], timeout=None,
- max_steps=100)
- self.assertEqual(
- ([1], False, bf.Status.STEP_LIMIT, 100),
- (er.output, er.success, er.failure_reason, er.steps))
-
- er = bf.evaluate('+.[-].', base=5, input_buffer=[], timeout=None,
- max_steps=100)
- self.assertEqual(
- ([1, 0], True, bf.Status.SUCCESS),
- (er.output, er.success, er.failure_reason))
- self.assertTrue(er.steps < 100)
-
- def testOutputMemory(self):
- er = bf.evaluate('+>++>+++>++++.', base=256, input_buffer=[],
- output_memory=True)
- self.assertEqual(
- ([4], True, bf.Status.SUCCESS),
- (er.output, er.success, er.failure_reason))
- self.assertEqual([1, 2, 3, 4], er.memory)
-
- def testProgramTrace(self):
- es = bf.ExecutionSnapshot
- er = bf.evaluate(',[.>,].', base=256, input_buffer=[2, 1], debug=True)
- self.assertEqual(
- [es(codeptr=0, codechar=',', memptr=0, memval=0, memory=[0],
- next_input=2, output_buffer=[]),
- es(codeptr=1, codechar='[', memptr=0, memval=2, memory=[2],
- next_input=1, output_buffer=[]),
- es(codeptr=2, codechar='.', memptr=0, memval=2, memory=[2],
- next_input=1, output_buffer=[]),
- es(codeptr=3, codechar='>', memptr=0, memval=2, memory=[2],
- next_input=1, output_buffer=[2]),
- es(codeptr=4, codechar=',', memptr=1, memval=0, memory=[2, 0],
- next_input=1, output_buffer=[2]),
- es(codeptr=5, codechar=']', memptr=1, memval=1, memory=[2, 1],
- next_input=0, output_buffer=[2]),
- es(codeptr=2, codechar='.', memptr=1, memval=1, memory=[2, 1],
- next_input=0, output_buffer=[2]),
- es(codeptr=3, codechar='>', memptr=1, memval=1, memory=[2, 1],
- next_input=0, output_buffer=[2, 1]),
- es(codeptr=4, codechar=',', memptr=2, memval=0, memory=[2, 1, 0],
- next_input=0, output_buffer=[2, 1]),
- es(codeptr=5, codechar=']', memptr=2, memval=0, memory=[2, 1, 0],
- next_input=0, output_buffer=[2, 1]),
- es(codeptr=6, codechar='.', memptr=2, memval=0, memory=[2, 1, 0],
- next_input=0, output_buffer=[2, 1]),
- es(codeptr=7, codechar='', memptr=2, memval=0, memory=[2, 1, 0],
- next_input=0, output_buffer=[2, 1, 0])],
- er.program_trace)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/common/config_lib.py b/research/brain_coder/common/config_lib.py
deleted file mode 100644
index 733fa202f2e500f964beff2111cb7445fa66a9e1..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/config_lib.py
+++ /dev/null
@@ -1,337 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Objects for storing configuration and passing config into binaries.
-
-Config class stores settings and hyperparameters for models, data, and anything
-else that may be specific to a particular run.
-"""
-
-import ast
-import itertools
-from six.moves import xrange
-
-
-class Config(dict):
- """Stores model configuration, hyperparameters, or dataset parameters."""
-
- def __getattr__(self, attr):
- return self[attr]
-
- def __setattr__(self, attr, value):
- self[attr] = value
-
- def pretty_str(self, new_lines=True, indent=2, final_indent=0):
- prefix = (' ' * indent) if new_lines else ''
- final_prefix = (' ' * final_indent) if new_lines else ''
- kv = ['%s%s=%s' % (prefix, k,
- (repr(v) if not isinstance(v, Config)
- else v.pretty_str(new_lines=new_lines,
- indent=indent+2,
- final_indent=indent)))
- for k, v in self.items()]
- if new_lines:
- return 'Config(\n%s\n%s)' % (',\n'.join(kv), final_prefix)
- else:
- return 'Config(%s)' % ', '.join(kv)
-
- def _update_iterator(self, *args, **kwargs):
- """Convert mixed input into an iterator over (key, value) tuples.
-
- Follows the dict.update call signature.
-
- Args:
- *args: (Optional) Pass a dict or iterable of (key, value) 2-tuples as
- an unnamed argument. Only one unnamed argument allowed.
- **kwargs: (Optional) Pass (key, value) pairs as named arguments, where the
- argument name is the key and the argument value is the value.
-
- Returns:
- An iterator over (key, value) tuples given in the input.
-
- Raises:
- TypeError: If more than one unnamed argument is given.
- """
- if len(args) > 1:
- raise TypeError('Expected at most 1 unnamed arguments, got %d'
- % len(args))
- obj = args[0] if args else dict()
- if isinstance(obj, dict):
- return itertools.chain(obj.items(), kwargs.items())
- # Assume obj is an iterable of 2-tuples.
- return itertools.chain(obj, kwargs.items())
-
- def make_default(self, keys=None):
- """Convert OneOf objects into their default configs.
-
- Recursively calls into Config objects.
-
- Args:
- keys: Iterable of key names to check. If None, all keys in self will be
- used.
- """
- if keys is None:
- keys = self.keys()
- for k in keys:
- # Replace OneOf with its default value.
- if isinstance(self[k], OneOf):
- self[k] = self[k].default()
- # Recursively call into all Config objects, even those that came from
- # OneOf objects in the previous code line (for nested OneOf objects).
- if isinstance(self[k], Config):
- self[k].make_default()
-
- def update(self, *args, **kwargs):
- """Same as dict.update except nested Config objects are updated.
-
- Args:
- *args: (Optional) Pass a dict or list of (key, value) 2-tuples as unnamed
- argument.
- **kwargs: (Optional) Pass (key, value) pairs as named arguments, where the
- argument name is the key and the argument value is the value.
- """
- key_set = set(self.keys())
- for k, v in self._update_iterator(*args, **kwargs):
- if k in key_set:
- key_set.remove(k) # This key is updated so exclude from make_default.
- if k in self and isinstance(self[k], Config) and isinstance(v, dict):
- self[k].update(v)
- elif k in self and isinstance(self[k], OneOf) and isinstance(v, dict):
- # Replace OneOf with the chosen config.
- self[k] = self[k].update(v)
- else:
- self[k] = v
- self.make_default(key_set)
-
- def strict_update(self, *args, **kwargs):
- """Same as Config.update except keys and types are not allowed to change.
-
- If a given key is not already in this instance, an exception is raised. If a
- given value does not have the same type as the existing value for the same
- key, an exception is raised. Use this method to catch config mistakes.
-
- Args:
- *args: (Optional) Pass a dict or list of (key, value) 2-tuples as unnamed
- argument.
- **kwargs: (Optional) Pass (key, value) pairs as named arguments, where the
- argument name is the key and the argument value is the value.
-
- Raises:
- TypeError: If more than one unnamed argument is given.
- TypeError: If new value type does not match existing type.
- KeyError: If a given key is not already defined in this instance.
- """
- key_set = set(self.keys())
- for k, v in self._update_iterator(*args, **kwargs):
- if k in self:
- key_set.remove(k) # This key is updated so exclude from make_default.
- if isinstance(self[k], Config):
- if not isinstance(v, dict):
- raise TypeError('dict required for Config value, got %s' % type(v))
- self[k].strict_update(v)
- elif isinstance(self[k], OneOf):
- if not isinstance(v, dict):
- raise TypeError('dict required for OneOf value, got %s' % type(v))
- # Replace OneOf with the chosen config.
- self[k] = self[k].strict_update(v)
- else:
- if not isinstance(v, type(self[k])):
- raise TypeError('Expecting type %s for key %s, got type %s'
- % (type(self[k]), k, type(v)))
- self[k] = v
- else:
- raise KeyError(
- 'Key %s does not exist. New key creation not allowed in '
- 'strict_update.' % k)
- self.make_default(key_set)
-
- @staticmethod
- def from_str(config_str):
- """Inverse of Config.__str__."""
- parsed = ast.literal_eval(config_str)
- assert isinstance(parsed, dict)
-
- def _make_config(dictionary):
- for k, v in dictionary.items():
- if isinstance(v, dict):
- dictionary[k] = _make_config(v)
- return Config(**dictionary)
- return _make_config(parsed)
-
- @staticmethod
- def parse(key_val_string):
- """Parse hyperparameter string into Config object.
-
- Format is 'key=val,key=val,...'
- Values can be any python literal, or another Config object encoded as
- 'c(key=val,key=val,...)'.
- c(...) expressions can be arbitrarily nested.
-
- Example:
- 'a=1,b=3e-5,c=[1,2,3],d="hello world",e={"a":1,"b":2},f=c(x=1,y=[10,20])'
-
- Args:
- key_val_string: The hyperparameter string.
-
- Returns:
- Config object parsed from the input string.
- """
- if not key_val_string.strip():
- return Config()
- def _pair_to_kv(pair):
- split_index = pair.find('=')
- key, val = pair[:split_index].strip(), pair[split_index+1:].strip()
- if val.startswith('c(') and val.endswith(')'):
- val = Config.parse(val[2:-1])
- else:
- val = ast.literal_eval(val)
- return key, val
- return Config(**dict([_pair_to_kv(pair)
- for pair in _comma_iterator(key_val_string)]))
-
-
-class OneOf(object):
- """Stores branching config.
-
- In some cases there may be options which each have their own set of config
- params. For example, if specifying config for an environment, each environment
- can have custom config options. OneOf is a way to organize branching config.
-
- Usage example:
- one_of = OneOf(
- [Config(a=1, b=2),
- Config(a=2, c='hello'),
- Config(a=3, d=10, e=-10)],
- a=1)
- config = one_of.strict_update(Config(a=3, d=20))
- config == {'a': 3, 'd': 20, 'e': -10}
- """
-
- def __init__(self, choices, **kwargs):
- """Constructor.
-
- Usage: OneOf([Config(...), Config(...), ...], attribute=default_value)
-
- Args:
- choices: An iterable of Config objects. When update/strict_update is
- called on this OneOf, one of these Config will be selected.
- **kwargs: Give exactly one config attribute to branch on. The value of
- this attribute during update/strict_update will determine which
- Config is used.
-
- Raises:
- ValueError: If kwargs does not contain exactly one entry. Should give one
- named argument which is used as the attribute to condition on.
- """
- if len(kwargs) != 1:
- raise ValueError(
- 'Incorrect usage. Must give exactly one named argument. The argument '
- 'name is the config attribute to condition on, and the argument '
- 'value is the default choice. Got %d named arguments.' % len(kwargs))
- key, default_value = kwargs.items()[0]
- self.key = key
- self.default_value = default_value
-
- # Make sure each choice is a Config object.
- for config in choices:
- if not isinstance(config, Config):
- raise TypeError('choices must be a list of Config objects. Got %s.'
- % type(config))
-
- # Map value for key to the config with that value.
- self.value_map = {config[key]: config for config in choices}
- self.default_config = self.value_map[self.default_value]
-
- # Make sure there are no duplicate values.
- if len(self.value_map) != len(choices):
- raise ValueError('Multiple choices given for the same value of %s.' % key)
-
- # Check that the default value is valid.
- if self.default_value not in self.value_map:
- raise ValueError(
- 'Default value is not an available choice. Got %s=%s. Choices are %s.'
- % (key, self.default_value, self.value_map.keys()))
-
- def default(self):
- return self.default_config
-
- def update(self, other):
- """Choose a config and update it.
-
- If `other` is a Config, one of the config choices is selected and updated.
- Otherwise `other` is returned.
-
- Args:
- other: Will update chosen config with this value by calling `update` on
- the config.
-
- Returns:
- The chosen config after updating it, or `other` if no config could be
- selected.
- """
- if not isinstance(other, Config):
- return other
- if self.key not in other or other[self.key] not in self.value_map:
- return other
- target = self.value_map[other[self.key]]
- target.update(other)
- return target
-
- def strict_update(self, config):
- """Choose a config and update it.
-
- `config` must be a Config object. `config` must have the key used to select
- among the config choices, and that key must have a value which one of the
- config choices has.
-
- Args:
- config: A Config object. the chosen config will be update by calling
- `strict_update`.
-
- Returns:
- The chosen config after updating it.
-
- Raises:
- TypeError: If `config` is not a Config instance.
- ValueError: If `config` does not have the branching key in its key set.
- ValueError: If the value of the config's branching key is not one of the
- valid choices.
- """
- if not isinstance(config, Config):
- raise TypeError('Expecting Config instance, got %s.' % type(config))
- if self.key not in config:
- raise ValueError(
- 'Branching key %s required but not found in %s' % (self.key, config))
- if config[self.key] not in self.value_map:
- raise ValueError(
- 'Value %s for key %s is not a possible choice. Choices are %s.'
- % (config[self.key], self.key, self.value_map.keys()))
- target = self.value_map[config[self.key]]
- target.strict_update(config)
- return target
-
-
-def _next_comma(string, start_index):
- """Finds the position of the next comma not used in a literal collection."""
- paren_count = 0
- for i in xrange(start_index, len(string)):
- c = string[i]
- if c == '(' or c == '[' or c == '{':
- paren_count += 1
- elif c == ')' or c == ']' or c == '}':
- paren_count -= 1
- if paren_count == 0 and c == ',':
- return i
- return -1
-
-
-def _comma_iterator(string):
- index = 0
- while 1:
- next_index = _next_comma(string, index)
- if next_index == -1:
- yield string[index:]
- return
- yield string[index:next_index]
- index = next_index + 1
diff --git a/research/brain_coder/common/config_lib_test.py b/research/brain_coder/common/config_lib_test.py
deleted file mode 100644
index cdc96f92d2428f06e780930979662fdfda92e3f5..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/config_lib_test.py
+++ /dev/null
@@ -1,425 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for common.config_lib."""
-
-import tensorflow as tf
-
-from common import config_lib # brain coder
-
-
-class ConfigLibTest(tf.test.TestCase):
-
- def testConfig(self):
- config = config_lib.Config(hello='world', foo='bar', num=123, f=56.7)
- self.assertEqual('world', config.hello)
- self.assertEqual('bar', config['foo'])
- config.hello = 'everyone'
- config['bar'] = 9000
- self.assertEqual('everyone', config['hello'])
- self.assertEqual(9000, config.bar)
- self.assertEqual(5, len(config))
-
- def testConfigUpdate(self):
- config = config_lib.Config(a=1, b=2, c=3)
- config.update({'b': 10, 'd': 4})
- self.assertEqual({'a': 1, 'b': 10, 'c': 3, 'd': 4}, config)
-
- config = config_lib.Config(a=1, b=2, c=3)
- config.update(b=10, d=4)
- self.assertEqual({'a': 1, 'b': 10, 'c': 3, 'd': 4}, config)
-
- config = config_lib.Config(a=1, b=2, c=3)
- config.update({'e': 5}, b=10, d=4)
- self.assertEqual({'a': 1, 'b': 10, 'c': 3, 'd': 4, 'e': 5}, config)
-
- config = config_lib.Config(
- a=1,
- b=2,
- x=config_lib.Config(
- l='a',
- y=config_lib.Config(m=1, n=2),
- z=config_lib.Config(
- q=config_lib.Config(a=10, b=20),
- r=config_lib.Config(s=1, t=2))))
- config.update(x={'y': {'m': 10}, 'z': {'r': {'s': 5}}})
- self.assertEqual(
- config_lib.Config(
- a=1, b=2,
- x=config_lib.Config(
- l='a',
- y=config_lib.Config(m=10, n=2),
- z=config_lib.Config(
- q=config_lib.Config(a=10, b=20),
- r=config_lib.Config(s=5, t=2)))),
- config)
-
- config = config_lib.Config(
- foo='bar',
- num=100,
- x=config_lib.Config(a=1, b=2, c=config_lib.Config(h=10, i=20, j=30)),
- y=config_lib.Config(qrs=5, tuv=10),
- d={'a': 1, 'b': 2},
- l=[1, 2, 3])
- config.update(
- config_lib.Config(
- foo='hat',
- num=50.5,
- x={'a': 5, 'z': -10},
- y=config_lib.Config(wxyz=-1)),
- d={'a': 10, 'c': 20},
- l=[3, 4, 5, 6])
- self.assertEqual(
- config_lib.Config(
- foo='hat',
- num=50.5,
- x=config_lib.Config(a=5, b=2, z=-10,
- c=config_lib.Config(h=10, i=20, j=30)),
- y=config_lib.Config(qrs=5, tuv=10, wxyz=-1),
- d={'a': 10, 'c': 20},
- l=[3, 4, 5, 6]),
- config)
- self.assertTrue(isinstance(config.x, config_lib.Config))
- self.assertTrue(isinstance(config.x.c, config_lib.Config))
- self.assertTrue(isinstance(config.y, config_lib.Config))
-
- config = config_lib.Config(
- foo='bar',
- num=100,
- x=config_lib.Config(a=1, b=2, c=config_lib.Config(h=10, i=20, j=30)),
- y=config_lib.Config(qrs=5, tuv=10),
- d={'a': 1, 'b': 2},
- l=[1, 2, 3])
- config.update(
- config_lib.Config(
- foo=1234,
- num='hello',
- x={'a': 5, 'z': -10, 'c': {'h': -5, 'k': 40}},
- y=[1, 2, 3, 4],
- d='stuff',
- l={'a': 1, 'b': 2}))
- self.assertEqual(
- config_lib.Config(
- foo=1234,
- num='hello',
- x=config_lib.Config(a=5, b=2, z=-10,
- c=config_lib.Config(h=-5, i=20, j=30, k=40)),
- y=[1, 2, 3, 4],
- d='stuff',
- l={'a': 1, 'b': 2}),
- config)
- self.assertTrue(isinstance(config.x, config_lib.Config))
- self.assertTrue(isinstance(config.x.c, config_lib.Config))
- self.assertTrue(isinstance(config.y, list))
-
- def testConfigStrictUpdate(self):
- config = config_lib.Config(a=1, b=2, c=3)
- config.strict_update({'b': 10, 'c': 20})
- self.assertEqual({'a': 1, 'b': 10, 'c': 20}, config)
-
- config = config_lib.Config(a=1, b=2, c=3)
- config.strict_update(b=10, c=20)
- self.assertEqual({'a': 1, 'b': 10, 'c': 20}, config)
-
- config = config_lib.Config(a=1, b=2, c=3, d=4)
- config.strict_update({'d': 100}, b=10, a=20)
- self.assertEqual({'a': 20, 'b': 10, 'c': 3, 'd': 100}, config)
-
- config = config_lib.Config(
- a=1,
- b=2,
- x=config_lib.Config(
- l='a',
- y=config_lib.Config(m=1, n=2),
- z=config_lib.Config(
- q=config_lib.Config(a=10, b=20),
- r=config_lib.Config(s=1, t=2))))
- config.strict_update(x={'y': {'m': 10}, 'z': {'r': {'s': 5}}})
- self.assertEqual(
- config_lib.Config(
- a=1, b=2,
- x=config_lib.Config(
- l='a',
- y=config_lib.Config(m=10, n=2),
- z=config_lib.Config(
- q=config_lib.Config(a=10, b=20),
- r=config_lib.Config(s=5, t=2)))),
- config)
-
- config = config_lib.Config(
- foo='bar',
- num=100,
- x=config_lib.Config(a=1, b=2, c=config_lib.Config(h=10, i=20, j=30)),
- y=config_lib.Config(qrs=5, tuv=10),
- d={'a': 1, 'b': 2},
- l=[1, 2, 3])
- config.strict_update(
- config_lib.Config(
- foo='hat',
- num=50,
- x={'a': 5, 'c': {'h': 100}},
- y=config_lib.Config(tuv=-1)),
- d={'a': 10, 'c': 20},
- l=[3, 4, 5, 6])
- self.assertEqual(
- config_lib.Config(
- foo='hat',
- num=50,
- x=config_lib.Config(a=5, b=2,
- c=config_lib.Config(h=100, i=20, j=30)),
- y=config_lib.Config(qrs=5, tuv=-1),
- d={'a': 10, 'c': 20},
- l=[3, 4, 5, 6]),
- config)
-
- def testConfigStrictUpdateFail(self):
- config = config_lib.Config(a=1, b=2, c=3, x=config_lib.Config(a=1, b=2))
- with self.assertRaises(KeyError):
- config.strict_update({'b': 10, 'c': 20, 'd': 50})
- with self.assertRaises(KeyError):
- config.strict_update(b=10, d=50)
- with self.assertRaises(KeyError):
- config.strict_update(x={'c': 3})
- with self.assertRaises(TypeError):
- config.strict_update(a='string')
- with self.assertRaises(TypeError):
- config.strict_update(x={'a': 'string'})
- with self.assertRaises(TypeError):
- config.strict_update(x=[1, 2, 3])
-
- def testConfigFromStr(self):
- config = config_lib.Config.from_str("{'c': {'d': 5}, 'b': 2, 'a': 1}")
- self.assertEqual(
- {'c': {'d': 5}, 'b': 2, 'a': 1}, config)
- self.assertTrue(isinstance(config, config_lib.Config))
- self.assertTrue(isinstance(config.c, config_lib.Config))
-
- def testConfigParse(self):
- config = config_lib.Config.parse(
- 'hello="world",num=1234.5,lst=[10,20.5,True,"hi",("a","b","c")],'
- 'dct={9:10,"stuff":"qwerty","subdict":{1:True,2:False}},'
- 'subconfig=c(a=1,b=[1,2,[3,4]],c=c(f="f",g="g"))')
- self.assertEqual(
- {'hello': 'world', 'num': 1234.5,
- 'lst': [10, 20.5, True, 'hi', ('a', 'b', 'c')],
- 'dct': {9: 10, 'stuff': 'qwerty', 'subdict': {1: True, 2: False}},
- 'subconfig': {'a': 1, 'b': [1, 2, [3, 4]], 'c': {'f': 'f', 'g': 'g'}}},
- config)
- self.assertTrue(isinstance(config, config_lib.Config))
- self.assertTrue(isinstance(config.subconfig, config_lib.Config))
- self.assertTrue(isinstance(config.subconfig.c, config_lib.Config))
- self.assertFalse(isinstance(config.dct, config_lib.Config))
- self.assertFalse(isinstance(config.dct['subdict'], config_lib.Config))
- self.assertTrue(isinstance(config.lst[4], tuple))
-
- def testConfigParseErrors(self):
- with self.assertRaises(SyntaxError):
- config_lib.Config.parse('a=[1,2,b="hello"')
- with self.assertRaises(SyntaxError):
- config_lib.Config.parse('a=1,b=c(x="a",y="b"')
- with self.assertRaises(SyntaxError):
- config_lib.Config.parse('a=1,b=c(x="a")y="b"')
- with self.assertRaises(SyntaxError):
- config_lib.Config.parse('a=1,b=c(x="a"),y="b",')
-
- def testOneOf(self):
- def make_config():
- return config_lib.Config(
- data=config_lib.OneOf(
- [config_lib.Config(task=1, a='hello'),
- config_lib.Config(task=2, a='world', b='stuff'),
- config_lib.Config(task=3, c=1234)],
- task=2),
- model=config_lib.Config(stuff=1))
-
- config = make_config()
- config.update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=1,a="hi")'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=1, a='hi'),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=2,a="hi")'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=2, a='hi', b='stuff'),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=3)'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=3, c=1234),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.update(config_lib.Config.parse(
- 'model=c(stuff=2)'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=2, a='world', b='stuff'),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=4,d=9999)'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=4, d=9999),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.update(config_lib.Config.parse(
- 'model=c(stuff=2),data=5'))
- self.assertEqual(
- config_lib.Config(
- data=5,
- model=config_lib.Config(stuff=2)),
- config)
-
- def testOneOfStrict(self):
- def make_config():
- return config_lib.Config(
- data=config_lib.OneOf(
- [config_lib.Config(task=1, a='hello'),
- config_lib.Config(task=2, a='world', b='stuff'),
- config_lib.Config(task=3, c=1234)],
- task=2),
- model=config_lib.Config(stuff=1))
-
- config = make_config()
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=1,a="hi")'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=1, a='hi'),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=2,a="hi")'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=2, a='hi', b='stuff'),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=3)'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=3, c=1234),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2)'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(task=2, a='world', b='stuff'),
- model=config_lib.Config(stuff=2)),
- config)
-
- def testNestedOneOf(self):
- def make_config():
- return config_lib.Config(
- data=config_lib.OneOf(
- [config_lib.Config(task=1, a='hello'),
- config_lib.Config(
- task=2,
- a=config_lib.OneOf(
- [config_lib.Config(x=1, y=2),
- config_lib.Config(x=-1, y=1000, z=4)],
- x=1)),
- config_lib.Config(task=3, c=1234)],
- task=2),
- model=config_lib.Config(stuff=1))
-
- config = make_config()
- config.update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=2,a=c(x=-1,z=8))'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(
- task=2,
- a=config_lib.Config(x=-1, y=1000, z=8)),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=2,a=c(x=-1,z=8))'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(
- task=2,
- a=config_lib.Config(x=-1, y=1000, z=8)),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.update(config_lib.Config.parse('model=c(stuff=2)'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(
- task=2,
- a=config_lib.Config(x=1, y=2)),
- model=config_lib.Config(stuff=2)),
- config)
-
- config = make_config()
- config.strict_update(config_lib.Config.parse('model=c(stuff=2)'))
- self.assertEqual(
- config_lib.Config(
- data=config_lib.Config(
- task=2,
- a=config_lib.Config(x=1, y=2)),
- model=config_lib.Config(stuff=2)),
- config)
-
- def testOneOfStrictErrors(self):
- def make_config():
- return config_lib.Config(
- data=config_lib.OneOf(
- [config_lib.Config(task=1, a='hello'),
- config_lib.Config(task=2, a='world', b='stuff'),
- config_lib.Config(task=3, c=1234)],
- task=2),
- model=config_lib.Config(stuff=1))
-
- config = make_config()
- with self.assertRaises(TypeError):
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2),data=[1,2,3]'))
-
- config = make_config()
- with self.assertRaises(KeyError):
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=3,c=5678,d=9999)'))
-
- config = make_config()
- with self.assertRaises(ValueError):
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2),data=c(task=4,d=9999)'))
-
- config = make_config()
- with self.assertRaises(TypeError):
- config.strict_update(config_lib.Config.parse(
- 'model=c(stuff=2),data=5'))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/common/reward.py b/research/brain_coder/common/reward.py
deleted file mode 100644
index 87e01c9c52e1ee22f2745dce12bc5e2726711ff7..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/reward.py
+++ /dev/null
@@ -1,390 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Reward functions, distance functions, and reward managers."""
-
-from abc import ABCMeta
-from abc import abstractmethod
-from math import log
-
-
-# All sequences here are assumed to be lists of ints bounded
-# between 0 and `base`-1 (inclusive).
-
-
-#################################
-### Scalar Distance Functions ###
-#################################
-
-
-def abs_diff(a, b, base=0):
- """Absolute value of difference between scalars.
-
- abs_diff is symmetric, i.e. `a` and `b` are interchangeable.
-
- Args:
- a: First argument. An int.
- b: Seconds argument. An int.
- base: Dummy argument so that the argument signature matches other scalar
- diff functions. abs_diff is the same in all bases.
-
- Returns:
- abs(a - b).
- """
- del base # Unused.
- return abs(a - b)
-
-
-def mod_abs_diff(a, b, base):
- """Shortest distance between `a` and `b` in the modular integers base `base`.
-
- The smallest distance between a and b is returned.
- Example: mod_abs_diff(1, 99, 100) ==> 2. It is not 98.
-
- mod_abs_diff is symmetric, i.e. `a` and `b` are interchangeable.
-
- Args:
- a: First argument. An int.
- b: Seconds argument. An int.
- base: The modulo base. A positive int.
-
- Returns:
- Shortest distance.
- """
- diff = abs(a - b)
- if diff >= base:
- diff %= base
- return min(diff, (-diff) + base)
-
-
-###############################
-### List Distance Functions ###
-###############################
-
-
-def absolute_distance(pred, target, base, scalar_diff_fn=abs_diff):
- """Asymmetric list distance function.
-
- List distance is the sum of element-wise distances, like Hamming distance, but
- where `pred` can be longer or shorter than `target`. For each position in both
- `pred` and `target`, distance between those elements is computed with
- `scalar_diff_fn`. For missing or extra elements in `pred`, the maximum
- distance is assigned, which is equal to `base`.
-
- Distance is 0 when `pred` and `target` are identical, and will be a positive
- integer when they are not.
-
- Args:
- pred: Prediction list. Distance from this list is computed.
- target: Target list. Distance to this list is computed.
- base: The integer base to use. For example, a list of chars would use base
- 256.
- scalar_diff_fn: Element-wise distance function.
-
- Returns:
- List distance between `pred` and `target`.
- """
- d = 0
- for i, target_t in enumerate(target):
- if i >= len(pred):
- d += base # A missing slot is worth the max distance.
- else:
- # Add element-wise distance for this slot.
- d += scalar_diff_fn(pred[i], target_t, base)
- if len(pred) > len(target):
- # Each extra slot is worth the max distance.
- d += (len(pred) - len(target)) * base
- return d
-
-
-def log_absolute_distance(pred, target, base):
- """Asymmetric list distance function that uses log distance.
-
- A list distance which computes sum of element-wise distances, similar to
- `absolute_distance`. Unlike `absolute_distance`, this scales the resulting
- distance to be a float.
-
- Element-wise distance are log-scale. Distance between two list changes
- relatively less for elements that are far apart, but changes a lot (goes to 0
- faster) when values get close together.
-
- Args:
- pred: List of ints. Computes distance from this list to the target.
- target: List of ints. This is the "correct" list which the prediction list
- is trying to match.
- base: Integer base.
-
- Returns:
- Float distance normalized so that when `pred` is at most as long as `target`
- the distance is between 0.0 and 1.0. Distance grows unboundedly large
- as `pred` grows past `target` in length.
- """
- if not target:
- length_normalizer = 1.0
- if not pred:
- # Distance between [] and [] is 0.0 since they are equal.
- return 0.0
- else:
- length_normalizer = float(len(target))
- # max_dist is the maximum element-wise distance, before taking log and
- # scaling. Since we use `mod_abs_diff`, it would be (base // 2), but we add
- # 1 to it so that missing or extra positions get the maximum penalty.
- max_dist = base // 2 + 1
-
- # The log-distance will be scaled by a factor.
- # Note: +1 is added to the numerator and denominator to avoid log(0). This
- # only has a translational effect, i.e. log(dist + 1) / log(max_dist + 1).
- factor = log(max_dist + 1)
-
- d = 0.0 # Total distance to be computed.
- for i, target_t in enumerate(target):
- if i >= len(pred):
- # Assign the max element-wise distance for missing positions. This is 1.0
- # after scaling.
- d += 1.0
- else:
- # Add the log-dist divided by a scaling factor.
- d += log(mod_abs_diff(pred[i], target_t, base) + 1) / factor
- if len(pred) > len(target):
- # Add the max element-wise distance for each extra position.
- # Since max dist after scaling is 1, this is just the difference in list
- # lengths.
- d += (len(pred) - len(target))
- return d / length_normalizer # Normalize again by the target length.
-
-
-########################
-### Reward Functions ###
-########################
-
-# Reward functions assign reward based on program output.
-# Warning: only use these functions as the terminal rewards in episodes, i.e.
-# for the "final" programs.
-
-
-def absolute_distance_reward(pred, target, base, scalar_diff_fn=abs_diff):
- """Reward function based on absolute_distance function.
-
- Maximum reward, 1.0, is given when the lists are equal. Reward is scaled
- so that 0.0 reward is given when `pred` is the empty list (assuming `target`
- is not empty). Reward can go negative when `pred` is longer than `target`.
-
- This is an asymmetric reward function, so which list is the prediction and
- which is the target matters.
-
- Args:
- pred: Prediction sequence. This should be the sequence outputted by the
- generated code. List of ints n, where 0 <= n < base.
- target: Target sequence. The correct sequence that the generated code needs
- to output. List of ints n, where 0 <= n < base.
- base: Base of the computation.
- scalar_diff_fn: Element-wise distance function.
-
- Returns:
- Reward computed based on `pred` and `target`. A float.
- """
- unit_dist = float(base * len(target))
- if unit_dist == 0:
- unit_dist = base
- dist = absolute_distance(pred, target, base, scalar_diff_fn=scalar_diff_fn)
- return (unit_dist - dist) / unit_dist
-
-
-def absolute_mod_distance_reward(pred, target, base):
- """Same as `absolute_distance_reward` but `mod_abs_diff` scalar diff is used.
-
- Args:
- pred: Prediction sequence. This should be the sequence outputted by the
- generated code. List of ints n, where 0 <= n < base.
- target: Target sequence. The correct sequence that the generated code needs
- to output. List of ints n, where 0 <= n < base.
- base: Base of the computation.
-
- Returns:
- Reward computed based on `pred` and `target`. A float.
- """
- return absolute_distance_reward(pred, target, base, mod_abs_diff)
-
-
-def absolute_log_distance_reward(pred, target, base):
- """Compute reward using `log_absolute_distance`.
-
- Maximum reward, 1.0, is given when the lists are equal. Reward is scaled
- so that 0.0 reward is given when `pred` is the empty list (assuming `target`
- is not empty). Reward can go negative when `pred` is longer than `target`.
-
- This is an asymmetric reward function, so which list is the prediction and
- which is the target matters.
-
- This reward function has the nice property that much more reward is given
- for getting the correct value (at each position) than for there being any
- value at all. For example, in base 100, lets say pred = [1] * 1000
- and target = [10] * 1000. A lot of reward would be given for being 80%
- accurate (worst element-wise distance is 50, distances here are 9) using
- `absolute_distance`. `log_absolute_distance` on the other hand will give
- greater and greater reward increments the closer each predicted value gets to
- the target. That makes the reward given for accuracy somewhat independant of
- the base.
-
- Args:
- pred: Prediction sequence. This should be the sequence outputted by the
- generated code. List of ints n, where 0 <= n < base.
- target: Target sequence. The correct sequence that the generated code needs
- to output. List of ints n, where 0 <= n < base.
- base: Base of the computation.
-
- Returns:
- Reward computed based on `pred` and `target`. A float.
- """
- return 1.0 - log_absolute_distance(pred, target, base)
-
-
-#######################
-### Reward Managers ###
-#######################
-
-# Reward managers assign reward to many code attempts throughout an episode.
-
-
-class RewardManager(object):
- """Reward managers administer reward across an episode.
-
- Reward managers are used for "editor" environments. These are environments
- where the agent has some way to edit its code over time, and run its code
- many time in the same episode, so that it can make incremental improvements.
-
- Reward managers are instantiated with a target sequence, which is the known
- correct program output. The manager is called on the output from a proposed
- code, and returns reward. If many proposal outputs are tried, reward may be
- some stateful function that takes previous tries into account. This is done,
- in part, so that an agent cannot accumulate unbounded reward just by trying
- junk programs as often as possible. So reward managers should not give the
- same reward twice if the next proposal is not better than the last.
- """
- __metaclass__ = ABCMeta
-
- def __init__(self, target, base, distance_fn=absolute_distance):
- self._target = list(target)
- self._base = base
- self._distance_fn = distance_fn
-
- @abstractmethod
- def __call__(self, sequence):
- """Call this reward manager like a function to get reward.
-
- Calls to reward manager are stateful, and will take previous sequences
- into account. Repeated calls with the same sequence may produce different
- rewards.
-
- Args:
- sequence: List of integers (each between 0 and base - 1). This is the
- proposal sequence. Reward will be computed based on the distance
- from this sequence to the target (distance function and target are
- given in the constructor), as well as previous sequences tried during
- the lifetime of this object.
-
- Returns:
- Float value. The reward received from this call.
- """
- return 0.0
-
-
-class DeltaRewardManager(RewardManager):
- """Simple reward manager that assigns reward for the net change in distance.
-
- Given some (possibly asymmetric) list distance function, gives reward for
- relative changes in prediction distance to the target.
-
- For example, if on the first call the distance is 3.0, the change in distance
- is -3 (from starting distance of 0). That relative change will be scaled to
- produce a negative reward for this step. On the next call, the distance is 2.0
- which is a +1 change, and that will be scaled to give a positive reward.
- If the final call has distance 0 (the target is achieved), that is another
- positive change of +2. The total reward across all 3 calls is then 0, which is
- the highest posible episode total.
-
- Reward is scaled so that the maximum element-wise distance is worth 1.0.
- Maximum total episode reward attainable is 0.
- """
-
- def __init__(self, target, base, distance_fn=absolute_distance):
- super(DeltaRewardManager, self).__init__(target, base, distance_fn)
- self._last_diff = 0
-
- def _diff(self, seq):
- return self._distance_fn(seq, self._target, self._base)
-
- def _delta_reward(self, seq):
- # Reward is relative to previous sequence diff.
- # Reward is scaled so that maximum token difference is worth 1.0.
- # Reward = (last_diff - this_diff) / self.base.
- # Reward is positive if this sequence is closer to the target than the
- # previous sequence, and negative if this sequence is further away.
- diff = self._diff(seq)
- reward = (self._last_diff - diff) / float(self._base)
- self._last_diff = diff
- return reward
-
- def __call__(self, seq):
- return self._delta_reward(seq)
-
-
-class FloorRewardManager(RewardManager):
- """Assigns positive reward for each step taken closer to the target.
-
- Given some (possibly asymmetric) list distance function, gives reward for
- whenever a new episode minimum distance is reached. No reward is given if
- the distance regresses to a higher value, so that the sum of rewards
- for the episode is positive.
-
- Reward is scaled so that the maximum element-wise distance is worth 1.0.
- Maximum total episode reward attainable is len(target).
-
- If the prediction sequence is longer than the target, a reward of -1 is given.
- Subsequence predictions which are also longer get 0 reward. The -1 penalty
- will be canceled out with a +1 reward when a prediction is given which is at
- most the length of the target.
- """
-
- def __init__(self, target, base, distance_fn=absolute_distance):
- super(FloorRewardManager, self).__init__(target, base, distance_fn)
- self._last_diff = 0
- self._min_diff = self._max_diff()
- self._too_long_penality_given = False
-
- def _max_diff(self):
- return self._distance_fn([], self._target, self._base)
-
- def _diff(self, seq):
- return self._distance_fn(seq, self._target, self._base)
-
- def _delta_reward(self, seq):
- # Reward is only given if this sequence is closer to the target than any
- # previous sequence.
- # Reward is scaled so that maximum token difference is worth 1.0
- # Reward = (min_diff - this_diff) / self.base
- # Reward is always positive.
- diff = self._diff(seq)
- if diff < self._min_diff:
- reward = (self._min_diff - diff) / float(self._base)
- self._min_diff = diff
- else:
- reward = 0.0
- return reward
-
- def __call__(self, seq):
- if len(seq) > len(self._target): # Output is too long.
- if not self._too_long_penality_given:
- self._too_long_penality_given = True
- reward = -1.0
- else:
- reward = 0.0 # Don't give this penalty more than once.
- return reward
-
- reward = self._delta_reward(seq)
- if self._too_long_penality_given:
- reward += 1.0 # Return the subtracted reward.
- self._too_long_penality_given = False
- return reward
-
diff --git a/research/brain_coder/common/reward_test.py b/research/brain_coder/common/reward_test.py
deleted file mode 100644
index 38a1d4ace38cbc945362e52adb90cc9dd62f1be7..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/reward_test.py
+++ /dev/null
@@ -1,311 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for common.reward."""
-
-from math import log
-import numpy as np
-import tensorflow as tf
-
-from common import reward # brain coder
-
-
-class RewardTest(tf.test.TestCase):
-
- def testAbsDiff(self):
- self.assertEqual(5, reward.abs_diff(15, 20))
- self.assertEqual(5, reward.abs_diff(20, 15))
-
- def testModAbsDiff(self):
- self.assertEqual(5, reward.mod_abs_diff(15, 20, 25))
- self.assertEqual(5, reward.mod_abs_diff(20, 15, 25))
- self.assertEqual(2, reward.mod_abs_diff(1, 24, 25))
- self.assertEqual(2, reward.mod_abs_diff(24, 1, 25))
-
- self.assertEqual(0, reward.mod_abs_diff(0, 0, 5))
- self.assertEqual(1, reward.mod_abs_diff(0, 1, 5))
- self.assertEqual(2, reward.mod_abs_diff(0, 2, 5))
- self.assertEqual(2, reward.mod_abs_diff(0, 3, 5))
- self.assertEqual(1, reward.mod_abs_diff(0, 4, 5))
-
- self.assertEqual(0, reward.mod_abs_diff(-1, 4, 5))
- self.assertEqual(1, reward.mod_abs_diff(-5, 4, 5))
- self.assertEqual(1, reward.mod_abs_diff(-7, 4, 5))
- self.assertEqual(1, reward.mod_abs_diff(13, 4, 5))
- self.assertEqual(1, reward.mod_abs_diff(15, 4, 5))
-
- def testAbsoluteDistance_AbsDiffMethod(self):
- self.assertEqual(
- 4,
- reward.absolute_distance([0], [4], 5, scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 0,
- reward.absolute_distance([4], [4], 5, scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 0,
- reward.absolute_distance([], [], 5, scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([1], [], 5, scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([], [1], 5, scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 0,
- reward.absolute_distance([1, 2, 3], [1, 2, 3], 5,
- scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 1,
- reward.absolute_distance([1, 2, 4], [1, 2, 3], 5,
- scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 1,
- reward.absolute_distance([1, 2, 2], [1, 2, 3], 5,
- scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([1, 2], [1, 2, 3], 5,
- scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([1, 2, 3, 4], [1, 2, 3], 5,
- scalar_diff_fn=reward.abs_diff))
- self.assertEqual(
- 6,
- reward.absolute_distance([4, 4, 4], [1, 2, 3], 5,
- scalar_diff_fn=reward.abs_diff))
-
- def testAbsoluteDistance_ModDiffMethod(self):
- self.assertEqual(
- 1,
- reward.absolute_distance([0], [4], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 0,
- reward.absolute_distance([4], [4], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 0,
- reward.absolute_distance([], [], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([1], [], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([], [1], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 0,
- reward.absolute_distance([1, 2, 3], [1, 2, 3], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 1,
- reward.absolute_distance([1, 2, 4], [1, 2, 3], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 1,
- reward.absolute_distance([1, 2, 2], [1, 2, 3], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([1, 2], [1, 2, 3], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([1, 2, 3, 4], [1, 2, 3], 5,
- scalar_diff_fn=reward.mod_abs_diff))
- self.assertEqual(
- 5,
- reward.absolute_distance([4, 4, 4], [1, 2, 3], 5,
- scalar_diff_fn=reward.mod_abs_diff))
-
- def testLogAbsoluteDistance(self):
- def log_diff(diff, base):
- return log(diff + 1) / log(base // 2 + 2)
-
- self.assertEqual(
- log_diff(1, 5),
- reward.log_absolute_distance([0], [4], 5))
- self.assertEqual(
- log_diff(2, 5),
- reward.log_absolute_distance([1], [4], 5))
- self.assertEqual(
- log_diff(2, 5),
- reward.log_absolute_distance([2], [4], 5))
- self.assertEqual(
- log_diff(1, 5),
- reward.log_absolute_distance([3], [4], 5))
- self.assertEqual(
- log_diff(3, 5), # max_dist = base // 2 + 1 = 3
- reward.log_absolute_distance([], [4], 5))
- self.assertEqual(
- 0 + log_diff(3, 5), # max_dist = base // 2 + 1 = 3
- reward.log_absolute_distance([4, 4], [4], 5))
- self.assertEqual(
- 0,
- reward.log_absolute_distance([4], [4], 5))
- self.assertEqual(
- 0,
- reward.log_absolute_distance([], [], 5))
- self.assertEqual(
- 1,
- reward.log_absolute_distance([1], [], 5))
- self.assertEqual(
- 1,
- reward.log_absolute_distance([], [1], 5))
-
- self.assertEqual(
- 0,
- reward.log_absolute_distance([1, 2, 3], [1, 2, 3], 5))
- self.assertEqual(
- log_diff(1, 5) / 3, # divided by target length.
- reward.log_absolute_distance([1, 2, 4], [1, 2, 3], 5))
- self.assertEqual(
- log_diff(1, 5) / 3,
- reward.log_absolute_distance([1, 2, 2], [1, 2, 3], 5))
- self.assertEqual(
- log_diff(3, 5) / 3, # max_dist
- reward.log_absolute_distance([1, 2], [1, 2, 3], 5))
- self.assertEqual(
- log_diff(3, 5) / 3, # max_dist
- reward.log_absolute_distance([1, 2, 3, 4], [1, 2, 3], 5))
- # Add log differences for each position.
- self.assertEqual(
- (log_diff(2, 5) + log_diff(2, 5) + log_diff(1, 5)) / 3,
- reward.log_absolute_distance([4, 4, 4], [1, 2, 3], 5))
-
- def testAbsoluteDistanceReward(self):
- self.assertEqual(
- 1,
- reward.absolute_distance_reward([1, 2, 3], [1, 2, 3], 5))
- self.assertEqual(
- 1 - 1 / (5 * 3.), # 1 - distance / (base * target_len)
- reward.absolute_distance_reward([1, 2, 4], [1, 2, 3], 5))
- self.assertEqual(
- 1 - 1 / (5 * 3.),
- reward.absolute_distance_reward([1, 2, 2], [1, 2, 3], 5))
- self.assertTrue(np.isclose(
- 1 - 5 / (5 * 3.),
- reward.absolute_distance_reward([1, 2], [1, 2, 3], 5)))
- self.assertTrue(np.isclose(
- 1 - 5 / (5 * 3.),
- reward.absolute_distance_reward([1, 2, 3, 4], [1, 2, 3], 5)))
- # Add log differences for each position.
- self.assertEqual(
- 1 - (3 + 2 + 1) / (5 * 3.),
- reward.absolute_distance_reward([4, 4, 4], [1, 2, 3], 5))
- self.assertEqual(
- 1,
- reward.absolute_distance_reward([], [], 5))
-
- def testAbsoluteModDistanceReward(self):
- self.assertEqual(
- 1,
- reward.absolute_mod_distance_reward([1, 2, 3], [1, 2, 3], 5))
- self.assertEqual(
- 1 - 1 / (5 * 3.), # 1 - distance / (base * target_len)
- reward.absolute_mod_distance_reward([1, 2, 4], [1, 2, 3], 5))
- self.assertEqual(
- 1 - 1 / (5 * 3.),
- reward.absolute_mod_distance_reward([1, 2, 2], [1, 2, 3], 5))
- self.assertTrue(np.isclose(
- 1 - 5 / (5 * 3.),
- reward.absolute_mod_distance_reward([1, 2], [1, 2, 3], 5)))
- self.assertTrue(np.isclose(
- 1 - 5 / (5 * 3.),
- reward.absolute_mod_distance_reward([1, 2, 3, 4], [1, 2, 3], 5)))
- # Add log differences for each position.
- self.assertTrue(np.isclose(
- 1 - (2 + 2 + 1) / (5 * 3.),
- reward.absolute_mod_distance_reward([4, 4, 4], [1, 2, 3], 5)))
- self.assertTrue(np.isclose(
- 1 - (1 + 2 + 2) / (5 * 3.),
- reward.absolute_mod_distance_reward([0, 1, 2], [4, 4, 4], 5)))
- self.assertEqual(
- 1,
- reward.absolute_mod_distance_reward([], [], 5))
-
- def testAbsoluteLogDistanceReward(self):
- def log_diff(diff, base):
- return log(diff + 1) / log(base // 2 + 2)
-
- self.assertEqual(
- 1,
- reward.absolute_log_distance_reward([1, 2, 3], [1, 2, 3], 5))
- self.assertEqual(
- 1 - log_diff(1, 5) / 3, # divided by target length.
- reward.absolute_log_distance_reward([1, 2, 4], [1, 2, 3], 5))
- self.assertEqual(
- 1 - log_diff(1, 5) / 3,
- reward.absolute_log_distance_reward([1, 2, 2], [1, 2, 3], 5))
- self.assertEqual(
- 1 - log_diff(3, 5) / 3, # max_dist
- reward.absolute_log_distance_reward([1, 2], [1, 2, 3], 5))
- self.assertEqual(
- 1 - log_diff(3, 5) / 3, # max_dist
- reward.absolute_log_distance_reward([1, 2, 3, 4], [1, 2, 3], 5))
- # Add log differences for each position.
- self.assertEqual(
- 1 - (log_diff(2, 5) + log_diff(2, 5) + log_diff(1, 5)) / 3,
- reward.absolute_log_distance_reward([4, 4, 4], [1, 2, 3], 5))
- self.assertEqual(
- 1 - (log_diff(1, 5) + log_diff(2, 5) + log_diff(2, 5)) / 3,
- reward.absolute_log_distance_reward([0, 1, 2], [4, 4, 4], 5))
- self.assertEqual(
- 1,
- reward.absolute_log_distance_reward([], [], 5))
-
- def testDeltaRewardManager(self):
- reward_manager = reward.DeltaRewardManager(
- [1, 2, 3, 4], base=5, distance_fn=reward.absolute_distance)
- self.assertEqual(-3, reward_manager([1]))
- self.assertEqual(0, reward_manager([1]))
- self.assertEqual(4 / 5., reward_manager([1, 3]))
- self.assertEqual(-4 / 5, reward_manager([1]))
- self.assertEqual(3, reward_manager([1, 2, 3, 4]))
- self.assertEqual(-1, reward_manager([1, 2, 3]))
- self.assertEqual(0, reward_manager([1, 2, 3, 4, 3]))
- self.assertEqual(-1, reward_manager([1, 2, 3, 4, 3, 2]))
- self.assertEqual(2, reward_manager([1, 2, 3, 4]))
- self.assertEqual(0, reward_manager([1, 2, 3, 4]))
- self.assertEqual(0, reward_manager([1, 2, 3, 4]))
-
- def testFloorRewardMananger(self):
- reward_manager = reward.FloorRewardManager(
- [1, 2, 3, 4], base=5, distance_fn=reward.absolute_distance)
- self.assertEqual(1, reward_manager([1]))
- self.assertEqual(0, reward_manager([1]))
- self.assertEqual(4 / 5., reward_manager([1, 3]))
- self.assertEqual(0, reward_manager([1]))
- self.assertEqual(1 / 5., reward_manager([1, 2]))
- self.assertEqual(0, reward_manager([0, 1]))
- self.assertEqual(0, reward_manager([]))
- self.assertEqual(0, reward_manager([1, 2]))
- self.assertEqual(2, reward_manager([1, 2, 3, 4]))
- self.assertEqual(0, reward_manager([1, 2, 3]))
- self.assertEqual(-1, reward_manager([1, 2, 3, 4, 3]))
- self.assertEqual(0, reward_manager([1, 2, 3, 4, 3, 2]))
- self.assertEqual(1, reward_manager([1, 2, 3, 4]))
- self.assertEqual(0, reward_manager([1, 2, 3, 4]))
- self.assertEqual(0, reward_manager([1, 2, 3, 4]))
-
- reward_manager = reward.FloorRewardManager(
- [1, 2, 3, 4], base=5, distance_fn=reward.absolute_distance)
- self.assertEqual(1, reward_manager([1]))
- self.assertEqual(-1, reward_manager([1, 0, 0, 0, 0, 0]))
- self.assertEqual(0, reward_manager([1, 2, 3, 4, 0, 0]))
- self.assertEqual(0, reward_manager([1, 2, 3, 4, 0]))
- self.assertEqual(1, reward_manager([]))
- self.assertEqual(0, reward_manager([]))
- self.assertEqual(0, reward_manager([1]))
- self.assertEqual(1, reward_manager([1, 2]))
- self.assertEqual(-1, reward_manager([1, 2, 3, 4, 0, 0]))
- self.assertEqual(0, reward_manager([1, 1, 1, 1, 1]))
- self.assertEqual(1 + 2, reward_manager([1, 2, 3, 4]))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/common/rollout.py b/research/brain_coder/common/rollout.py
deleted file mode 100644
index e377aa662db640dfa907de83d32875cc096c4295..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/rollout.py
+++ /dev/null
@@ -1,306 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Utilities related to computing training batches from episode rollouts.
-
-Implementations here are based on code from Open AI:
-https://github.com/openai/universe-starter-agent/blob/master/a3c.py.
-"""
-
-from collections import namedtuple
-import numpy as np
-import scipy.signal
-
-from common import utils # brain coder
-
-
-class Rollout(object):
- """Holds a rollout for an episode.
-
- A rollout is a record of the states observed in some environment and actions
- taken by the agent to arrive at those states. Other information includes
- rewards received after each action, values estimated for each state, whether
- the rollout concluded the episide, and total reward received. Everything
- should be given in time order.
-
- At each time t, the agent sees state s_t, takes action a_t, and then receives
- reward r_t. The agent may optionally estimate a state value V(s_t) for each
- state.
-
- For an episode of length T:
- states = [s_0, ..., s_(T-1)]
- actions = [a_0, ..., a_(T-1)]
- rewards = [r_0, ..., r_(T-1)]
- values = [V(s_0), ..., V(s_(T-1))]
-
- Note that there is an extra state s_T observed after taking action a_(T-1),
- but this is not included in the rollout.
-
- Rollouts have an `terminated` attribute which is True when the rollout is
- "finalized", i.e. it holds a full episode. terminated will be False when
- time steps are still being added to it.
- """
-
- def __init__(self):
- self.states = []
- self.actions = []
- self.rewards = []
- self.values = []
- self.total_reward = 0.0
- self.terminated = False
-
- def add(self, state, action, reward, value=0.0, terminated=False):
- """Add the next timestep to this rollout.
-
- Args:
- state: The state observed at the start of this timestep.
- action: The action taken after observing the given state.
- reward: The reward received for taking the given action.
- value: The value estimated for the given state.
- terminated: Whether this timestep ends the episode.
-
- Raises:
- ValueError: If this.terminated is already True, meaning that the episode
- has already ended.
- """
- if self.terminated:
- raise ValueError(
- 'Trying to add timestep to an already terminal rollout.')
- self.states += [state]
- self.actions += [action]
- self.rewards += [reward]
- self.values += [value]
- self.terminated = terminated
- self.total_reward += reward
-
- def add_many(self, states, actions, rewards, values=None, terminated=False):
- """Add many timesteps to this rollout.
-
- Arguments are the same as `add`, but are lists of equal size.
-
- Args:
- states: The states observed.
- actions: The actions taken.
- rewards: The rewards received.
- values: The values estimated for the given states.
- terminated: Whether this sequence ends the episode.
-
- Raises:
- ValueError: If the lengths of all the input lists are not equal.
- ValueError: If this.terminated is already True, meaning that the episode
- has already ended.
- """
- if len(states) != len(actions):
- raise ValueError(
- 'Number of states and actions must be the same. Got %d states and '
- '%d actions' % (len(states), len(actions)))
- if len(states) != len(rewards):
- raise ValueError(
- 'Number of states and rewards must be the same. Got %d states and '
- '%d rewards' % (len(states), len(rewards)))
- if values is not None and len(states) != len(values):
- raise ValueError(
- 'Number of states and values must be the same. Got %d states and '
- '%d values' % (len(states), len(values)))
- if self.terminated:
- raise ValueError(
- 'Trying to add timesteps to an already terminal rollout.')
- self.states += states
- self.actions += actions
- self.rewards += rewards
- self.values += values if values is not None else [0.0] * len(states)
- self.terminated = terminated
- self.total_reward += sum(rewards)
-
- def extend(self, other):
- """Append another rollout to this rollout."""
- assert not self.terminated
- self.states.extend(other.states)
- self.actions.extend(other.actions)
- self.rewards.extend(other.rewards)
- self.values.extend(other.values)
- self.terminated = other.terminated
- self.total_reward += other.total_reward
-
-
-def discount(x, gamma):
- """Returns discounted sums for each value in x, with discount factor gamma.
-
- This can be used to compute the return (discounted sum of rewards) at each
- timestep given a sequence of rewards. See the definitions for return and
- REINFORCE in section 3 of https://arxiv.org/pdf/1602.01783.pdf.
-
- Let g^k mean gamma ** k.
- For list [x_0, ..., x_N], the following list of discounted sums is computed:
- [x_0 + g^1 * x_1 + g^2 * x_2 + ... g^N * x_N,
- x_1 + g^1 * x_2 + g^2 * x_3 + ... g^(N-1) * x_N,
- x_2 + g^1 * x_3 + g^2 * x_4 + ... g^(N-2) * x_N,
- ...,
- x_(N-1) + g^1 * x_N,
- x_N]
-
- Args:
- x: List of numbers [x_0, ..., x_N].
- gamma: Float between 0 and 1 (inclusive). This is the discount factor.
-
- Returns:
- List of discounted sums.
- """
- return scipy.signal.lfilter([1], [1, -gamma], x[::-1], axis=0)[::-1]
-
-
-def discounted_advantage_and_rewards(rewards, values, gamma, lambda_=1.0):
- """Compute advantages and returns (discounted sum of rewards).
-
- For an episode of length T, rewards = [r_0, ..., r_(T-1)].
- Each reward r_t is observed after taking action a_t at state s_t. A final
- state s_T is observed but no reward is given at this state since no action
- a_T is taken (otherwise there would be a new state s_(T+1)).
-
- `rewards` and `values` are for a single episode. Return R_t is the discounted
- sum of future rewards starting at time t, where `gamma` is the discount
- factor.
- R_t = r_t + gamma * r_(t+1) + gamma**2 * r_(t+2) + ...
- + gamma**(T-1-t) * r_(T-1)
-
- Advantage A(a_t, s_t) is approximated by computing A(a_t, s_t) = R_t - V(s_t)
- where V(s_t) is an approximation of the value at that state, given in the
- `values` list. Returns R_t are needed for all REINFORCE algorithms. Advantage
- is used for the advantage actor critic variant of REINFORCE.
- See algorithm S3 in https://arxiv.org/pdf/1602.01783.pdf.
-
- Additionally another parameter `lambda_` controls the bias-variance tradeoff.
- See "Generalized Advantage Estimation": https://arxiv.org/abs/1506.02438.
- lambda_ = 1 reduces to regular advantage.
- 0 <= lambda_ < 1 trades off variance for bias, with lambda_ = 0 being the
- most biased.
-
- Bootstrapping is also supported. If an episode does not end in a terminal
- state (either because the episode was ended early, or the environment does not
- have end states), the true return cannot be computed from the rewards alone.
- However, it can be estimated by computing the value (an approximation of
- return) of the last state s_T. Thus the `values` list will have an extra item:
- values = [V(s_0), ..., V(s_(T-1)), V(s_T)].
-
- Args:
- rewards: List of observed rewards [r_0, ..., r_(T-1)].
- values: List of estimated values [V(s_0), ..., V(s_(T-1))] with an optional
- extra V(s_T) item.
- gamma: Discount factor. Number between 0 and 1. 1 means no discount.
- If not 1, gamma is typically near 1, like 0.99.
- lambda_: Bias-variance tradeoff factor. Between 0 and 1.
-
- Returns:
- empirical_values: Returns at each timestep.
- generalized_advantage: Avantages at each timestep.
-
- Raises:
- ValueError: If shapes of `rewards` and `values` are not rank 1.
- ValueError: If len(values) not in (len(rewards), len(rewards) + 1).
- """
- rewards = np.asarray(rewards, dtype=np.float32)
- values = np.asarray(values, dtype=np.float32)
- if rewards.ndim != 1:
- raise ValueError('Single episode only. rewards must be rank 1.')
- if values.ndim != 1:
- raise ValueError('Single episode only. values must be rank 1.')
- if len(values) == len(rewards):
- # No bootstrapping.
- values = np.append(values, 0)
- empirical_values = discount(rewards, gamma)
- elif len(values) == len(rewards) + 1:
- # With bootstrapping.
- # Last value is for the terminal state (final state after last action was
- # taken).
- empirical_values = discount(np.append(rewards, values[-1]), gamma)[:-1]
- else:
- raise ValueError('values should contain the same number of items or one '
- 'more item than rewards')
- delta = rewards + gamma * values[1:] - values[:-1]
- generalized_advantage = discount(delta, gamma * lambda_)
-
- # empirical_values is the discounted sum of rewards into the future.
- # generalized_advantage is the target for each policy update.
- return empirical_values, generalized_advantage
-
-
-"""Batch holds a minibatch of episodes.
-
-Let bi = batch_index, i.e. the index of each episode in the minibatch.
-Let t = time.
-
-Attributes:
- states: States for each timestep in each episode. Indexed by states[bi, t].
- actions: Actions for each timestep in each episode. Indexed by actions[bi, t].
- discounted_adv: Advantages (computed by discounted_advantage_and_rewards)
- for each timestep in each episode. Indexed by discounted_adv[bi, t].
- discounted_r: Returns (discounted sum of rewards computed by
- discounted_advantage_and_rewards) for each timestep in each episode.
- Indexed by discounted_r[bi, t].
- total_rewards: Total reward for each episode, i.e. sum of rewards across all
- timesteps (not discounted). Indexed by total_rewards[bi].
- episode_lengths: Number of timesteps in each episode. If an episode has
- N actions, N rewards, and N states, then its length is N. Indexed by
- episode_lengths[bi].
- batch_size: Number of episodes in this minibatch. An integer.
- max_time: Maximum episode length in the batch. An integer.
-""" # pylint: disable=pointless-string-statement
-Batch = namedtuple(
- 'Batch',
- ['states', 'actions', 'discounted_adv', 'discounted_r', 'total_rewards',
- 'episode_lengths', 'batch_size', 'max_time'])
-
-
-def process_rollouts(rollouts, gamma, lambda_=1.0):
- """Convert a batch of rollouts into tensors ready to be fed into a model.
-
- Lists from each episode are stacked into 2D tensors and padded with 0s up to
- the maximum timestep in the batch.
-
- Args:
- rollouts: A list of Rollout instances.
- gamma: The discount factor. A number between 0 and 1 (inclusive). See gamma
- argument in discounted_advantage_and_rewards.
- lambda_: See lambda_ argument in discounted_advantage_and_rewards.
-
- Returns:
- Batch instance. states, actions, discounted_adv, and discounted_r are
- numpy arrays with shape (batch_size, max_episode_length). episode_lengths
- is a list of ints. total_rewards is a list of floats (total reward in each
- episode). batch_size and max_time are ints.
-
- Raises:
- ValueError: If any of the rollouts are not terminal.
- """
- for ro in rollouts:
- if not ro.terminated:
- raise ValueError('Can only process terminal rollouts.')
-
- episode_lengths = [len(ro.states) for ro in rollouts]
- batch_size = len(rollouts)
- max_time = max(episode_lengths)
-
- states = utils.stack_pad([ro.states for ro in rollouts], 0, max_time)
- actions = utils.stack_pad([ro.actions for ro in rollouts], 0, max_time)
-
- discounted_rewards = [None] * batch_size
- discounted_adv = [None] * batch_size
- for i, ro in enumerate(rollouts):
- disc_r, disc_adv = discounted_advantage_and_rewards(
- ro.rewards, ro.values, gamma, lambda_)
- discounted_rewards[i] = disc_r
- discounted_adv[i] = disc_adv
- discounted_rewards = utils.stack_pad(discounted_rewards, 0, max_time)
- discounted_adv = utils.stack_pad(discounted_adv, 0, max_time)
-
- total_rewards = [sum(ro.rewards) for ro in rollouts]
-
- return Batch(states=states,
- actions=actions,
- discounted_adv=discounted_adv,
- discounted_r=discounted_rewards,
- total_rewards=total_rewards,
- episode_lengths=episode_lengths,
- batch_size=batch_size,
- max_time=max_time)
diff --git a/research/brain_coder/common/rollout_test.py b/research/brain_coder/common/rollout_test.py
deleted file mode 100644
index 5be4cb0fafd8a2e94004c17b41e189d989a3a851..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/rollout_test.py
+++ /dev/null
@@ -1,129 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for common.rollout."""
-
-import numpy as np
-import tensorflow as tf
-
-from common import rollout as rollout_lib # brain coder
-
-
-class RolloutTest(tf.test.TestCase):
-
- def MakeRollout(self, states, actions, rewards, values=None, terminated=True):
- rollout = rollout_lib.Rollout()
- rollout.add_many(
- states=states, actions=actions, rewards=rewards, values=values,
- terminated=terminated)
- return rollout
-
- def testDiscount(self):
- discounted = np.array([1.0 / 2 ** n for n in range(4, -1, -1)])
- discounted[:2] += [1.0 / 2 ** n for n in range(1, -1, -1)]
-
- self.assertTrue(np.array_equal(
- rollout_lib.discount([0.0, 1.0, 0.0, 0.0, 1.0], 0.50),
- discounted))
- self.assertTrue(np.array_equal(
- rollout_lib.discount(np.array([0.0, 1.0, 0.0, 0.0, 1.0]), 0.50),
- discounted))
-
- def testDiscountedAdvantageAndRewards(self):
- # lambda=1, No bootstrapping.
- values = [0.1, 0.5, 0.5, 0.25]
- (empirical_values,
- generalized_advantage) = rollout_lib.discounted_advantage_and_rewards(
- [0.0, 0.0, 0.0, 1.0],
- values,
- gamma=0.75,
- lambda_=1.0)
- expected_discounted_r = (
- np.array([1.0 * 0.75 ** n for n in range(3, -1, -1)]))
- expected_adv = expected_discounted_r - values
- self.assertTrue(np.array_equal(empirical_values, expected_discounted_r))
- self.assertTrue(np.allclose(generalized_advantage, expected_adv))
-
- # lambda=1, With bootstrapping.
- values = [0.1, 0.5, 0.5, 0.25, 0.75]
- (empirical_values,
- generalized_advantage) = rollout_lib.discounted_advantage_and_rewards(
- [0.0, 0.0, 0.0, 1.0],
- values,
- gamma=0.75,
- lambda_=1.0)
- expected_discounted_r = (
- np.array([0.75 * 0.75 ** n for n in range(4, 0, -1)])
- + np.array([1.0 * 0.75 ** n for n in range(3, -1, -1)]))
- expected_adv = expected_discounted_r - values[:-1]
- self.assertTrue(np.array_equal(empirical_values, expected_discounted_r))
- self.assertTrue(np.allclose(generalized_advantage, expected_adv))
-
- # lambda=0.5, With bootstrapping.
- values = [0.1, 0.5, 0.5, 0.25, 0.75]
- rewards = [0.0, 0.0, 0.0, 1.0]
- l = 0.5 # lambda
- g = 0.75 # gamma
- (empirical_values,
- generalized_advantage) = rollout_lib.discounted_advantage_and_rewards(
- rewards,
- values,
- gamma=g,
- lambda_=l)
- expected_discounted_r = (
- np.array([0.75 * g ** n for n in range(4, 0, -1)])
- + np.array([1.0 * g ** n for n in range(3, -1, -1)]))
- expected_adv = [0.0] * len(values)
- for t in range(3, -1, -1):
- delta_t = rewards[t] + g * values[t + 1] - values[t]
- expected_adv[t] = delta_t + g * l * expected_adv[t + 1]
- expected_adv = expected_adv[:-1]
- self.assertTrue(np.array_equal(empirical_values, expected_discounted_r))
- self.assertTrue(np.allclose(generalized_advantage, expected_adv))
-
- def testProcessRollouts(self):
- g = 0.95
- rollouts = [
- self.MakeRollout(
- states=[3, 6, 9],
- actions=[1, 2, 3],
- rewards=[1.0, -1.0, 0.5],
- values=[0.5, 0.5, 0.1]),
- self.MakeRollout(
- states=[10],
- actions=[5],
- rewards=[1.0],
- values=[0.5])]
- batch = rollout_lib.process_rollouts(rollouts, gamma=g)
-
- self.assertEqual(2, batch.batch_size)
- self.assertEqual(3, batch.max_time)
- self.assertEqual([3, 1], batch.episode_lengths)
- self.assertEqual([0.5, 1.0], batch.total_rewards)
- self.assertEqual(
- [[3, 6, 9], [10, 0, 0]],
- batch.states.tolist())
- self.assertEqual(
- [[1, 2, 3], [5, 0, 0]],
- batch.actions.tolist())
-
- rew1, rew2 = rollouts[0].rewards, rollouts[1].rewards
- expected_discounted_rewards = [
- [rew1[0] + g * rew1[1] + g * g * rew1[2],
- rew1[1] + g * rew1[2],
- rew1[2]],
- [rew2[0], 0.0, 0.0]]
- expected_advantages = [
- [dr - v
- for dr, v
- in zip(expected_discounted_rewards[0], rollouts[0].values)],
- [expected_discounted_rewards[1][0] - rollouts[1].values[0], 0.0, 0.0]]
- self.assertTrue(
- np.allclose(expected_discounted_rewards, batch.discounted_r))
- self.assertTrue(
- np.allclose(expected_advantages, batch.discounted_adv))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/common/schedules.py b/research/brain_coder/common/schedules.py
deleted file mode 100644
index fff2481e536d65f154ad2d9dc3972657d860abf8..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/schedules.py
+++ /dev/null
@@ -1,301 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Schedule functions for controlling hparams over time."""
-
-from abc import ABCMeta
-from abc import abstractmethod
-import math
-
-from common import config_lib # brain coder
-
-
-class Schedule(object):
- """Schedule is a function which sets a hyperparameter's value over time.
-
- For example, a schedule can be used to decay an hparams, or oscillate it over
- time.
-
- This object is constructed with an instance of config_lib.Config (will be
- specific to each class implementation). For example if this is a decay
- schedule, the config may specify the rate of decay and decay start time. Then
- the object instance is called like a function, mapping global step (an integer
- counting how many calls to the train op have been made) to the hparam value.
-
- Properties of a schedule function f(t):
- 0) Domain of t is the non-negative integers (t may be 0).
- 1) Range of f is the reals.
- 2) Schedule functions can assume that they will be called in time order. This
- allows schedules to be stateful.
- 3) Schedule functions should be deterministic. Two schedule instances with the
- same config must always give the same value for each t, and regardless of
- what t's it was previously called on. Users may call f(t) on arbitrary
- (positive) time jumps. Essentially, multiple schedule instances used in
- replica training will behave the same.
- 4) Duplicate successive calls on the same time are allowed.
- """
- __metaclass__ = ABCMeta
-
- @abstractmethod
- def __init__(self, config):
- """Construct this schedule with a config specific to each class impl.
-
- Args:
- config: An instance of config_lib.Config.
- """
- pass
-
- @abstractmethod
- def __call__(self, global_step):
- """Map `global_step` to a value.
-
- `global_step` is an integer counting how many calls to the train op have
- been made across all replicas (hence why it is global). Implementations
- may assume calls to be made in time order, i.e. `global_step` now >=
- previous `global_step` values.
-
- Args:
- global_step: Non-negative integer.
-
- Returns:
- Hparam value at this step. A number.
- """
- pass
-
-
-class ConstSchedule(Schedule):
- """Constant function.
-
- config:
- const: Constant value at every step.
-
- f(t) = const.
- """
-
- def __init__(self, config):
- super(ConstSchedule, self).__init__(config)
- self.const = config.const
-
- def __call__(self, global_step):
- return self.const
-
-
-class LinearDecaySchedule(Schedule):
- """Linear decay function.
-
- config:
- initial: Decay starts from this value.
- final: Decay ends at this value.
- start_time: Step when decay starts. Constant before it.
- end_time: When decay ends. Constant after it.
-
- f(t) is a linear function when start_time <= t <= end_time, with slope of
- (final - initial) / (end_time - start_time). f(t) = initial
- when t <= start_time. f(t) = final when t >= end_time.
-
- If start_time == end_time, this becomes a step function.
- """
-
- def __init__(self, config):
- super(LinearDecaySchedule, self).__init__(config)
- self.initial = config.initial
- self.final = config.final
- self.start_time = config.start_time
- self.end_time = config.end_time
-
- if self.end_time < self.start_time:
- raise ValueError('start_time must be before end_time.')
-
- # Linear interpolation.
- self._time_diff = float(self.end_time - self.start_time)
- self._diff = float(self.final - self.initial)
- self._slope = (
- self._diff / self._time_diff if self._time_diff > 0 else float('inf'))
-
- def __call__(self, global_step):
- if global_step <= self.start_time:
- return self.initial
- if global_step > self.end_time:
- return self.final
- return self.initial + (global_step - self.start_time) * self._slope
-
-
-class ExponentialDecaySchedule(Schedule):
- """Exponential decay function.
-
- See https://en.wikipedia.org/wiki/Exponential_decay.
-
- Use this decay function to decay over orders of magnitude. For example, to
- decay learning rate from 1e-2 to 1e-6. Exponential decay will decay the
- exponent linearly.
-
- config:
- initial: Decay starts from this value.
- final: Decay ends at this value.
- start_time: Step when decay starts. Constant before it.
- end_time: When decay ends. Constant after it.
-
- f(t) is an exponential decay function when start_time <= t <= end_time. The
- decay rate and amplitude are chosen so that f(t) = initial when
- t = start_time, and f(t) = final when t = end_time. f(t) is constant for
- t < start_time or t > end_time. initial and final must be positive values.
-
- If start_time == end_time, this becomes a step function.
- """
-
- def __init__(self, config):
- super(ExponentialDecaySchedule, self).__init__(config)
- self.initial = config.initial
- self.final = config.final
- self.start_time = config.start_time
- self.end_time = config.end_time
-
- if self.initial <= 0 or self.final <= 0:
- raise ValueError('initial and final must be positive numbers.')
-
- # Linear interpolation in log space.
- self._linear_fn = LinearDecaySchedule(
- config_lib.Config(
- initial=math.log(self.initial),
- final=math.log(self.final),
- start_time=self.start_time,
- end_time=self.end_time))
-
- def __call__(self, global_step):
- return math.exp(self._linear_fn(global_step))
-
-
-class SmootherstepDecaySchedule(Schedule):
- """Smootherstep decay function.
-
- A sigmoidal like transition from initial to final values. A smoother
- transition than linear and exponential decays, hence the name.
- See https://en.wikipedia.org/wiki/Smoothstep.
-
- config:
- initial: Decay starts from this value.
- final: Decay ends at this value.
- start_time: Step when decay starts. Constant before it.
- end_time: When decay ends. Constant after it.
-
- f(t) is fully defined here:
- https://en.wikipedia.org/wiki/Smoothstep#Variations.
-
- f(t) is smooth, as in its first-derivative exists everywhere.
- """
-
- def __init__(self, config):
- super(SmootherstepDecaySchedule, self).__init__(config)
- self.initial = config.initial
- self.final = config.final
- self.start_time = config.start_time
- self.end_time = config.end_time
-
- if self.end_time < self.start_time:
- raise ValueError('start_time must be before end_time.')
-
- self._time_diff = float(self.end_time - self.start_time)
- self._diff = float(self.final - self.initial)
-
- def __call__(self, global_step):
- if global_step <= self.start_time:
- return self.initial
- if global_step > self.end_time:
- return self.final
- x = (global_step - self.start_time) / self._time_diff
-
- # Smootherstep
- return self.initial + x * x * x * (x * (x * 6 - 15) + 10) * self._diff
-
-
-class HardOscillatorSchedule(Schedule):
- """Hard oscillator function.
-
- config:
- high: Max value of the oscillator. Value at constant plateaus.
- low: Min value of the oscillator. Value at constant valleys.
- start_time: Global step when oscillation starts. Constant before this.
- period: Width of one oscillation, i.e. number of steps over which the
- oscillation takes place.
- transition_fraction: Fraction of the period spent transitioning between high
- and low values. 50% of this time is spent rising, and 50% of this time
- is spent falling. 50% of the remaining time is spent constant at the
- high value, and 50% of the remaining time is spent constant at the low
- value. transition_fraction = 1.0 means the entire period is spent
- rising and falling. transition_fraction = 0.0 means no time is spent
- rising and falling, i.e. the function jumps instantaneously between
- high and low.
-
- f(t) = high when t < start_time.
- f(t) is periodic when t >= start_time, with f(t + period) = f(t).
- f(t) is linear with positive slope when rising, and negative slope when
- falling. At the start of the period t0, f(t0) = high and begins to descend.
- At the middle of the period f is low and is constant until the ascension
- begins. f then rises from low to high and is constant again until the period
- repeats.
-
- Note: when transition_fraction is 0, f starts the period low and ends high.
- """
-
- def __init__(self, config):
- super(HardOscillatorSchedule, self).__init__(config)
- self.high = config.high
- self.low = config.low
- self.start_time = config.start_time
- self.period = float(config.period)
- self.transition_fraction = config.transition_fraction
- self.half_transition_fraction = config.transition_fraction / 2.0
-
- if self.transition_fraction < 0 or self.transition_fraction > 1.0:
- raise ValueError('transition_fraction must be between 0 and 1.0')
- if self.period <= 0:
- raise ValueError('period must be positive')
-
- self._slope = (
- float(self.high - self.low) / self.half_transition_fraction
- if self.half_transition_fraction > 0 else float('inf'))
-
- def __call__(self, global_step):
- if global_step < self.start_time:
- return self.high
- period_pos = ((global_step - self.start_time) / self.period) % 1.0
- if period_pos >= 0.5:
- # ascending
- period_pos -= 0.5
- if period_pos < self.half_transition_fraction:
- return self.low + period_pos * self._slope
- else:
- return self.high
- else:
- # descending
- if period_pos < self.half_transition_fraction:
- return self.high - period_pos * self._slope
- else:
- return self.low
-
-
-_NAME_TO_CONFIG = {
- 'const': ConstSchedule,
- 'linear_decay': LinearDecaySchedule,
- 'exp_decay': ExponentialDecaySchedule,
- 'smooth_decay': SmootherstepDecaySchedule,
- 'hard_osc': HardOscillatorSchedule,
-}
-
-
-def make_schedule(config):
- """Schedule factory.
-
- Given `config` containing a `fn` property, a Schedule implementation is
- instantiated with `config`. See `_NAME_TO_CONFIG` for `fn` options.
-
- Args:
- config: Config with a `fn` option that specifies which Schedule
- implementation to use. `config` is passed into the constructor.
-
- Returns:
- A Schedule impl instance.
- """
- schedule_class = _NAME_TO_CONFIG[config.fn]
- return schedule_class(config)
diff --git a/research/brain_coder/common/schedules_test.py b/research/brain_coder/common/schedules_test.py
deleted file mode 100644
index b17022f45a833fb3aa219fd06225f77fbd1b1055..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/schedules_test.py
+++ /dev/null
@@ -1,139 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for common.schedules."""
-
-from math import exp
-from math import sqrt
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from common import config_lib # brain coder
-from common import schedules # brain coder
-
-
-class SchedulesTest(tf.test.TestCase):
-
- def ScheduleTestHelper(self, config, schedule_subtype, io_values):
- """Run common checks for schedules.
-
- Args:
- config: Config object which is passed into schedules.make_schedule.
- schedule_subtype: The expected schedule type to be instantiated.
- io_values: List of (input, output) pairs. Must be in ascending input
- order. No duplicate inputs.
- """
-
- # Check that make_schedule makes the correct type.
- f = schedules.make_schedule(config)
- self.assertTrue(isinstance(f, schedule_subtype))
-
- # Check that multiple instances returned from make_schedule behave the same.
- fns = [schedules.make_schedule(config) for _ in xrange(3)]
-
- # Check that all the inputs map to the right outputs.
- for i, o in io_values:
- for f in fns:
- f_out = f(i)
- self.assertTrue(
- np.isclose(o, f_out),
- 'Wrong value at input %d. Expected %s, got %s' % (i, o, f_out))
-
- # Check that a subset of the io_values are still correct.
- f = schedules.make_schedule(config)
- subseq = [io_values[i**2] for i in xrange(int(sqrt(len(io_values))))]
- if subseq[-1] != io_values[-1]:
- subseq.append(io_values[-1])
- for i, o in subseq:
- f_out = f(i)
- self.assertTrue(
- np.isclose(o, f_out),
- 'Wrong value at input %d. Expected %s, got %s' % (i, o, f_out))
-
- # Check duplicate calls.
- f = schedules.make_schedule(config)
- for i, o in io_values:
- for _ in xrange(3):
- f_out = f(i)
- self.assertTrue(
- np.isclose(o, f_out),
- 'Duplicate calls at input %d are not equal. Expected %s, got %s'
- % (i, o, f_out))
-
- def testConstSchedule(self):
- self.ScheduleTestHelper(
- config_lib.Config(fn='const', const=5),
- schedules.ConstSchedule,
- [(0, 5), (1, 5), (10, 5), (20, 5), (100, 5), (1000000, 5)])
-
- def testLinearDecaySchedule(self):
- self.ScheduleTestHelper(
- config_lib.Config(fn='linear_decay', initial=2, final=0, start_time=10,
- end_time=20),
- schedules.LinearDecaySchedule,
- [(0, 2), (1, 2), (10, 2), (11, 1.8), (15, 1), (19, 0.2), (20, 0),
- (100000, 0)])
-
- # Test step function.
- self.ScheduleTestHelper(
- config_lib.Config(fn='linear_decay', initial=2, final=0, start_time=10,
- end_time=10),
- schedules.LinearDecaySchedule,
- [(0, 2), (1, 2), (10, 2), (11, 0), (15, 0)])
-
- def testExponentialDecaySchedule(self):
- self.ScheduleTestHelper(
- config_lib.Config(fn='exp_decay', initial=exp(-1), final=exp(-6),
- start_time=10, end_time=20),
- schedules.ExponentialDecaySchedule,
- [(0, exp(-1)), (1, exp(-1)), (10, exp(-1)), (11, exp(-1/2. - 1)),
- (15, exp(-5/2. - 1)), (19, exp(-9/2. - 1)), (20, exp(-6)),
- (100000, exp(-6))])
-
- # Test step function.
- self.ScheduleTestHelper(
- config_lib.Config(fn='exp_decay', initial=exp(-1), final=exp(-6),
- start_time=10, end_time=10),
- schedules.ExponentialDecaySchedule,
- [(0, exp(-1)), (1, exp(-1)), (10, exp(-1)), (11, exp(-6)),
- (15, exp(-6))])
-
- def testSmootherstepDecaySchedule(self):
- self.ScheduleTestHelper(
- config_lib.Config(fn='smooth_decay', initial=2, final=0, start_time=10,
- end_time=20),
- schedules.SmootherstepDecaySchedule,
- [(0, 2), (1, 2), (10, 2), (11, 1.98288), (15, 1), (19, 0.01712),
- (20, 0), (100000, 0)])
-
- # Test step function.
- self.ScheduleTestHelper(
- config_lib.Config(fn='smooth_decay', initial=2, final=0, start_time=10,
- end_time=10),
- schedules.SmootherstepDecaySchedule,
- [(0, 2), (1, 2), (10, 2), (11, 0), (15, 0)])
-
- def testHardOscillatorSchedule(self):
- self.ScheduleTestHelper(
- config_lib.Config(fn='hard_osc', high=2, low=0, start_time=100,
- period=10, transition_fraction=0.5),
- schedules.HardOscillatorSchedule,
- [(0, 2), (1, 2), (10, 2), (100, 2), (101, 1.2), (102, 0.4), (103, 0),
- (104, 0), (105, 0), (106, 0.8), (107, 1.6), (108, 2), (109, 2),
- (110, 2), (111, 1.2), (112, 0.4), (115, 0), (116, 0.8), (119, 2),
- (120, 2), (100001, 1.2), (100002, 0.4), (100005, 0), (100006, 0.8),
- (100010, 2)])
-
- # Test instantaneous step.
- self.ScheduleTestHelper(
- config_lib.Config(fn='hard_osc', high=2, low=0, start_time=100,
- period=10, transition_fraction=0),
- schedules.HardOscillatorSchedule,
- [(0, 2), (1, 2), (10, 2), (99, 2), (100, 0), (104, 0), (105, 2),
- (106, 2), (109, 2), (110, 0)])
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/common/utils.py b/research/brain_coder/common/utils.py
deleted file mode 100644
index fa5f1c50768986ee10eee6120a0bca392b1d9d0e..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/utils.py
+++ /dev/null
@@ -1,558 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Configuration class."""
-
-import bisect
-from collections import deque
-import cPickle
-import heapq
-import random
-
-from absl import logging
-import numpy as np
-import six
-from six.moves import xrange
-import tensorflow as tf
-
-
-def tuple_to_record(tuple_, record_type):
- return record_type(**dict(zip(record_type.__slots__, tuple_)))
-
-
-def make_record(type_name, attributes, defaults=None):
- """Factory for mutable record classes.
-
- A record acts just like a collections.namedtuple except slots are writable.
- One exception is that record classes are not equivalent to tuples or other
- record classes of the same length.
-
- Note, each call to `make_record` produces a unique type. Two calls will make
- different types even if `type_name` is the same each time.
-
- Args:
- type_name: Name of the record type to create.
- attributes: List of names of each record attribute. The order of the list
- is preserved.
- defaults: (optional) default values for attributes. A dict mapping attribute
- names to values.
-
- Returns:
- A new record type.
-
- Raises:
- ValueError: If,
- `defaults` is not a dict,
- `attributes` contains duplicate names,
- `defaults` keys are not contained in `attributes`.
- """
- if defaults is None:
- defaults = {}
- if not isinstance(defaults, dict):
- raise ValueError('defaults must be a dict.')
- attr_set = set(attributes)
- if len(attr_set) < len(attributes):
- raise ValueError('No duplicate attributes allowed.')
- if not set(defaults.keys()).issubset(attr_set):
- raise ValueError('Default attributes must be given in the attributes list.')
-
- class RecordClass(object):
- """A record type.
-
- Acts like mutable tuple with named slots.
- """
- __slots__ = list(attributes)
- _defaults = dict(defaults)
-
- def __init__(self, *args, **kwargs):
- if len(args) > len(self.__slots__):
- raise ValueError('Too many arguments. %s has length %d.'
- % (type(self).__name__, len(self.__slots__)))
- for attr, val in self._defaults.items():
- setattr(self, attr, val)
- for i, arg in enumerate(args):
- setattr(self, self.__slots__[i], arg)
- for attr, val in kwargs.items():
- setattr(self, attr, val)
- for attr in self.__slots__:
- if not hasattr(self, attr):
- raise ValueError('Required attr "%s" is not set.' % attr)
-
- def __len__(self):
- return len(self.__slots__)
-
- def __iter__(self):
- for attr in self.__slots__:
- yield getattr(self, attr)
-
- def __getitem__(self, index):
- return getattr(self, self.__slots__[index])
-
- def __setitem__(self, index, value):
- return setattr(self, self.__slots__[index], value)
-
- def __eq__(self, other):
- # Types must be equal as well as values.
- return (isinstance(other, type(self))
- and all(a == b for a, b in zip(self, other)))
-
- def __str__(self):
- return '%s(%s)' % (
- type(self).__name__,
- ', '.join(attr + '=' + str(getattr(self, attr))
- for attr in self.__slots__))
-
- def __repr__(self):
- return str(self)
-
- RecordClass.__name__ = type_name
- return RecordClass
-
-
-# Making minibatches.
-def stack_pad(tensors, pad_axes=None, pad_to_lengths=None, dtype=np.float32,
- pad_value=0):
- """Stack tensors along 0-th dim and pad them to be the same shape.
-
- Args:
- tensors: Any list of iterables (python list, numpy array, etc). Can be 1D
- or multi-D iterables.
- pad_axes: An int or list of ints. Axes to pad along.
- pad_to_lengths: Length in each dimension. If pad_axes was an int, this is an
- int or None. If pad_axes was a list of ints, this is a list of mixed int
- and None types with the same length, or None. A None length means the
- maximum length among the given tensors is used.
- dtype: Type of output numpy array. Defaults to np.float32.
- pad_value: Value to use for padding. Defaults to 0.
-
- Returns:
- Numpy array containing the tensors stacked along the 0-th dimension and
- padded along the specified dimensions.
-
- Raises:
- ValueError: If the tensors do not have equal shapes along non-padded
- dimensions.
- """
- tensors = [np.asarray(t) for t in tensors]
- max_lengths = [max(l) for l in zip(*[t.shape for t in tensors])]
- same_axes = dict(enumerate(max_lengths))
- if pad_axes is None:
- pad_axes = []
- if isinstance(pad_axes, six.integer_types):
- if pad_to_lengths is not None:
- max_lengths[pad_axes] = pad_to_lengths
- del same_axes[pad_axes]
- else:
- if pad_to_lengths is None:
- pad_to_lengths = [None] * len(pad_axes)
- for i, l in zip(pad_axes, pad_to_lengths):
- if l is not None:
- max_lengths[i] = l
- del same_axes[i]
- same_axes_items = same_axes.items()
- dest = np.full([len(tensors)] + max_lengths, pad_value, dtype=dtype)
- for i, t in enumerate(tensors):
- for j, l in same_axes_items:
- if t.shape[j] != l:
- raise ValueError(
- 'Tensor at index %d does not have size %d along axis %d'
- % (i, l, j))
- dest[[i] + [slice(0, d) for d in t.shape]] = t
- return dest
-
-
-class RandomQueue(deque):
-
- def __init__(self, capacity):
- super(RandomQueue, self).__init__([], capacity)
- self.capacity = capacity
-
- def random_sample(self, sample_size):
- idx = np.random.choice(len(self), sample_size)
- return [self[i] for i in idx]
-
- def push(self, item):
- # Append to right. Oldest element will be popped from left.
- self.append(item)
-
-
-class MPQItemContainer(object):
- """Class for holding an item with its score.
-
- Defines a comparison function for use in the heap-queue.
- """
-
- def __init__(self, score, item, extra_data):
- self.item = item
- self.score = score
- self.extra_data = extra_data
-
- def __cmp__(self, other):
- assert isinstance(other, type(self))
- return cmp(self.score, other.score)
-
- def __iter__(self):
- """Allows unpacking like a tuple."""
- yield self.score
- yield self.item
- yield self.extra_data
-
- def __repr__(self):
- """String representation of this item.
-
- `extra_data` is not included in the representation. We are assuming that
- `extra_data` is not easily interpreted by a human (if it was, it should be
- hashable, like a string or tuple).
-
- Returns:
- String representation of `self`.
- """
- return str((self.score, self.item))
-
- def __str__(self):
- return repr(self)
-
-
-class MaxUniquePriorityQueue(object):
- """A maximum priority queue where duplicates are not added.
-
- The top items by score remain in the queue. When the capacity is reached,
- the lowest scored item in the queue will be dropped.
-
- This implementation differs from a typical priority queue, in that the minimum
- score is popped, instead of the maximum. Largest scores remain stuck in the
- queue. This is useful for accumulating the best known items from a population.
-
- The items used to determine uniqueness must be hashable, but additional
- non-hashable data may be stored with each item.
- """
-
- def __init__(self, capacity):
- self.capacity = capacity
- self.heap = []
- self.unique_items = set()
-
- def push(self, score, item, extra_data=None):
- """Push an item onto the queue.
-
- If the queue is at capacity, the item with the smallest score will be
- dropped. Note that it is assumed each item has exactly one score. The same
- item with a different score will still be dropped.
-
- Args:
- score: Number used to prioritize items in the queue. Largest scores are
- kept in the queue.
- item: A hashable item to be stored. Duplicates of this item will not be
- added to the queue.
- extra_data: An extra (possible not hashable) data to store with the item.
- """
- if item in self.unique_items:
- return
- if len(self.heap) >= self.capacity:
- _, popped_item, _ = heapq.heappushpop(
- self.heap, MPQItemContainer(score, item, extra_data))
- self.unique_items.add(item)
- self.unique_items.remove(popped_item)
- else:
- heapq.heappush(self.heap, MPQItemContainer(score, item, extra_data))
- self.unique_items.add(item)
-
- def pop(self):
- """Pop the item with the lowest score.
-
- Returns:
- score: Item's score.
- item: The item that was popped.
- extra_data: Any extra data stored with the item.
- """
- if not self.heap:
- return ()
- score, item, extra_data = heapq.heappop(self.heap)
- self.unique_items.remove(item)
- return score, item, extra_data
-
- def get_max(self):
- """Peek at the item with the highest score.
-
- Returns:
- Same as `pop`.
- """
- if not self.heap:
- return ()
- score, item, extra_data = heapq.nlargest(1, self.heap)[0]
- return score, item, extra_data
-
- def get_min(self):
- """Peek at the item with the lowest score.
-
- Returns:
- Same as `pop`.
- """
- if not self.heap:
- return ()
- score, item, extra_data = heapq.nsmallest(1, self.heap)[0]
- return score, item, extra_data
-
- def random_sample(self, sample_size):
- """Randomly select items from the queue.
-
- This does not modify the queue.
-
- Items are drawn from a uniform distribution, and not weighted by score.
-
- Args:
- sample_size: Number of random samples to draw. The same item can be
- sampled multiple times.
-
- Returns:
- List of sampled items (of length `sample_size`). Each element in the list
- is a tuple: (item, extra_data).
- """
- idx = np.random.choice(len(self.heap), sample_size)
- return [(self.heap[i].item, self.heap[i].extra_data) for i in idx]
-
- def iter_in_order(self):
- """Iterate over items in the queue from largest score to smallest.
-
- Yields:
- item: Hashable item.
- extra_data: Extra data stored with the item.
- """
- for _, item, extra_data in heapq.nlargest(len(self.heap), self.heap):
- yield item, extra_data
-
- def __len__(self):
- return len(self.heap)
-
- def __iter__(self):
- for _, item, _ in self.heap:
- yield item
-
- def __repr__(self):
- return '[' + ', '.join(repr(c) for c in self.heap) + ']'
-
- def __str__(self):
- return repr(self)
-
-
-class RouletteWheel(object):
- """Randomly samples stored objects proportionally to their given weights.
-
- Stores objects and weights. Acts like a roulette wheel where each object is
- given a slice of the roulette disk proportional to its weight.
-
- This can be used as a replay buffer where past experiences are sampled
- proportionally to their weights. A good choice of "weight" for reinforcement
- learning is exp(reward / temperature) where temperature -> inf makes the
- distribution more uniform and temperature -> 0 makes the distribution more
- peaky.
-
- To prevent experiences from being overweighted by appearing in the replay
- buffer multiple times, a "unique mode" is supported where duplicate
- experiences are ignored. In unique mode, weights can be quickly retrieved from
- keys.
- """
-
- def __init__(self, unique_mode=False, save_file=None):
- """Construct empty RouletteWheel.
-
- If `save_file` is not None, and the file already exists on disk, whatever
- is in the file will be loaded into this instance. This allows jobs using
- RouletteWheel to resume after preemption.
-
- Args:
- unique_mode: If True, puts this RouletteWheel into unique mode, where
- objects are added with hashable keys, so that duplicates are ignored.
- save_file: Optional file path to save to. Must be a string containing
- an absolute path to a file, or None. File will be Python pickle
- format.
- """
- self.unique_mode = unique_mode
- self.objects = []
- self.weights = []
- self.partial_sums = []
- if self.unique_mode:
- self.keys_to_weights = {}
- self.save_file = save_file
- self.save_to_disk_buffer = []
-
- if save_file is not None and tf.gfile.Exists(save_file):
- # Load from disk.
- with tf.gfile.OpenFast(save_file, 'r') as f:
- count = 0
- while 1:
- try:
- obj, weight, key = cPickle.load(f)
- except EOFError:
- break
- else:
- self.add(obj, weight, key)
- count += 1
- logging.info('Loaded %d samples from disk.', count)
- # Clear buffer since these items are already on disk.
- self.save_to_disk_buffer = []
-
- def __iter__(self):
- return iter(zip(self.objects, self.weights))
-
- def __len__(self):
- return len(self.objects)
-
- def is_empty(self):
- """Returns whether there is anything in the roulette wheel."""
- return not self.partial_sums
-
- @property
- def total_weight(self):
- """Total cumulative weight across all objects."""
- if self.partial_sums:
- return self.partial_sums[-1]
- return 0.0
-
- def has_key(self, key):
- if self.unique_mode:
- RuntimeError('has_key method can only be called in unique mode.')
- return key in self.keys_to_weights
-
- def get_weight(self, key):
- if self.unique_mode:
- RuntimeError('get_weight method can only be called in unique mode.')
- return self.keys_to_weights[key]
-
- def add(self, obj, weight, key=None):
- """Add one object and its weight to the roulette wheel.
-
- Args:
- obj: Any object to be stored.
- weight: A non-negative float. The given object will be drawn with
- probability proportional to this weight when sampling.
- key: This argument is only used when in unique mode. To allow `obj` to
- be an unhashable type, like list, a separate hashable key is given.
- Each `key` should be unique to each `obj`. `key` is used to check if
- `obj` has been added to the roulette wheel before.
-
- Returns:
- True if the object was added, False if it was not added due to it being
- a duplicate (this only happens in unique mode).
-
- Raises:
- ValueError: If `weight` is negative.
- ValueError: If `key` is not given when in unique mode, or if `key` is
- given when not in unique mode.
- """
- if weight < 0:
- raise ValueError('Weight must be non-negative')
- if self.unique_mode:
- if key is None:
- raise ValueError(
- 'Hashable key required for objects when unique mode is enabled.')
- if key in self.keys_to_weights:
- # Weight updates are not allowed. Ignore the given value of `weight`.
- return False
- self.keys_to_weights[key] = weight
- elif key is not None:
- raise ValueError(
- 'key argument should not be used when unique mode is disabled.')
- self.objects.append(obj)
- self.weights.append(weight)
- self.partial_sums.append(self.total_weight + weight)
- if self.save_file is not None:
- # Record new item in buffer.
- self.save_to_disk_buffer.append((obj, weight, key))
- return True
-
- def add_many(self, objs, weights, keys=None):
- """Add many object and their weights to the roulette wheel.
-
- Arguments are the same as the `add` method, except each is a list. Lists
- must all be the same length.
-
- Args:
- objs: List of objects to be stored.
- weights: List of non-negative floats. See `add` method.
- keys: List of hashable keys. This argument is only used when in unique
- mode. See `add` method.
-
- Returns:
- Number of objects added. This number will be less than the number of
- objects provided if we are in unique mode and some keys are already
- in the roulette wheel.
-
- Raises:
- ValueError: If `keys` argument is provided when unique_mode == False, or
- is not provided when unique_mode == True.
- ValueError: If any of the lists are not the same length.
- ValueError: If any of the weights are negative.
- """
- if keys is not None and not self.unique_mode:
- raise ValueError('Not in unique mode. Do not provide keys.')
- elif keys is None and self.unique_mode:
- raise ValueError('In unique mode. You must provide hashable keys.')
- if keys and len(objs) != len(keys):
- raise ValueError('Number of objects does not equal number of keys.')
- if len(objs) != len(weights):
- raise ValueError('Number of objects does not equal number of weights.')
- return sum([self.add(obj, weights[i], key=keys[i] if keys else None)
- for i, obj in enumerate(objs)])
-
- def sample(self):
- """Spin the roulette wheel.
-
- Randomly select an object with probability proportional to its weight.
-
- Returns:
- object: The selected object.
- weight: The weight of the selected object.
-
- Raises:
- RuntimeError: If the roulette wheel is empty.
- """
- if self.is_empty():
- raise RuntimeError('Trying to sample from empty roulette wheel.')
- spin = random.random() * self.total_weight
-
- # Binary search.
- i = bisect.bisect_right(self.partial_sums, spin)
- if i == len(self.partial_sums):
- # This should not happen since random.random() will always be strictly
- # less than 1.0, and the last partial sum equals self.total_weight().
- # However it may happen due to rounding error. In that case it is easy to
- # handle this, just select the last object.
- i -= 1
-
- return self.objects[i], self.weights[i]
-
- def sample_many(self, count):
- """Spin the roulette wheel `count` times and return the results."""
- if self.is_empty():
- raise RuntimeError('Trying to sample from empty roulette wheel.')
- return [self.sample() for _ in xrange(count)]
-
- def incremental_save(self, log_info=False):
- """Write new entries to disk.
-
- This performs an append operation on the `save_file` given in the
- constructor. Any entries added since the last call to `incremental_save`
- will be appended to the file.
-
- If a new RouletteWheel is constructed with the same `save_file`, all the
- entries written there will be automatically loaded into the instance.
- This is useful when a job resumes after preemption.
-
- Args:
- log_info: If True, info about this operation will be logged.
-
- Raises:
- RuntimeError: If `save_file` given in the constructor is None.
- """
- if self.save_file is None:
- raise RuntimeError('Cannot call incremental_save. `save_file` is None.')
- if log_info:
- logging.info('Saving %d new samples to disk.',
- len(self.save_to_disk_buffer))
- with tf.gfile.OpenFast(self.save_file, 'a') as f:
- for entry in self.save_to_disk_buffer:
- cPickle.dump(entry, f)
- # Clear the buffer.
- self.save_to_disk_buffer = []
diff --git a/research/brain_coder/common/utils_test.py b/research/brain_coder/common/utils_test.py
deleted file mode 100644
index 569c2877d17bf7707616029cdd2a5eac55df7f60..0000000000000000000000000000000000000000
--- a/research/brain_coder/common/utils_test.py
+++ /dev/null
@@ -1,382 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for common.utils.
-"""
-
-from collections import Counter
-import random
-import tempfile
-import numpy as np
-import tensorflow as tf
-
-from common import utils # brain coder
-
-
-class UtilsTest(tf.test.TestCase):
-
- def testStackPad(self):
- # 1D.
- tensors = [[1, 2, 3], [4, 5, 6, 7, 8], [9]]
- result = utils.stack_pad(tensors, pad_axes=0, pad_to_lengths=6)
- self.assertTrue(np.array_equal(
- result,
- np.asarray([[1, 2, 3, 0, 0, 0],
- [4, 5, 6, 7, 8, 0],
- [9, 0, 0, 0, 0, 0]], dtype=np.float32)))
-
- # 3D.
- tensors = [[[[1, 2, 3], [4, 5, 6]]],
- [[[7, 8, 9], [0, 1, 2]], [[3, 4, 5], [6, 7, 8]]],
- [[[0, 1, 2]], [[3, 4, 5]]]]
- result = utils.stack_pad(tensors, pad_axes=[0, 1], pad_to_lengths=[2, 2])
- self.assertTrue(np.array_equal(
- result,
- np.asarray([[[[1, 2, 3], [4, 5, 6]],
- [[0, 0, 0], [0, 0, 0]]],
- [[[7, 8, 9], [0, 1, 2]],
- [[3, 4, 5], [6, 7, 8]]],
- [[[0, 1, 2], [0, 0, 0]],
- [[3, 4, 5], [0, 0, 0]]]], dtype=np.float32)))
-
- def testStackPadNoAxes(self):
- # 2D.
- tensors = [[[1, 2, 3], [4, 5, 6]],
- [[7, 8, 9], [1, 2, 3]],
- [[4, 5, 6], [7, 8, 9]]]
- result = utils.stack_pad(tensors)
- self.assertTrue(np.array_equal(
- result,
- np.asarray(tensors)))
-
- def testStackPadNoneLength(self):
- # 1D.
- tensors = [[1, 2, 3], [4, 5, 6, 7, 8], [9]]
- result = utils.stack_pad(tensors, pad_axes=0, pad_to_lengths=None)
- self.assertTrue(np.array_equal(
- result,
- np.asarray([[1, 2, 3, 0, 0],
- [4, 5, 6, 7, 8],
- [9, 0, 0, 0, 0]], dtype=np.float32)))
-
- # 3D.
- tensors = [[[[1, 2, 3], [4, 5, 6]]],
- [[[7, 8, 9], [0, 1, 2]], [[3, 4, 5], [6, 7, 8]]],
- [[[0, 1, 2]], [[3, 4, 5]]]]
- result = utils.stack_pad(tensors, pad_axes=[0, 1], pad_to_lengths=None)
- self.assertTrue(np.array_equal(
- result,
- np.asarray([[[[1, 2, 3], [4, 5, 6]],
- [[0, 0, 0], [0, 0, 0]]],
- [[[7, 8, 9], [0, 1, 2]],
- [[3, 4, 5], [6, 7, 8]]],
- [[[0, 1, 2], [0, 0, 0]],
- [[3, 4, 5], [0, 0, 0]]]], dtype=np.float32)))
-
- # 3D with partial pad_to_lengths.
- tensors = [[[[1, 2, 3], [4, 5, 6]]],
- [[[7, 8, 9], [0, 1, 2]], [[3, 4, 5], [6, 7, 8]]],
- [[[0, 1, 2]], [[3, 4, 5]]]]
- result = utils.stack_pad(tensors, pad_axes=[0, 1], pad_to_lengths=[None, 3])
- self.assertTrue(np.array_equal(
- result,
- np.asarray([[[[1, 2, 3], [4, 5, 6], [0, 0, 0]],
- [[0, 0, 0], [0, 0, 0], [0, 0, 0]]],
- [[[7, 8, 9], [0, 1, 2], [0, 0, 0]],
- [[3, 4, 5], [6, 7, 8], [0, 0, 0]]],
- [[[0, 1, 2], [0, 0, 0], [0, 0, 0]],
- [[3, 4, 5], [0, 0, 0], [0, 0, 0]]]], dtype=np.float32)))
-
- def testStackPadValueError(self):
- # 3D.
- tensors = [[[[1, 2, 3], [4, 5, 6]]],
- [[[7, 8, 9], [0, 1, 2]], [[3, 4, 5], [6, 7, 8]]],
- [[[0, 1, 2]], [[3, 4, 5]]],
- [[[1, 2, 3, 4]]]]
-
- # Not all tensors have the same shape along axis 2.
- with self.assertRaises(ValueError):
- utils.stack_pad(tensors, pad_axes=[0, 1], pad_to_lengths=[2, 2])
-
- def testRecord(self):
- my_record = utils.make_record('my_record', ['a', 'b', 'c'], {'b': 55})
- inst = my_record(a=1, b=2, c=3)
- self.assertEqual(1, inst.a)
- self.assertEqual(2, inst.b)
- self.assertEqual(3, inst.c)
- self.assertEqual(1, inst[0])
- self.assertEqual(2, inst[1])
- self.assertEqual(3, inst[2])
- self.assertEqual([1, 2, 3], list(iter(inst)))
- self.assertEqual(3, len(inst))
-
- inst.b = 999
- self.assertEqual(999, inst.b)
- self.assertEqual(999, inst[1])
-
- inst2 = my_record(1, 999, 3)
- self.assertTrue(inst == inst2)
- inst2[1] = 3
- self.assertFalse(inst == inst2)
-
- inst3 = my_record(a=1, c=3)
- inst.b = 55
- self.assertEqual(inst, inst3)
-
- def testRecordUnique(self):
- record1 = utils.make_record('record1', ['a', 'b', 'c'])
- record2 = utils.make_record('record2', ['a', 'b', 'c'])
- self.assertNotEqual(record1(1, 2, 3), record2(1, 2, 3))
- self.assertEqual(record1(1, 2, 3), record1(1, 2, 3))
-
- def testTupleToRecord(self):
- my_record = utils.make_record('my_record', ['a', 'b', 'c'])
- inst = utils.tuple_to_record((5, 6, 7), my_record)
- self.assertEqual(my_record(5, 6, 7), inst)
-
- def testRecordErrors(self):
- my_record = utils.make_record('my_record', ['a', 'b', 'c'], {'b': 10})
-
- with self.assertRaises(ValueError):
- my_record(c=5) # Did not provide required argument 'a'.
- with self.assertRaises(ValueError):
- my_record(1, 2, 3, 4) # Too many arguments.
-
- def testRandomQueue(self):
- np.random.seed(567890)
- queue = utils.RandomQueue(5)
- queue.push(5)
- queue.push(6)
- queue.push(7)
- queue.push(8)
- queue.push(9)
- queue.push(10)
- self.assertTrue(5 not in queue)
- sample = queue.random_sample(1000)
- self.assertEqual(1000, len(sample))
- self.assertEqual([6, 7, 8, 9, 10], sorted(np.unique(sample).tolist()))
-
- def testMaxUniquePriorityQueue(self):
- queue = utils.MaxUniquePriorityQueue(5)
- queue.push(1.0, 'string 1')
- queue.push(-0.5, 'string 2')
- queue.push(0.5, 'string 3')
- self.assertEqual((-0.5, 'string 2', None), queue.pop())
- queue.push(0.1, 'string 4')
- queue.push(1.5, 'string 5')
- queue.push(0.0, 'string 6')
- queue.push(0.2, 'string 7')
- self.assertEqual((1.5, 'string 5', None), queue.get_max())
- self.assertEqual((0.1, 'string 4', None), queue.get_min())
- self.assertEqual(
- [('string 5', None), ('string 1', None), ('string 3', None),
- ('string 7', None), ('string 4', None)],
- list(queue.iter_in_order()))
-
- def testMaxUniquePriorityQueue_Duplicates(self):
- queue = utils.MaxUniquePriorityQueue(5)
- queue.push(0.0, 'string 1')
- queue.push(0.0, 'string 2')
- queue.push(0.0, 'string 3')
- self.assertEqual((0.0, 'string 1', None), queue.pop())
- self.assertEqual((0.0, 'string 2', None), queue.pop())
- self.assertEqual((0.0, 'string 3', None), queue.pop())
- self.assertEqual(0, len(queue))
- queue.push(0.1, 'string 4')
- queue.push(1.5, 'string 5')
- queue.push(0.3, 'string 6')
- queue.push(0.2, 'string 7')
- queue.push(0.0, 'string 8')
- queue.push(1.5, 'string 5')
- queue.push(1.5, 'string 5')
- self.assertEqual((1.5, 'string 5', None), queue.get_max())
- self.assertEqual((0.0, 'string 8', None), queue.get_min())
- self.assertEqual(
- [('string 5', None), ('string 6', None), ('string 7', None),
- ('string 4', None), ('string 8', None)],
- list(queue.iter_in_order()))
-
- def testMaxUniquePriorityQueue_ExtraData(self):
- queue = utils.MaxUniquePriorityQueue(5)
- queue.push(1.0, 'string 1', [1, 2, 3])
- queue.push(0.5, 'string 2', [4, 5, 6])
- queue.push(0.5, 'string 3', [7, 8, 9])
- queue.push(0.5, 'string 2', [10, 11, 12])
- self.assertEqual((0.5, 'string 2', [4, 5, 6]), queue.pop())
- self.assertEqual((0.5, 'string 3', [7, 8, 9]), queue.pop())
- self.assertEqual((1.0, 'string 1', [1, 2, 3]), queue.pop())
- self.assertEqual(0, len(queue))
- queue.push(0.5, 'string 2', [10, 11, 12])
- self.assertEqual((0.5, 'string 2', [10, 11, 12]), queue.pop())
-
- def testRouletteWheel(self):
- random.seed(12345678987654321)
- r = utils.RouletteWheel()
- self.assertTrue(r.is_empty())
- with self.assertRaises(RuntimeError):
- r.sample() # Cannot sample when empty.
- self.assertEqual(0, r.total_weight)
- self.assertEqual(True, r.add('a', 0.1))
- self.assertFalse(r.is_empty())
- self.assertEqual(0.1, r.total_weight)
- self.assertEqual(True, r.add('b', 0.01))
- self.assertEqual(0.11, r.total_weight)
- self.assertEqual(True, r.add('c', 0.5))
- self.assertEqual(True, r.add('d', 0.1))
- self.assertEqual(True, r.add('e', 0.05))
- self.assertEqual(True, r.add('f', 0.03))
- self.assertEqual(True, r.add('g', 0.001))
- self.assertEqual(0.791, r.total_weight)
- self.assertFalse(r.is_empty())
-
- # Check that sampling is correct.
- obj, weight = r.sample()
- self.assertTrue(isinstance(weight, float), 'Type: %s' % type(weight))
- self.assertTrue((obj, weight) in r)
- for obj, weight in r.sample_many(100):
- self.assertTrue(isinstance(weight, float), 'Type: %s' % type(weight))
- self.assertTrue((obj, weight) in r)
-
- # Check that sampling distribution is correct.
- n = 1000000
- c = Counter(r.sample_many(n))
- for obj, w in r:
- estimated_w = c[(obj, w)] / float(n) * r.total_weight
- self.assertTrue(
- np.isclose(w, estimated_w, atol=1e-3),
- 'Expected %s, got %s, for object %s' % (w, estimated_w, obj))
-
- def testRouletteWheel_AddMany(self):
- random.seed(12345678987654321)
- r = utils.RouletteWheel()
- self.assertTrue(r.is_empty())
- with self.assertRaises(RuntimeError):
- r.sample() # Cannot sample when empty.
- self.assertEqual(0, r.total_weight)
- count = r.add_many(
- ['a', 'b', 'c', 'd', 'e', 'f', 'g'],
- [0.1, 0.01, 0.5, 0.1, 0.05, 0.03, 0.001])
- self.assertEqual(7, count)
- self.assertFalse(r.is_empty())
- self.assertEqual(0.791, r.total_weight)
-
- # Adding no items is allowed.
- count = r.add_many([], [])
- self.assertEqual(0, count)
- self.assertFalse(r.is_empty())
- self.assertEqual(0.791, r.total_weight)
-
- # Check that sampling is correct.
- obj, weight = r.sample()
- self.assertTrue(isinstance(weight, float), 'Type: %s' % type(weight))
- self.assertTrue((obj, weight) in r)
- for obj, weight in r.sample_many(100):
- self.assertTrue(isinstance(weight, float), 'Type: %s' % type(weight))
- self.assertTrue((obj, weight) in r)
-
- # Check that sampling distribution is correct.
- n = 1000000
- c = Counter(r.sample_many(n))
- for obj, w in r:
- estimated_w = c[(obj, w)] / float(n) * r.total_weight
- self.assertTrue(
- np.isclose(w, estimated_w, atol=1e-3),
- 'Expected %s, got %s, for object %s' % (w, estimated_w, obj))
-
- def testRouletteWheel_AddZeroWeights(self):
- r = utils.RouletteWheel()
- self.assertEqual(True, r.add('a', 0))
- self.assertFalse(r.is_empty())
- self.assertEqual(4, r.add_many(['b', 'c', 'd', 'e'], [0, 0.1, 0, 0]))
- self.assertEqual(
- [('a', 0.0), ('b', 0.0), ('c', 0.1), ('d', 0.0), ('e', 0.0)],
- list(r))
-
- def testRouletteWheel_UniqueMode(self):
- random.seed(12345678987654321)
- r = utils.RouletteWheel(unique_mode=True)
- self.assertEqual(True, r.add([1, 2, 3], 1, 'a'))
- self.assertEqual(True, r.add([4, 5], 0.5, 'b'))
- self.assertEqual(False, r.add([1, 2, 3], 1.5, 'a'))
- self.assertEqual(
- [([1, 2, 3], 1.0), ([4, 5], 0.5)],
- list(r))
- self.assertEqual(1.5, r.total_weight)
- self.assertEqual(
- 2,
- r.add_many(
- [[5, 6, 2, 3], [1, 2, 3], [8], [1, 2, 3]],
- [0.1, 0.2, 0.1, 2.0],
- ['c', 'a', 'd', 'a']))
- self.assertEqual(
- [([1, 2, 3], 1.0), ([4, 5], 0.5), ([5, 6, 2, 3], 0.1), ([8], 0.1)],
- list(r))
- self.assertTrue(np.isclose(1.7, r.total_weight))
- self.assertEqual(0, r.add_many([], [], [])) # Adding no items is allowed.
- with self.assertRaises(ValueError):
- # Key not given.
- r.add([7, 8, 9], 2.0)
- with self.assertRaises(ValueError):
- # Keys not given.
- r.add_many([[7, 8, 9], [10]], [2.0, 2.0])
- self.assertEqual(True, r.has_key('a'))
- self.assertEqual(True, r.has_key('b'))
- self.assertEqual(False, r.has_key('z'))
- self.assertEqual(1.0, r.get_weight('a'))
- self.assertEqual(0.5, r.get_weight('b'))
-
- r = utils.RouletteWheel(unique_mode=False)
- self.assertEqual(True, r.add([1, 2, 3], 1))
- self.assertEqual(True, r.add([4, 5], 0.5))
- self.assertEqual(True, r.add([1, 2, 3], 1.5))
- self.assertEqual(
- [([1, 2, 3], 1.0), ([4, 5], 0.5), ([1, 2, 3], 1.5)],
- list(r))
- self.assertEqual(3, r.total_weight)
- self.assertEqual(
- 4,
- r.add_many(
- [[5, 6, 2, 3], [1, 2, 3], [8], [1, 2, 3]],
- [0.1, 0.2, 0.1, 0.2]))
- self.assertEqual(
- [([1, 2, 3], 1.0), ([4, 5], 0.5), ([1, 2, 3], 1.5),
- ([5, 6, 2, 3], 0.1), ([1, 2, 3], 0.2), ([8], 0.1), ([1, 2, 3], 0.2)],
- list(r))
- self.assertTrue(np.isclose(3.6, r.total_weight))
- with self.assertRaises(ValueError):
- # Key is given.
- r.add([7, 8, 9], 2.0, 'a')
- with self.assertRaises(ValueError):
- # Keys are given.
- r.add_many([[7, 8, 9], [10]], [2.0, 2.0], ['a', 'b'])
-
- def testRouletteWheel_IncrementalSave(self):
- f = tempfile.NamedTemporaryFile()
- r = utils.RouletteWheel(unique_mode=True, save_file=f.name)
- entries = [
- ([1, 2, 3], 0.1, 'a'),
- ([4, 5], 0.2, 'b'),
- ([6], 0.3, 'c'),
- ([7, 8, 9, 10], 0.25, 'd'),
- ([-1, -2], 0.15, 'e'),
- ([-3, -4, -5], 0.5, 'f')]
-
- self.assertTrue(r.is_empty())
- for i in range(0, len(entries), 2):
- r.add(*entries[i])
- r.add(*entries[i + 1])
- r.incremental_save()
-
- r2 = utils.RouletteWheel(unique_mode=True, save_file=f.name)
- self.assertEqual(i + 2, len(r2))
- count = 0
- for j, (obj, weight) in enumerate(r2):
- self.assertEqual(entries[j][0], obj)
- self.assertEqual(entries[j][1], weight)
- self.assertEqual(weight, r2.get_weight(entries[j][2]))
- count += 1
- self.assertEqual(i + 2, count)
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/single_task/BUILD b/research/brain_coder/single_task/BUILD
deleted file mode 100644
index 47e91b12b8ba40a2a9916a89375fbb773758d7cf..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/BUILD
+++ /dev/null
@@ -1,244 +0,0 @@
-licenses(["notice"])
-
-package(default_visibility = [
- "//learning/brain/research/neural_coder:__subpackages__",
-])
-
-load("@subpar//:subpar.bzl", "par_binary")
-
-par_binary(
- name = "run",
- srcs = ["run.py"],
- deps = [
- ":defaults",
- ":ga_train",
- ":pg_train",
- # absl dep :app
- # absl dep /flags
- # absl dep /logging
- ],
-)
-
-par_binary(
- name = "tune",
- srcs = ["tune.py"],
- deps = [
- ":defaults",
- ":run",
- # file dep
- # absl dep :app
- # absl dep /flags
- # absl dep /logging
- # numpy dep
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "ga_train",
- srcs = ["ga_train.py"],
- deps = [
- ":data",
- ":defaults",
- ":ga_lib",
- ":results_lib",
- # file dep
- # absl dep /flags
- # absl dep /logging
- # numpy dep
- # tensorflow dep
- "//common:utils", # project
- ],
-)
-
-py_library(
- name = "ga_lib",
- srcs = ["ga_lib.py"],
- deps = [
- ":misc",
- # absl dep /flags
- # absl dep /logging
- # numpy dep
- "//common:bf", # project
- "//common:utils", # project
- ],
-)
-
-py_test(
- name = "ga_train_test",
- srcs = ["ga_train_test.py"],
- deps = [
- ":defaults",
- ":run",
- # absl dep /flags
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "pg_train",
- srcs = ["pg_train.py"],
- deps = [
- ":data",
- ":defaults",
- ":pg_agent",
- ":results_lib",
- # file dep
- # absl dep /flags
- # absl dep /logging
- # tensorflow dep
- # tensorflow internal dep # build_cleaner: keep
- ],
-)
-
-py_library(
- name = "pg_agent",
- srcs = ["pg_agent.py"],
- deps = [
- ":misc",
- # file dep
- # absl dep /logging
- # numpy dep
- # tensorflow dep
- "//common:rollout", # project
- "//common:utils", # project
- ],
-)
-
-py_test(
- name = "pg_agent_test",
- srcs = ["pg_agent_test.py"],
- deps = [
- ":data",
- ":defaults",
- ":misc",
- ":pg_agent",
- ":pg_train",
- # absl dep /logging
- # numpy dep
- # tensorflow dep
- "//common:utils", # project
- ],
-)
-
-py_library(
- name = "defaults",
- srcs = ["defaults.py"],
- deps = [
- # absl dep /logging
- "//common:config_lib", # project
- ],
-)
-
-py_library(
- name = "misc",
- srcs = ["misc.py"],
-)
-
-py_library(
- name = "data",
- srcs = ["data.py"],
- deps = [
- ":code_tasks",
- # absl dep /logging
- ],
-)
-
-py_library(
- name = "code_tasks",
- srcs = ["code_tasks.py"],
- deps = [
- ":misc",
- ":test_tasks",
- # absl dep /logging
- # numpy dep
- "//common:bf", # project
- "//common:reward", # project
- ],
-)
-
-py_test(
- name = "code_tasks_test",
- srcs = ["code_tasks_test.py"],
- deps = [
- ":code_tasks",
- ":defaults",
- # numpy dep
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "test_tasks",
- srcs = ["test_tasks.py"],
- deps = [
- ":misc",
- "//common:reward", # project
- ],
-)
-
-py_test(
- name = "test_tasks_test",
- srcs = ["test_tasks_test.py"],
- deps = [
- ":misc",
- ":test_tasks",
- # numpy dep
- # tensorflow dep
- ],
-)
-
-py_test(
- name = "pg_train_test",
- size = "large",
- srcs = ["pg_train_test.py"],
- deps = [
- ":defaults",
- ":run",
- # absl dep /logging
- # tensorflow dep
- ],
-)
-
-py_library(
- name = "results_lib",
- srcs = ["results_lib.py"],
- deps = [
- # file dep
- # tensorflow dep
- ],
-)
-
-py_test(
- name = "results_lib_test",
- srcs = ["results_lib_test.py"],
- deps = [
- ":results_lib",
- # tensorflow dep
- ],
-)
-
-par_binary(
- name = "aggregate_experiment_results",
- srcs = ["aggregate_experiment_results.py"],
- deps = [
- ":misc",
- ":results_lib",
- # file dep
- # absl dep :app
- # absl dep /flags
- # numpy dep
- # tensorflow dep
- ],
-)
-
-par_binary(
- name = "aggregate_tuning_results",
- srcs = ["aggregate_tuning_results.py"],
- deps = [
- # file dep
- # absl dep :app
- # absl dep /flags
- # tensorflow dep
- ],
-)
diff --git a/research/brain_coder/single_task/README.md b/research/brain_coder/single_task/README.md
deleted file mode 100644
index 69eaabcc6ccabada838a0a2a3f12fd7eed69744c..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/README.md
+++ /dev/null
@@ -1,192 +0,0 @@
-# Experiments for ICLR 2018 paper.
-
-[Neural Program Synthesis with Priority Queue Training](https://arxiv.org/abs/1801.03526).
-
-Runs policy gradient (REINFORCE), priority queue training, genetic algorithm,
-and uniform random search.
-
-Run all examples below out of your top-level repo directory, i.e. where your git
-clone resides.
-
-
-## Just tell me how to run something and see results
-```bash
-# These tasks are the fastest to learn. 'echo' and 'count-down' are very
-# easy. run_eval_tasks.py will do most of the work to run all the jobs.
-# Should take between 10 and 30 minutes.
-
-# How many repetitions each experiment will run. In the paper, we use 25. Less
-# reps means faster experiments, but noisier results.
-REPS=25
-
-# Extra description in the job names for these experiments. Use this description
-# to distinguish between multiple runs of the same experiment.
-DESC="demo"
-
-# The tasks to run.
-TASKS="reverse echo-second-seq"
-
-# The model types and max NPE.
-EXPS=( pg-20M topk-20M ga-20M rand-20M )
-
-# Where training data is saved. This is chosen by launch_training.sh. Custom
-# implementations of launch_training.sh may use different locations.
-MODELS_DIR="/tmp/models"
-
-# Run run_eval_tasks.py for each experiment name in EXPS.
-for exp in "${EXPS[@]}"
-do
- ./single_task/run_eval_tasks.py \
- --exp "$exp" --tasks $TASKS --desc "$DESC" --reps $REPS
-done
-
-# During training or after completion, run this to aggregate results into a
-# table. This is also useful for seeing how much progress has been made.
-# Make sure the arguments here match the settings used above.
-# Note: This can take a few minutes because it reads from every experiment
-# directory.
-bazel run single_task:aggregate_experiment_results -- \
- --models_dir="$MODELS_DIR" \
- --max_npe="20M" \
- --task_list="$TASKS" \
- --model_types="[('pg', '$DESC'), ('topk', '$DESC'), ('ga', '$DESC'),
- ('rand', '$DESC')]" \
- --csv_file="/tmp/results_table.csv"
-```
-
-
-## Reproduce tuning results in paper
-```bash
-bazel build -c opt single_task:tune.par
-
-# PG and TopK Tuning.
-MAX_NPE=5000000
-CONFIG="
-env=c(task_cycle=['reverse-tune','remove-tune']),
-agent=c(
- algorithm='pg',
- grad_clip_threshold=50.0,param_init_factor=0.5,entropy_beta=0.05,lr=1e-5,
- optimizer='rmsprop',ema_baseline_decay=0.99,topk_loss_hparam=0.0,topk=0,
- replay_temperature=1.0,alpha=0.0,eos_token=False),
-timestep_limit=50,batch_size=64"
-
-./single_task/launch_tuning.sh \
- --job_name="iclr_pg_gridsearch.reverse-remove" \
- --config="$CONFIG" \
- --max_npe="$MAX_NPE" \
- --num_workers_per_tuner=1 \
- --num_ps_per_tuner=0 \
- --num_tuners=1 \
- --num_repetitions=50 \
- --hparam_space_type="pg" \
- --stop_on_success=true
-./single_task/launch_tuning.sh \
- --job_name="iclr_pg_topk_gridsearch.reverse-remove" \
- --config="$CONFIG" \
- --max_npe="$MAX_NPE" \
- --num_workers_per_tuner=1 \
- --num_ps_per_tuner=0 \
- --num_tuners=1 \
- --num_repetitions=50 \
- --hparam_space_type="pg-topk" \
- --fixed_hparams="topk=10" \
- --stop_on_success=true
-./single_task/launch_tuning.sh \
- --job_name="iclr_topk_gridsearch.reverse-remove" \
- --config="$CONFIG" \
- --max_npe="$MAX_NPE" \
- --num_workers_per_tuner=1 \
- --num_ps_per_tuner=0 \
- --num_tuners=1 \
- --num_repetitions=50 \
- --hparam_space_type="topk" \
- --fixed_hparams="topk=10" \
- --stop_on_success=true
-
-# GA Tuning.
-CONFIG="
-env=c(task_cycle=['reverse-tune','remove-char-tune']),
-agent=c(algorithm='ga'),
-timestep_limit=50"
-./single_task/launch_tuning.sh \
- --job_name="iclr_ga_gridsearch.reverse-remove" \
- --config="$CONFIG" \
- --max_npe="$MAX_NPE" \
- --num_workers_per_tuner=25 \
- --num_ps_per_tuner=0 \
- --num_tuners=1 \
- --num_repetitions=50 \
- --hparam_space_type="ga" \
- --stop_on_success=true
-
-# Aggregate tuning results. Run after tuning jobs complete.
-bazel run -c opt single_task:aggregate_tuning_results -- \
- --tuning_dir="$MODELS_DIR/iclr_pg_gridsearch.reverse-remove"
-bazel run -c opt single_task:aggregate_tuning_results -- \
- --tuning_dir="$MODELS_DIR/iclr_pg_topk_gridsearch.reverse-remove"
-bazel run -c opt single_task:aggregate_tuning_results -- \
- --tuning_dir="$MODELS_DIR/iclr_topk_gridsearch.reverse-remove"
-bazel run -c opt single_task:aggregate_tuning_results -- \
- --tuning_dir="$MODELS_DIR/iclr_ga_gridsearch.reverse-remove"
-```
-
-## Reproduce eval results in paper
-```bash
-DESC="v0" # Description for each experiment. "Version 0" is a good default.
-EXPS=( pg-5M topk-5M ga-5M rand-5M pg-20M topk-20M ga-20M rand-20M )
-for exp in "${EXPS[@]}"
-do
- ./single_task/run_eval_tasks.py \
- --exp "$exp" --iclr_tasks --desc "$DESC"
-done
-```
-
-## Run single experiment
-```bash
-EXP="topk-20M" # Learning algorithm + max-NPE
-TASK="reverse" # Coding task
-DESC="v0" # Description for each experiment. "Version 0" is a good default.
-./single_task/run_eval_tasks.py \
- --exp "$EXP" --task "$TASK" --desc "$DESC"
-```
-
-## Fetch eval results into a table
-```bash
-# These arguments should match the settings you used to run the experiments.
-MODELS_DIR="/tmp/models"
-MAX_NPE="20M"
-DESC="v0" # Same description used in the experiments.
-# MODEL_TYPES specifies each model type and the description used in their
-# experiments.
-MODEL_TYPES="[('pg', '$DESC'), ('topk', '$DESC'),
- ('ga', '$DESC'), ('rand', '$DESC')]"
-TASKS="" # Empty string will default to all ICLR tasks.
-# To specify custom task list, give task names separated by spaces. Example:
-# TASKS="reverse remove-char"
-bazel run single_task:aggregate_experiment_results -- \
- --models_dir="$MODELS_DIR" \
- --max_npe="$MAX_NPE" \
- --task_list="$TASKS" \
- --model_types="$MODEL_TYPES" \
- --csv_file="/tmp/results_table.csv"
-```
-
-## Reproduce shortest code examples in paper
-```bash
-# Maximum NPE is higher here. We only do 1 repetition, and the algorithm needs
-# time to simplify its solution.
-MODELS_DIR="/tmp/models"
-NPE="500M"
-DESC="short-code"
-./single_task/run_eval_tasks.py \
- --exp "simpl-$NPE" --desc "$DESC" --iclr_tasks --reps 1
-
-# Aggregate best code strings. Run after training completes.
-TASKS="" # Empty string. Will default to all ICLR tasks.
-bazel run single_task:aggregate_experiment_results -- \
- --models_dir="$MODELS_DIR" \
- --max_npe="$NPE" \
- --task_list="$TASKS" \
- --model_types="[('topk', '$DESC')]" \
- --data=code
-```
diff --git a/research/brain_coder/single_task/aggregate_experiment_results.py b/research/brain_coder/single_task/aggregate_experiment_results.py
deleted file mode 100644
index f106253004b3bbe1ff32443c41b8999b1c9e96f6..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/aggregate_experiment_results.py
+++ /dev/null
@@ -1,380 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-r"""This script crawls experiment directories for results and aggregates them.
-
-Usage example:
-
-MODELS_DIR="/tmp/models"
-bazel run single_task:aggregate_experiment_results -- \
- --models_dir="$MODELS_DIR" \
- --max_npe="20M" \
- --task_list="add echo" \
- --model_types="[('topk', 'v0'), ('ga', 'v0')]" \
- --csv_file=/tmp/results_table.csv
-"""
-
-import ast
-from collections import namedtuple
-import csv
-import os
-import re
-import StringIO
-import sys
-
-from absl import app
-from absl import flags
-import numpy as np
-import tensorflow as tf
-
-from single_task import misc # brain coder
-from single_task import results_lib # brain coder
-
-DEFAULT_MODELS = [('pg', 'v0'), ('topk', 'v0'), ('ga', 'v0'), ('rand', 'v0')]
-DEFAULT_TASKS = [
- 'reverse', 'remove-char', 'count-char', 'add', 'bool-logic', 'print-hello',
- 'echo-twice', 'echo-thrice', 'copy-reverse', 'zero-cascade', 'cascade',
- 'shift-left', 'shift-right', 'riffle', 'unriffle', 'middle-char',
- 'remove-last', 'remove-last-two', 'echo-alternating', 'echo-half', 'length',
- 'echo-second-seq', 'echo-nth-seq', 'substring', 'divide-2', 'dedup']
-
-FLAGS = flags.FLAGS
-flags.DEFINE_string(
- 'models_dir', '',
- 'Absolute path where results folders are found.')
-flags.DEFINE_string(
- 'exp_prefix', 'bf_rl_iclr',
- 'Prefix for all experiment folders.')
-flags.DEFINE_string(
- 'max_npe', '5M',
- 'String representation of max NPE of the experiments.')
-flags.DEFINE_spaceseplist(
- 'task_list', DEFAULT_TASKS,
- 'List of task names separated by spaces. If empty string, defaults to '
- '`DEFAULT_TASKS`. These are the rows of the results table.')
-flags.DEFINE_string(
- 'model_types', str(DEFAULT_MODELS),
- 'String representation of a python list of 2-tuples, each a model_type + '
- 'job description pair. Descriptions allow you to choose among different '
- 'runs of the same experiment. These are the columns of the results table.')
-flags.DEFINE_string(
- 'csv_file', '/tmp/results_table.csv',
- 'Where to write results table. Format is CSV.')
-flags.DEFINE_enum(
- 'data', 'success_rates', ['success_rates', 'code'],
- 'What type of data to aggregate.')
-
-
-def make_csv_string(table):
- """Convert 2D list to CSV string."""
- s = StringIO.StringIO()
- writer = csv.writer(s)
- writer.writerows(table)
- value = s.getvalue()
- s.close()
- return value
-
-
-def process_results(metrics):
- """Extract useful information from given metrics.
-
- Args:
- metrics: List of results dicts. These should have been written to disk by
- training jobs.
-
- Returns:
- Dict mapping stats names to values.
-
- Raises:
- ValueError: If max_npe or max_global_repetitions values are inconsistant
- across dicts in the `metrics` list.
- """
- count = len(metrics)
- success_count = 0
- total_npe = 0 # Counting NPE across all runs.
- success_npe = 0 # Counting NPE in successful runs only.
- max_npe = 0
- max_repetitions = 0
- for metric_dict in metrics:
- if not max_npe:
- max_npe = metric_dict['max_npe']
- elif max_npe != metric_dict['max_npe']:
- raise ValueError(
- 'Invalid experiment. Different reps have different max-NPE settings.')
- if not max_repetitions:
- max_repetitions = metric_dict['max_global_repetitions']
- elif max_repetitions != metric_dict['max_global_repetitions']:
- raise ValueError(
- 'Invalid experiment. Different reps have different num-repetition '
- 'settings.')
- if metric_dict['found_solution']:
- success_count += 1
- success_npe += metric_dict['npe']
- total_npe += metric_dict['npe']
- stats = {}
- stats['max_npe'] = max_npe
- stats['max_repetitions'] = max_repetitions
- stats['repetitions'] = count
- stats['successes'] = success_count # successful reps
- stats['failures'] = count - success_count # failed reps
- stats['success_npe'] = success_npe
- stats['total_npe'] = total_npe
- if success_count:
- # Only successful runs counted.
- stats['avg_success_npe'] = stats['success_npe'] / float(success_count)
- else:
- stats['avg_success_npe'] = 0.0
- if count:
- stats['success_rate'] = success_count / float(count)
- stats['avg_total_npe'] = stats['total_npe'] / float(count)
- else:
- stats['success_rate'] = 0.0
- stats['avg_total_npe'] = 0.0
-
- return stats
-
-
-ProcessedResults = namedtuple('ProcessedResults', ['metrics', 'processed'])
-
-
-def get_results_for_experiment(
- models_dir, task_name, model_type='pg', max_npe='5M', desc='v0',
- name_prefix='bf_rl_paper', extra_desc=''):
- """Get and process results for a given experiment.
-
- An experiment is a set of runs with the same hyperparameters and environment.
- It is uniquely specified by a (task_name, model_type, max_npe) triple, as
- well as an optional description.
-
- We assume that each experiment has a folder with the same name as the job that
- ran the experiment. The name is computed by
- "%name_prefix%.%desc%-%max_npe%_%task_name%".
-
- Args:
- models_dir: Parent directory containing experiment folders.
- task_name: String name of task (the coding env). See code_tasks.py or
- run_eval_tasks.py
- model_type: Name of the algorithm, such as 'pg', 'topk', 'ga', 'rand'.
- max_npe: String SI unit representation of the maximum NPE threshold for the
- experiment. For example, "5M" means 5 million.
- desc: Description.
- name_prefix: Prefix of job names. Normally leave this as default.
- extra_desc: Optional extra description at the end of the job name.
-
- Returns:
- ProcessedResults namedtuple instance, containing
- metrics: Raw dicts read from disk.
- processed: Stats computed by `process_results`.
-
- Raises:
- ValueError: If max_npe in the metrics does not match NPE in the experiment
- folder name.
- """
- folder = name_prefix + '.{0}.{1}-{2}_{3}'.format(desc, model_type, max_npe,
- task_name)
- if extra_desc:
- folder += '.' + extra_desc
-
- results = results_lib.Results(os.path.join(models_dir, folder))
- metrics, _ = results.read_all()
- processed = process_results(metrics)
- if (not np.isclose(processed['max_npe'], misc.si_to_int(max_npe))
- and processed['repetitions']):
- raise ValueError(
- 'Invalid experiment. Max-NPE setting does not match expected max-NPE '
- 'in experiment name.')
- return ProcessedResults(metrics=metrics, processed=processed)
-
-
-BestCodeResults = namedtuple(
- 'BestCodeResults',
- ['code', 'reward', 'npe', 'folder', 'finished', 'error'])
-
-
-class BestCodeResultError(object):
- success = 0
- no_solution_found = 1
- experiment_does_not_exist = 2
-
-
-def get_best_code_for_experiment(
- models_dir, task_name, model_type='pg', max_npe='5M', desc=0,
- name_prefix='bf_rl_paper', extra_desc=''):
- """Like `get_results_for_experiment`, but fetches the code solutions."""
- folder = name_prefix + '.{0}.{1}-{2}_{3}'.format(desc, model_type, max_npe,
- task_name)
- if extra_desc:
- folder += '.' + extra_desc
-
- log_dir = os.path.join(models_dir, folder, 'logs')
- search_regex = r'^solutions_([0-9])+\.txt$'
- try:
- all_children = tf.gfile.ListDirectory(log_dir)
- except tf.errors.NotFoundError:
- return BestCodeResults(
- code=None, reward=0.0, npe=0, folder=folder, finished=False,
- error=BestCodeResultError.experiment_does_not_exist)
- solution_files = [
- fname for fname in all_children if re.search(search_regex, fname)]
- max_reward = 0.0
- npe = 0
- best_code = None
- for fname in solution_files:
- with tf.gfile.FastGFile(os.path.join(log_dir, fname), 'r') as reader:
- results = [ast.literal_eval(entry) for entry in reader]
- for res in results:
- if res['reward'] > max_reward:
- best_code = res['code']
- max_reward = res['reward']
- npe = res['npe']
- error = (
- BestCodeResultError.success if best_code
- else BestCodeResultError.no_solution_found)
- try:
- # If there is a status.txt file, check if it contains the status of the job.
- with tf.gfile.FastGFile(os.path.join(log_dir, 'status.txt'), 'r') as f:
- # Job is done, so mark this experiment as finished.
- finished = f.read().lower().strip() == 'done'
- except tf.errors.NotFoundError:
- # No status file has been written, so the experiment is not done. No need to
- # report an error here, because we do not require that experiment jobs write
- # out a status.txt file until they have finished.
- finished = False
- return BestCodeResults(
- code=best_code, reward=max_reward, npe=npe, folder=folder,
- finished=finished, error=error)
-
-
-def make_results_table(
- models=None,
- tasks=None,
- max_npe='5M',
- name_prefix='bf_rl_paper',
- extra_desc='',
- models_dir='/tmp'):
- """Creates a table of results: algorithm + version by tasks.
-
- Args:
- models: The table columns. A list of (algorithm, desc) tuples.
- tasks: The table rows. List of task names.
- max_npe: String SI unit representation of the maximum NPE threshold for the
- experiment. For example, "5M" means 5 million. All entries in the table
- share the same max-NPE.
- name_prefix: Name prefix used in logging directory for the experiment.
- extra_desc: Extra description added to name of logging directory for the
- experiment.
- models_dir: Parent directory containing all experiment folders.
-
- Returns:
- A 2D list holding the table cells.
- """
- if models is None:
- models = DEFAULT_MODELS
- if tasks is None:
- tasks = DEFAULT_TASKS
- model_results = {}
- for model_type, desc in models:
- model_results[model_type] = {
- tname: get_results_for_experiment(
- models_dir, tname, model_type, max_npe, desc,
- name_prefix=name_prefix, extra_desc=extra_desc
- ).processed
- for tname in tasks}
-
- def info(stats):
- return [str(stats['repetitions']),
- '%.2f' % stats['success_rate'],
- str(int(stats['avg_total_npe']))]
-
- rows = [['max NPE: ' + max_npe]
- + misc.flatten([['{0} ({1})'.format(m, d), '', '']
- for m, d in models])]
- rows.append(
- [''] + misc.flatten([['reps', 'success rate', 'avg NPE']
- for _ in models]))
- for tname in tasks:
- rows.append(
- [tname]
- + misc.flatten([info(model_results[model][tname])
- for model, _ in models]))
-
- return rows
-
-
-def print_results_table(results_table):
- """Print human readable results table to stdout."""
- print('')
- print('=== Results Table ===')
- print('Format: # reps [success rate, avg total NPE]')
-
- def info_str(info_row):
- # num_runs (success_rate, avg_total_npe)
- if not info_row[0]:
- return '0'
- return '%s [%s, %s]' % (str(info_row[0]).ljust(2), info_row[1], info_row[2])
-
- nc = len(results_table[0]) # num cols
- out_table = [
- [results_table[0][0]] + [results_table[0][i] for i in range(1, nc, 3)]]
- for row in results_table[2:]:
- out_table.append([row[0]] + [info_str(row[i:i+3]) for i in range(1, nc, 3)])
-
- nc = len(out_table[0]) # num cols
- col_widths = [max(len(row[col]) for row in out_table) for col in range(nc)]
-
- table_string = ''
- for row in out_table:
- table_string += ''.join(
- [row[c].ljust(col_widths[c] + 2) for c in range(nc)]) + '\n'
-
- print(table_string)
-
-
-def main(argv):
- del argv # Unused.
-
- name_prefix = FLAGS.exp_prefix
- print('Experiments prefix: %s' % name_prefix)
-
- model_types = ast.literal_eval(FLAGS.model_types)
-
- if FLAGS.data == 'success_rates':
- results_table = make_results_table(
- models=model_types, tasks=FLAGS.task_list, max_npe=FLAGS.max_npe,
- models_dir=FLAGS.models_dir,
- name_prefix=name_prefix, extra_desc='')
- with tf.gfile.FastGFile(FLAGS.csv_file, 'w') as f:
- f.write(make_csv_string(results_table))
-
- print_results_table(results_table)
- else:
- # Best code
- print('* = experiment is still running')
- print('')
- print('=== Best Synthesized Code ===')
- for model_type, desc in model_types:
- print('%s (%s)' % (model_type, desc))
- sys.stdout.flush()
- for tname in FLAGS.task_list:
- res = get_best_code_for_experiment(
- FLAGS.models_dir, tname, model_type, FLAGS.max_npe, desc,
- name_prefix=name_prefix, extra_desc='')
- unfinished_mark = '' if res.finished else ' *'
- tname += unfinished_mark
- if res.error == BestCodeResultError.success:
- print(' %s' % tname)
- print(' %s' % res.code)
- print(' R=%.6f, NPE=%s' % (res.reward, misc.int_to_si(res.npe)))
- elif res.error == BestCodeResultError.experiment_does_not_exist:
- print(' Experiment does not exist. Check arguments.')
- print(' Experiment folder: %s' % res.folder)
- break
- else:
- print(' %s' % tname)
- print(' (none)')
- sys.stdout.flush()
-
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/brain_coder/single_task/aggregate_tuning_results.py b/research/brain_coder/single_task/aggregate_tuning_results.py
deleted file mode 100644
index bb2e008ce583afbea8acabfe1ed8ccf264698f5e..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/aggregate_tuning_results.py
+++ /dev/null
@@ -1,71 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-r"""After running tuning, use this script to aggregate the results.
-
-Usage:
-
-OUT_DIR=""
-bazel run -c opt single_task:aggregate_tuning_results -- \
- --alsologtostderr \
- --tuning_dir="$OUT_DIR"
-"""
-
-import ast
-import os
-
-from absl import app
-from absl import flags
-import tensorflow as tf
-
-
-FLAGS = flags.FLAGS
-flags.DEFINE_string(
- 'tuning_dir', '',
- 'Absolute path where results tuning trial folders are found.')
-
-
-def main(argv):
- del argv # Unused.
-
- try:
- trial_dirs = tf.gfile.ListDirectory(FLAGS.tuning_dir)
- except tf.errors.NotFoundError:
- print('Tuning directory %s does not exist.' % (FLAGS.tuning_dir,))
- return
-
- metrics = []
- for trial_dir in trial_dirs:
- tuning_results_file = os.path.join(
- FLAGS.tuning_dir, trial_dir, 'tuning_results.txt')
- if tf.gfile.Exists(tuning_results_file):
- with tf.gfile.FastGFile(tuning_results_file, 'r') as reader:
- for line in reader:
- metrics.append(ast.literal_eval(line.replace(': nan,', ': 0.0,')))
-
- if not metrics:
- print('No trials found.')
- return
-
- num_trials = [m['num_trials'] for m in metrics]
- assert all(n == num_trials[0] for n in num_trials)
- num_trials = num_trials[0]
- print('Found %d completed trials out of %d' % (len(metrics), num_trials))
-
- # Sort by objective descending.
- sorted_trials = sorted(metrics, key=lambda m: -m['objective'])
-
- for i, metrics in enumerate(sorted_trials):
- hparams = metrics['hparams']
- keys = sorted(hparams.keys())
- print(
- str(i).ljust(4) + ': '
- + '{0:.2f}'.format(metrics['objective']).ljust(10)
- + '['
- + ','.join(['{}={}'.format(k, hparams[k]).ljust(24) for k in keys])
- + ']')
-
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/brain_coder/single_task/code_tasks.py b/research/brain_coder/single_task/code_tasks.py
deleted file mode 100644
index 27cc7ecd1c76f2d765692ce0a94acd1df04ff681..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/code_tasks.py
+++ /dev/null
@@ -1,1381 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tasks for RL."""
-
-import abc
-import copy
-import itertools
-import random
-
-from absl import logging
-import numpy as np
-from six.moves import xrange
-
-from common import bf # brain coder
-from common import reward as r # brain coder
-from single_task import misc # brain coder
-from single_task import test_tasks # brain coder
-
-
-MAX_EXECUTION_STEPS = 5000
-
-
-def make_task(task_name, override_kwargs=None, max_code_length=100,
- require_correct_syntax=False,
- do_code_simplification=False,
- correct_bonus=2.0, code_length_bonus=1.0):
- """Make tasks with setting from paper."""
- logging.info('Making paper-config task.')
- n = 16 # Number of test cases.
- task_mapping = {
- 'print-hello': (
- PrintTask, dict(base=27, fixed_string=[8, 5, 12, 12, 15])),
- 'print': (PrintIntTask, dict(base=256, fixed_string=[1, 2, 3, 4, 5])),
- 'echo': (EchoTask, dict(base=27, min_length=1, max_length=6)),
- 'remove-char': (
- RemoveCharTask, dict(base=256, n=n, min_len=1, max_len=6)),
- 'reverse': (
- ReverseTask, dict(base=256, n=n, min_len=1, max_len=6)),
- 'reverse-tune': (
- ReverseTaskV2, dict(base=256, reward_type='static-bylen')),
- 'remove-char-tune': (RemoveCharTaskV2, dict(base=27)),
- 'prefix': (CommonPrefixTask, dict(base=27)),
- 'find': (FindSubStrTask, dict(base=27)),
- 'sort3': (SortFixedTaskV2, dict(base=27, n=150, length=3)),
- 'count-char': (CountCharTaskV2, dict(n=n, max_len=6)),
- 'bool-logic': (BooleanLogicTask, dict()),
- 'add': (AddTask, dict(n=9)),
- 'echo-twice': (EchoTwiceTask, dict(n=n)),
- 'echo-thrice': (EchoThriceTask, dict(n=n)),
- 'copy-reverse': (CopyReverseTask, dict(n=n)),
- 'zero-cascade': (EchoZeroCascadeTask, dict(n=n)),
- 'cascade': (EchoCascadeTask, dict(n=n)),
- 'shift-left': (ShiftLeftTask, dict(n=n)),
- 'shift-right': (ShiftRightTask, dict(n=n)),
- 'riffle': (RiffleTask, dict(n=n)),
- 'unriffle': (UnriffleTask, dict(n=n)),
- 'middle-char': (MiddleCharTask, dict(n=n)),
- 'remove-last': (RemoveLastTask, dict(n=n)),
- 'remove-last-two': (RemoveLastTwoTask, dict(n=n)),
- 'echo-alternating': (EchoAlternatingTask, dict(n=n)),
- 'echo-half': (EchoHalfTask, dict(n=n)),
- 'length': (LengthTask, dict(n=n)),
- 'echo-second-seq': (EchoSecondSequenceTask, dict(n=n)),
- 'echo-nth-seq': (EchoNthSequenceTask, dict(n=n)),
- 'substring': (SubstringTask, dict(n=n)),
- 'divide-2': (Divide2Task, dict(n=n)),
- 'dedup': (DedupTask, dict(n=n)),
- 'remove-target-char': (RemoveTargetCharTask, dict(n=n)),
- 'list-index': (ListIndexTask, dict(n=n)),
- 'fib': (FibonacciTask, dict()),
- 'count-down': (BottlesOfBeerTask, dict()),
- 'split': (SplitTask, dict()),
- 'trim-left': (TrimLeftTask, dict()),
- 'circle-route': (
- JudgeRouteCircleTask, dict(n=100, max_len=32)),
- 'multiply': (MultiplyTask, dict(n=100)),
- 'divmod': (DivModTask, dict(n=100)),
- }
-
- if task_name not in task_mapping:
- # Test tasks.
- if task_name == 'test-hill-climb':
- return test_tasks.BasicTaskManager(test_tasks.HillClimbingTask())
- raise ValueError('Unknown task type "%s"' % task_name)
- task_cls, kwargs = task_mapping[task_name]
-
- if override_kwargs:
- if not isinstance(override_kwargs, dict):
- raise ValueError(
- 'override_kwargs must be a dict, got: %s', override_kwargs)
- kwargs.update(override_kwargs)
-
- task = task_cls(**kwargs)
-
- reward_fn = r.absolute_distance_reward
- # reward_fn = r.absolute_mod_distance_reward
- # reward_fn = r.absolute_log_distance_reward
- logging.info('Using reward function: %s', reward_fn.__name__)
-
- # We want reward with and without code simplification to be scaled the same
- # way. Without code simplification, give the maximum code length bonus
- # every time.
- min_code_length = 0.0 if do_code_simplification else max_code_length
-
- return MultiIOTaskManager(
- task=task, correct_bonus=correct_bonus,
- code_length_bonus=code_length_bonus,
- max_code_length=max_code_length, min_code_length=min_code_length,
- reward_fn=reward_fn, require_correct_syntax=require_correct_syntax)
-
-
-def concat(lists):
- if not lists:
- return []
- l = lists[0]
- for k in lists[1:]:
- l += k
- return l
-
-
-def concat_join(lists, sep):
- if not lists:
- return []
- l = lists[0]
- for k in lists[1:]:
- l += [sep] + k
- return l
-
-
-def clipped_linear(x, x0, y0, slope, y_range):
- min_y, max_y = y_range
- return min(max(slope * (x - x0) + y0, min_y), max_y)
-
-
-class MultiIOTaskManager(object):
- """Supports tasks which test the code with multiple I/O examples."""
-
- def __init__(self, task, max_code_length=32, min_code_length=0,
- max_execution_steps=MAX_EXECUTION_STEPS, correct_bonus=1.0,
- code_length_bonus=1.0, failure_reward=-2.0, reward_fn=None,
- require_correct_syntax=False):
- assert isinstance(task, BaseTask)
- self.task = task
- self.max_code_length = max_code_length
- self.min_code_length = min_code_length
- self.max_execution_steps = max_execution_steps
- self.require_correct_syntax = require_correct_syntax
- self.correct_bonus = correct_bonus
- self.code_length_bonus = code_length_bonus
- self.failure_reward = failure_reward
- self.time_penalty = (
- 1.0 / (max_code_length - min_code_length)
- if max_code_length > min_code_length else 0.0)
- if reward_fn is None:
- self.reward_fn = r.absolute_distance_reward
- else:
- self.reward_fn = reward_fn
- self.input_type = (
- task.input_type if hasattr(task, 'input_type') else misc.IOType.integer)
- self.output_type = (
- task.output_type if hasattr(task, 'output_type')
- else misc.IOType.integer)
- self._compute_best_reward()
-
- def _compute_best_reward(self):
- io_seqs = self.task.make_io_set()
- reward = 0.0
- for _, output_seq in io_seqs:
- reward += self.reward_fn(output_seq, output_seq, self.task.base)
- reward += self.correct_bonus
- reward += self.code_length_bonus # Bonus for shortest code.
- self.best_reward = reward
- self.good_reward = 0.75 * reward
- logging.info('Known best reward: %.4f', self.best_reward)
-
- def _score_batch(self, code_strings):
- return [self._score_code(code) for code in code_strings]
-
- def _score_code(self, code):
- """Run test cases on code and compute reward.
-
- Args:
- code: A single BF code string.
-
- Returns:
- misc.RewardInfo namedtuple instance containing reward and code execution
- information, including inputs, expected outputs, code outputs, input
- and output types, and reason for the reward obtained.
- """
- # Get list of 2-tuples, each containing an input sequence and an output
- # sequence.
- io_seqs = self.task.make_io_set()
- terminal_reward = 0.0
- results = []
- reason = 'correct'
- for input_seq, output_seq in io_seqs:
- eval_result = bf.evaluate(
- code, input_buffer=input_seq, timeout=0.1,
- max_steps=self.max_execution_steps,
- base=self.task.base,
- require_correct_syntax=self.require_correct_syntax)
- result, success = eval_result.output, eval_result.success
- if not success:
- # Code execution timed out.
- terminal_reward = self.failure_reward
- results = []
- reason = eval_result.failure_reason
- break
- else:
- terminal_reward += self.reward_fn(result, output_seq, self.task.base)
- if result == output_seq:
- terminal_reward += self.correct_bonus # Bonus for correct answer.
-
- # Only add additional reward for shorter code. Subtracting reward
- # interferes with the main objective. Only optimize for length once
- # any solution is found.
- if self.min_code_length == self.max_code_length:
- terminal_reward += self.code_length_bonus
- else:
- terminal_reward += self.code_length_bonus * clipped_linear(
- x=len(code), x0=self.min_code_length, y0=1.0,
- slope=-self.time_penalty, y_range=(0.0, 1.0))
-
- # reason remains 'correct' if it is already
- elif reason == 'correct':
- reason = 'wrong'
- results.append(result)
-
- # Return list of rewards, one for each char in the code. All are 0 except
- # for the terminal reward.
- terminal_reward /= self.best_reward
- return misc.RewardInfo(
- episode_rewards=[0.0] * (len(code) - 1) + [terminal_reward],
- input_case=misc.IOTuple(i for i, o in io_seqs),
- correct_output=misc.IOTuple(o for i, o in io_seqs),
- code_output=misc.IOTuple(results),
- input_type=self.input_type,
- output_type=self.output_type,
- reason=reason)
-
- def rl_batch(self, batch_size):
- """Produces list of reward functions. One for each program in the batch."""
- return [self._score_code] * batch_size
-
-
-def conditional_overwrite(current_value, new_value, allowed_overwrite_values):
- if current_value in allowed_overwrite_values:
- return new_value
- return current_value
-
-
-class BaseTask(object):
- """A coding task.
-
- All coding tasks should inherit this class.
- """
- __metaclass__ = abc.ABCMeta
-
- def __init__(self, base=256):
- self.base = base # All tasks must set the integer base that the expect.
-
- @abc.abstractmethod
- def make_io_set(self):
- """Generate a set of test cases for the task.
-
- Returns:
- List of tuples, where each tuple is (input_case, output_case).
- input_case and output_case are lists of integers.
- """
- pass
-
-
-# ==============================================================================
-# ICLR tasks.
-# ==============================================================================
-
-
-class PrintTask(BaseTask):
- """Print string coding task.
-
- Code needs to output a fixed string (given as a hyperparameter to the
- task constructor). Program input is ignored.
- """
-
- def __init__(self, base, fixed_string=None):
- super(type(self), self).__init__()
- self.base = base # base includes EOS
- self.eos = 0
- if fixed_string:
- self.fixed_string = fixed_string
- else:
- self.fixed_string = [1, 2, 3, 0] # ABC
- self.min_length = self.max_length = len(self.fixed_string)
-
- def make_io_set(self):
- return [(list(), list(self.fixed_string))]
-
-
-class RemoveCharTaskV2(BaseTask):
- """Remove character coding task (version 2).
-
- Code needs to pipe input to output, but with all the 'A' (value 1) chars
- removed. 'A' appears exactly once in each input.
-
- Test cases are hard-coded.
- """
-
- def __init__(self, base):
- super(type(self), self).__init__()
- self.base = base
- self.eos = 0
- self.remove_char = 1
- assert base >= 27
-
- def make_io_set(self):
- rm = self.remove_char
- return [
- ([rm, 0], [0]),
- ([20, rm, 0], [20, 0]),
- ([rm, 13, 0], [13, 0]),
- ([6, rm, 17, 0], [6, 17, 0]),
- ([rm, 11, 24, 0], [11, 24, 0]),
- ([2, 16, 21, rm, 0], [2, 16, 21, 0]),
- ([18, rm, 12, 26, 7, 0], [18, 12, 26, 7, 0]),
- ([9, 10, 22, rm, 4, 0], [9, 10, 22, 4, 0])]
-
-
-class RemoveCharTask(BaseTask):
- """Remove character coding task.
-
- Code needs to pipe input to output, but with all the 'A' (value 1) chars
- removed. 'A' appears at least once in each input.
-
- Test cases are dynamically generated, allowing for the number of test cases
- to be a hyperparameter.
- """
-
- def __init__(self, base, n, min_len, max_len):
- super(type(self), self).__init__()
- self.base = base
- self.eos = 0
- self.remove_char = 1
- assert base >= 27
- self._io_pairs = self._make_io_examples(n, min_len, max_len)
-
- def _make_io_examples(self, n, min_len, max_len):
- """Generate test cases for the task."""
- rand = random.Random(6849275409234) # Test cases are fixed, but varied.
- io_examples = []
- for _ in xrange(n):
- length = rand.randrange(min_len, max_len + 1)
- rm_char_pos = rand.randrange(0, length)
- input_seq = [rand.randrange(1, self.base) for _ in xrange(length)]
- input_seq[rm_char_pos] = self.remove_char
- output_seq = list(input_seq)
- del output_seq[rm_char_pos]
- output_seq.append(0)
- io_examples.append((input_seq, output_seq))
- return io_examples
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class ReverseTaskV2(BaseTask):
- """Reverse string coding task (version 2).
-
- Code needs to pipe input to output, but in reverse order.
-
- Stochastic test case = new test case randomly generated for every run of
- `make_io_set`, i.e. different test cases every time code is scored.
-
- Task supports different types of test cases:
- rand-one: Code is scored on one stochastic test case.
- rand-many: Code is scored on 5 stochastic test cases.
- static-bylen: Code is scored on 5 static test cases. There is one test
- case for string lengths 1 through 5.
- rand-bylen: Code is scored on 5 stochastic test cases, where there is one
- test case for string lengths 1 through 5.
- """
-
- def __init__(self, base, reward_type):
- super(type(self), self).__init__()
- self.base = base # base includes EOS
- assert base >= 27
- self.eos = 0
- self.io_pair_fn = {
- # One random example at a time.
- 'rand-one': lambda: self._io_rand(1),
- # K randomy examples at a time (any lengths).
- 'rand-many': lambda: self._io_rand(5),
- # Static examples, one for each length.
- 'static-bylen': self._io_static_by_len,
- # Random examples, one for each length.
- 'rand-bylen': self._io_rand_by_len}[reward_type]
-
- def _make_io_examples(self, sequences):
- outputs = [list(i) for i in sequences]
- for o in outputs:
- o.reverse()
- o.append(0)
- inputs = [i + [0] for i in sequences]
- return zip(inputs, outputs)
-
- def _io_rand(self, k):
- inputs = [(np.random.choice(26, random.randrange(1, 6)) + 1).tolist()
- for _ in xrange(k)]
- return self._make_io_examples(inputs)
-
- def _io_rand_by_len(self, k=5):
- inputs = [(np.random.choice(26, length) + 1).tolist()
- for length in xrange(1, k + 1)]
- return self._make_io_examples(inputs)
-
- def _io_static_by_len(self):
- return [
- ([7, 0], [7, 0]),
- ([6, 2, 0], [2, 6, 0]),
- ([5, 1, 10, 0], [10, 1, 5, 0]),
- ([8, 6, 5, 15, 0], [15, 5, 6, 8, 0]),
- ([10, 12, 5, 2, 7, 0], [7, 2, 5, 12, 10, 0])]
-
- def make_io_set(self):
- return self.io_pair_fn()
-
-
-class ReverseTask(BaseTask):
- """Reverse string coding task.
-
- Code needs to pipe input to output, but in reverse order.
-
- Test cases are dynamically generated, allowing for the number of test cases
- to be a hyperparameter.
- """
-
- def __init__(self, base, n, min_len, max_len):
- super(type(self), self).__init__()
- self.base = base # base includes EOS
- assert base >= 27
- self.eos = 0
- self._io_pairs = self._make_io_examples(n, min_len, max_len)
-
- def _make_io_examples(self, n, min_len, max_len):
- """Generate test cases for the task."""
- rand = random.Random(6849275409234) # Test cases are fixed, but varied.
- io_examples = []
- for _ in xrange(n):
- length = rand.randrange(min_len, max_len + 1)
- input_seq = [rand.randrange(1, self.base) for _ in xrange(length)]
- output_seq = list(input_seq)
- output_seq.reverse()
- output_seq.append(0)
- io_examples.append((input_seq, output_seq))
- return io_examples
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class CommonPrefixTask(BaseTask):
- """Common prefix coding task.
-
- Code needs to output the common prefix between two input lists. Input lists
- are variable length, where each list ends with a 0. A common prefix is a
- sequence which both lists start with.
- """
-
- def __init__(self, base):
- super(type(self), self).__init__()
- assert base >= 27
- self.base = base
- self.eos = 0
-
- def make_io_set(self):
- return [
- ([12, 24, 18, 0, 12, 5, 0], [12, 0]),
- ([1, 2, 3, 0, 1, 2, 17, 14, 0], [1, 2, 0]),
- ([15, 2, 1, 9, 2, 0, 15, 2, 1, 25, 8, 14, 0], [15, 2, 1, 0]),
- ([14, 9, 7, 8, 6, 16, 0, 14, 9, 7, 8, 8, 6, 8, 26, 0],
- [14, 9, 7, 8, 0]),
- ([12, 4, 16, 22, 1, 17, 0, 12, 4, 16, 22, 1, 8, 10, 0],
- [12, 4, 16, 22, 1, 0])]
-
-
-class CountCharTask(BaseTask):
-
- def __init__(self):
- super(type(self), self).__init__()
- self.base = 27
- self.eos = 0
- self.char = 1
- self.input_type = misc.IOType.string
- self.output_type = misc.IOType.integer
-
- def make_io_set(self):
- return [
- ([10, 0], [0]),
- ([1, 0], [1]),
- ([1, 1, 0], [2]),
- ([11, 1, 0], [1]),
- ([1, 24, 0], [1]),
- ([13, 6, 0], [0]),
- ([9, 2, 7, 0], [0]),
- ([1, 24, 11, 0], [1]),
- ([19, 1, 1, 0], [2]),
- ([1, 6, 1, 0], [2]),
- ([22, 16, 17, 9, 0], [0]),
- ([1, 1, 1, 19, 0], [3]),
- ([1, 1, 1, 1, 0], [4]),
- ([9, 4, 19, 11, 5, 0], [0]),
- ([24, 11, 26, 1, 15, 0], [1]),
- ([1, 1, 20, 1, 1, 0], [4]),
- ([1, 1, 1, 1, 1, 0], [5])]
-
-
-class CountCharTaskV2(BaseTask):
- """Count char coding task (version 2).
-
- Code must output the number of occurances of character 'A' (value 1) in an
- input string.
-
- Test cases are dynamically generated, allowing for the number of test cases
- to be a hyperparameter.
- """
-
- def __init__(self, n, max_len):
- super(type(self), self).__init__()
- self.base = 27
- self.eos = 0
- self.char = 1
- self.other_chars = [c for c in xrange(self.base)
- if c not in (self.eos, self.char)]
- self.input_type = misc.IOType.string
- self.output_type = misc.IOType.integer
- self._io_pairs = self._make_io_examples(n, max_len)
-
- def _make_io_examples(self, n, max_len):
- """Generate test cases for the task."""
- rand = random.Random(6849275409234) # Test cases are fixed, but varied.
- io_examples = []
- io_examples.append(([10, 0], [0]))
- io_examples.append(([1, 0], [1]))
- io_examples.append(([1, 1, 0], [2]))
- io_examples.append(([9, 4, 19, 11, 5, 0], [0]))
- io_examples.append(([24, 11, 26, 1, 15, 0], [1]))
- for _ in xrange(n - 5):
- length = rand.randrange(2, max_len + 1)
- num_chars = rand.randrange(0, max_len + 1)
- input_seq = [self.char] * num_chars + [0] * (length - num_chars)
- rand.shuffle(input_seq)
- for i in xrange(len(input_seq)):
- if not input_seq[i]:
- input_seq[i] = self.other_chars[rand.randrange(len(self.other_chars))]
- output_seq = [num_chars]
- io_examples.append((input_seq, output_seq))
- return io_examples
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class AddTask(BaseTask):
- """Addition coding task.
-
- Code needs to read in two integers and output their sum mod the BF base,
- followed by a terminating 0.
- """
-
- def __init__(self, n=16):
- super(type(self), self).__init__()
- self.base = 256
- self.input_type = misc.IOType.integer
- self.output_type = misc.IOType.integer
- self._io_pairs = self._make_io_examples(n)
-
- def _make_io_examples(self, n):
- """Generate test cases for the task."""
- rand = random.Random(6849275409234) # Test cases are fixed, but varied.
- io_examples = [
- ([4, 0], [4, 0]),
- ([0, 5], [5, 0]),
- ([1, 2], [3, 0]),
- ([67, 21], [88, 0]),
- ([55, 56], [111, 0]),
- ([128, 33], [161, 0]),
- ([221, 251], [216, 0]),
- ([130, 127], [1, 0]),
- ([255, 1], [0, 0])]
- extra_examples = max(n - len(io_examples), 0)
- for _ in xrange(extra_examples):
- a = rand.randrange(256)
- b = rand.randrange(256)
- input_seq = [a, b]
- output_seq = [(a + b) % 256, 0]
- io_examples.append((input_seq, output_seq))
- return io_examples
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class BooleanLogicTask(BaseTask):
- """Boolean logic (truth table) coding task.
-
- Code needs to memorize a boolean truth table. Specifically, it must encode a
- mapping from triple of bools to a single bool.
- """
-
- def __init__(self):
- super(type(self), self).__init__()
- self.base = 2
- self.input_type = misc.IOType.boolean
- self.output_type = misc.IOType.boolean
- # X(~Z) + (~Y)(~Z) + (~X)YZ
- self._truth_fn = (
- lambda x, y, z: # pylint: disable=g-long-lambda
- (x and not z) or (not y and not z) or (not x and y and z))
- self._test_cases = [
- ([x, y, z], [int(self._truth_fn(x, y, z))])
- for x, y, z in itertools.product(range(2), range(2), range(2))]
-
- def make_io_set(self):
- return copy.deepcopy(self._test_cases)
-
-
-# ------------------------------------------------------------------------------
-# The following tasks are generated from known BF solutions. This guarantees
-# that each task can be solved within the maximum code length, and maximum
-# execution steps.
-# ------------------------------------------------------------------------------
-
-
-def default_input_fn_factory(min_length=1, max_length=6, base=256):
- def _input_gen(rand):
- l = rand.randrange(min_length, max_length + 1)
- return [rand.randrange(base) for _ in xrange(l)]
- return _input_gen
-
-
-class KnownCodeBaseTask(BaseTask):
- """These tasks generate their test cases from a known BF solution.
-
- This ensures that each task has a solution which is under the max character
- length, and that it solves the test cases under the max number of execution
- steps.
- """
-
- def __init__(self, code_solution, make_input_fn, n=100, base=256,
- max_steps=5000, seed=6849275409234):
- super(KnownCodeBaseTask, self).__init__()
- # Make sure known solution is less than the code length used in experiments.
- assert len(code_solution) < 100
- self.code_solution = code_solution
- self.make_input_fn = make_input_fn
- self.n = n
- self.base = base
- self.max_steps = max_steps
- self.seed = seed
- self._test_cases = list(self._test_case_generator(code_solution))
-
- def _test_case_generator(self, code_solution):
- rand = random.Random(self.seed)
- for _ in xrange(self.n):
- input_case = self.make_input_fn(rand)
- result = bf.evaluate(
- code_solution, input_buffer=input_case, max_steps=self.max_steps,
- base=self.base, require_correct_syntax=False)
- if not result.success:
- raise RuntimeError(
- 'Program must succeed. Failed on input: %s' % input_case)
- yield input_case, result.output
-
- def make_io_set(self):
- return copy.deepcopy(self._test_cases)
-
-
-class EchoTwiceTask(KnownCodeBaseTask):
- """Echo twice."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>,.[>,.]<[<]>[.>].',
- default_input_fn_factory(),
- **kwargs)
-
-
-class EchoThriceTask(KnownCodeBaseTask):
- """Echo three times."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>,.[>,.]<[<]>[.>].<[<]>[.>].',
- default_input_fn_factory(),
- **kwargs)
-
-
-class CopyReverseTask(KnownCodeBaseTask):
- """Echo forwards, backwards, and then forwards again."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>,.[>,.]<[.<].>[.>].',
- default_input_fn_factory(),
- **kwargs)
-
-
-class EchoZeroCascadeTask(KnownCodeBaseTask):
- """Print k-th char with k zeros inbetween (1-indexed)."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- ',[.>[->+>.<<]>+[-<+>]<<,]',
- default_input_fn_factory(),
- **kwargs)
-
-
-class EchoCascadeTask(KnownCodeBaseTask):
- """Print k-th char k times (1-indexed)."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- ',>>+<<[>>[-<+>]<[->+<<.>]>+<<,].',
- default_input_fn_factory(base=20),
- **kwargs)
-
-
-class ShiftLeftTask(KnownCodeBaseTask):
- """Circulate shift input left."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- ',>,[.,]<.,.',
- default_input_fn_factory(),
- **kwargs)
-
-
-class ShiftRightTask(KnownCodeBaseTask):
- """Circular shift input right."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>,[>,]<.[-]<[<]>[.>].',
- default_input_fn_factory(),
- **kwargs)
-
-
-class RiffleTask(KnownCodeBaseTask):
- """Shuffle like a deck of cards.
-
- For input of length N, output values in the following index order:
- N-1, 0, N-2, 1, N-3, 2, ...
- """
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>,[>,]<[.[-]<[<]>.[-]>[>]<]',
- default_input_fn_factory(base=20, max_length=8),
- **kwargs)
-
-
-class UnriffleTask(KnownCodeBaseTask):
- """Inverse of riffle."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>,[>,[.[-]],]<[.<].',
- default_input_fn_factory(base=20, max_length=8),
- **kwargs)
-
-
-class MiddleCharTask(KnownCodeBaseTask):
- """Print middle char if length is odd, or 0 if even."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>,[>,]<<[[>]<[,<[<]>,>[>]][>]<<]>.',
- default_input_fn_factory(max_length=10),
- **kwargs)
-
-
-class RemoveLastTask(KnownCodeBaseTask):
- """Remove last character."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- ',>,[[<.[-]>[-<+>]],].',
- default_input_fn_factory(base=20),
- **kwargs)
-
-
-class RemoveLastTwoTask(KnownCodeBaseTask):
- """Remove last two characters."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- ',>,>,[[<<.[-]>[-<+>]>[-<+>]],].',
- default_input_fn_factory(base=10),
- **kwargs)
-
-
-class EchoAlternatingTask(KnownCodeBaseTask):
- # Print even numbered chars first (0-indexed), then odd numbered chars
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>,[.,>,]<<[<]>[.>].',
- default_input_fn_factory(base=20, max_length=8),
- **kwargs)
-
-
-class EchoHalfTask(KnownCodeBaseTask):
- """Echo only first half of the input (round down when odd lengthed)."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>>+>,[[<]>+[>],]<[<]>-[-[-<<+>]<[>]>]<<[->+<]>[[>]>.,<+[<]>-].',
- default_input_fn_factory(base=20, max_length=9),
- **kwargs)
-
-
-class LengthTask(KnownCodeBaseTask):
- """Print length of the input sequence."""
-
- def __init__(self, **kwargs):
- super(type(self), self).__init__(
- '>+>,[[<]>+[>],]<[<]>-.',
- default_input_fn_factory(max_length=14),
- **kwargs)
-
-
-class EchoSecondSequenceTask(KnownCodeBaseTask):
- """Echo second sequence. Sequences are separated by 0."""
-
- def __init__(self, **kwargs):
- def echo_second_gen(rand):
- l = rand.randrange(1, 6)
- x = [rand.randrange(256) for _ in xrange(l)]
- l = rand.randrange(1, 6)
- y = [rand.randrange(256) for _ in xrange(l)]
- return x + [0] + y + [0]
- super(type(self), self).__init__(
- ',[,],[.,].',
- echo_second_gen,
- **kwargs)
-
-
-class EchoNthSequenceTask(KnownCodeBaseTask):
- """Echo n-th sequence (1-indexed). Sequences are separated by 0."""
-
- def __init__(self, **kwargs):
- def echo_nth_gen(rand):
- k = rand.randrange(1, 7)
- n = rand.randrange(1, k + 1)
- x = []
- for _ in xrange(k):
- l = rand.randrange(0, 4)
- x += [rand.randrange(256) for _ in xrange(l)] + [0]
- return [n] + x
- super(type(self), self).__init__(
- ',-[->,[,]<],[.,].',
- echo_nth_gen,
- **kwargs)
-
-
-class SubstringTask(KnownCodeBaseTask):
- """Echo substring.
-
- First two inputs are i and l, where i is the starting index (0-indexed)
- and l is the length of the substring.
- """
-
- def __init__(self, **kwargs):
- def substring_gen(rand):
- l = rand.randrange(2, 16)
- i, j = sorted([rand.randrange(l), rand.randrange(l)])
- n = j - i
- x = [rand.randrange(256) for _ in xrange(l)] + [0]
- return [i, n] + x
- super(type(self), self).__init__(
- '>,<,>[->,<]>,<<[->>.,<<]',
- substring_gen,
- **kwargs)
-
-
-class Divide2Task(KnownCodeBaseTask):
- """Divide by 2 (integer floor division)."""
-
- def __init__(self, **kwargs):
- def int_input_gen(rand):
- return [rand.randrange(256)]
- super(type(self), self).__init__(
- ',[-[->>+<]>[<]<]>>.',
- int_input_gen,
- **kwargs)
-
-
-class DedupTask(KnownCodeBaseTask):
- """Deduplicate adjacent duplicate chars."""
-
- def __init__(self, **kwargs):
- def dedup_input_gen(rand):
- np_random = np.random.RandomState(rand.randrange(2147483647))
- num_unique = rand.randrange(1, 5)
- unique = np_random.choice(6, num_unique, replace=False) + 1
- return [v for v in unique for _ in xrange(rand.randrange(1, 5))] + [0]
- super(type(self), self).__init__(
- '>>,.[[-<+<+>>],[-<->]<[[-<->]<.>]<[->>+<<]>>]',
- dedup_input_gen,
- **kwargs)
-
-
-# ==============================================================================
-# Extra tasks.
-# ==============================================================================
-
-
-class PrintIntTask(BaseTask):
- """Print integer coding task.
-
- Code needs to output a fixed single value (given as a hyperparameter to the
- task constructor). Program input is ignored.
- """
-
- def __init__(self, base, fixed_string):
- super(type(self), self).__init__()
- self.base = base
- self.eos = 0
- self.fixed_string = fixed_string
- self.input_type = misc.IOType.integer
- self.output_type = misc.IOType.integer
-
- def make_io_set(self):
- return [(list(), list(self.fixed_string))]
-
-
-class EchoTask(BaseTask):
- """Echo string coding task.
-
- Code needs to pipe input to putput (without any modifications).
- """
-
- def __init__(self, base, min_length=1, max_length=5):
- super(type(self), self).__init__()
- self.base = base # base includes EOS
- self.eos = 0
- self.min_length = min_length
- self.max_length = max_length
- self._io_pairs = self._make_io_examples(25)
-
- def _make_io_examples(self, n):
- # Test cases are fixed, but varied.
- np_random = np.random.RandomState(1234567890)
- io_pairs = []
- for _ in xrange(n):
- length = np_random.randint(self.min_length, self.max_length + 1)
- input_seq = np_random.randint(1, self.base, length).tolist() + [self.eos]
- output_seq = list(input_seq)
- io_pairs.append((input_seq, output_seq))
- return io_pairs
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class JudgeRouteCircleTask(BaseTask):
- """Judge route circle coding task.
-
- Code needs to determine if the given route makes a closed loop.
- Encoding: U = 1, R = 2, D = 3, L = 4.
-
- Based on
- https://leetcode.com/problems/judge-route-circle/description/
- """
- base = 256
- input_type = misc.IOType.integer
- output_type = misc.IOType.integer
-
- def __init__(self, n, max_len=12):
- super(type(self), self).__init__()
- self.eos = 0
- self._io_pairs = self._make_io_examples(n, max_len)
- self.input_type = misc.IOType.integer
- self.output_type = misc.IOType.integer
-
- def _solve(self, input_seq):
- assert input_seq[-1] == 0
- pos = [0, 0] # (x, y)
- for move in input_seq[:-1]:
- assert 0 < move <= 4
- if move & 1 == 0: # Left or Right.
- pos[0] += 3 - move # Add or subtract 1.
- else:
- pos[1] += 2 - move # Add or subtract 1.
- return [int(not pos[0] and not pos[1])]
-
- def _make_io_examples(self, n, max_len):
- """Generate test cases for the task."""
- rand = random.Random(6849275409234) # Test cases are fixed, but varied.
- io_examples = []
- io_examples.append(([0], [1]))
- io_examples.append(([4, 2, 0], [1]))
- io_examples.append(([2, 4, 0], [1]))
- io_examples.append(([3, 1, 0], [1]))
- io_examples.append(([1, 3, 0], [1]))
- io_examples.append(([1, 0], [0]))
- io_examples.append(([2, 0], [0]))
- io_examples.append(([3, 0], [0]))
- io_examples.append(([4, 0], [0]))
- for _ in xrange(n):
- is_true = rand.randrange(2)
- length = rand.randrange(1, max_len + 1)
- if is_true:
- # Make a true case.
- length = (length >> 1) << 1 # Make even.
- partition = (rand.randrange(length + 1) >> 1) << 1
- a = partition >> 1
- b = (length - partition) >> 1
- counts = {1: a, 2: b, 3: a, 4: b}
- else:
- # Make a false case.
- partitions = (
- [0]
- + sorted([rand.randrange(length + 1) for _ in range(3)])
- + [length])
- counts = {n: partitions[n] - partitions[n - 1] for n in range(1, 5)}
- if counts[1] == counts[3] and counts[2] == counts[4]:
- # By chance we sampled a true case. Make it false by exchanging
- # one count between even and odd pairs.
- base = 1 + 2 * rand.randrange(2)
- a, b = (base, base + 1) if rand.randrange(2) else (base + 1, base)
- if counts[a] == length or counts[b] == 0:
- # If counts are at their extreme values, then swap who gets
- # incremented and decremented.
- a, b = b, a
- counts[a] += 1
- counts[b] -= 1
- assert counts[a] <= length and counts[b] >= 0
- assert sum(counts.values()) == length
- input_seq = [n for n in xrange(1, 5) for _ in xrange(counts[n])]
- rand.shuffle(input_seq)
- input_seq += [0]
- output_seq = self._solve(input_seq)
- assert output_seq[0] == is_true
- io_examples.append((input_seq, output_seq))
- return io_examples
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class MultiplyTask(BaseTask):
- """Multiply coding task.
-
- Code needs to multiple two ints.
-
- Solution:
- http://robl.co/brief-look-at-brainfuck/
- ,>,><<[->[->+>+<<]>>[-<<+>>]<<<]>>.
- """
- base = 512
- input_type = misc.IOType.integer
- output_type = misc.IOType.integer
-
- def __init__(self, n):
- super(type(self), self).__init__()
- self.eos = 0
- self._io_pairs = self._make_io_examples(n)
- self.input_type = misc.IOType.integer
- self.output_type = misc.IOType.integer
-
- def _factors(self, n):
- return set(i for i in range(1, int(n**0.5) + 1) if n % i == 0)
-
- def _make_io_examples(self, n):
- """Generate test cases for the task."""
- rand = random.Random(6849275409234) # Test cases are fixed, but varied.
- io_examples = []
- for _ in xrange(n):
- n = rand.randrange(self.base)
- if n == 0:
- a, b = 0, rand.randrange(self.base)
- else:
- f = list(self._factors(n))
- a = f[rand.randrange(len(f))]
- b = n // a
- if rand.randrange(2):
- a, b = b, a
- io_examples.append(([a, b], [n]))
- return io_examples
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class DivModTask(BaseTask):
- """Divmod coding task.
-
- Code needs to take the quotient and remainder of two ints.
-
- Solution:
- http://robl.co/brief-look-at-brainfuck/
- ,>,><<[>[->+>+<<]>[-<<-[>]>>>[<[-<->]<[>]>>[[-]>>+<]>-<]<<]>>>+<<[-<<+>>]<<<]>
- >>>>[-<<<<<+>>>>>]<<<<<.>.>
- """
- base = 512
- input_type = misc.IOType.integer
- output_type = misc.IOType.integer
-
- def __init__(self, n):
- super(type(self), self).__init__()
- self.eos = 0
- self._io_pairs = self._make_io_examples(n)
- self.input_type = misc.IOType.integer
- self.output_type = misc.IOType.integer
-
- def _make_io_examples(self, n):
- rand = random.Random(6849275409234) # Test cases are fixed, but varied.
- io_examples = []
- for _ in xrange(n):
- n = rand.randrange(0, self.base)
- k = rand.randrange(1, self.base) # Divisor cannot be 0.
- io_examples.append(([n, k], list(divmod(n, k))))
- return io_examples
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class FibonacciTask(BaseTask):
-
- def __init__(self):
- super(type(self), self).__init__()
- self.base = 256
- self.input_type = misc.IOType.integer
- self.output_type = misc.IOType.integer
-
- def make_io_set(self):
- return [
- ([0], [0, 1]),
- ([1], [1, 1]),
- ([2], [1, 2]),
- ([3], [2, 3]),
- ([4], [3, 5]),
- ([5], [5, 8]),
- ([6], [8, 13]),
- ([7], [13, 21]),
- ([8], [21, 34]),
- ([9], [34, 55]),
- ([10], [55, 89]),
- ([11], [89, 144]),
- ([12], [144, 233]),
- ([13], [233, 121])]
-
-
-class FindSubStrTask(BaseTask):
- """Find sub-string coding task.
-
- Code needs to output a bool: True if the input string contains a hard-coded
- substring, 'AB' (values [1, 2]).
- """
-
- def __init__(self, base):
- super(type(self), self).__init__()
- assert base >= 27
- self.base = base
- self.eos = 0
- self.find_str = [1, 2]
- self.input_type = misc.IOType.string
- self.output_type = misc.IOType.boolean
-
- def make_io_set(self):
- return [
- ([1, 1, 23, 0], [0]),
- ([21, 3, 2, 0], [0]),
- ([2, 1, 19, 0], [0]),
- ([2, 24, 15, 3, 0], [0]),
- ([24, 6, 10, 16, 4, 0], [0]),
- ([1, 2, 12, 0], [1]),
- ([7, 1, 2, 0], [1]),
- ([1, 2, 11, 3, 0], [1]),
- ([1, 1, 2, 18, 0], [1]),
- ([7, 25, 1, 2, 0], [1]),
- ([3, 1, 2, 11, 8, 0], [1]),
- ([15, 16, 20, 1, 2, 0], [1])]
-
-
-class SortFixedTask(BaseTask):
- """Sort list coding task.
-
- Code needs to output a sorted input list. The task consists of lists of the
- same length L, where L is provided to this task's constructor as a
- hyperparameter.
- """
-
- def __init__(self, base, length=3):
- super(type(self), self).__init__()
- assert base >= 27
- self.base = base
- self.eos = 0
- self.length = length
- assert length == 3 # More lengths will be supported.
-
- def make_io_set(self):
- if self.length == 3:
- return [
- ([1, 20, 6], [1, 6, 20]),
- ([13, 6, 7], [6, 7, 13]),
- ([24, 2, 23], [2, 23, 24]),
- ([16, 12, 3], [3, 12, 16]),
- ([11, 24, 4], [4, 11, 24]),
- ([10, 1, 19], [1, 10, 19])]
-
-
-class SortFixedTaskV2(BaseTask):
- """Sort list coding task (version 2).
-
- Code needs to output a sorted input list. The task consists of lists of the
- same length L, where L is provided to this task's constructor as a
- hyperparameter.
-
- Test cases are dynamically generated, allowing for the number of test cases
- to be a hyperparameter.
- """
-
- def __init__(self, base, n, length=3):
- super(type(self), self).__init__()
- assert base >= 27
- self.base = base
- self.eos = 0
- self._io_pairs = self._make_io_examples(n, length)
- self.input_type = misc.IOType.integer
- self.output_type = misc.IOType.integer
-
- def _make_io_examples(self, n, length):
- rand = random.Random(6849275409234) # Test cases are fixed, but varied.
- io_examples = []
- for _ in xrange(n):
- input_seq = [rand.randrange(1, self.base) for _ in xrange(length)]
- output_seq = sorted(input_seq)
- io_examples.append((input_seq, output_seq))
- return io_examples
-
- def make_io_set(self):
- return copy.deepcopy(self._io_pairs)
-
-
-class RemoveTargetCharTask(KnownCodeBaseTask):
- """Remove target character from string, where first input is the target.
-
- Target can appear multiple times.
- """
-
- def __init__(self, **kwargs):
- def randrange_hole(rand, a, hole, b):
- x = rand.randrange(a, b - 1)
- if x >= hole:
- return x + 1
- return x
- def remove_target_char_gen(rand):
- char = rand.randrange(1, 6)
- l = rand.randrange(1, 8)
- input_seq = [randrange_hole(rand, 1, char, 256) for _ in xrange(l)]
- idx = range(l)
- rand.shuffle(idx)
- num_targets = rand.randrange(0, l)
- for pos in idx[:num_targets]:
- input_seq[pos] = char
- return [char] + input_seq + [0]
- super(type(self), self).__init__(
- ',>>>,[<<<[->+>+<<]>>[->->+<<]>[>[-<+>]<.[-]]>[-]<<<[-<+>]>>,].',
- remove_target_char_gen,
- **kwargs)
-
-
-class ListIndexTask(KnownCodeBaseTask):
- """Echo i-th value in the given list."""
-
- def __init__(self, **kwargs):
- def array_index_gen(rand):
- l = rand.randrange(1, 16)
- i = rand.randrange(l)
- return [i] + [rand.randrange(256) for _ in xrange(l)] + [0]
- super(type(self), self).__init__(
- ',[->,<]>,.',
- array_index_gen,
- **kwargs)
-
-
-# ==============================================================================
-# Tasks based on primaryobjects paper.
-# ==============================================================================
-
-
-def string2tokens(string):
- return [ord(c) for c in string]
-
-
-def stringlist2tokens(strings):
- return [string2tokens(string) for string in strings]
-
-
-def string2tokens_b27(string):
- return [ord(c.lower()) - ord('a') + 1 for c in string]
-
-
-def stringlist2tokens_b27(strings):
- return [string2tokens_b27(string) for string in strings]
-
-
-class BottlesOfBeerTask(BaseTask):
- """Bottles of beer coding task.
-
- This is a counting task. Code needs to read in an int N and then output
- every int from N to 0, each separated by a 0.
- """
- base = 256
- input_type = misc.IOType.integer
- output_type = misc.IOType.integer
-
- def make_io_set(self):
- return [
- ([1], [1, 0]),
- ([2], [2, 0, 1, 0]),
- ([3], [3, 0, 2, 0, 1, 0]),
- ([4], [4, 0, 3, 0, 2, 0, 1, 0]),
- ([5], [5, 0, 4, 0, 3, 0, 2, 0, 1, 0]),
- ([6], [6, 0, 5, 0, 4, 0, 3, 0, 2, 0, 1, 0])]
-
-
-class SplitTask(BaseTask):
- """Split coding task.
-
- Code needs to pipe input strings to output, but insert a 0 after every 3
- characters. This is in essence splitting the string into intervals of length
- 3.
- """
- base = 28
- input_type = misc.IOType.string
- output_type = misc.IOType.integer
-
- def _splicer(self, lst, insert, interval=3):
- for i, item in enumerate(lst):
- yield item
- if (i + 1) % interval == 0 and i < len(lst) - 1:
- yield insert
-
- def __init__(self):
- super(type(self), self).__init__()
- inputs = stringlist2tokens_b27(
- ['hello', 'orange', 'spaghetti', 'wins', 'one'])
- targets = [list(self._splicer(i, 27)) for i in inputs]
- self._test_cases = list(zip(inputs, targets))
-
- def make_io_set(self):
- return copy.deepcopy(self._test_cases)
-
-
-class TrimLeftTask(BaseTask):
- """Trim left coding task.
-
- Code needs to pipe input strings to output, but remove everything before the
- first quotation char (").
- """
- base = 256
- input_type = misc.IOType.integer
- output_type = misc.IOType.integer
-
- def __init__(self):
- super(type(self), self).__init__()
- inputs = stringlist2tokens(
- ['a "inside" over', 'xy "test" rights', 'ca6 "foresting" service',
- 'abc"def"yz.', 'A"B"'])
- targets = stringlist2tokens(
- ['"inside" over', '"test" rights', '"foresting" service', '"def"yz.',
- '"B"'])
- self._test_cases = list(zip(inputs, targets))
-
- def make_io_set(self):
- return copy.deepcopy(self._test_cases)
diff --git a/research/brain_coder/single_task/code_tasks_test.py b/research/brain_coder/single_task/code_tasks_test.py
deleted file mode 100644
index d3260a1a56ec0f7c36363d558122f7f7e49198e6..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/code_tasks_test.py
+++ /dev/null
@@ -1,108 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for code_tasks."""
-
-import numpy as np
-import tensorflow as tf
-
-from single_task import code_tasks # brain coder
-from single_task import defaults # brain coder
-
-
-def pad(string, pad_length, pad_char):
- return string + pad_char * (pad_length - len(string))
-
-
-class CodeTasksTest(tf.test.TestCase):
-
- def assertClose(self, a, b):
- self.assertTrue(
- np.isclose(a, b, atol=1e-4),
- 'Expecting approximately equal values. Got: %s, %s' % (a, b))
-
- def testMultiIOTaskManager(self):
- maxlen = 100
- padchr = '['
- task = code_tasks.make_paper_task(
- 'print', timestep_limit=maxlen, do_code_simplification=False)
- reward_fns = task.rl_batch(1)
- r = reward_fns[0]
- self.assertClose(
- r(pad('++++++++.---.+++++++...', maxlen, padchr)).episode_rewards[-1],
- 0.2444)
- self.assertClose(
- r(pad('++++++++.---.+++++++..+++.',
- maxlen, padchr)).episode_rewards[-1],
- 1.0)
-
- task = code_tasks.make_paper_task(
- 'print', timestep_limit=maxlen, do_code_simplification=True)
- reward_fns = task.rl_batch(1)
- r = reward_fns[0]
- self.assertClose(
- r('++++++++.---.+++++++...').episode_rewards[-1],
- 0.2444)
- self.assertClose(
- r('++++++++.---.+++++++..+++.').episode_rewards[-1],
- 0.935)
- self.assertClose(
- r(pad('++++++++.---.+++++++..+++.',
- maxlen, padchr)).episode_rewards[-1],
- 0.75)
-
- task = code_tasks.make_paper_task(
- 'reverse', timestep_limit=maxlen, do_code_simplification=False)
- reward_fns = task.rl_batch(1)
- r = reward_fns[0]
- self.assertClose(
- r(pad('>,>,>,.<.<.<.', maxlen, padchr)).episode_rewards[-1],
- 0.1345)
- self.assertClose(
- r(pad(',[>,]+[,<.]', maxlen, padchr)).episode_rewards[-1],
- 1.0)
-
- task = code_tasks.make_paper_task(
- 'reverse', timestep_limit=maxlen, do_code_simplification=True)
- reward_fns = task.rl_batch(1)
- r = reward_fns[0]
- self.assertClose(r('>,>,>,.<.<.<.').episode_rewards[-1], 0.1324)
- self.assertClose(r(',[>,]+[,<.]').episode_rewards[-1], 0.9725)
- self.assertClose(
- r(pad(',[>,]+[,<.]', maxlen, padchr)).episode_rewards[-1],
- 0.75)
-
- def testMakeTask(self):
- maxlen = 100
- padchr = '['
- config = defaults.default_config_with_updates(
- 'env=c(config_for_iclr=False,fixed_string=[8,5,12,12,15])')
- task = code_tasks.make_task(config.env, 'print', timestep_limit=maxlen)
- reward_fns = task.rl_batch(1)
- r = reward_fns[0]
- self.assertClose(
- r('++++++++.---.+++++++...').episode_rewards[-1],
- 0.2444)
- self.assertClose(
- r('++++++++.---.+++++++..+++.').episode_rewards[-1],
- 0.935)
- self.assertClose(
- r(pad('++++++++.---.+++++++..+++.',
- maxlen, padchr)).episode_rewards[-1],
- 0.75)
-
- def testKnownCodeBaseTask(self):
- maxlen = 100
- padchr = '['
- task = code_tasks.make_paper_task(
- 'shift-left', timestep_limit=maxlen, do_code_simplification=False)
- reward_fns = task.rl_batch(1)
- r = reward_fns[0]
- self.assertClose(
- r(pad(',>,[.,]<.,.', maxlen, padchr)).episode_rewards[-1],
- 1.0)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/single_task/data.py b/research/brain_coder/single_task/data.py
deleted file mode 100644
index 8f34464f5a3e1c403b0f253f1520920c303b0819..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/data.py
+++ /dev/null
@@ -1,89 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Manage data for pretraining and RL tasks."""
-
-import ast
-from collections import namedtuple
-
-from absl import logging
-
-from single_task import code_tasks # brain coder
-
-
-RLBatch = namedtuple('RLBatch', ['reward_fns', 'batch_size', 'good_reward'])
-
-
-class DataManager(object):
- """Interface between environment and model."""
-
- def __init__(self, global_config, run_number=None,
- do_code_simplification=False):
- """Constructs a DataManager.
-
- Args:
- global_config: A config_lib.Config instance containing all config. See
- config in defaults.py.
- run_number: Which run this is (of the same experiment). This should be set
- when a task cycle is defined in the config. A task cycle is a list of
- tasks to cycle through repeatedly, and the selected task is a function
- of the run number, i.e. 0-th run, 1-st run, 2-nd run, etc...
- This can be None if only a single task is set in the config.
- do_code_simplification: When global_config.env.config_for_iclr is True,
- use this option to create code simplification (code golf) tasks, vs
- fixed length coding tasks. If True, a task with code simplification
- reward will be constructed.
-
- Raises:
- ValueError: If global_config.env.task and global_config.env.task_cycle
- are both set, or both not set. Only one should be given.
- ValueError: If global_config.env.task_cycle is set but run_number is None.
- """
- env_config = global_config.env
- self.batch_size = global_config.batch_size
-
- if env_config.task_cycle:
- if env_config.task:
- raise ValueError('Do not set both `task` and `task_cycle`.')
- if run_number is None:
- raise ValueError('Do not use task_cycle for single-run experiment.')
- index = run_number % len(env_config.task_cycle)
- self.task_name = env_config.task_cycle[index]
- logging.info('run_number: %d, task_cycle index: %d', run_number, index)
- logging.info('task_cycle: %s', env_config.task_cycle)
- elif env_config.task:
- self.task_name = env_config.task
- else:
- raise ValueError('Either `task` or `task_cycle` must be set.')
- logging.info('Task for this run: "%s"', self.task_name)
-
- logging.info('config_for_iclr=True; do_code_simplification=%s',
- do_code_simplification)
- self.rl_task = code_tasks.make_task(
- task_name=self.task_name,
- override_kwargs=ast.literal_eval(env_config.task_kwargs),
- max_code_length=global_config.timestep_limit,
- require_correct_syntax=env_config.correct_syntax,
- do_code_simplification=do_code_simplification,
- correct_bonus=env_config.task_manager_config.correct_bonus,
- code_length_bonus=env_config.task_manager_config.code_length_bonus)
-
- def sample_rl_batch(self):
- """Create reward functions from the current task.
-
- Returns:
- RLBatch namedtuple instance, which holds functions and information for
- a minibatch of episodes.
- * reward_fns: A reward function for each episode. Maps code string to
- reward.
- * batch_size: Number of episodes in this minibatch.
- * good_reward: Estimated threshold of rewards which indicate the algorithm
- is starting to solve the task. This is a heuristic that tries to
- reduce the amount of stuff written to disk.
- """
- reward_fns = self.rl_task.rl_batch(self.batch_size)
- return RLBatch(
- reward_fns=reward_fns,
- batch_size=self.batch_size,
- good_reward=self.rl_task.good_reward)
diff --git a/research/brain_coder/single_task/defaults.py b/research/brain_coder/single_task/defaults.py
deleted file mode 100644
index d9bd8b942532dfffcf06d90d331e58725c4d82a9..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/defaults.py
+++ /dev/null
@@ -1,82 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Default configuration for agent and environment."""
-
-from absl import logging
-
-from common import config_lib # brain coder
-
-
-def default_config():
- return config_lib.Config(
- agent=config_lib.OneOf(
- [config_lib.Config(
- algorithm='pg',
- policy_lstm_sizes=[35,35],
- # Set value_lstm_sizes to None to share weights with policy.
- value_lstm_sizes=[35,35],
- obs_embedding_size=10,
- grad_clip_threshold=10.0,
- param_init_factor=1.0,
- lr=5e-5,
- pi_loss_hparam=1.0,
- vf_loss_hparam=0.5,
- entropy_beta=1e-2,
- regularizer=0.0,
- softmax_tr=1.0, # Reciprocal temperature.
- optimizer='rmsprop', # 'adam', 'sgd', 'rmsprop'
- topk=0, # Top-k unique codes will be stored.
- topk_loss_hparam=0.0, # off policy loss multiplier.
- # Uniformly sample this many episodes from topk buffer per batch.
- # If topk is 0, this has no effect.
- topk_batch_size=1,
- # Exponential moving average baseline for REINFORCE.
- # If zero, A2C is used.
- # If non-zero, should be close to 1, like .99, .999, etc.
- ema_baseline_decay=0.99,
- # Whether agent can emit EOS token. If true, agent can emit EOS
- # token which ends the episode early (ends the sequence).
- # If false, agent must emit tokens until the timestep limit is
- # reached. e.g. True means variable length code, False means fixed
- # length code.
- # WARNING: Making this false slows things down.
- eos_token=False,
- replay_temperature=1.0,
- # Replay probability. 1 = always replay, 0 = always on policy.
- alpha=0.0,
- # Whether to normalize importance weights in each minibatch.
- iw_normalize=True),
- config_lib.Config(
- algorithm='ga',
- crossover_rate=0.99,
- mutation_rate=0.086),
- config_lib.Config(
- algorithm='rand')],
- algorithm='pg',
- ),
- env=config_lib.Config(
- # If True, task-specific settings are not needed.
- task='', # 'print', 'echo', 'reverse', 'remove', ...
- task_cycle=[], # If non-empty, reptitions will cycle through tasks.
- task_kwargs='{}', # Python dict literal.
- task_manager_config=config_lib.Config(
- # Reward recieved per test case. These bonuses will be scaled
- # based on how many test cases there are.
- correct_bonus=2.0, # Bonus for code getting correct answer.
- code_length_bonus=1.0), # Maximum bonus for short code.
- correct_syntax=False,
- ),
- batch_size=64,
- timestep_limit=32)
-
-
-def default_config_with_updates(config_string, do_logging=True):
- if do_logging:
- logging.info('Config string: "%s"', config_string)
- config = default_config()
- config.strict_update(config_lib.Config.parse(config_string))
- if do_logging:
- logging.info('Config:\n%s', config.pretty_str())
- return config
diff --git a/research/brain_coder/single_task/ga_lib.py b/research/brain_coder/single_task/ga_lib.py
deleted file mode 100644
index fadb96482b21a5c65c0d6d6cf4a3aec3b5708235..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/ga_lib.py
+++ /dev/null
@@ -1,472 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Genetic algorithm for BF tasks.
-
-Inspired by https://github.com/primaryobjects/AI-Programmer.
-GA function code borrowed from https://github.com/DEAP/deap.
-"""
-
-from collections import namedtuple
-import random
-
-from absl import flags
-from absl import logging
-import numpy as np
-from six.moves import xrange
-
-from common import bf # brain coder
-from common import utils # brain coder
-from single_task import misc # brain coder
-
-FLAGS = flags.FLAGS
-
-# Saving reward of previous programs saves computation if a program appears
-# again.
-USE_REWARD_CACHE = True # Disable this if GA is using up too much memory.
-GENES = bf.CHARS
-MAX_PROGRAM_STEPS = 500
-STEP_BONUS = True
-
-ALPHANUM_CHARS = (
- ['_'] +
- [chr(ord('a') + i_) for i_ in range(26)] +
- [chr(ord('A') + i_) for i_ in range(26)] +
- [chr(ord('0') + i_) for i_ in range(10)])
-
-Result = namedtuple(
- 'Result',
- ['reward', 'inputs', 'code_outputs', 'target_outputs', 'type_in',
- 'type_out', 'base', 'correct'])
-
-
-class IOType(object):
- string = 'string'
- integer = 'integer'
-
-
-class CustomType(object):
-
- def __init__(self, to_str_fn):
- self.to_str_fn = to_str_fn
-
- def __call__(self, obj):
- return self.to_str_fn(obj)
-
-
-def tokens_list_repr(tokens, repr_type, base):
- """Make human readable representation of program IO."""
- if isinstance(repr_type, CustomType):
- return repr_type(tokens)
- elif repr_type == IOType.string:
- chars = (
- [ALPHANUM_CHARS[t] for t in tokens] if base < len(ALPHANUM_CHARS)
- else [chr(t) for t in tokens])
- return ''.join(chars)
- elif repr_type == IOType.integer:
- return str(tokens)
- raise ValueError('No such representation type "%s"', repr_type)
-
-
-def io_repr(result):
- """Make human readable representation of test cases."""
- inputs = ','.join(
- tokens_list_repr(tokens, result.type_in, result.base)
- for tokens in result.inputs)
- code_outputs = ','.join(
- tokens_list_repr(tokens, result.type_out, result.base)
- for tokens in result.code_outputs)
- target_outputs = ','.join(
- tokens_list_repr(tokens, result.type_out, result.base)
- for tokens in result.target_outputs)
- return inputs, target_outputs, code_outputs
-
-
-def make_task_eval_fn(task_manager):
- """Returns a wrapper that converts an RL task into a GA task.
-
- Args:
- task_manager: Is a task manager object from code_tasks.py
-
- Returns:
- A function that takes as input a single list of a code chars, and outputs
- a Result namedtuple instance containing the reward and information about
- code execution.
- """
- def to_data_list(single_or_tuple):
- if isinstance(single_or_tuple, misc.IOTuple):
- return list(single_or_tuple)
- return [single_or_tuple]
-
- def to_ga_type(rl_type):
- if rl_type == misc.IOType.string:
- return IOType.string
- return IOType.integer
-
- # Wrapper function.
- def evalbf(bf_chars):
- result = task_manager._score_code(''.join(bf_chars))
- reward = sum(result.episode_rewards)
- correct = result.reason == 'correct'
- return Result(
- reward=reward,
- inputs=to_data_list(result.input_case),
- code_outputs=to_data_list(result.code_output),
- target_outputs=to_data_list(result.correct_output),
- type_in=to_ga_type(result.input_type),
- type_out=to_ga_type(result.output_type),
- correct=correct,
- base=task_manager.task.base)
-
- return evalbf
-
-
-def debug_str(individual, task_eval_fn):
- res = task_eval_fn(individual)
- input_str, target_output_str, code_output_str = io_repr(res)
- return (
- ''.join(individual) +
- ' | ' + input_str +
- ' | ' + target_output_str +
- ' | ' + code_output_str +
- ' | ' + str(res.reward) +
- ' | ' + str(res.correct))
-
-
-def mutate_single(code_tokens, mutation_rate):
- """Mutate a single code string.
-
- Args:
- code_tokens: A string/list/Individual of BF code chars. Must end with EOS
- symbol '_'.
- mutation_rate: Float between 0 and 1 which sets the probability of each char
- being mutated.
-
- Returns:
- An Individual instance containing the mutated code string.
-
- Raises:
- ValueError: If `code_tokens` does not end with EOS symbol.
- """
- if len(code_tokens) <= 1:
- return code_tokens
- if code_tokens[-1] == '_':
- # Do this check to ensure that the code strings have not been corrupted.
- raise ValueError('`code_tokens` must end with EOS symbol.')
- else:
- cs = Individual(code_tokens)
- eos = []
- mutated = False
- for pos in range(len(cs)):
- if random.random() < mutation_rate:
- mutated = True
- new_char = GENES[random.randrange(len(GENES))]
- x = random.random()
- if x < 0.25 and pos != 0 and pos != len(cs) - 1:
- # Insertion mutation.
- if random.random() < 0.50:
- # Shift up.
- cs = cs[:pos] + [new_char] + cs[pos:-1]
- else:
- # Shift down.
- cs = cs[1:pos] + [new_char] + cs[pos:]
- elif x < 0.50:
- # Deletion mutation.
- if random.random() < 0.50:
- # Shift down.
- cs = cs[:pos] + cs[pos + 1:] + [new_char]
- else:
- # Shift up.
- cs = [new_char] + cs[:pos] + cs[pos + 1:]
- elif x < 0.75:
- # Shift rotate mutation (position invariant).
- if random.random() < 0.50:
- # Shift down.
- cs = cs[1:] + [cs[0]]
- else:
- # Shift up.
- cs = [cs[-1]] + cs[:-1]
- else:
- # Replacement mutation.
- cs = cs[:pos] + [new_char] + cs[pos + 1:]
- assert len(cs) + len(eos) == len(code_tokens)
- if mutated:
- return Individual(cs + eos)
- else:
- return Individual(code_tokens)
-
-
-def crossover(parent1, parent2):
- """Performs crossover mating between two code strings.
-
- Crossover mating is where a random position is selected, and the chars
- after that point are swapped. The resulting new code strings are returned.
-
- Args:
- parent1: First code string.
- parent2: Second code string.
-
- Returns:
- A 2-tuple of children, i.e. the resulting code strings after swapping.
- """
- max_parent, min_parent = (
- (parent1, parent2) if len(parent1) > len(parent2)
- else (parent2, parent1))
- pos = random.randrange(len(max_parent))
- if pos >= len(min_parent):
- child1 = max_parent[:pos]
- child2 = min_parent + max_parent[pos:]
- else:
- child1 = max_parent[:pos] + min_parent[pos:]
- child2 = min_parent[:pos] + max_parent[pos:]
- return Individual(child1), Individual(child2)
-
-
-def _make_even(n):
- """Return largest even integer less than or equal to `n`."""
- return (n >> 1) << 1
-
-
-def mutate_and_crossover(population, mutation_rate, crossover_rate):
- """Take a generational step over a population.
-
- Transforms population of parents into population of children (of the same
- size) via crossover mating and then mutation on the resulting children.
-
- Args:
- population: Parent population. A list of Individual objects.
- mutation_rate: Probability of mutation. See `mutate_single`.
- crossover_rate: Probability that two parents will mate.
-
- Returns:
- Child population. A list of Individual objects.
- """
- children = [None] * len(population)
- for i in xrange(0, _make_even(len(population)), 2):
- p1 = population[i]
- p2 = population[i + 1]
- if random.random() < crossover_rate:
- p1, p2 = crossover(p1, p2)
- c1 = mutate_single(p1, mutation_rate)
- c2 = mutate_single(p2, mutation_rate)
- children[i] = c1
- children[i + 1] = c2
- if children[-1] is None:
- children[-1] = population[-1]
- return children
-
-
-def ga_loop(population, cxpb, mutpb, ngen, task_eval_fn, halloffame=None,
- checkpoint_writer=None):
- """A bare bones genetic algorithm.
-
- Similar to chapter 7 of Back, Fogel and Michalewicz, "Evolutionary
- Computation 1 : Basic Algorithms and Operators", 2000.
-
- Args:
- population: A list of individuals.
- cxpb: The probability of mating two individuals.
- mutpb: The probability of mutating a gene.
- ngen: The number of generation. Unlimited if zero.
- task_eval_fn: A python function which maps an Individual to a Result
- namedtuple.
- halloffame: (optional) a utils.MaxUniquePriorityQueue object that will be
- used to aggregate the best individuals found during search.
- checkpoint_writer: (optional) an object that can save and load populations.
- Needs to have `write`, `load`, and `has_checkpoint` methods. Used to
- periodically save progress. In event of a restart, the population will
- be loaded from disk.
-
- Returns:
- GaResult namedtuple instance. This contains information about the GA run,
- including the resulting population, best reward (fitness) obtained, and
- the best code string found.
- """
-
- has_checkpoint = False
- if checkpoint_writer and checkpoint_writer.has_checkpoint():
- try:
- gen, population, halloffame = checkpoint_writer.load()
- except EOFError: # Data was corrupted. Start over.
- pass
- else:
- has_checkpoint = True
- logging.info(
- 'Loaded population from checkpoint. Starting at generation %d', gen)
-
- # Evaluate the individuals with an invalid fitness
- invalid_ind = [ind for ind in population if not ind.fitness.valid]
- for ind in invalid_ind:
- ind.fitness.values = task_eval_fn(ind).reward,
- for _, ind in halloffame.iter_in_order():
- ind.fitness.values = task_eval_fn(ind).reward,
-
- if not has_checkpoint:
- # Evaluate the individuals with an invalid fitness
- invalid_ind = [ind for ind in population if not ind.fitness.valid]
- for ind in invalid_ind:
- ind.fitness.values = task_eval_fn(ind).reward,
-
- if halloffame is not None:
- for ind in population:
- halloffame.push(ind.fitness.values, tuple(ind), ind)
-
- logging.info('Initialized new population.')
-
- gen = 1
-
- pop_size = len(population)
- program_reward_cache = {} if USE_REWARD_CACHE else None
-
- # Begin the generational process
- while ngen == 0 or gen <= ngen:
- # Select the next generation individuals
- offspring = roulette_selection(population, pop_size - len(halloffame))
-
- # Vary the pool of individuals
- # offspring = varAnd(offspring, toolbox, cxpb, mutpb)
- offspring = mutate_and_crossover(
- offspring, mutation_rate=mutpb, crossover_rate=cxpb)
-
- # Evaluate the individuals with an invalid fitness
- invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
- for ind in invalid_ind:
- str_repr = ''.join(ind)
- if program_reward_cache is not None and str_repr in program_reward_cache:
- ind.fitness.values = (program_reward_cache[str_repr],)
- else:
- eval_result = task_eval_fn(ind)
- ind.fitness.values = (eval_result.reward,)
- if program_reward_cache is not None:
- program_reward_cache[str_repr] = eval_result.reward
-
- # Replace the current population by the offspring
- population = list(offspring)
-
- # Update the hall of fame with the generated individuals
- if halloffame is not None:
- for ind in population:
- halloffame.push(ind.fitness.values, tuple(ind), ind)
-
- # elitism
- population.extend([ind for _, ind in halloffame.iter_in_order()])
-
- if gen % 100 == 0:
- top_code = '\n'.join([debug_str(ind, task_eval_fn)
- for ind in topk(population, k=4)])
- logging.info('gen: %d\nNPE: %d\n%s\n\n', gen, gen * pop_size, top_code)
-
- best_code = ''.join(halloffame.get_max()[1])
- res = task_eval_fn(best_code)
-
- # Write population and hall-of-fame to disk.
- if checkpoint_writer:
- checkpoint_writer.write(gen, population, halloffame)
-
- if res.correct:
- logging.info('Solution found:\n%s\nreward = %s\n',
- best_code, res.reward)
- break
-
- gen += 1
-
- best_code = ''.join(halloffame.get_max()[1])
- res = task_eval_fn(best_code)
-
- return GaResult(
- population=population, best_code=best_code, reward=res.reward,
- solution_found=res.correct, generations=gen,
- num_programs=gen * len(population),
- max_generations=ngen, max_num_programs=ngen * len(population))
-
-
-GaResult = namedtuple(
- 'GaResult',
- ['population', 'best_code', 'reward', 'generations', 'num_programs',
- 'solution_found', 'max_generations', 'max_num_programs'])
-
-
-def reward_conversion(reward):
- """Convert real value into positive value."""
- if reward <= 0:
- return 0.05
- return reward + 0.05
-
-
-def roulette_selection(population, k):
- """Select `k` individuals with prob proportional to fitness.
-
- Each of the `k` selections is independent.
-
- Warning:
- The roulette selection by definition cannot be used for minimization
- or when the fitness can be smaller or equal to 0.
-
- Args:
- population: A list of Individual objects to select from.
- k: The number of individuals to select.
-
- Returns:
- A list of selected individuals.
- """
- fitnesses = np.asarray(
- [reward_conversion(ind.fitness.values[0])
- for ind in population])
- assert np.all(fitnesses > 0)
-
- sum_fits = fitnesses.sum()
- chosen = [None] * k
- for i in xrange(k):
- u = random.random() * sum_fits
- sum_ = 0
- for ind, fitness in zip(population, fitnesses):
- sum_ += fitness
- if sum_ > u:
- chosen[i] = Individual(ind)
- break
- if not chosen[i]:
- chosen[i] = Individual(population[-1])
-
- return chosen
-
-
-def make_population(make_individual_fn, n):
- return [make_individual_fn() for _ in xrange(n)]
-
-
-def best(population):
- best_ind = None
- for ind in population:
- if best_ind is None or best_ind.fitness.values < ind.fitness.values:
- best_ind = ind
- return best_ind
-
-
-def topk(population, k):
- q = utils.MaxUniquePriorityQueue(k)
- for ind in population:
- q.push(ind.fitness.values, tuple(ind), ind)
- return [ind for _, ind in q.iter_in_order()]
-
-
-class Fitness(object):
-
- def __init__(self):
- self.values = ()
-
- @property
- def valid(self):
- """Assess if a fitness is valid or not."""
- return bool(self.values)
-
-
-class Individual(list):
-
- def __init__(self, *args):
- super(Individual, self).__init__(*args)
- self.fitness = Fitness()
-
-
-def random_individual(genome_size):
- return lambda: Individual(np.random.choice(GENES, genome_size).tolist())
diff --git a/research/brain_coder/single_task/ga_train.py b/research/brain_coder/single_task/ga_train.py
deleted file mode 100644
index 630eca427e478dbadad58bd94b56e89a5a747526..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/ga_train.py
+++ /dev/null
@@ -1,324 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Genetic algorithm for BF tasks.
-
-Also contains the uniform random search algorithm.
-
-Inspired by https://github.com/primaryobjects/AI-Programmer.
-GA function code borrowed from https://github.com/DEAP/deap.
-"""
-
-import cPickle
-import os
-import sys
-from time import sleep
-
-from absl import flags
-from absl import logging
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from common import utils # brain coder
-from single_task import data # brain coder
-from single_task import defaults # brain coder
-from single_task import ga_lib # brain coder
-from single_task import results_lib # brain coder
-
-FLAGS = flags.FLAGS
-
-
-def define_tuner_hparam_space(hparam_space_type):
- """Define tunable hparams for grid search."""
- if hparam_space_type != 'ga':
- raise ValueError('Hparam space is not valid: "%s"' % hparam_space_type)
- return {
- 'population_size': [10, 25, 50, 100, 500],
- 'crossover_rate': [0.2, 0.5, 0.7, 0.9, 0.95],
- 'mutation_rate': [0.01, 0.03, 0.05, 0.1, 0.15]}
-
-
-def write_hparams_to_config(config, hparams, hparam_space_type):
- """Write hparams given by the tuner into the Config object."""
- if hparam_space_type != 'ga':
- raise ValueError('Hparam space is not valid: "%s"' % hparam_space_type)
- config.batch_size = hparams.population_size
- config.agent.crossover_rate = hparams.crossover_rate
- config.agent.mutation_rate = hparams.mutation_rate
-
-
-class CheckpointWriter(object):
- """Manages loading and saving GA populations to disk.
-
- This object is used by the genetic algorithm to save progress periodically
- so that a recent population can be loaded from disk in the event of a restart.
- """
-
- def __init__(self, checkpoint_dir, population_size):
- self.checkpoint_file = os.path.join(checkpoint_dir, 'checkpoint.pickle')
- self.population_size = population_size
-
- def write(self, gen, population, halloffame):
- """Write GA state to disk.
-
- Overwrites previous saved state.
-
- Args:
- gen: Generation number.
- population: List of Individual objects.
- halloffame: Hall-of-fame buffer. Typically a priority queue.
- """
- raw = cPickle.dumps((gen, population, halloffame))
- with tf.gfile.FastGFile(self.checkpoint_file, 'w') as f:
- f.write(raw)
-
- def load(self):
- """Loads GA state from disk.
-
- Loads whatever is on disk, which will be whatever the most recent call
- to `write` wrote.
-
- Returns:
- gen: Generation number.
- population: List of Individual objects.
- halloffame: Hall-of-fame buffer. Typically a priority queue.
- """
- with tf.gfile.FastGFile(self.checkpoint_file, 'r') as f:
- raw = f.read()
- objs = cPickle.loads(raw)
- # Validate data.
- assert isinstance(objs, tuple) and len(objs) == 3, (
- 'Expecting a 3-tuple, but got %s instead.' % (objs,))
- gen, population, halloffame = objs
- assert isinstance(gen, int), (
- 'Expecting `gen` to be an integer, got %s' % (gen,))
- assert (
- isinstance(population, list)
- and len(population) == self.population_size
- ), (
- 'Expecting `population` to be a list with size %d, got %s'
- % (self.population_size, population))
- assert halloffame is None or len(halloffame) == 2, (
- 'Expecting hall-of-fame object to have length two, got length %d'
- % len(halloffame))
- logging.info('Loaded pop from checkpoint file: "%s".',
- self.checkpoint_file)
- return gen, population, halloffame
-
- def has_checkpoint(self):
- """Checks if a checkpoint exists on disk, and if so returns True."""
- return tf.gfile.Exists(self.checkpoint_file)
-
-
-def run_training(config=None, tuner=None, logdir=None, trial_name=None, # pylint: disable=unused-argument
- is_chief=True):
- """Do all training runs.
-
- This is the top level training function for policy gradient based models.
- Run this from the main function.
-
- Args:
- config: config_lib.Config instance containing global config (agent and
- environment hparams). If None, config will be parsed from FLAGS.config.
- tuner: (unused) A tuner instance. Leave as None if not tuning.
- logdir: Parent directory where all data from all runs will be written. If
- None, FLAGS.logdir will be used.
- trial_name: (unused) If tuning, set this to a unique string that identifies
- this trial. If `tuner` is not None, this also must be set.
- is_chief: True if this worker is the chief.
-
- Returns:
- List of results dicts which were written to disk. Each training run gets a
- results dict. Results dict contains metrics, i.e. (name, value) pairs which
- give information about the training run.
-
- Raises:
- ValueError: If FLAGS.num_workers does not divide FLAGS.num_repetitions.
- ValueError: If results dicts read from disk contain invalid data.
- """
- if not config:
- # If custom config is not given, get it from flags.
- config = defaults.default_config_with_updates(FLAGS.config)
- if not logdir:
- logdir = FLAGS.logdir
-
- if FLAGS.num_repetitions % FLAGS.num_workers != 0:
- raise ValueError('Number of workers must divide number of repetitions')
- num_local_reps = FLAGS.num_repetitions // FLAGS.num_workers
- logging.info('Running %d reps globally.', FLAGS.num_repetitions)
- logging.info('This worker will run %d local reps.', num_local_reps)
- if FLAGS.max_npe:
- max_generations = FLAGS.max_npe // config.batch_size
- logging.info('Max samples per rep: %d', FLAGS.max_npe)
- logging.info('Max generations per rep: %d', max_generations)
- else:
- max_generations = sys.maxint
- logging.info('Running unlimited generations.')
-
- assert FLAGS.num_workers > 0
- logging.info('Starting experiment. Directory: "%s"', logdir)
- results = results_lib.Results(logdir, FLAGS.task_id)
- local_results_list = results.read_this_shard()
- if local_results_list:
- if local_results_list[0]['max_npe'] != FLAGS.max_npe:
- raise ValueError(
- 'Cannot resume training. Max-NPE changed. Was %s, now %s',
- local_results_list[0]['max_npe'], FLAGS.max_npe)
- if local_results_list[0]['max_global_repetitions'] != FLAGS.num_repetitions:
- raise ValueError(
- 'Cannot resume training. Number of repetitions changed. Was %s, '
- 'now %s',
- local_results_list[0]['max_global_repetitions'],
- FLAGS.num_repetitions)
- start_rep = len(local_results_list)
-
- for rep in xrange(start_rep, num_local_reps):
- global_rep = num_local_reps * FLAGS.task_id + rep
- logging.info(
- 'Starting repetition: Rep = %d. (global rep = %d)',
- rep, global_rep)
-
- # Save data for each rep, like checkpoints, goes into separate folders.
- run_dir = os.path.join(logdir, 'run_%d' % global_rep)
-
- if not tf.gfile.IsDirectory(run_dir):
- tf.gfile.MakeDirs(run_dir)
- checkpoint_writer = CheckpointWriter(run_dir,
- population_size=config.batch_size)
-
- data_manager = data.DataManager(config, run_number=global_rep)
- task_eval_fn = ga_lib.make_task_eval_fn(data_manager.rl_task)
-
- if config.agent.algorithm == 'rand':
- logging.info('Running random search.')
- assert FLAGS.max_npe
- result = run_random_search(
- FLAGS.max_npe, run_dir, task_eval_fn, config.timestep_limit)
- else:
- assert config.agent.algorithm == 'ga'
- logging.info('Running genetic algorithm.')
- pop = ga_lib.make_population(
- ga_lib.random_individual(config.timestep_limit),
- n=config.batch_size)
- hof = utils.MaxUniquePriorityQueue(2) # Hall of fame.
- result = ga_lib.ga_loop(
- pop,
- cxpb=config.agent.crossover_rate, mutpb=config.agent.mutation_rate,
- task_eval_fn=task_eval_fn,
- ngen=max_generations, halloffame=hof,
- checkpoint_writer=checkpoint_writer)
-
- logging.info('Finished rep. Num gens: %d', result.generations)
-
- results_dict = {
- 'max_npe': FLAGS.max_npe,
- 'batch_size': config.batch_size,
- 'max_batches': FLAGS.max_npe // config.batch_size,
- 'npe': result.num_programs,
- 'max_global_repetitions': FLAGS.num_repetitions,
- 'max_local_repetitions': num_local_reps,
- 'code_solution': result.best_code if result.solution_found else '',
- 'best_reward': result.reward,
- 'num_batches': result.generations,
- 'found_solution': result.solution_found,
- 'task': data_manager.task_name,
- 'global_rep': global_rep}
- logging.info('results_dict: %s', results_dict)
- results.append(results_dict)
-
- if is_chief:
- logging.info(
- 'Worker is chief. Waiting for all workers to finish so that results '
- 'can be reported to the tuner.')
-
- global_results_list, shard_stats = results.read_all(
- num_shards=FLAGS.num_workers)
- while not all(s.finished for s in shard_stats):
- logging.info(
- 'Still waiting on these workers: %s',
- ', '.join(
- ['%d (%d reps left)'
- % (i, s.max_local_reps - s.num_local_reps_completed)
- for i, s in enumerate(shard_stats)
- if not s.finished]))
- sleep(60)
- global_results_list, shard_stats = results.read_all(
- num_shards=FLAGS.num_workers)
-
- logging.info(
- '%d results obtained. Chief worker is exiting the experiment.',
- len(global_results_list))
-
- return global_results_list
-
-
-def run_random_search(max_num_programs, checkpoint_dir, task_eval_fn,
- timestep_limit):
- """Run uniform random search routine.
-
- Randomly samples programs from a uniform distribution until either a valid
- program is found, or the maximum NPE is reached. Results are written to disk
- and returned.
-
- Args:
- max_num_programs: Maximum NPE (number of programs executed). If no solution
- is found after this many programs are tried, the run is stopped and
- considered a failure.
- checkpoint_dir: Where to save state during the run.
- task_eval_fn: Function that maps code string to result containing total
- reward and info about success.
- timestep_limit: Maximum length of code strings.
-
- Returns:
- ga_lib.GaResult namedtuple instance. This contains the best code and highest
- reward found.
- """
- checkpoint_file = os.path.join(checkpoint_dir, 'random_search.txt')
- num_programs_seen = 0
- found_solution = False
- best_code = ''
- best_reward = 0.0
- if tf.gfile.Exists(checkpoint_file):
- try:
- with tf.gfile.FastGFile(checkpoint_file, 'r') as f:
- lines = list(f)
- num_programs_seen = int(lines[0])
- found_solution = bool(int(lines[1]))
- if found_solution:
- best_code = lines[2]
- best_reward = float(lines[3])
- except: # pylint: disable=bare-except
- pass
-
- while not found_solution and num_programs_seen < max_num_programs:
- if num_programs_seen % 1000 == 0:
- logging.info('num_programs_seen = %d', num_programs_seen)
- with tf.gfile.FastGFile(checkpoint_file, 'w') as f:
- f.write(str(num_programs_seen) + '\n')
- f.write(str(int(found_solution)) + '\n')
-
- code = np.random.choice(ga_lib.GENES, timestep_limit).tolist()
- res = task_eval_fn(code)
- found_solution = res.correct
- num_programs_seen += 1
-
- if found_solution:
- best_code = ''.join(code)
- best_reward = res.reward
-
- logging.info('num_programs_seen = %d', num_programs_seen)
- logging.info('found solution: %s', found_solution)
- with tf.gfile.FastGFile(checkpoint_file, 'w') as f:
- f.write(str(num_programs_seen) + '\n')
- f.write(str(int(found_solution)) + '\n')
- if found_solution:
- f.write(best_code + '\n')
- f.write(str(best_reward) + '\n')
-
- return ga_lib.GaResult(
- population=[], best_code=best_code, reward=best_reward,
- solution_found=found_solution, generations=num_programs_seen,
- num_programs=num_programs_seen, max_generations=max_num_programs,
- max_num_programs=max_num_programs)
diff --git a/research/brain_coder/single_task/ga_train_test.py b/research/brain_coder/single_task/ga_train_test.py
deleted file mode 100644
index ff69ad84952a3fb90cad28b3cf8e67ff55c96e95..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/ga_train_test.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for ga_train.
-
-Tests that ga runs for a few generations without crashing.
-"""
-
-from absl import flags
-import tensorflow as tf
-
-from single_task import defaults # brain coder
-from single_task import run # brain coder
-
-FLAGS = flags.FLAGS
-
-
-class GaTest(tf.test.TestCase):
-
- def RunTrainingSteps(self, config_string, num_steps=10):
- """Run a few training steps with the given config.
-
- Just check that nothing crashes.
-
- Args:
- config_string: Config encoded in a string. See
- $REPO_PATH/common/config_lib.py
- num_steps: Number of training steps to run. Defaults to 10.
- """
- config = defaults.default_config_with_updates(config_string)
- FLAGS.max_npe = num_steps * config.batch_size
- FLAGS.logdir = tf.test.get_temp_dir()
- FLAGS.config = config_string
- run.main(None)
-
- def testGeneticAlgorithm(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="ga"),'
- 'timestep_limit=40,batch_size=64')
-
- def testUniformRandomSearch(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="rand"),'
- 'timestep_limit=40,batch_size=64')
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/single_task/launch_training.sh b/research/brain_coder/single_task/launch_training.sh
deleted file mode 100755
index a4a4688ed2912792185aa8f3134b1680fed6f006..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/launch_training.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-# Launches training jobs.
-# Modify this file to launch workers with your prefered cloud API.
-# The following implementation runs each worker as a subprocess on the local
-# machine.
-
-MODELS_DIR="/tmp/models"
-
-# Get command line options.
-OPTS=$(getopt -n "$0" -o "" --long "job_name:,config:,num_workers:,num_ps:,max_npe:,num_repetitions:,stop_on_success:" -- "$@")
-if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi
-
-eval set -- "$OPTS"
-
-JOB_NAME="" # Name of the process and the logs directory.
-CONFIG="" # Model and environment hparams.
-# NUM_WORKERS: Number of workers to launch for this training job. If using
-# neural networks, each worker will be 1 replica.
-NUM_WORKERS=1
-# NUM_PS: Number of parameter servers to launch for this training job. Only set
-# this if using neural networks. For 1 worker, no parameter servers are needed.
-# For more than 1 worker, at least 1 parameter server is needed to store the
-# global model.
-NUM_PS=0
-# MAX_NPE: Maximum number of programs executed. Training will quit once this
-# threshold is reached. If 0, the threshold is infinite.
-MAX_NPE=0
-NUM_REPETITIONS=1 # How many times to run this experiment.
-STOP_ON_SUCCESS=true # Whether to halt training when a solution is found.
-
-# Parse options into variables.
-while true; do
- case "$1" in
- --job_name ) JOB_NAME="$2"; shift; shift ;;
- --config ) CONFIG="$2"; shift; shift ;;
- --num_workers ) NUM_WORKERS="$2"; shift; shift ;;
- --num_ps ) NUM_PS="$2"; shift; shift ;;
- --max_npe ) MAX_NPE="$2"; shift; shift ;;
- --num_repetitions ) NUM_REPETITIONS="$2"; shift; shift ;;
- --stop_on_success ) STOP_ON_SUCCESS="$2"; shift; shift ;;
- -- ) shift; break ;;
- * ) break ;;
- esac
-done
-
-# Launch jobs.
-# TODO: multi-worker RL training
-
-LOGDIR="$MODELS_DIR/$JOB_NAME"
-mkdir -p $LOGDIR
-
-BIN_DIR="bazel-bin/single_task"
-for (( i=0; i "$LOGDIR/task_$i.log" & # Run as subprocess
- echo "Launched task $i. Logs: $LOGDIR/task_$i.log"
-done
-
-
-# Use "pidof run.par" to find jobs.
-# Kill with "pkill run.par"
diff --git a/research/brain_coder/single_task/launch_tuning.sh b/research/brain_coder/single_task/launch_tuning.sh
deleted file mode 100755
index 97ce51b543e13d4b1c412656a93197b5b47373bb..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/launch_tuning.sh
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/bin/bash
-# Launches tuning jobs.
-# Modify this file to launch workers with your prefered cloud API.
-# The following implementation runs each worker as a subprocess on the local
-# machine.
-
-MODELS_DIR="/tmp/models"
-
-# Get command line options.
-OPTS=$(getopt -n "$0" -o "" --long "job_name:,config:,num_tuners:,num_workers_per_tuner:,num_ps_per_tuner:,max_npe:,num_repetitions:,stop_on_success:,fixed_hparams:,hparam_space_type:" -- "$@")
-if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi
-
-eval set -- "$OPTS"
-
-JOB_NAME="" # Name of the process and the logs directory.
-CONFIG="" # Model and environment hparams.
-# NUM_TUNERS: Number of tuning jobs to launch. Each tuning job can train a
-# hparam combination. So more tuners means more hparams tried in parallel.
-NUM_TUNERS=1
-# NUM_WORKERS_PER_TUNER: Number of workers to launch for each tuning job. If
-# using neural networks, each worker will be 1 replica.
-NUM_WORKERS_PER_TUNER=1
-# NUM_PS_PER_TUNER: Number of parameter servers to launch for this tuning job.
-# Only set this if using neural networks. For 1 worker per tuner, no parameter
-# servers are needed. For more than 1 worker per tuner, at least 1 parameter
-# server per tuner is needed to store the global model for each tuner.
-NUM_PS_PER_TUNER=0
-# MAX_NPE: Maximum number of programs executed. Training will quit once this
-# threshold is reached. If 0, the threshold is infinite.
-MAX_NPE=0
-NUM_REPETITIONS=25 # How many times to run this experiment.
-STOP_ON_SUCCESS=true # Whether to halt training when a solution is found.
-# FIXED_HPARAMS: Hold hparams fixed in the grid search. This reduces the search
-# space.
-FIXED_HPARAMS=""
-# HPARAM_SPACE_TYPE: Specifies the hparam search space. See
-# `define_tuner_hparam_space` functions defined in pg_train.py and ga_train.py.
-HPARAM_SPACE_TYPE="pg"
-
-# Parse options into variables.
-while true; do
- case "$1" in
- --job_name ) JOB_NAME="$2"; shift; shift ;;
- --config ) CONFIG="$2"; shift; shift ;;
- --num_tuners ) NUM_TUNERS="$2"; shift; shift ;;
- --num_workers_per_tuner ) NUM_WORKERS_PER_TUNER="$2"; shift; shift ;;
- --num_ps_per_tuner ) NUM_PS_PER_TUNER="$2"; shift; shift ;;
- --max_npe ) MAX_NPE="$2"; shift; shift ;;
- --num_repetitions ) NUM_REPETITIONS="$2"; shift; shift ;;
- --stop_on_success ) STOP_ON_SUCCESS="$2"; shift; shift ;;
- --fixed_hparams ) FIXED_HPARAMS="$2"; shift; shift ;;
- --hparam_space_type ) HPARAM_SPACE_TYPE="$2"; shift; shift ;;
- -- ) shift; break ;;
- * ) break ;;
- esac
-done
-
-# Launch jobs.
-# TODO: multi-worker RL training
-
-LOGDIR="$MODELS_DIR/$JOB_NAME"
-mkdir -p $LOGDIR
-
-BIN_DIR="bazel-bin/single_task"
-for ((tuner=0;tuner "$LOGDIR/tuner_$tuner.task_$i.log" & # Run as subprocess
- echo "Launched tuner $tuner, task $i. Logs: $LOGDIR/tuner_$tuner.task_$i.log"
- done
-done
-
-# Use "pidof tune.par" to find jobs.
-# Kill with "pkill tune.par"
diff --git a/research/brain_coder/single_task/misc.py b/research/brain_coder/single_task/misc.py
deleted file mode 100644
index 07061d81c8aaafd4d97efc11ecca451528c6e9dd..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/misc.py
+++ /dev/null
@@ -1,149 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Utilities specific to this project."""
-
-from collections import namedtuple
-from six import string_types
-
-
-#####################
-# BF-lang utilities #
-#####################
-
-
-BF_EOS_INT = 0 # Also used as SOS (start of sequence).
-BF_EOS_CHAR = TEXT_EOS_CHAR = '_'
-BF_LANG_INTS = range(1, 9)
-BF_INT_TO_CHAR = [BF_EOS_CHAR, '>', '<', '+', '-', '[', ']', '.', ',']
-BF_CHAR_TO_INT = dict([(c, i) for i, c in enumerate(BF_INT_TO_CHAR)])
-
-
-RewardInfo = namedtuple('RewardInfo', ['episode_rewards', 'input_case',
- 'correct_output',
- 'code_output', 'reason', 'input_type',
- 'output_type'])
-
-
-class IOType(object):
- string = 'string'
- integer = 'integer'
- boolean = 'boolean'
-
-
-class IOTuple(tuple):
- pass
-
-
-def flatten(lst):
- return [item for row in lst for item in row]
-
-
-def bf_num_tokens():
- # BF tokens plus EOS.
- return len(BF_INT_TO_CHAR)
-
-
-def bf_char2int(bf_char):
- """Convert BF code char to int token."""
- return BF_CHAR_TO_INT[bf_char]
-
-
-def bf_int2char(bf_int):
- """Convert BF int token to code char."""
- return BF_INT_TO_CHAR[bf_int]
-
-
-def bf_tokens_to_string(bf_tokens, truncate=True):
- """Convert token list to code string. Will truncate at EOS token.
-
- Args:
- bf_tokens: Python list of ints representing the code string.
- truncate: If true, the output string will end at the first EOS token.
- If false, the entire token list is converted to string.
-
- Returns:
- String representation of the tokens.
-
- Raises:
- ValueError: If bf_tokens is not a python list.
- """
- if not isinstance(bf_tokens, list):
- raise ValueError('Only python list supported here.')
- if truncate:
- try:
- eos_index = bf_tokens.index(BF_EOS_INT)
- except ValueError:
- eos_index = len(bf_tokens)
- else:
- eos_index = len(bf_tokens)
- return ''.join([BF_INT_TO_CHAR[t] for t in bf_tokens[:eos_index]])
-
-
-def bf_string_to_tokens(bf_string):
- """Convert string to token list. Will strip and append EOS token."""
- tokens = [BF_CHAR_TO_INT[char] for char in bf_string.strip()]
- tokens.append(BF_EOS_INT)
- return tokens
-
-
-def tokens_to_text(tokens):
- """Convert token list to human readable text."""
- return ''.join(
- [TEXT_EOS_CHAR if t == 0 else chr(t - 1 + ord('A')) for t in tokens])
-
-
-###################################
-# Number representation utilities #
-###################################
-
-
-# https://en.wikipedia.org/wiki/Metric_prefix
-si_magnitudes = {
- 'k': 1e3,
- 'm': 1e6,
- 'g': 1e9}
-
-
-def si_to_int(s):
- """Convert string ending with SI magnitude to int.
-
- Examples: 5K ==> 5000, 12M ==> 12000000.
-
- Args:
- s: String in the form 'xx..xP' where x is a digit and P is an SI prefix.
-
- Returns:
- Integer equivalent to the string.
- """
- if isinstance(s, string_types) and s[-1].lower() in si_magnitudes.keys():
- return int(int(s[:-1]) * si_magnitudes[s[-1].lower()])
- return int(s)
-
-
-def int_to_si(n):
- """Convert integer to string with SI magnitude.
-
- `n` will be truncated.
-
- Examples: 5432 ==> 5k, 12345678 ==> 12M
-
- Args:
- n: Integer to represent as a string.
-
- Returns:
- String representation of `n` containing SI magnitude.
- """
- m = abs(n)
- sign = -1 if n < 0 else 1
- if m < 1e3:
- return str(n)
- if m < 1e6:
- return '{0}K'.format(sign*int(m / 1e3))
- if m < 1e9:
- return '{0}M'.format(sign*int(m / 1e6))
- if m < 1e12:
- return '{0}G'.format(sign*int(m / 1e9))
- return str(m)
-
diff --git a/research/brain_coder/single_task/pg_agent.py b/research/brain_coder/single_task/pg_agent.py
deleted file mode 100644
index 13fc7da2dc89a1fbcc7fa5efbbce87008580aa92..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/pg_agent.py
+++ /dev/null
@@ -1,1297 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Language model agent.
-
-Agent outputs code in a sequence just like a language model. Can be trained
-as a language model or using RL, or a combination of the two.
-"""
-
-from collections import namedtuple
-from math import exp
-from math import log
-import time
-
-from absl import logging
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from common import rollout as rollout_lib # brain coder
-from common import utils # brain coder
-from single_task import misc # brain coder
-
-
-# Experiments in the ICLR 2018 paper used reduce_sum instead of reduce_mean for
-# some losses. We make all loses be batch_size independent, and multiply the
-# changed losses by 64, which was the fixed batch_size when the experiments
-# where run. The loss hyperparameters still match what is reported in the paper.
-MAGIC_LOSS_MULTIPLIER = 64
-
-
-def rshift_time(tensor_2d, fill=misc.BF_EOS_INT):
- """Right shifts a 2D tensor along the time dimension (axis-1)."""
- dim_0 = tf.shape(tensor_2d)[0]
- fill_tensor = tf.fill([dim_0, 1], fill)
- return tf.concat([fill_tensor, tensor_2d[:, :-1]], axis=1)
-
-
-def join(a, b):
- # Concat a and b along 0-th dim.
- if a is None or len(a) == 0: # pylint: disable=g-explicit-length-test
- return b
- if b is None or len(b) == 0: # pylint: disable=g-explicit-length-test
- return a
- return np.concatenate((a, b))
-
-
-def make_optimizer(kind, lr):
- if kind == 'sgd':
- return tf.train.GradientDescentOptimizer(lr)
- elif kind == 'adam':
- return tf.train.AdamOptimizer(lr)
- elif kind == 'rmsprop':
- return tf.train.RMSPropOptimizer(learning_rate=lr, decay=0.99)
- else:
- raise ValueError('Optimizer type "%s" not recognized.' % kind)
-
-
-class LinearWrapper(tf.contrib.rnn.RNNCell):
- """RNNCell wrapper that adds a linear layer to the output."""
-
- def __init__(self, cell, output_size, dtype=tf.float32, suppress_index=None):
- self.cell = cell
- self._output_size = output_size
- self._dtype = dtype
- self._suppress_index = suppress_index
- self.smallest_float = -2.4e38
-
- def __call__(self, inputs, state, scope=None):
- with tf.variable_scope(type(self).__name__):
- outputs, state = self.cell(inputs, state, scope=scope)
- logits = tf.matmul(
- outputs,
- tf.get_variable('w_output',
- [self.cell.output_size, self.output_size],
- dtype=self._dtype))
- if self._suppress_index is not None:
- # Replace the target index with -inf, so that it never gets selected.
- batch_size = tf.shape(logits)[0]
- logits = tf.concat(
- [logits[:, :self._suppress_index],
- tf.fill([batch_size, 1], self.smallest_float),
- logits[:, self._suppress_index + 1:]],
- axis=1)
-
- return logits, state
-
- @property
- def output_size(self):
- return self._output_size
-
- @property
- def state_size(self):
- return self.cell.state_size
-
- def zero_state(self, batch_size, dtype):
- return self.cell.zero_state(batch_size, dtype)
-
-
-UpdateStepResult = namedtuple(
- 'UpdateStepResult',
- ['global_step', 'global_npe', 'summaries_list', 'gradients_dict'])
-
-
-class AttrDict(dict):
- """Dict with attributes as keys.
-
- https://stackoverflow.com/a/14620633
- """
-
- def __init__(self, *args, **kwargs):
- super(AttrDict, self).__init__(*args, **kwargs)
- self.__dict__ = self
-
-
-class LMAgent(object):
- """Language model agent."""
- action_space = misc.bf_num_tokens()
- observation_space = misc.bf_num_tokens()
-
- def __init__(self, global_config, task_id=0,
- logging_file=None,
- experience_replay_file=None,
- global_best_reward_fn=None,
- found_solution_op=None,
- assign_code_solution_fn=None,
- program_count=None,
- do_iw_summaries=False,
- stop_on_success=True,
- dtype=tf.float32,
- verbose_level=0,
- is_local=True):
- self.config = config = global_config.agent
- self.logging_file = logging_file
- self.experience_replay_file = experience_replay_file
- self.task_id = task_id
- self.verbose_level = verbose_level
- self.global_best_reward_fn = global_best_reward_fn
- self.found_solution_op = found_solution_op
- self.assign_code_solution_fn = assign_code_solution_fn
- self.parent_scope_name = tf.get_variable_scope().name
- self.dtype = dtype
- self.allow_eos_token = config.eos_token
- self.stop_on_success = stop_on_success
- self.pi_loss_hparam = config.pi_loss_hparam
- self.vf_loss_hparam = config.vf_loss_hparam
- self.is_local = is_local
-
- self.top_reward = 0.0
- self.embeddings_trainable = True
-
- self.no_op = tf.no_op()
-
- self.learning_rate = tf.constant(
- config.lr, dtype=dtype, name='learning_rate')
- self.initializer = tf.contrib.layers.variance_scaling_initializer(
- factor=config.param_init_factor,
- mode='FAN_AVG',
- uniform=True,
- dtype=dtype) # TF's default initializer.
- tf.get_variable_scope().set_initializer(self.initializer)
-
- self.a2c = config.ema_baseline_decay == 0
- if not self.a2c:
- logging.info('Using exponential moving average REINFORCE baselines.')
- self.ema_baseline_decay = config.ema_baseline_decay
- self.ema_by_len = [0.0] * global_config.timestep_limit
- else:
- logging.info('Using advantage (a2c) with learned value function.')
- self.ema_baseline_decay = 0.0
- self.ema_by_len = None
-
- # Top-k
- if config.topk and config.topk_loss_hparam:
- self.topk_loss_hparam = config.topk_loss_hparam
- self.topk_batch_size = config.topk_batch_size
- if self.topk_batch_size <= 0:
- raise ValueError('topk_batch_size must be a positive integer. Got %s',
- self.topk_batch_size)
- self.top_episodes = utils.MaxUniquePriorityQueue(config.topk)
- logging.info('Made max-priorty-queue with capacity %d',
- self.top_episodes.capacity)
- else:
- self.top_episodes = None
- self.topk_loss_hparam = 0.0
- logging.info('No max-priorty-queue')
-
- # Experience replay.
- self.replay_temperature = config.replay_temperature
- self.num_replay_per_batch = int(global_config.batch_size * config.alpha)
- self.num_on_policy_per_batch = (
- global_config.batch_size - self.num_replay_per_batch)
- self.replay_alpha = (
- self.num_replay_per_batch / float(global_config.batch_size))
- logging.info('num_replay_per_batch: %d', self.num_replay_per_batch)
- logging.info('num_on_policy_per_batch: %d', self.num_on_policy_per_batch)
- logging.info('replay_alpha: %s', self.replay_alpha)
- if self.num_replay_per_batch > 0:
- # Train with off-policy episodes from replay buffer.
- start_time = time.time()
- self.experience_replay = utils.RouletteWheel(
- unique_mode=True, save_file=experience_replay_file)
- logging.info('Took %s sec to load replay buffer from disk.',
- int(time.time() - start_time))
- logging.info('Replay buffer file location: "%s"',
- self.experience_replay.save_file)
- else:
- # Only train on-policy.
- self.experience_replay = None
-
- if program_count is not None:
- self.program_count = program_count
- self.program_count_add_ph = tf.placeholder(
- tf.int64, [], 'program_count_add_ph')
- self.program_count_add_op = self.program_count.assign_add(
- self.program_count_add_ph)
-
- ################################
- # RL policy and value networks #
- ################################
- batch_size = global_config.batch_size
- logging.info('batch_size: %d', batch_size)
-
- self.policy_cell = LinearWrapper(
- tf.contrib.rnn.MultiRNNCell(
- [tf.contrib.rnn.BasicLSTMCell(cell_size)
- for cell_size in config.policy_lstm_sizes]),
- self.action_space,
- dtype=dtype,
- suppress_index=None if self.allow_eos_token else misc.BF_EOS_INT)
- self.value_cell = LinearWrapper(
- tf.contrib.rnn.MultiRNNCell(
- [tf.contrib.rnn.BasicLSTMCell(cell_size)
- for cell_size in config.value_lstm_sizes]),
- 1,
- dtype=dtype)
-
- obs_embedding_scope = 'obs_embed'
- with tf.variable_scope(
- obs_embedding_scope,
- initializer=tf.random_uniform_initializer(minval=-1.0, maxval=1.0)):
- obs_embeddings = tf.get_variable(
- 'embeddings',
- [self.observation_space, config.obs_embedding_size],
- dtype=dtype, trainable=self.embeddings_trainable)
- self.obs_embeddings = obs_embeddings
-
- ################################
- # RL policy and value networks #
- ################################
-
- initial_state = tf.fill([batch_size], misc.BF_EOS_INT)
- def loop_fn(loop_time, cell_output, cell_state, loop_state):
- """Function called by tf.nn.raw_rnn to instantiate body of the while_loop.
-
- See https://www.tensorflow.org/api_docs/python/tf/nn/raw_rnn for more
- information.
-
- When time is 0, and cell_output, cell_state, loop_state are all None,
- `loop_fn` will create the initial input, internal cell state, and loop
- state. When time > 0, `loop_fn` will operate on previous cell output,
- state, and loop state.
-
- Args:
- loop_time: A scalar tensor holding the current timestep (zero based
- counting).
- cell_output: Output of the raw_rnn cell at the current timestep.
- cell_state: Cell internal state at the current timestep.
- loop_state: Additional loop state. These tensors were returned by the
- previous call to `loop_fn`.
-
- Returns:
- elements_finished: Bool tensor of shape [batch_size] which marks each
- sequence in the batch as being finished or not finished.
- next_input: A tensor containing input to be fed into the cell at the
- next timestep.
- next_cell_state: Cell internal state to be fed into the cell at the
- next timestep.
- emit_output: Tensor to be added to the TensorArray returned by raw_rnn
- as output from the while_loop.
- next_loop_state: Additional loop state. These tensors will be fed back
- into the next call to `loop_fn` as `loop_state`.
- """
- if cell_output is None: # 0th time step.
- next_cell_state = self.policy_cell.zero_state(batch_size, dtype)
- elements_finished = tf.zeros([batch_size], tf.bool)
- output_lengths = tf.ones([batch_size], dtype=tf.int32)
- next_input = tf.gather(obs_embeddings, initial_state)
- emit_output = None
- next_loop_state = (
- tf.TensorArray(dtype=tf.int32, size=0, dynamic_size=True),
- output_lengths,
- elements_finished
- )
- else:
- scaled_logits = cell_output * config.softmax_tr # Scale temperature.
- prev_chosen, prev_output_lengths, prev_elements_finished = loop_state
- next_cell_state = cell_state
- chosen_outputs = tf.to_int32(tf.where(
- tf.logical_not(prev_elements_finished),
- tf.multinomial(logits=scaled_logits, num_samples=1)[:, 0],
- tf.zeros([batch_size], dtype=tf.int64)))
- elements_finished = tf.logical_or(
- tf.equal(chosen_outputs, misc.BF_EOS_INT),
- loop_time >= global_config.timestep_limit)
- output_lengths = tf.where(
- elements_finished,
- prev_output_lengths,
- # length includes EOS token. empty seq has len 1.
- tf.tile(tf.expand_dims(loop_time + 1, 0), [batch_size])
- )
- next_input = tf.gather(obs_embeddings, chosen_outputs)
- emit_output = scaled_logits
- next_loop_state = (prev_chosen.write(loop_time - 1, chosen_outputs),
- output_lengths,
- tf.logical_or(prev_elements_finished,
- elements_finished))
- return (elements_finished, next_input, next_cell_state, emit_output,
- next_loop_state)
-
- with tf.variable_scope('policy'):
- (decoder_outputs_ta,
- _, # decoder_state
- (sampled_output_ta, output_lengths, _)) = tf.nn.raw_rnn(
- cell=self.policy_cell,
- loop_fn=loop_fn)
- policy_logits = tf.transpose(decoder_outputs_ta.stack(), (1, 0, 2),
- name='policy_logits')
- sampled_tokens = tf.transpose(sampled_output_ta.stack(), (1, 0),
- name='sampled_tokens')
- # Add SOS to beginning of the sequence.
- rshift_sampled_tokens = rshift_time(sampled_tokens, fill=misc.BF_EOS_INT)
-
- # Initial state is 0, 2nd state is first token.
- # Note: If value of last state is computed, this will be used as bootstrap.
- if self.a2c:
- with tf.variable_scope('value'):
- value_output, _ = tf.nn.dynamic_rnn(
- self.value_cell,
- tf.gather(obs_embeddings, rshift_sampled_tokens),
- sequence_length=output_lengths,
- dtype=dtype)
- value = tf.squeeze(value_output, axis=[2])
- else:
- value = tf.zeros([], dtype=dtype)
-
- # for sampling actions from the agent, and which told tensors for doing
- # gradient updates on the agent.
- self.sampled_batch = AttrDict(
- logits=policy_logits,
- value=value,
- tokens=sampled_tokens,
- episode_lengths=output_lengths,
- probs=tf.nn.softmax(policy_logits),
- log_probs=tf.nn.log_softmax(policy_logits))
-
- # adjusted_lengths can be less than the full length of each episode.
- # Use this to train on only part of an episode (starting from t=0).
- self.adjusted_lengths = tf.placeholder(
- tf.int32, [None], name='adjusted_lengths')
- self.policy_multipliers = tf.placeholder(
- dtype,
- [None, None],
- name='policy_multipliers')
- # Empirical value, i.e. discounted sum of observed future rewards from each
- # time step in the episode.
- self.empirical_values = tf.placeholder(
- dtype,
- [None, None],
- name='empirical_values')
-
- # Off-policy training. Just add supervised loss to the RL loss.
- self.off_policy_targets = tf.placeholder(
- tf.int32,
- [None, None],
- name='off_policy_targets')
- self.off_policy_target_lengths = tf.placeholder(
- tf.int32, [None], name='off_policy_target_lengths')
-
- self.actions = tf.placeholder(tf.int32, [None, None], name='actions')
- # Add SOS to beginning of the sequence.
- inputs = rshift_time(self.actions, fill=misc.BF_EOS_INT)
- with tf.variable_scope('policy', reuse=True):
- logits, _ = tf.nn.dynamic_rnn(
- self.policy_cell, tf.gather(obs_embeddings, inputs),
- sequence_length=self.adjusted_lengths,
- dtype=dtype)
-
- if self.a2c:
- with tf.variable_scope('value', reuse=True):
- value_output, _ = tf.nn.dynamic_rnn(
- self.value_cell,
- tf.gather(obs_embeddings, inputs),
- sequence_length=self.adjusted_lengths,
- dtype=dtype)
- value2 = tf.squeeze(value_output, axis=[2])
- else:
- value2 = tf.zeros([], dtype=dtype)
-
- self.given_batch = AttrDict(
- logits=logits,
- value=value2,
- tokens=sampled_tokens,
- episode_lengths=self.adjusted_lengths,
- probs=tf.nn.softmax(logits),
- log_probs=tf.nn.log_softmax(logits))
-
- # Episode masks.
- max_episode_length = tf.shape(self.actions)[1]
- # range_row shape: [1, max_episode_length]
- range_row = tf.expand_dims(tf.range(max_episode_length), 0)
- episode_masks = tf.cast(
- tf.less(range_row, tf.expand_dims(self.given_batch.episode_lengths, 1)),
- dtype=dtype)
- episode_masks_3d = tf.expand_dims(episode_masks, 2)
-
- # Length adjusted episodes.
- self.a_probs = a_probs = self.given_batch.probs * episode_masks_3d
- self.a_log_probs = a_log_probs = (
- self.given_batch.log_probs * episode_masks_3d)
- self.a_value = a_value = self.given_batch.value * episode_masks
- self.a_policy_multipliers = a_policy_multipliers = (
- self.policy_multipliers * episode_masks)
- if self.a2c:
- self.a_empirical_values = a_empirical_values = (
- self.empirical_values * episode_masks)
-
- # pi_loss is scalar
- acs_onehot = tf.one_hot(self.actions, self.action_space, dtype=dtype)
- self.acs_onehot = acs_onehot
- chosen_masked_log_probs = acs_onehot * a_log_probs
- pi_target = tf.expand_dims(a_policy_multipliers, -1)
- pi_loss_per_step = chosen_masked_log_probs * pi_target # Maximize.
- self.pi_loss = pi_loss = (
- -tf.reduce_mean(tf.reduce_sum(pi_loss_per_step, axis=[1, 2]), axis=0)
- * MAGIC_LOSS_MULTIPLIER) # Minimize.
- assert len(self.pi_loss.shape) == 0 # pylint: disable=g-explicit-length-test
-
- # shape: [batch_size, time]
- self.chosen_log_probs = tf.reduce_sum(chosen_masked_log_probs, axis=2)
- self.chosen_probs = tf.reduce_sum(acs_onehot * a_probs, axis=2)
-
- # loss of value function
- if self.a2c:
- vf_loss_per_step = tf.square(a_value - a_empirical_values)
- self.vf_loss = vf_loss = (
- tf.reduce_mean(tf.reduce_sum(vf_loss_per_step, axis=1), axis=0)
- * MAGIC_LOSS_MULTIPLIER) # Minimize.
- assert len(self.vf_loss.shape) == 0 # pylint: disable=g-explicit-length-test
- else:
- self.vf_loss = vf_loss = 0.0
-
- # Maximize entropy regularizer
- self.entropy = entropy = (
- -tf.reduce_mean(
- tf.reduce_sum(a_probs * a_log_probs, axis=[1, 2]), axis=0)
- * MAGIC_LOSS_MULTIPLIER) # Maximize
- self.negentropy = -entropy # Minimize negentropy.
- assert len(self.negentropy.shape) == 0 # pylint: disable=g-explicit-length-test
-
- # off-policy loss
- self.offp_switch = tf.placeholder(dtype, [], name='offp_switch')
- if self.top_episodes is not None:
- # Add SOS to beginning of the sequence.
- offp_inputs = tf.gather(obs_embeddings,
- rshift_time(self.off_policy_targets,
- fill=misc.BF_EOS_INT))
- with tf.variable_scope('policy', reuse=True):
- offp_logits, _ = tf.nn.dynamic_rnn(
- self.policy_cell, offp_inputs, self.off_policy_target_lengths,
- dtype=dtype) # shape: [batch_size, time, action_space]
- topk_loss_per_step = tf.nn.sparse_softmax_cross_entropy_with_logits(
- labels=self.off_policy_targets,
- logits=offp_logits,
- name='topk_loss_per_logit')
- # Take mean over batch dimension so that the loss multiplier strength is
- # independent of batch size. Sum over time dimension.
- topk_loss = tf.reduce_mean(
- tf.reduce_sum(topk_loss_per_step, axis=1), axis=0)
- assert len(topk_loss.shape) == 0 # pylint: disable=g-explicit-length-test
- self.topk_loss = topk_loss * self.offp_switch
- logging.info('Including off policy loss.')
- else:
- self.topk_loss = topk_loss = 0.0
-
- self.entropy_hparam = tf.constant(
- config.entropy_beta, dtype=dtype, name='entropy_beta')
-
- self.pi_loss_term = pi_loss * self.pi_loss_hparam
- self.vf_loss_term = vf_loss * self.vf_loss_hparam
- self.entropy_loss_term = self.negentropy * self.entropy_hparam
- self.topk_loss_term = self.topk_loss_hparam * topk_loss
- self.loss = (
- self.pi_loss_term
- + self.vf_loss_term
- + self.entropy_loss_term
- + self.topk_loss_term)
-
- params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
- tf.get_variable_scope().name)
- self.trainable_variables = params
- self.sync_variables = self.trainable_variables
- non_embedding_params = [p for p in params
- if obs_embedding_scope not in p.name]
- self.non_embedding_params = non_embedding_params
- self.params = params
-
- if config.regularizer:
- logging.info('Adding L2 regularizer with scale %.2f.',
- config.regularizer)
- self.regularizer = config.regularizer * sum(
- tf.nn.l2_loss(w) for w in non_embedding_params)
- self.loss += self.regularizer
- else:
- logging.info('Skipping regularizer.')
- self.regularizer = 0.0
-
- # Only build gradients graph for local model.
- if self.is_local:
- unclipped_grads = tf.gradients(self.loss, params)
- self.dense_unclipped_grads = [
- tf.convert_to_tensor(g) for g in unclipped_grads]
- self.grads, self.global_grad_norm = tf.clip_by_global_norm(
- unclipped_grads, config.grad_clip_threshold)
- self.gradients_dict = dict(zip(params, self.grads))
- self.optimizer = make_optimizer(config.optimizer, self.learning_rate)
- self.all_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
- tf.get_variable_scope().name)
-
- self.do_iw_summaries = do_iw_summaries
- if self.do_iw_summaries:
- b = None
- self.log_iw_replay_ph = tf.placeholder(tf.float32, [b],
- 'log_iw_replay_ph')
- self.log_iw_policy_ph = tf.placeholder(tf.float32, [b],
- 'log_iw_policy_ph')
- self.log_prob_replay_ph = tf.placeholder(tf.float32, [b],
- 'log_prob_replay_ph')
- self.log_prob_policy_ph = tf.placeholder(tf.float32, [b],
- 'log_prob_policy_ph')
- self.log_norm_replay_weights_ph = tf.placeholder(
- tf.float32, [b], 'log_norm_replay_weights_ph')
- self.iw_summary_op = tf.summary.merge([
- tf.summary.histogram('is/log_iw_replay', self.log_iw_replay_ph),
- tf.summary.histogram('is/log_iw_policy', self.log_iw_policy_ph),
- tf.summary.histogram('is/log_prob_replay', self.log_prob_replay_ph),
- tf.summary.histogram('is/log_prob_policy', self.log_prob_policy_ph),
- tf.summary.histogram(
- 'is/log_norm_replay_weights', self.log_norm_replay_weights_ph),
- ])
-
- def make_summary_ops(self):
- """Construct summary ops for the model."""
- # size = number of timesteps across entire batch. Number normalized by size
- # will not be affected by the amount of padding at the ends of sequences
- # in the batch.
- size = tf.cast(
- tf.reduce_sum(self.given_batch.episode_lengths), dtype=self.dtype)
- offp_size = tf.cast(tf.reduce_sum(self.off_policy_target_lengths),
- dtype=self.dtype)
- scope_prefix = self.parent_scope_name
-
- def _remove_prefix(prefix, name):
- assert name.startswith(prefix)
- return name[len(prefix):]
-
- # RL summaries.
- self.rl_summary_op = tf.summary.merge(
- [tf.summary.scalar('model/policy_loss', self.pi_loss / size),
- tf.summary.scalar('model/value_loss', self.vf_loss / size),
- tf.summary.scalar('model/topk_loss', self.topk_loss / offp_size),
- tf.summary.scalar('model/entropy', self.entropy / size),
- tf.summary.scalar('model/loss', self.loss / size),
- tf.summary.scalar('model/grad_norm',
- tf.global_norm(self.grads)),
- tf.summary.scalar('model/unclipped_grad_norm', self.global_grad_norm),
- tf.summary.scalar('model/non_embedding_var_norm',
- tf.global_norm(self.non_embedding_params)),
- tf.summary.scalar('hparams/entropy_beta', self.entropy_hparam),
- tf.summary.scalar('hparams/topk_loss_hparam', self.topk_loss_hparam),
- tf.summary.scalar('hparams/learning_rate', self.learning_rate),
- tf.summary.scalar('model/trainable_var_norm',
- tf.global_norm(self.trainable_variables)),
- tf.summary.scalar('loss/loss', self.loss),
- tf.summary.scalar('loss/entropy', self.entropy_loss_term),
- tf.summary.scalar('loss/vf', self.vf_loss_term),
- tf.summary.scalar('loss/policy', self.pi_loss_term),
- tf.summary.scalar('loss/offp', self.topk_loss_term)] +
- [tf.summary.scalar(
- 'param_norms/' + _remove_prefix(scope_prefix + '/', p.name),
- tf.norm(p))
- for p in self.params] +
- [tf.summary.scalar(
- 'grad_norms/' + _remove_prefix(scope_prefix + '/', p.name),
- tf.norm(g))
- for p, g in zip(self.params, self.grads)] +
- [tf.summary.scalar(
- 'unclipped_grad_norms/' + _remove_prefix(scope_prefix + '/',
- p.name),
- tf.norm(g))
- for p, g in zip(self.params, self.dense_unclipped_grads)])
-
- self.text_summary_placeholder = tf.placeholder(tf.string, shape=[])
- self.rl_text_summary_op = tf.summary.text('rl',
- self.text_summary_placeholder)
-
- def _rl_text_summary(self, session, step, npe, tot_r, num_steps,
- input_case, code_output, code, reason):
- """Logs summary about a single episode and creates a text_summary for TB.
-
- Args:
- session: tf.Session instance.
- step: Global training step.
- npe: Number of programs executed so far.
- tot_r: Total reward.
- num_steps: Number of timesteps in the episode (i.e. code length).
- input_case: Inputs for test cases.
- code_output: Outputs produced by running the code on the inputs.
- code: String representation of the code.
- reason: Reason for the reward assigned by the task.
-
- Returns:
- Serialized text summary data for tensorboard.
- """
- if not input_case:
- input_case = ' '
- if not code_output:
- code_output = ' '
- if not code:
- code = ' '
- text = (
- 'Tot R: **%.2f**; Len: **%d**; Reason: **%s**\n\n'
- 'Input: **`%s`**; Output: **`%s`**\n\nCode: **`%s`**'
- % (tot_r, num_steps, reason, input_case, code_output, code))
- text_summary = session.run(self.rl_text_summary_op,
- {self.text_summary_placeholder: text})
- logging.info(
- 'Step %d.\t NPE: %d\t Reason: %s.\t Tot R: %.2f.\t Length: %d. '
- '\tInput: %s \tOutput: %s \tProgram: %s',
- step, npe, reason, tot_r, num_steps, input_case,
- code_output, code)
- return text_summary
-
- def _rl_reward_summary(self, total_rewards):
- """Create summary ops that report on episode rewards.
-
- Creates summaries for average, median, max, and min rewards in the batch.
-
- Args:
- total_rewards: Tensor of shape [batch_size] containing the total reward
- from each episode in the batch.
-
- Returns:
- tf.Summary op.
- """
- tr = np.asarray(total_rewards)
- reward_summary = tf.Summary(value=[
- tf.Summary.Value(
- tag='reward/avg',
- simple_value=np.mean(tr)),
- tf.Summary.Value(
- tag='reward/med',
- simple_value=np.median(tr)),
- tf.Summary.Value(
- tag='reward/max',
- simple_value=np.max(tr)),
- tf.Summary.Value(
- tag='reward/min',
- simple_value=np.min(tr))])
- return reward_summary
-
- def _iw_summary(self, session, replay_iw, replay_log_probs,
- norm_replay_weights, on_policy_iw,
- on_policy_log_probs):
- """Compute summaries for importance weights at a given batch.
-
- Args:
- session: tf.Session instance.
- replay_iw: Importance weights for episodes from replay buffer.
- replay_log_probs: Total log probabilities of the replay episodes under the
- current policy.
- norm_replay_weights: Normalized replay weights, i.e. values in `replay_iw`
- divided by the total weight in the entire replay buffer. Note, this is
- also the probability of selecting each episode from the replay buffer
- (in a roulette wheel replay buffer).
- on_policy_iw: Importance weights for episodes sampled from the current
- policy.
- on_policy_log_probs: Total log probabilities of the on-policy episodes
- under the current policy.
-
- Returns:
- Serialized TF summaries. Use a summary writer to write these summaries to
- disk.
- """
- return session.run(
- self.iw_summary_op,
- {self.log_iw_replay_ph: np.log(replay_iw),
- self.log_iw_policy_ph: np.log(on_policy_iw),
- self.log_norm_replay_weights_ph: np.log(norm_replay_weights),
- self.log_prob_replay_ph: replay_log_probs,
- self.log_prob_policy_ph: on_policy_log_probs})
-
- def _compute_iw(self, policy_log_probs, replay_weights):
- """Compute importance weights for a batch of episodes.
-
- Arguments are iterables of length batch_size.
-
- Args:
- policy_log_probs: Log probability of each episode under the current
- policy.
- replay_weights: Weight of each episode in the replay buffer. 0 for
- episodes not sampled from the replay buffer (i.e. sampled from the
- policy).
-
- Returns:
- Numpy array of shape [batch_size] containing the importance weight for
- each episode in the batch.
- """
- log_total_replay_weight = log(self.experience_replay.total_weight)
-
- # importance weight
- # = 1 / [(1 - a) + a * exp(log(replay_weight / total_weight / p))]
- # = 1 / ((1-a) + a*q/p)
- a = float(self.replay_alpha)
- a_com = 1.0 - a # compliment of a
- importance_weights = np.asarray(
- [1.0 / (a_com
- + a * exp((log(replay_weight) - log_total_replay_weight)
- - log_p))
- if replay_weight > 0 else 1.0 / a_com
- for log_p, replay_weight
- in zip(policy_log_probs, replay_weights)])
- return importance_weights
-
- def update_step(self, session, rl_batch, train_op, global_step_op,
- return_gradients=False):
- """Perform gradient update on the model.
-
- Args:
- session: tf.Session instance.
- rl_batch: RLBatch instance from data.py. Use DataManager to create a
- RLBatch for each call to update_step. RLBatch contains a batch of
- tasks.
- train_op: A TF op which will perform the gradient update. LMAgent does not
- own its training op, so that trainers can do distributed training
- and construct a specialized training op.
- global_step_op: A TF op which will return the current global step when
- run (should not increment it).
- return_gradients: If True, the gradients will be saved and returned from
- this method call. This is useful for testing.
-
- Returns:
- Results from the update step in a UpdateStepResult namedtuple, including
- global step, global NPE, serialized summaries, and optionally gradients.
- """
- assert self.is_local
-
- # Do update for REINFORCE or REINFORCE + replay buffer.
- if self.experience_replay is None:
- # Train with on-policy REINFORCE.
-
- # Sample new programs from the policy.
- num_programs_from_policy = rl_batch.batch_size
- (batch_actions,
- batch_values,
- episode_lengths) = session.run(
- [self.sampled_batch.tokens, self.sampled_batch.value,
- self.sampled_batch.episode_lengths])
- if episode_lengths.size == 0:
- # This should not happen.
- logging.warn(
- 'Shapes:\n'
- 'batch_actions.shape: %s\n'
- 'batch_values.shape: %s\n'
- 'episode_lengths.shape: %s\n',
- batch_actions.shape, batch_values.shape, episode_lengths.shape)
-
- # Compute rewards.
- code_scores = compute_rewards(
- rl_batch, batch_actions, episode_lengths)
- code_strings = code_scores.code_strings
- batch_tot_r = code_scores.total_rewards
- test_cases = code_scores.test_cases
- code_outputs = code_scores.code_outputs
- reasons = code_scores.reasons
-
- # Process on-policy samples.
- batch_targets, batch_returns = process_episodes(
- code_scores.batch_rewards, episode_lengths, a2c=self.a2c,
- baselines=self.ema_by_len,
- batch_values=batch_values)
- batch_policy_multipliers = batch_targets
- batch_emp_values = batch_returns if self.a2c else [[]]
- adjusted_lengths = episode_lengths
-
- if self.top_episodes:
- assert len(self.top_episodes) > 0 # pylint: disable=g-explicit-length-test
- off_policy_targets = [
- item for item, _
- in self.top_episodes.random_sample(self.topk_batch_size)]
- off_policy_target_lengths = [len(t) for t in off_policy_targets]
- off_policy_targets = utils.stack_pad(off_policy_targets, pad_axes=0,
- dtype=np.int32)
- offp_switch = 1
- else:
- off_policy_targets = [[0]]
- off_policy_target_lengths = [1]
- offp_switch = 0
-
- fetches = {
- 'global_step': global_step_op,
- 'program_count': self.program_count,
- 'summaries': self.rl_summary_op,
- 'train_op': train_op,
- 'gradients': self.gradients_dict if return_gradients else self.no_op}
- fetched = session.run(
- fetches,
- {self.actions: batch_actions,
- self.empirical_values: batch_emp_values,
- self.policy_multipliers: batch_policy_multipliers,
- self.adjusted_lengths: adjusted_lengths,
- self.off_policy_targets: off_policy_targets,
- self.off_policy_target_lengths: off_policy_target_lengths,
- self.offp_switch: offp_switch})
-
- combined_adjusted_lengths = adjusted_lengths
- combined_returns = batch_returns
- else:
- # Train with REINFORCE + off-policy replay buffer by using importance
- # sampling.
-
- # Sample new programs from the policy.
- # Note: batch size is constant. A full batch will be sampled, but not all
- # programs will be executed and added to the replay buffer. Those which
- # are not executed will be discarded and not counted.
- batch_actions, batch_values, episode_lengths, log_probs = session.run(
- [self.sampled_batch.tokens, self.sampled_batch.value,
- self.sampled_batch.episode_lengths, self.sampled_batch.log_probs])
- if episode_lengths.size == 0:
- # This should not happen.
- logging.warn(
- 'Shapes:\n'
- 'batch_actions.shape: %s\n'
- 'batch_values.shape: %s\n'
- 'episode_lengths.shape: %s\n',
- batch_actions.shape, batch_values.shape, episode_lengths.shape)
-
- # Sample from experince replay buffer
- empty_replay_buffer = (
- self.experience_replay.is_empty()
- if self.experience_replay is not None else True)
- num_programs_from_replay_buff = (
- self.num_replay_per_batch if not empty_replay_buffer else 0)
- num_programs_from_policy = (
- rl_batch.batch_size - num_programs_from_replay_buff)
- if (not empty_replay_buffer) and num_programs_from_replay_buff:
- result = self.experience_replay.sample_many(
- num_programs_from_replay_buff)
- experience_samples, replay_weights = zip(*result)
- (replay_actions,
- replay_rewards,
- _, # log probs
- replay_adjusted_lengths) = zip(*experience_samples)
-
- replay_batch_actions = utils.stack_pad(replay_actions, pad_axes=0,
- dtype=np.int32)
-
- # compute log probs for replay samples under current policy
- all_replay_log_probs, = session.run(
- [self.given_batch.log_probs],
- {self.actions: replay_batch_actions,
- self.adjusted_lengths: replay_adjusted_lengths})
- replay_log_probs = [
- np.choose(replay_actions[i], all_replay_log_probs[i, :l].T).sum()
- for i, l in enumerate(replay_adjusted_lengths)]
- else:
- # Replay buffer is empty. Do not sample from it.
- replay_actions = None
- replay_policy_multipliers = None
- replay_adjusted_lengths = None
- replay_log_probs = None
- replay_weights = None
- replay_returns = None
- on_policy_weights = [0] * num_programs_from_replay_buff
-
- assert not self.a2c # TODO(danabo): Support A2C with importance sampling.
-
- # Compute rewards.
- code_scores = compute_rewards(
- rl_batch, batch_actions, episode_lengths,
- batch_size=num_programs_from_policy)
- code_strings = code_scores.code_strings
- batch_tot_r = code_scores.total_rewards
- test_cases = code_scores.test_cases
- code_outputs = code_scores.code_outputs
- reasons = code_scores.reasons
-
- # Process on-policy samples.
- p = num_programs_from_policy
- batch_targets, batch_returns = process_episodes(
- code_scores.batch_rewards, episode_lengths[:p], a2c=False,
- baselines=self.ema_by_len)
- batch_policy_multipliers = batch_targets
- batch_emp_values = [[]]
- on_policy_returns = batch_returns
-
- # Process off-policy samples.
- if (not empty_replay_buffer) and num_programs_from_replay_buff:
- offp_batch_rewards = [
- [0.0] * (l - 1) + [r]
- for l, r in zip(replay_adjusted_lengths, replay_rewards)]
- assert len(offp_batch_rewards) == num_programs_from_replay_buff
- assert len(replay_adjusted_lengths) == num_programs_from_replay_buff
- replay_batch_targets, replay_returns = process_episodes(
- offp_batch_rewards, replay_adjusted_lengths, a2c=False,
- baselines=self.ema_by_len)
- # Convert 2D array back into ragged 2D list.
- replay_policy_multipliers = [
- replay_batch_targets[i, :l]
- for i, l
- in enumerate(
- replay_adjusted_lengths[:num_programs_from_replay_buff])]
-
- adjusted_lengths = episode_lengths[:num_programs_from_policy]
-
- if self.top_episodes:
- assert len(self.top_episodes) > 0 # pylint: disable=g-explicit-length-test
- off_policy_targets = [
- item for item, _
- in self.top_episodes.random_sample(self.topk_batch_size)]
- off_policy_target_lengths = [len(t) for t in off_policy_targets]
- off_policy_targets = utils.stack_pad(off_policy_targets, pad_axes=0,
- dtype=np.int32)
- offp_switch = 1
- else:
- off_policy_targets = [[0]]
- off_policy_target_lengths = [1]
- offp_switch = 0
-
- # On-policy episodes.
- if num_programs_from_policy:
- separate_actions = [
- batch_actions[i, :l]
- for i, l in enumerate(adjusted_lengths)]
- chosen_log_probs = [
- np.choose(separate_actions[i], log_probs[i, :l].T)
- for i, l in enumerate(adjusted_lengths)]
- new_experiences = [
- (separate_actions[i],
- batch_tot_r[i],
- chosen_log_probs[i].sum(), l)
- for i, l in enumerate(adjusted_lengths)]
- on_policy_policy_multipliers = [
- batch_policy_multipliers[i, :l]
- for i, l in enumerate(adjusted_lengths)]
- (on_policy_actions,
- _, # rewards
- on_policy_log_probs,
- on_policy_adjusted_lengths) = zip(*new_experiences)
- else:
- new_experiences = []
- on_policy_policy_multipliers = []
- on_policy_actions = []
- on_policy_log_probs = []
- on_policy_adjusted_lengths = []
-
- if (not empty_replay_buffer) and num_programs_from_replay_buff:
- # Look for new experiences in replay buffer. Assign weight if an episode
- # is in the buffer.
- on_policy_weights = [0] * num_programs_from_policy
- for i, cs in enumerate(code_strings):
- if self.experience_replay.has_key(cs):
- on_policy_weights[i] = self.experience_replay.get_weight(cs)
-
- # Randomly select on-policy or off policy episodes to train on.
- combined_actions = join(replay_actions, on_policy_actions)
- combined_policy_multipliers = join(
- replay_policy_multipliers, on_policy_policy_multipliers)
- combined_adjusted_lengths = join(
- replay_adjusted_lengths, on_policy_adjusted_lengths)
- combined_returns = join(replay_returns, on_policy_returns)
- combined_actions = utils.stack_pad(combined_actions, pad_axes=0)
- combined_policy_multipliers = utils.stack_pad(combined_policy_multipliers,
- pad_axes=0)
- # P
- combined_on_policy_log_probs = join(replay_log_probs, on_policy_log_probs)
- # Q
- # Assume weight is zero for all sequences sampled from the policy.
- combined_q_weights = join(replay_weights, on_policy_weights)
-
- # Importance adjustment. Naive formulation:
- # E_{x~p}[f(x)] ~= 1/N sum_{x~p}(f(x)) ~= 1/N sum_{x~q}(f(x) * p(x)/q(x)).
- # p(x) is the policy, and q(x) is the off-policy distribution, i.e. replay
- # buffer distribution. Importance weight w(x) = p(x) / q(x).
-
- # Instead of sampling from the replay buffer only, we sample from a
- # mixture distribution of the policy and replay buffer.
- # We are sampling from the mixture a*q(x) + (1-a)*p(x), where 0 <= a <= 1.
- # Thus the importance weight w(x) = p(x) / (a*q(x) + (1-a)*p(x))
- # = 1 / ((1-a) + a*q(x)/p(x)) where q(x) is 0 for x sampled from the
- # policy.
- # Note: a = self.replay_alpha
- if empty_replay_buffer:
- # The replay buffer is empty.
- # Do no gradient update this step. The replay buffer will have stuff in
- # it next time.
- combined_policy_multipliers *= 0
- elif not num_programs_from_replay_buff:
- combined_policy_multipliers = np.ones([len(combined_actions), 1],
- dtype=np.float32)
- else:
- # If a < 1 compute importance weights
- # importance weight
- # = 1 / [(1 - a) + a * exp(log(replay_weight / total_weight / p))]
- # = 1 / ((1-a) + a*q/p)
- importance_weights = self._compute_iw(combined_on_policy_log_probs,
- combined_q_weights)
- if self.config.iw_normalize:
- importance_weights *= (
- float(rl_batch.batch_size) / importance_weights.sum())
- combined_policy_multipliers *= importance_weights.reshape(-1, 1)
-
- # Train on replay batch, top-k MLE.
- assert self.program_count is not None
- fetches = {
- 'global_step': global_step_op,
- 'program_count': self.program_count,
- 'summaries': self.rl_summary_op,
- 'train_op': train_op,
- 'gradients': self.gradients_dict if return_gradients else self.no_op}
- fetched = session.run(
- fetches,
- {self.actions: combined_actions,
- self.empirical_values: [[]], # replay_emp_values,
- self.policy_multipliers: combined_policy_multipliers,
- self.adjusted_lengths: combined_adjusted_lengths,
- self.off_policy_targets: off_policy_targets,
- self.off_policy_target_lengths: off_policy_target_lengths,
- self.offp_switch: offp_switch})
-
- # Add to experience replay buffer.
- self.experience_replay.add_many(
- objs=new_experiences,
- weights=[exp(r / self.replay_temperature) for r in batch_tot_r],
- keys=code_strings)
-
- # Update program count.
- session.run(
- [self.program_count_add_op],
- {self.program_count_add_ph: num_programs_from_policy})
-
- # Update EMA baselines on the mini-batch which we just did traning on.
- if not self.a2c:
- for i in xrange(rl_batch.batch_size):
- episode_length = combined_adjusted_lengths[i]
- empirical_returns = combined_returns[i, :episode_length]
- for j in xrange(episode_length):
- # Update ema_baselines in place.
- self.ema_by_len[j] = (
- self.ema_baseline_decay * self.ema_by_len[j]
- + (1 - self.ema_baseline_decay) * empirical_returns[j])
-
- global_step = fetched['global_step']
- global_npe = fetched['program_count']
- core_summaries = fetched['summaries']
- summaries_list = [core_summaries]
-
- if num_programs_from_policy:
- s_i = 0
- text_summary = self._rl_text_summary(
- session,
- global_step,
- global_npe,
- batch_tot_r[s_i],
- episode_lengths[s_i], test_cases[s_i],
- code_outputs[s_i], code_strings[s_i], reasons[s_i])
- reward_summary = self._rl_reward_summary(batch_tot_r)
-
- is_best = False
- if self.global_best_reward_fn:
- # Save best reward.
- best_reward = np.max(batch_tot_r)
- is_best = self.global_best_reward_fn(session, best_reward)
-
- if self.found_solution_op is not None and 'correct' in reasons:
- session.run(self.found_solution_op)
-
- # Save program to disk for record keeping.
- if self.stop_on_success:
- solutions = [
- {'code': code_strings[i], 'reward': batch_tot_r[i],
- 'npe': global_npe}
- for i in xrange(len(reasons)) if reasons[i] == 'correct']
- elif is_best:
- solutions = [
- {'code': code_strings[np.argmax(batch_tot_r)],
- 'reward': np.max(batch_tot_r),
- 'npe': global_npe}]
- else:
- solutions = []
- if solutions:
- if self.assign_code_solution_fn:
- self.assign_code_solution_fn(session, solutions[0]['code'])
- with tf.gfile.FastGFile(self.logging_file, 'a') as writer:
- for solution_dict in solutions:
- writer.write(str(solution_dict) + '\n')
-
- max_i = np.argmax(batch_tot_r)
- max_tot_r = batch_tot_r[max_i]
- if max_tot_r >= self.top_reward:
- if max_tot_r >= self.top_reward:
- self.top_reward = max_tot_r
- logging.info('Top code: r=%.2f, \t%s', max_tot_r, code_strings[max_i])
- if self.top_episodes is not None:
- self.top_episodes.push(
- max_tot_r, tuple(batch_actions[max_i, :episode_lengths[max_i]]))
-
- summaries_list += [text_summary, reward_summary]
-
- if self.do_iw_summaries and not empty_replay_buffer:
- # prob of replay samples under replay buffer sampling.
- norm_replay_weights = [
- w / self.experience_replay.total_weight
- for w in replay_weights]
- replay_iw = self._compute_iw(replay_log_probs, replay_weights)
- on_policy_iw = self._compute_iw(on_policy_log_probs, on_policy_weights)
- summaries_list.append(
- self._iw_summary(
- session, replay_iw, replay_log_probs, norm_replay_weights,
- on_policy_iw, on_policy_log_probs))
-
- return UpdateStepResult(
- global_step=global_step,
- global_npe=global_npe,
- summaries_list=summaries_list,
- gradients_dict=fetched['gradients'])
-
-
-def io_to_text(io_case, io_type):
- if isinstance(io_case, misc.IOTuple):
- # If there are many strings, join them with ','.
- return ','.join([io_to_text(e, io_type) for e in io_case])
- if io_type == misc.IOType.string:
- # There is one string. Return it.
- return misc.tokens_to_text(io_case)
- if (io_type == misc.IOType.integer
- or io_type == misc.IOType.boolean):
- if len(io_case) == 1:
- return str(io_case[0])
- return str(io_case)
-
-
-CodeScoreInfo = namedtuple(
- 'CodeScoreInfo',
- ['code_strings', 'batch_rewards', 'total_rewards', 'test_cases',
- 'code_outputs', 'reasons'])
-
-
-def compute_rewards(rl_batch, batch_actions, episode_lengths, batch_size=None):
- """Compute rewards for each episode in the batch.
-
- Args:
- rl_batch: A data.RLBatch instance. This holds information about the task
- each episode is solving, and a reward function for each episode.
- batch_actions: Contains batch of episodes. Each sequence of actions will be
- converted into a BF program and then scored. A numpy array of shape
- [batch_size, max_sequence_length].
- episode_lengths: The sequence length of each episode in the batch. Iterable
- of length batch_size.
- batch_size: (optional) number of programs to score. Use this to limit the
- number of programs executed from this batch. For example, when doing
- importance sampling some of the on-policy episodes will be discarded
- and they should not be executed. `batch_size` can be less than or equal
- to the size of the input batch.
-
- Returns:
- CodeScoreInfo namedtuple instance. This holds not just the computed rewards,
- but additional information computed during code execution which can be used
- for debugging and monitoring. this includes: BF code strings, test cases
- the code was executed on, code outputs from those test cases, and reasons
- for success or failure.
- """
- code_strings = [
- ''.join([misc.bf_int2char(a) for a in action_sequence[:l]])
- for action_sequence, l in zip(batch_actions, episode_lengths)]
- if batch_size is None:
- batch_size = len(code_strings)
- else:
- assert batch_size <= len(code_strings)
- code_strings = code_strings[:batch_size]
-
- if isinstance(rl_batch.reward_fns, (list, tuple)):
- # reward_fns is a list of functions, same length as code_strings.
- assert len(rl_batch.reward_fns) >= batch_size
- r_fn_results = [
- rl_batch.reward_fns[i](code_strings[i]) for i in xrange(batch_size)]
- else:
- # reward_fns is allowed to be one function which processes a batch of code
- # strings. This is useful for efficiency and batch level computation.
- r_fn_results = rl_batch.reward_fns(code_strings)
-
- # Expecting that r_fn returns a list of rewards. Length of list equals
- # length of the code string (including EOS char).
-
- batch_rewards = [r.episode_rewards for r in r_fn_results]
- total_rewards = [sum(b) for b in batch_rewards]
- test_cases = [io_to_text(r.input_case, r.input_type) for r in r_fn_results]
- code_outputs = [io_to_text(r.code_output, r.output_type)
- for r in r_fn_results]
- reasons = [r.reason for r in r_fn_results]
- return CodeScoreInfo(
- code_strings=code_strings,
- batch_rewards=batch_rewards,
- total_rewards=total_rewards,
- test_cases=test_cases,
- code_outputs=code_outputs,
- reasons=reasons)
-
-
-def process_episodes(
- batch_rewards, episode_lengths, a2c=False, baselines=None,
- batch_values=None):
- """Compute REINFORCE targets.
-
- REINFORCE here takes the form:
- grad_t = grad[log(pi(a_t|c_t))*target_t]
- where c_t is context: i.e. RNN state or environment state (or both).
-
- Two types of targets are supported:
- 1) Advantage actor critic (a2c).
- 2) Vanilla REINFORCE with baseline.
-
- Args:
- batch_rewards: Rewards received in each episode in the batch. A numpy array
- of shape [batch_size, max_sequence_length]. Note, these are per-timestep
- rewards, not total reward.
- episode_lengths: Length of each episode. An iterable of length batch_size.
- a2c: A bool. Whether to compute a2c targets (True) or vanilla targets
- (False).
- baselines: If a2c is False, provide baselines for each timestep. This is a
- list (or indexable container) of length max_time. Note: baselines are
- shared across all episodes, which is why there is no batch dimension.
- It is up to the caller to update baselines accordingly.
- batch_values: If a2c is True, provide values computed by a value estimator.
- A numpy array of shape [batch_size, max_sequence_length].
-
- Returns:
- batch_targets: REINFORCE targets for each episode and timestep. A numpy
- array of shape [batch_size, max_sequence_length].
- batch_returns: Returns computed for each episode and timestep. This is for
- reference, and is not used in the REINFORCE gradient update (but was
- used to compute the targets). A numpy array of shape
- [batch_size, max_sequence_length].
- """
- num_programs = len(batch_rewards)
- assert num_programs <= len(episode_lengths)
- batch_returns = [None] * num_programs
- batch_targets = [None] * num_programs
- for i in xrange(num_programs):
- episode_length = episode_lengths[i]
- assert len(batch_rewards[i]) == episode_length
- # Compute target for each timestep.
- # If we are computing A2C:
- # target_t = advantage_t = R_t - V(c_t)
- # where V(c_t) is a learned value function (provided as `values`).
- # Otherwise:
- # target_t = R_t - baselines[t]
- # where `baselines` are provided.
- # In practice we use a more generalized formulation of advantage. See docs
- # for `discounted_advantage_and_rewards`.
- if a2c:
- # Compute advantage.
- assert batch_values is not None
- episode_values = batch_values[i, :episode_length]
- episode_rewards = batch_rewards[i]
- emp_val, gen_adv = rollout_lib.discounted_advantage_and_rewards(
- episode_rewards, episode_values, gamma=1.0, lambda_=1.0)
- batch_returns[i] = emp_val
- batch_targets[i] = gen_adv
- else:
- # Compute return for each timestep. See section 3 of
- # https://arxiv.org/pdf/1602.01783.pdf
- assert baselines is not None
- empirical_returns = rollout_lib.discount(batch_rewards[i], gamma=1.0)
- targets = [None] * episode_length
- for j in xrange(episode_length):
- targets[j] = empirical_returns[j] - baselines[j]
- batch_returns[i] = empirical_returns
- batch_targets[i] = targets
- batch_returns = utils.stack_pad(batch_returns, 0)
- if num_programs:
- batch_targets = utils.stack_pad(batch_targets, 0)
- else:
- batch_targets = np.array([], dtype=np.float32)
-
- return (batch_targets, batch_returns)
diff --git a/research/brain_coder/single_task/pg_agent_test.py b/research/brain_coder/single_task/pg_agent_test.py
deleted file mode 100644
index 503d37ecacbf968b0786b3553e6a97667569bf7d..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/pg_agent_test.py
+++ /dev/null
@@ -1,395 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for pg_agent."""
-
-from collections import Counter
-
-from absl import logging
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from common import utils # brain coder
-from single_task import data # brain coder
-from single_task import defaults # brain coder
-from single_task import misc # brain coder
-from single_task import pg_agent as agent_lib # brain coder
-from single_task import pg_train # brain coder
-
-
-# Symmetric mean absolute percentage error (SMAPE).
-# https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error
-def smape(a, b):
- return 2.0 * abs(a - b) / float(a + b)
-
-
-def onehot(dim, num_dims):
- value = np.zeros(num_dims, dtype=np.float32)
- value[dim] = 1
- return value
-
-
-def random_sequence(max_length, num_tokens, eos=0):
- length = np.random.randint(1, max_length - 1)
- return np.append(np.random.randint(1, num_tokens, length), eos)
-
-
-def repeat_and_pad(v, rep, total_len):
- return [v] * rep + [0.0] * (total_len - rep)
-
-
-class AgentTest(tf.test.TestCase):
-
- def testProcessEpisodes(self):
- batch_size = 3
-
- def reward_fn(code_string):
- return misc.RewardInfo(
- episode_rewards=[float(ord(c)) for c in code_string],
- input_case=[],
- correct_output=[],
- code_output=[],
- input_type=misc.IOType.integer,
- output_type=misc.IOType.integer,
- reason='none')
-
- rl_batch = data.RLBatch(
- reward_fns=[reward_fn for _ in range(batch_size)],
- batch_size=batch_size,
- good_reward=10.0)
- batch_actions = np.asarray([
- [4, 5, 3, 6, 8, 1, 0, 0],
- [1, 2, 3, 4, 0, 0, 0, 0],
- [8, 7, 6, 5, 4, 3, 2, 1]], dtype=np.int32)
- batch_values = np.asarray([
- [0, 1, 2, 1, 0, 1, 1, 0],
- [0, 2, 1, 2, 1, 0, 0, 0],
- [0, 1, 1, 0, 0, 0, 1, 1]], dtype=np.float32)
- episode_lengths = np.asarray([7, 5, 8], dtype=np.int32)
-
- scores = agent_lib.compute_rewards(
- rl_batch, batch_actions, episode_lengths)
- batch_targets, batch_returns = agent_lib.process_episodes(
- scores.batch_rewards, episode_lengths, a2c=True,
- batch_values=batch_values)
- self.assertEqual(
- [[473.0, 428.0, 337.0, 294.0, 201.0, 157.0, 95.0, 0.0],
- [305.0, 243.0, 183.0, 140.0, 95.0, 0.0, 0.0, 0.0],
- [484.0, 440.0, 394.0, 301.0, 210.0, 165.0, 122.0, 62.0]],
- batch_returns.tolist())
- self.assertEqual(
- [[473.0, 427.0, 335.0, 293.0, 201.0, 156.0, 94.0, 0.0],
- [305.0, 241.0, 182.0, 138.0, 94.0, 0.0, 0.0, 0.0],
- [484.0, 439.0, 393.0, 301.0, 210.0, 165.0, 121.0, 61.0]],
- batch_targets.tolist())
-
- def testVarUpdates(self):
- """Tests that variables get updated as expected.
-
- For the RL update, check that gradients are non-zero and that the global
- model gets updated.
- """
- config = defaults.default_config_with_updates(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="pg",eos_token=True,optimizer="sgd",lr=1.0)')
- lr = config.agent.lr
-
- tf.reset_default_graph()
- trainer = pg_train.AsyncTrainer(
- config, task_id=0, ps_tasks=0, num_workers=1)
- global_init_op = tf.variables_initializer(
- tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'global'))
- with tf.Session() as sess:
- sess.run(global_init_op) # Initialize global copy.
- trainer.initialize(sess)
- model = trainer.model
- global_vars = sess.run(trainer.global_model.trainable_variables)
- local_vars = sess.run(model.trainable_variables)
-
- # Make sure names match.
- g_prefix = 'global/'
- l_prefix = 'local/'
- for g, l in zip(trainer.global_model.trainable_variables,
- model.trainable_variables):
- self.assertEqual(g.name[len(g_prefix):], l.name[len(l_prefix):])
-
- # Assert that shapes and values are the same between global and local
- # models.
- for g, l in zip(global_vars, local_vars):
- self.assertEqual(g.shape, l.shape)
- self.assertTrue(np.array_equal(g, l))
-
- # Make all gradients dense tensors.
- for param, grad in model.gradients_dict.items():
- if isinstance(grad, tf.IndexedSlices):
- # Converts to dense tensor.
- model.gradients_dict[param] = tf.multiply(grad, 1.0)
-
- # Perform update.
- results = model.update_step(
- sess, trainer.data_manager.sample_rl_batch(), trainer.train_op,
- trainer.global_step, return_gradients=True)
- grads_dict = results.gradients_dict
- for grad in grads_dict.values():
- self.assertIsNotNone(grad)
- self.assertTrue(np.count_nonzero(grad) > 0)
- global_update = sess.run(trainer.global_model.trainable_variables)
- for tf_var, var_before, var_after in zip(
- model.trainable_variables, local_vars, global_update):
- # Check that the params were updated.
- self.assertTrue(np.allclose(
- var_after,
- var_before - grads_dict[tf_var] * lr))
-
- # Test that global to local sync works.
- sess.run(trainer.sync_op)
- global_vars = sess.run(trainer.global_model.trainable_variables)
- local_vars = sess.run(model.trainable_variables)
- for l, g in zip(local_vars, global_vars):
- self.assertTrue(np.allclose(l, g))
-
- def testMonteCarloGradients(self):
- """Test Monte Carlo estimate of REINFORCE gradient.
-
- Test that the Monte Carlo estimate of the REINFORCE gradient is
- approximately equal to the true gradient. We compute the true gradient for a
- toy environment with a very small action space.
-
- Similar to section 5 of https://arxiv.org/pdf/1505.00521.pdf.
- """
- # Test may have different outcome on different machines due to different
- # rounding behavior of float arithmetic.
- tf.reset_default_graph()
- tf.set_random_seed(12345678987654321)
- np.random.seed(1294024302)
- max_length = 2
- num_tokens = misc.bf_num_tokens()
- eos = misc.BF_EOS_INT
- assert eos == 0
- def sequence_iterator(max_length):
- """Iterates through all sequences up to the given length."""
- yield [eos]
- for a in xrange(1, num_tokens):
- if max_length > 1:
- for sub_seq in sequence_iterator(max_length - 1):
- yield [a] + sub_seq
- else:
- yield [a]
- actions = list(sequence_iterator(max_length))
-
- # This batch contains all possible episodes up to max_length.
- actions_batch = utils.stack_pad(actions, 0)
- lengths_batch = [len(s) for s in actions]
-
- reward_map = {tuple(a): np.random.randint(-1, 7) for a in actions_batch}
- # reward_map = {tuple(a): np.random.normal(3, 1)
- # for a in actions_batch} # normal distribution
- # reward_map = {tuple(a): 1.0
- # for a in actions_batch} # expected reward is 1
-
- n = 100000 # MC sample size.
- config = defaults.default_config_with_updates(
- 'env=c(task="print"),'
- 'agent=c(algorithm="pg",optimizer="sgd",lr=1.0,ema_baseline_decay=0.99,'
- 'entropy_beta=0.0,topk_loss_hparam=0.0,regularizer=0.0,'
- 'policy_lstm_sizes=[10],eos_token=True),'
- 'batch_size='+str(n)+',timestep_limit='+str(max_length))
-
- dtype = tf.float64
- trainer = pg_train.AsyncTrainer(
- config, task_id=0, ps_tasks=0, num_workers=1, dtype=dtype)
- model = trainer.model
- actions_ph = model.actions
- lengths_ph = model.adjusted_lengths
- multipliers_ph = model.policy_multipliers
-
- global_init_op = tf.variables_initializer(
- tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'global'))
- with tf.Session() as sess, sess.graph.as_default():
- sess.run(global_init_op) # Initialize global copy.
- trainer.initialize(sess)
-
- # Compute exact gradients.
- # exact_grads = sum(P(a) * grad(log P(a)) * R(a) for a in actions_batch)
- true_loss_unnormalized = 0.0
- exact_grads = [np.zeros(v.shape) for v in model.trainable_variables]
- episode_probs_map = {}
- grads_map = {}
- for a_idx in xrange(len(actions_batch)):
- a = actions_batch[a_idx]
- grads_result, probs_result, loss = sess.run(
- [model.dense_unclipped_grads, model.chosen_probs, model.loss],
- {actions_ph: [a],
- lengths_ph: [lengths_batch[a_idx]],
- multipliers_ph: [
- repeat_and_pad(reward_map[tuple(a)],
- lengths_batch[a_idx],
- max_length)]})
- # Take product over time axis.
- episode_probs_result = np.prod(probs_result[0, :lengths_batch[a_idx]])
- for i in range(0, len(exact_grads)):
- exact_grads[i] += grads_result[i] * episode_probs_result
- episode_probs_map[tuple(a)] = episode_probs_result
- reward_map[tuple(a)] = reward_map[tuple(a)]
- grads_map[tuple(a)] = grads_result
- true_loss_unnormalized += loss
- # Normalize loss. Since each episode is feed into the model one at a time,
- # normalization needs to be done manually.
- true_loss = true_loss_unnormalized / float(len(actions_batch))
-
- # Compute Monte Carlo gradients.
- # E_a~P[grad(log P(a)) R(a)] is aprox. eq. to
- # sum(grad(log P(a)) R(a) for a in actions_sampled_from_P) / n
- # where len(actions_sampled_from_P) == n.
- #
- # In other words, sample from the policy and compute the gradients of the
- # log probs weighted by the returns. This will excersize the code in
- # agent.py
- sampled_actions, sampled_lengths = sess.run(
- [model.sampled_tokens, model.episode_lengths])
- pi_multipliers = [
- repeat_and_pad(reward_map[tuple(a)], l, max_length)
- for a, l in zip(sampled_actions, sampled_lengths)]
- mc_grads_unnormalized, sampled_probs, mc_loss_unnormalized = sess.run(
- [model.dense_unclipped_grads, model.chosen_probs, model.loss],
- {actions_ph: sampled_actions,
- multipliers_ph: pi_multipliers,
- lengths_ph: sampled_lengths})
- # Loss is already normalized across the minibatch, so no normalization
- # is needed.
- mc_grads = mc_grads_unnormalized
- mc_loss = mc_loss_unnormalized
-
- # Make sure true loss and MC loss are similar.
- loss_error = smape(true_loss, mc_loss)
- self.assertTrue(loss_error < 0.15, msg='actual: %s' % loss_error)
-
- # Check that probs computed for episodes sampled from the model are the same
- # as the recorded true probs.
- for i in range(100):
- acs = tuple(sampled_actions[i].tolist())
- sampled_prob = np.prod(sampled_probs[i, :sampled_lengths[i]])
- self.assertTrue(np.isclose(episode_probs_map[acs], sampled_prob))
-
- # Make sure MC estimates of true probs are close.
- counter = Counter(tuple(e) for e in sampled_actions)
- for acs, count in counter.iteritems():
- mc_prob = count / float(len(sampled_actions))
- true_prob = episode_probs_map[acs]
- error = smape(mc_prob, true_prob)
- self.assertTrue(
- error < 0.15,
- msg='actual: %s; count: %s; mc_prob: %s; true_prob: %s'
- % (error, count, mc_prob, true_prob))
-
- # Manually recompute MC gradients and make sure they match MC gradients
- # computed in TF.
- mc_grads_recompute = [np.zeros(v.shape) for v in model.trainable_variables]
- for i in range(n):
- acs = tuple(sampled_actions[i].tolist())
- for i in range(0, len(mc_grads_recompute)):
- mc_grads_recompute[i] += grads_map[acs][i]
- for i in range(0, len(mc_grads_recompute)):
- self.assertTrue(np.allclose(mc_grads[i], mc_grads_recompute[i] / n))
-
- # Check angle between gradients as fraction of pi.
- for index in range(len(mc_grads)):
- v1 = mc_grads[index].reshape(-1)
- v2 = exact_grads[index].reshape(-1)
- # angle = arccos(v1 . v2 / (|v1|*|v2|))
- angle_rad = np.arccos(
- np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2)))
- logging.info('angle / pi: %s', angle_rad / np.pi)
- angle_frac = angle_rad / np.pi
- self.assertTrue(angle_frac < 0.02, msg='actual: %s' % angle_frac)
- # Check norms.
- for index in range(len(mc_grads)):
- v1_norm = np.linalg.norm(mc_grads[index].reshape(-1))
- v2_norm = np.linalg.norm(exact_grads[index].reshape(-1))
- error = smape(v1_norm, v2_norm)
- self.assertTrue(error < 0.02, msg='actual: %s' % error)
-
- # Check expected rewards.
- # E_a~P[R(a)] approx eq sum(P(a) * R(a) for a in actions)
- mc_expected_reward = np.mean(
- [reward_map[tuple(a)] for a in sampled_actions])
- exact_expected_reward = np.sum(
- [episode_probs_map[k] * reward_map[k] for k in reward_map])
- error = smape(mc_expected_reward, exact_expected_reward)
- self.assertTrue(error < 0.005, msg='actual: %s' % angle_frac)
-
- def testNumericalGradChecking(self):
- # Similar to
- # http://ufldl.stanford.edu/wiki/index.php/Gradient_checking_and_advanced_optimization.
- epsilon = 1e-4
- eos = misc.BF_EOS_INT
- self.assertEqual(0, eos)
- config = defaults.default_config_with_updates(
- 'env=c(task="print"),'
- 'agent=c(algorithm="pg",optimizer="sgd",lr=1.0,ema_baseline_decay=0.99,'
- 'entropy_beta=0.0,topk_loss_hparam=0.0,policy_lstm_sizes=[10],'
- 'eos_token=True),'
- 'batch_size=64')
- dtype = tf.float64
- tf.reset_default_graph()
- tf.set_random_seed(12345678987654321)
- np.random.seed(1294024302)
- trainer = pg_train.AsyncTrainer(
- config, task_id=0, ps_tasks=0, num_workers=1, dtype=dtype)
- model = trainer.model
- actions_ph = model.actions
- lengths_ph = model.adjusted_lengths
- multipliers_ph = model.policy_multipliers
- loss = model.pi_loss
- global_init_op = tf.variables_initializer(
- tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'global'))
-
- assign_add_placeholders = [None] * len(model.trainable_variables)
- assign_add_ops = [None] * len(model.trainable_variables)
- param_shapes = [None] * len(model.trainable_variables)
- for i, param in enumerate(model.trainable_variables):
- param_shapes[i] = param.get_shape().as_list()
- assign_add_placeholders[i] = tf.placeholder(dtype,
- np.prod(param_shapes[i]))
- assign_add_ops[i] = param.assign_add(
- tf.reshape(assign_add_placeholders[i], param_shapes[i]))
-
- with tf.Session() as sess:
- sess.run(global_init_op) # Initialize global copy.
- trainer.initialize(sess)
-
- actions_raw = [random_sequence(10, 9) for _ in xrange(16)]
- actions_batch = utils.stack_pad(actions_raw, 0)
- lengths_batch = [len(l) for l in actions_raw]
- feed = {actions_ph: actions_batch,
- multipliers_ph: np.ones_like(actions_batch),
- lengths_ph: lengths_batch}
-
- estimated_grads = [None] * len(model.trainable_variables)
- for i, param in enumerate(model.trainable_variables):
- param_size = np.prod(param_shapes[i])
- estimated_grads[i] = np.zeros(param_size, dtype=np.float64)
- for index in xrange(param_size):
- e = onehot(index, param_size) * epsilon
- sess.run(assign_add_ops[i],
- {assign_add_placeholders[i]: e})
- j_plus = sess.run(loss, feed)
- sess.run(assign_add_ops[i],
- {assign_add_placeholders[i]: -2 * e})
- j_minus = sess.run(loss, feed)
- sess.run(assign_add_ops[i],
- {assign_add_placeholders[i]: e})
- estimated_grads[i][index] = (j_plus - j_minus) / (2 * epsilon)
- estimated_grads[i] = estimated_grads[i].reshape(param_shapes[i])
-
- analytic_grads = sess.run(model.dense_unclipped_grads, feed)
-
- for g1, g2 in zip(estimated_grads[1:], analytic_grads[1:]):
- logging.info('norm (g1-g2): %s', np.abs(g1 - g2).mean())
- self.assertTrue(np.allclose(g1, g2))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/single_task/pg_train.py b/research/brain_coder/single_task/pg_train.py
deleted file mode 100644
index fde7cc84729a56002e8688d268a2085432ee124e..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/pg_train.py
+++ /dev/null
@@ -1,782 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-r"""Train RL agent on coding tasks."""
-
-import contextlib
-import cPickle
-import cProfile
-import marshal
-import os
-import time
-
-from absl import flags
-from absl import logging
-import tensorflow as tf
-
-# internal session lib import
-
-from single_task import data # brain coder
-from single_task import defaults # brain coder
-from single_task import pg_agent as agent_lib # brain coder
-from single_task import results_lib # brain coder
-
-
-FLAGS = flags.FLAGS
-flags.DEFINE_string(
- 'master', '',
- 'URL of the TensorFlow master to use.')
-flags.DEFINE_integer(
- 'ps_tasks', 0,
- 'Number of parameter server tasks. Only set to 0 for '
- 'single worker training.')
-flags.DEFINE_integer(
- 'summary_interval', 10,
- 'How often to write summaries.')
-flags.DEFINE_integer(
- 'summary_tasks', 16,
- 'If greater than 0 only tasks 0 through summary_tasks - 1 '
- 'will write summaries. If 0, all tasks will write '
- 'summaries.')
-flags.DEFINE_bool(
- 'stop_on_success', True,
- 'If True, training will stop as soon as a solution is found. '
- 'If False, training will continue indefinitely until another '
- 'stopping condition is reached.')
-flags.DEFINE_bool(
- 'do_profiling', False,
- 'If True, cProfile profiler will run and results will be '
- 'written to logdir. WARNING: Results will not be written if '
- 'the code crashes. Make sure it exists successfully.')
-flags.DEFINE_integer('model_v', 0, 'Model verbosity level.')
-flags.DEFINE_bool(
- 'delayed_graph_cleanup', True,
- 'If true, container for n-th run will not be reset until the (n+1)-th run '
- 'is complete. This greatly reduces the chance that a worker is still '
- 'using the n-th container when it is cleared.')
-
-
-def define_tuner_hparam_space(hparam_space_type):
- """Define tunable hparams for grid search."""
- if hparam_space_type not in ('pg', 'pg-topk', 'topk', 'is'):
- raise ValueError('Hparam space is not valid: "%s"' % hparam_space_type)
-
- # Discrete hparam space is stored as a dict from hparam name to discrete
- # values.
- hparam_space = {}
-
- if hparam_space_type in ('pg', 'pg-topk', 'is'):
- # Add a floating point parameter named learning rate.
- hparam_space['lr'] = [1e-5, 1e-4, 1e-3]
- hparam_space['entropy_beta'] = [0.005, 0.01, 0.05, 0.10]
- else: # 'topk'
- # Add a floating point parameter named learning rate.
- hparam_space['lr'] = [1e-5, 1e-4, 1e-3]
- hparam_space['entropy_beta'] = [0.0, 0.005, 0.01, 0.05, 0.10]
-
- if hparam_space_type in ('topk', 'pg-topk'):
- # topk tuning will be enabled.
- hparam_space['topk'] = [10]
- hparam_space['topk_loss_hparam'] = [1.0, 10.0, 50.0, 200.0]
-
- elif hparam_space_type == 'is':
- # importance sampling tuning will be enabled.
- hparam_space['replay_temperature'] = [0.25, 0.5, 1.0, 2.0]
- hparam_space['alpha'] = [0.5, 0.75, 63/64.]
-
- return hparam_space
-
-
-def write_hparams_to_config(config, hparams, hparam_space_type):
- """Write hparams given by the tuner into the Config object."""
- if hparam_space_type not in ('pg', 'pg-topk', 'topk', 'is'):
- raise ValueError('Hparam space is not valid: "%s"' % hparam_space_type)
-
- config.agent.lr = hparams.lr
- config.agent.entropy_beta = hparams.entropy_beta
-
- if hparam_space_type in ('topk', 'pg-topk'):
- # topk tuning will be enabled.
- config.agent.topk = hparams.topk
- config.agent.topk_loss_hparam = hparams.topk_loss_hparam
- elif hparam_space_type == 'is':
- # importance sampling tuning will be enabled.
- config.agent.replay_temperature = hparams.replay_temperature
- config.agent.alpha = hparams.alpha
-
-
-def make_initialized_variable(value, name, shape=None, dtype=tf.float32):
- """Create a tf.Variable with a constant initializer.
-
- Args:
- value: Constant value to initialize the variable with. This is the value
- that the variable starts with.
- name: Name of the variable in the TF graph.
- shape: Shape of the variable. If None, variable will be a scalar.
- dtype: Data type of the variable. Should be a TF dtype. Defaults to
- tf.float32.
-
- Returns:
- tf.Variable instance.
- """
- if shape is None:
- shape = []
- return tf.get_variable(
- name=name, shape=shape, initializer=tf.constant_initializer(value),
- dtype=dtype, trainable=False)
-
-
-class AsyncTrainer(object):
- """Manages graph creation and training.
-
- This async trainer creates a global model on the parameter server, and a local
- model (for this worker). Gradient updates are sent to the global model, and
- the updated weights are synced to the local copy.
- """
-
- def __init__(self, config, task_id, ps_tasks, num_workers, is_chief=True,
- summary_writer=None,
- dtype=tf.float32,
- summary_interval=1,
- run_number=0,
- logging_dir='/tmp', model_v=0):
- self.config = config
- self.data_manager = data.DataManager(
- config, run_number=run_number,
- do_code_simplification=not FLAGS.stop_on_success)
- self.task_id = task_id
- self.ps_tasks = ps_tasks
- self.is_chief = is_chief
- if ps_tasks == 0:
- assert task_id == 0, 'No parameter servers specified. Expecting 1 task.'
- assert num_workers == 1, (
- 'No parameter servers specified. Expecting 1 task.')
- worker_device = '/job:localhost/replica:%d/task:0/cpu:0' % task_id
- # worker_device = '/cpu:0'
- # ps_device = '/cpu:0'
- else:
- assert num_workers > 0, 'There must be at least 1 training worker.'
- worker_device = '/job:worker/replica:%d/task:0/cpu:0' % task_id
- # ps_device = '/job:ps/replica:0/task:0/cpu:0'
- logging.info('worker_device: %s', worker_device)
-
- logging_file = os.path.join(
- logging_dir, 'solutions_%d.txt' % task_id)
- experience_replay_file = os.path.join(
- logging_dir, 'replay_buffer_%d.pickle' % task_id)
- self.topk_file = os.path.join(
- logging_dir, 'topk_buffer_%d.pickle' % task_id)
-
- tf.get_variable_scope().set_use_resource(True)
-
- # global model
- with tf.device(tf.train.replica_device_setter(ps_tasks,
- ps_device='/job:ps/replica:0',
- worker_device=worker_device)):
- with tf.variable_scope('global'):
- global_model = agent_lib.LMAgent(config, dtype=dtype, is_local=False)
- global_params_dict = {p.name: p
- for p in global_model.sync_variables}
- self.global_model = global_model
- self.global_step = make_initialized_variable(
- 0, 'global_step', dtype=tf.int64)
-
- self.global_best_reward = make_initialized_variable(
- -10.0, 'global_best_reward', dtype=tf.float64)
- self.is_best_model = make_initialized_variable(
- False, 'is_best_model', dtype=tf.bool)
- self.reset_is_best_model = self.is_best_model.assign(False)
- self.global_best_reward_placeholder = tf.placeholder(
- tf.float64, [], name='global_best_reward_placeholder')
- self.assign_global_best_reward_op = tf.group(
- self.global_best_reward.assign(
- self.global_best_reward_placeholder),
- self.is_best_model.assign(True))
- def assign_global_best_reward_fn(session, reward):
- reward = round(reward, 10)
- best_reward = round(session.run(self.global_best_reward), 10)
- is_best = reward > best_reward
- if is_best:
- session.run(self.assign_global_best_reward_op,
- {self.global_best_reward_placeholder: reward})
- return is_best
- self.assign_global_best_reward_fn = assign_global_best_reward_fn
-
- # Any worker will set to true when it finds a solution.
- self.found_solution_flag = make_initialized_variable(
- False, 'found_solution_flag', dtype=tf.bool)
- self.found_solution_op = self.found_solution_flag.assign(True)
-
- self.run_number = make_initialized_variable(
- run_number, 'run_number', dtype=tf.int32)
-
- # Store a solution when found.
- self.code_solution_variable = tf.get_variable(
- 'code_solution', [], tf.string,
- initializer=tf.constant_initializer(''))
- self.code_solution_ph = tf.placeholder(
- tf.string, [], name='code_solution_ph')
- self.code_solution_assign_op = self.code_solution_variable.assign(
- self.code_solution_ph)
- def assign_code_solution_fn(session, code_solution_string):
- session.run(self.code_solution_assign_op,
- {self.code_solution_ph: code_solution_string})
- self.assign_code_solution_fn = assign_code_solution_fn
-
- # Count all programs sampled from policy. This does not include
- # programs sampled from replay buffer.
- # This equals NPE (number of programs executed). Only programs sampled
- # from the policy need to be executed.
- self.program_count = make_initialized_variable(
- 0, 'program_count', dtype=tf.int64)
-
- # local model
- with tf.device(worker_device):
- with tf.variable_scope('local'):
- self.model = model = agent_lib.LMAgent(
- config,
- task_id=task_id,
- logging_file=logging_file,
- experience_replay_file=experience_replay_file,
- dtype=dtype,
- global_best_reward_fn=self.assign_global_best_reward_fn,
- found_solution_op=self.found_solution_op,
- assign_code_solution_fn=self.assign_code_solution_fn,
- program_count=self.program_count,
- stop_on_success=FLAGS.stop_on_success,
- verbose_level=model_v)
- local_params = model.trainable_variables
- local_params_dict = {p.name: p for p in local_params}
-
- # Pull global params to local model.
- def _global_to_local_scope(name):
- assert name.startswith('global/')
- return 'local' + name[6:]
- sync_dict = {
- local_params_dict[_global_to_local_scope(p_name)]: p
- for p_name, p in global_params_dict.items()}
- self.sync_op = tf.group(*[v_local.assign(v_global)
- for v_local, v_global
- in sync_dict.items()])
-
- # Pair local gradients with global params.
- grad_var_dict = {
- gradient: sync_dict[local_var]
- for local_var, gradient in model.gradients_dict.items()}
-
- # local model
- model.make_summary_ops() # Don't put summaries under 'local' scope.
- with tf.variable_scope('local'):
- self.train_op = model.optimizer.apply_gradients(
- grad_var_dict.items(), global_step=self.global_step)
- self.local_init_op = tf.variables_initializer(
- tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
- tf.get_variable_scope().name))
-
- self.local_step = 0
- self.last_summary_time = time.time()
- self.summary_interval = summary_interval
- self.summary_writer = summary_writer
- self.cached_global_step = -1
- self.cached_global_npe = -1
-
- logging.info('summary_interval: %d', self.summary_interval)
-
- # Load top-k buffer.
- if self.model.top_episodes is not None and tf.gfile.Exists(self.topk_file):
- try:
- with tf.gfile.FastGFile(self.topk_file, 'r') as f:
- self.model.top_episodes = cPickle.loads(f.read())
- logging.info(
- 'Loaded top-k buffer from disk with %d items. Location: "%s"',
- len(self.model.top_episodes), self.topk_file)
- except (cPickle.UnpicklingError, EOFError) as e:
- logging.warn(
- 'Failed to load existing top-k buffer from disk. Removing bad file.'
- '\nLocation: "%s"\nException: %s', self.topk_file, str(e))
- tf.gfile.Remove(self.topk_file)
-
- def initialize(self, session):
- """Run initialization ops."""
- session.run(self.local_init_op)
- session.run(self.sync_op)
- self.cached_global_step, self.cached_global_npe = session.run(
- [self.global_step, self.program_count])
-
- def update_global_model(self, session):
- """Run an update step.
-
- 1) Asynchronously copy global weights to local model.
- 2) Call into local model's update_step method, which does the following:
- a) Sample batch of programs from policy.
- b) Compute rewards.
- c) Compute gradients and update the global model asynchronously.
- 3) Write tensorboard summaries to disk.
-
- Args:
- session: tf.Session instance.
- """
- session.run(self.sync_op) # Copy weights from global to local.
-
- with session.as_default():
- result = self.model.update_step(
- session, self.data_manager.sample_rl_batch(), self.train_op,
- self.global_step)
- global_step = result.global_step
- global_npe = result.global_npe
- summaries = result.summaries_list
- self.cached_global_step = global_step
- self.cached_global_npe = global_npe
- self.local_step += 1
-
- if self.summary_writer and self.local_step % self.summary_interval == 0:
- if not isinstance(summaries, (tuple, list)):
- summaries = [summaries]
- summaries.append(self._local_step_summary())
- if self.is_chief:
- (global_best_reward,
- found_solution_flag,
- program_count) = session.run(
- [self.global_best_reward,
- self.found_solution_flag,
- self.program_count])
- summaries.append(
- tf.Summary(
- value=[tf.Summary.Value(
- tag='model/best_reward',
- simple_value=global_best_reward)]))
- summaries.append(
- tf.Summary(
- value=[tf.Summary.Value(
- tag='model/solution_found',
- simple_value=int(found_solution_flag))]))
- summaries.append(
- tf.Summary(
- value=[tf.Summary.Value(
- tag='model/program_count',
- simple_value=program_count)]))
- for s in summaries:
- self.summary_writer.add_summary(s, global_step)
- self.last_summary_time = time.time()
-
- def _local_step_summary(self):
- """Compute number of local steps per time increment."""
- dt = time.time() - self.last_summary_time
- steps_per_time = self.summary_interval / float(dt)
- return tf.Summary(value=[
- tf.Summary.Value(
- tag='local_step/per_sec',
- simple_value=steps_per_time),
- tf.Summary.Value(
- tag='local_step/step',
- simple_value=self.local_step)])
-
- def maybe_save_best_model(self, session, saver, checkpoint_file):
- """Check if this model got the highest reward and save to disk if so."""
- if self.is_chief and session.run(self.is_best_model):
- logging.info('Saving best model to "%s"', checkpoint_file)
- saver.save(session, checkpoint_file)
- session.run(self.reset_is_best_model)
-
- def save_replay_buffer(self):
- """Save replay buffer to disk.
-
- Call this periodically so that training can recover if jobs go down.
- """
- if self.model.experience_replay is not None:
- logging.info('Saving experience replay buffer to "%s".',
- self.model.experience_replay.save_file)
- self.model.experience_replay.incremental_save(True)
-
- def delete_replay_buffer(self):
- """Delete replay buffer from disk.
-
- Call this at the end of training to clean up. Replay buffer can get very
- large.
- """
- if self.model.experience_replay is not None:
- logging.info('Deleting experience replay buffer at "%s".',
- self.model.experience_replay.save_file)
- tf.gfile.Remove(self.model.experience_replay.save_file)
-
- def save_topk_buffer(self):
- """Save top-k buffer to disk.
-
- Call this periodically so that training can recover if jobs go down.
- """
- if self.model.top_episodes is not None:
- logging.info('Saving top-k buffer to "%s".', self.topk_file)
- # Overwrite previous data each time.
- with tf.gfile.FastGFile(self.topk_file, 'w') as f:
- f.write(cPickle.dumps(self.model.top_episodes))
-
-
-@contextlib.contextmanager
-def managed_session(sv, master='', config=None,
- start_standard_services=True,
- close_summary_writer=True,
- max_wait_secs=7200):
- # Same as Supervisor.managed_session, but with configurable timeout.
- try:
- sess = sv.prepare_or_wait_for_session(
- master=master, config=config,
- start_standard_services=start_standard_services,
- max_wait_secs=max_wait_secs)
- yield sess
- except tf.errors.DeadlineExceededError:
- raise
- except Exception as e: # pylint: disable=broad-except
- sv.request_stop(e)
- finally:
- try:
- # Request all the threads to stop and wait for them to do so. Any
- # exception raised by the threads is raised again from stop().
- # Passing stop_grace_period_secs is for blocked enqueue/dequeue
- # threads which are not checking for `should_stop()`. They
- # will be stopped when we close the session further down.
- sv.stop(close_summary_writer=close_summary_writer)
- finally:
- # Close the session to finish up all pending calls. We do not care
- # about exceptions raised when closing. This takes care of
- # blocked enqueue/dequeue calls.
- try:
- sess.close()
- except Exception: # pylint: disable=broad-except
- # Silently ignore exceptions raised by close().
- pass
-
-
-def train(config, is_chief, tuner=None, run_dir=None, run_number=0,
- results_writer=None):
- """Run training loop.
-
- Args:
- config: config_lib.Config instance containing global config (agent and env).
- is_chief: True if this worker is chief. Chief worker manages writing some
- data to disk and initialization of the global model.
- tuner: A tuner instance. If not tuning, leave as None.
- run_dir: Directory where all data for this run will be written. If None,
- run_dir = FLAGS.logdir. Set this argument when doing multiple runs.
- run_number: Which run is this.
- results_writer: Managest writing training results to disk. Results are a
- dict of metric names and values.
-
- Returns:
- The trainer object used to run training updates.
- """
- logging.info('Will run asynchronous training.')
-
- if run_dir is None:
- run_dir = FLAGS.logdir
- train_dir = os.path.join(run_dir, 'train')
- best_model_checkpoint = os.path.join(train_dir, 'best.ckpt')
- events_dir = '%s/events_%d' % (run_dir, FLAGS.task_id)
- logging.info('Events directory: %s', events_dir)
-
- logging_dir = os.path.join(run_dir, 'logs')
- if not tf.gfile.Exists(logging_dir):
- tf.gfile.MakeDirs(logging_dir)
- status_file = os.path.join(logging_dir, 'status.txt')
-
- if FLAGS.summary_tasks and FLAGS.task_id < FLAGS.summary_tasks:
- summary_writer = tf.summary.FileWriter(events_dir)
- else:
- summary_writer = None
-
- # Only profile task 0.
- if FLAGS.do_profiling:
- logging.info('Profiling enabled')
- profiler = cProfile.Profile()
- profiler.enable()
- else:
- profiler = None
-
- trainer = AsyncTrainer(
- config, FLAGS.task_id, FLAGS.ps_tasks, FLAGS.num_workers,
- is_chief=is_chief,
- summary_interval=FLAGS.summary_interval,
- summary_writer=summary_writer,
- logging_dir=logging_dir,
- run_number=run_number,
- model_v=FLAGS.model_v)
-
- variables_to_save = [v for v in tf.global_variables()
- if v.name.startswith('global')]
- global_init_op = tf.variables_initializer(variables_to_save)
- saver = tf.train.Saver(variables_to_save)
-
- var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
- tf.get_variable_scope().name)
- logging.info('Trainable vars:')
- for v in var_list:
- logging.info(' %s, %s, %s', v.name, v.device, v.get_shape())
-
- logging.info('All vars:')
- for v in tf.global_variables():
- logging.info(' %s, %s, %s', v.name, v.device, v.get_shape())
-
- def init_fn(unused_sess):
- logging.info('No checkpoint found. Initialized global params.')
-
- sv = tf.train.Supervisor(is_chief=is_chief,
- logdir=train_dir,
- saver=saver,
- summary_op=None,
- init_op=global_init_op,
- init_fn=init_fn,
- summary_writer=summary_writer,
- ready_op=tf.report_uninitialized_variables(
- variables_to_save),
- ready_for_local_init_op=None,
- global_step=trainer.global_step,
- save_model_secs=30,
- save_summaries_secs=30)
-
- # Add a thread that periodically checks if this Trial should stop
- # based on an early stopping policy.
- if tuner:
- sv.Loop(60, tuner.check_for_stop, (sv.coord,))
-
- last_replay_save_time = time.time()
-
- global_step = -1
- logging.info(
- 'Starting session. '
- 'If this hangs, we\'re mostly likely waiting to connect '
- 'to the parameter server. One common cause is that the parameter '
- 'server DNS name isn\'t resolving yet, or is misspecified.')
- should_retry = True
- supervisor_deadline_exceeded = False
- while should_retry:
- try:
- with managed_session(
- sv, FLAGS.master, max_wait_secs=60) as session, session.as_default():
- should_retry = False
- do_training = True
-
- try:
- trainer.initialize(session)
- if session.run(trainer.run_number) != run_number:
- # If we loaded existing model from disk, and the saved run number is
- # different, throw an exception.
- raise RuntimeError(
- 'Expecting to be on run %d, but is actually on run %d. '
- 'run_dir: "%s"'
- % (run_number, session.run(trainer.run_number), run_dir))
- global_step = trainer.cached_global_step
- logging.info('Starting training at step=%d', global_step)
- while do_training:
- trainer.update_global_model(session)
-
- if is_chief:
- trainer.maybe_save_best_model(
- session, saver, best_model_checkpoint)
- global_step = trainer.cached_global_step
- global_npe = trainer.cached_global_npe
-
- if time.time() - last_replay_save_time >= 30:
- trainer.save_replay_buffer()
- trainer.save_topk_buffer()
- last_replay_save_time = time.time()
-
- # Stopping conditions.
- if tuner and tuner.should_trial_stop():
- logging.info('Tuner requested early stopping. Finishing.')
- do_training = False
- if is_chief and FLAGS.stop_on_success:
- found_solution = session.run(trainer.found_solution_flag)
- if found_solution:
- do_training = False
- logging.info('Solution found. Finishing.')
- if FLAGS.max_npe and global_npe >= FLAGS.max_npe:
- # Max NPE (number of programs executed) reached.
- logging.info('Max NPE reached. Finishing.')
- do_training = False
- if sv.should_stop():
- logging.info('Supervisor issued stop. Finishing.')
- do_training = False
-
- except tf.errors.NotFoundError:
- # Catch "Error while reading resource variable".
- # The chief worker likely destroyed the container, so do not retry.
- logging.info('Caught NotFoundError. Quitting.')
- do_training = False
- should_retry = False
- break
- except tf.errors.InternalError as e:
- # Catch "Invalid variable reference."
- if str(e).startswith('Invalid variable reference.'):
- # The chief worker likely destroyed the container, so do not
- # retry.
- logging.info(
- 'Caught "InternalError: Invalid variable reference.". '
- 'Quitting.')
- do_training = False
- should_retry = False
- break
- else:
- # Pass exception through.
- raise
-
- # Exited training loop. Write results to disk.
- if is_chief and results_writer:
- assert not should_retry
- with tf.gfile.FastGFile(status_file, 'w') as f:
- f.write('done')
- (program_count,
- found_solution,
- code_solution,
- best_reward,
- global_step) = session.run(
- [trainer.program_count,
- trainer.found_solution_flag,
- trainer.code_solution_variable,
- trainer.global_best_reward,
- trainer.global_step])
- results_dict = {
- 'max_npe': FLAGS.max_npe,
- 'batch_size': config.batch_size,
- 'max_batches': FLAGS.max_npe // config.batch_size,
- 'npe': program_count,
- 'max_global_repetitions': FLAGS.num_repetitions,
- 'max_local_repetitions': FLAGS.num_repetitions,
- 'code_solution': code_solution,
- 'best_reward': best_reward,
- 'num_batches': global_step,
- 'found_solution': found_solution,
- 'task': trainer.data_manager.task_name,
- 'global_rep': run_number}
- logging.info('results_dict: %s', results_dict)
- results_writer.append(results_dict)
-
- except tf.errors.AbortedError:
- # Catch "Graph handle is not found" error due to preempted jobs.
- logging.info('Caught AbortedError. Retying.')
- should_retry = True
- except tf.errors.DeadlineExceededError:
- supervisor_deadline_exceeded = True
- should_retry = False
-
- if is_chief:
- logging.info('This is chief worker. Stopping all workers.')
- sv.stop()
-
- if supervisor_deadline_exceeded:
- logging.info('Supervisor timed out. Quitting.')
- else:
- logging.info('Reached %s steps. Worker stopped.', global_step)
-
- # Dump profiling.
- """
- How to use profiling data.
-
- Download the profiler dump to your local machine, say to PROF_FILE_PATH.
- In a separate script, run something like the following:
-
- import pstats
- p = pstats.Stats(PROF_FILE_PATH)
- p.strip_dirs().sort_stats('cumtime').print_stats()
-
- This will sort by 'cumtime', which "is the cumulative time spent in this and
- all subfunctions (from invocation till exit)."
- https://docs.python.org/2/library/profile.html#instant-user-s-manual
- """ # pylint: disable=pointless-string-statement
- if profiler:
- prof_file = os.path.join(run_dir, 'task_%d.prof' % FLAGS.task_id)
- logging.info('Done profiling.\nDumping to "%s".', prof_file)
- profiler.create_stats()
- with tf.gfile.Open(prof_file, 'w') as f:
- f.write(marshal.dumps(profiler.stats))
-
- return trainer
-
-
-def run_training(config=None, tuner=None, logdir=None, trial_name=None,
- is_chief=True):
- """Do all training runs.
-
- This is the top level training function for policy gradient based models.
- Run this from the main function.
-
- Args:
- config: config_lib.Config instance containing global config (agent and
- environment hparams). If None, config will be parsed from FLAGS.config.
- tuner: A tuner instance. Leave as None if not tuning.
- logdir: Parent directory where all data from all runs will be written. If
- None, FLAGS.logdir will be used.
- trial_name: If tuning, set this to a unique string that identifies this
- trial. If `tuner` is not None, this also must be set.
- is_chief: True if this worker is the chief.
-
- Returns:
- List of results dicts which were written to disk. Each training run gets a
- results dict. Results dict contains metrics, i.e. (name, value) pairs which
- give information about the training run.
-
- Raises:
- ValueError: If results dicts read from disk contain invalid data.
- """
- if not config:
- # If custom config is not given, get it from flags.
- config = defaults.default_config_with_updates(FLAGS.config)
- if not logdir:
- logdir = FLAGS.logdir
- if not tf.gfile.Exists(logdir):
- tf.gfile.MakeDirs(logdir)
- assert FLAGS.num_repetitions > 0
- results = results_lib.Results(logdir)
- results_list, _ = results.read_all()
-
- logging.info('Starting experiment. Directory: "%s"', logdir)
-
- if results_list:
- if results_list[0]['max_npe'] != FLAGS.max_npe:
- raise ValueError(
- 'Cannot resume training. Max-NPE changed. Was %s, now %s',
- results_list[0]['max_npe'], FLAGS.max_npe)
- if results_list[0]['max_global_repetitions'] != FLAGS.num_repetitions:
- raise ValueError(
- 'Cannot resume training. Number of repetitions changed. Was %s, '
- 'now %s',
- results_list[0]['max_global_repetitions'],
- FLAGS.num_repetitions)
-
- while len(results_list) < FLAGS.num_repetitions:
- run_number = len(results_list)
- rep_container_name = trial_name if trial_name else 'container'
- if FLAGS.num_repetitions > 1:
- rep_dir = os.path.join(logdir, 'run_%d' % run_number)
- rep_container_name = rep_container_name + '_run_' + str(run_number)
- else:
- rep_dir = logdir
-
- logging.info(
- 'Starting repetition %d (%d out of %d)', run_number, run_number + 1,
- FLAGS.num_repetitions)
-
- # Train will write result to disk.
- with tf.container(rep_container_name):
- trainer = train(config, is_chief, tuner, rep_dir, run_number, results)
- logging.info('Done training.')
-
- if is_chief:
- # Destroy current container immediately (clears current graph).
- logging.info('Clearing shared variables.')
- tf.Session.reset(FLAGS.master, containers=[rep_container_name])
- logging.info('Shared variables cleared.')
-
- # Delete replay buffer on disk.
- assert trainer
- trainer.delete_replay_buffer()
- else:
- # Give chief worker time to clean up.
- sleep_sec = 30.0
- logging.info('Sleeping for %s sec.', sleep_sec)
- time.sleep(sleep_sec)
- tf.reset_default_graph()
- logging.info('Default graph reset.')
-
- # Expecting that train wrote new result to disk before returning.
- results_list, _ = results.read_all()
- return results_list
diff --git a/research/brain_coder/single_task/pg_train_test.py b/research/brain_coder/single_task/pg_train_test.py
deleted file mode 100644
index 0a562e5331e638cab82bc8033bfa2c1fc355e960..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/pg_train_test.py
+++ /dev/null
@@ -1,87 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for pg_train.
-
-These tests excersize code paths available through configuration options.
-Training will be run for just a few steps with the goal being to check that
-nothing crashes.
-"""
-
-from absl import flags
-import tensorflow as tf
-
-from single_task import defaults # brain coder
-from single_task import run # brain coder
-
-FLAGS = flags.FLAGS
-
-
-class TrainTest(tf.test.TestCase):
-
- def RunTrainingSteps(self, config_string, num_steps=10):
- """Run a few training steps with the given config.
-
- Just check that nothing crashes.
-
- Args:
- config_string: Config encoded in a string. See
- $REPO_PATH/common/config_lib.py
- num_steps: Number of training steps to run. Defaults to 10.
- """
- config = defaults.default_config_with_updates(config_string)
- FLAGS.master = ''
- FLAGS.max_npe = num_steps * config.batch_size
- FLAGS.summary_interval = 1
- FLAGS.logdir = tf.test.get_temp_dir()
- FLAGS.config = config_string
- tf.reset_default_graph()
- run.main(None)
-
- def testVanillaPolicyGradient(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="pg"),'
- 'timestep_limit=90,batch_size=64')
-
- def testVanillaPolicyGradient_VariableLengthSequences(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="pg",eos_token=False),'
- 'timestep_limit=90,batch_size=64')
-
- def testVanillaActorCritic(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="pg",ema_baseline_decay=0.0),'
- 'timestep_limit=90,batch_size=64')
-
- def testPolicyGradientWithTopK(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="pg",topk_loss_hparam=1.0,topk=10),'
- 'timestep_limit=90,batch_size=64')
-
- def testVanillaActorCriticWithTopK(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="pg",ema_baseline_decay=0.0,topk_loss_hparam=1.0,'
- 'topk=10),'
- 'timestep_limit=90,batch_size=64')
-
- def testPolicyGradientWithTopK_VariableLengthSequences(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="pg",topk_loss_hparam=1.0,topk=10,eos_token=False),'
- 'timestep_limit=90,batch_size=64')
-
- def testPolicyGradientWithImportanceSampling(self):
- self.RunTrainingSteps(
- 'env=c(task="reverse"),'
- 'agent=c(algorithm="pg",alpha=0.5),'
- 'timestep_limit=90,batch_size=64')
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/single_task/results_lib.py b/research/brain_coder/single_task/results_lib.py
deleted file mode 100644
index fd28fdd49ba3200dc9faa18d1722235ee4bf2ac2..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/results_lib.py
+++ /dev/null
@@ -1,155 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Results object manages distributed reading and writing of results to disk."""
-
-import ast
-from collections import namedtuple
-import os
-import re
-from six.moves import xrange
-import tensorflow as tf
-
-
-ShardStats = namedtuple(
- 'ShardStats',
- ['num_local_reps_completed', 'max_local_reps', 'finished'])
-
-
-def ge_non_zero(a, b):
- return a >= b and b > 0
-
-
-def get_shard_id(file_name):
- assert file_name[-4:].lower() == '.txt'
- return int(file_name[file_name.rfind('_') + 1: -4])
-
-
-class Results(object):
- """Manages reading and writing training results to disk asynchronously.
-
- Each worker writes to its own file, so that there are no race conditions when
- writing happens. However any worker may read any file, as is the case for
- `read_all`. Writes are expected to be atomic so that workers will never
- read incomplete data, and this is likely to be the case on Unix systems.
- Reading out of date data is fine, as workers calling `read_all` will wait
- until data from every worker has been written before proceeding.
- """
- file_template = 'experiment_results_{0}.txt'
- search_regex = r'^experiment_results_([0-9])+\.txt$'
-
- def __init__(self, log_dir, shard_id=0):
- """Construct `Results` instance.
-
- Args:
- log_dir: Where to write results files.
- shard_id: Unique id for this file (i.e. shard). Each worker that will
- be writing results should use a different shard id. If there are
- N shards, each shard should be numbered 0 through N-1.
- """
- # Use different files for workers so that they can write to disk async.
- assert 0 <= shard_id
- self.file_name = self.file_template.format(shard_id)
- self.log_dir = log_dir
- self.results_file = os.path.join(self.log_dir, self.file_name)
-
- def append(self, metrics):
- """Append results to results list on disk."""
- with tf.gfile.FastGFile(self.results_file, 'a') as writer:
- writer.write(str(metrics) + '\n')
-
- def read_this_shard(self):
- """Read only from this shard."""
- return self._read_shard(self.results_file)
-
- def _read_shard(self, results_file):
- """Read only from the given shard file."""
- try:
- with tf.gfile.FastGFile(results_file, 'r') as reader:
- results = [ast.literal_eval(entry) for entry in reader]
- except tf.errors.NotFoundError:
- # No results written to disk yet. Return empty list.
- return []
- return results
-
- def _get_max_local_reps(self, shard_results):
- """Get maximum number of repetitions the given shard needs to complete.
-
- Worker working on each shard needs to complete a certain number of runs
- before it finishes. This method will return that number so that we can
- determine which shards are still not done.
-
- We assume that workers are including a 'max_local_repetitions' value in
- their results, which should be the total number of repetitions it needs to
- run.
-
- Args:
- shard_results: Dict mapping metric names to values. This should be read
- from a shard on disk.
-
- Returns:
- Maximum number of repetitions the given shard needs to complete.
- """
- mlrs = [r['max_local_repetitions'] for r in shard_results]
- if not mlrs:
- return 0
- for n in mlrs[1:]:
- assert n == mlrs[0], 'Some reps have different max rep.'
- return mlrs[0]
-
- def read_all(self, num_shards=None):
- """Read results across all shards, i.e. get global results list.
-
- Args:
- num_shards: (optional) specifies total number of shards. If the caller
- wants information about which shards are incomplete, provide this
- argument (so that shards which have yet to be created are still
- counted as incomplete shards). Otherwise, no information about
- incomplete shards will be returned.
-
- Returns:
- aggregate: Global list of results (across all shards).
- shard_stats: List of ShardStats instances, one for each shard. Or None if
- `num_shards` is None.
- """
- try:
- all_children = tf.gfile.ListDirectory(self.log_dir)
- except tf.errors.NotFoundError:
- if num_shards is None:
- return [], None
- return [], [[] for _ in xrange(num_shards)]
- shard_ids = {
- get_shard_id(fname): fname
- for fname in all_children if re.search(self.search_regex, fname)}
-
- if num_shards is None:
- aggregate = []
- shard_stats = None
- for results_file in shard_ids.values():
- aggregate.extend(self._read_shard(
- os.path.join(self.log_dir, results_file)))
- else:
- results_per_shard = [None] * num_shards
- for shard_id in xrange(num_shards):
- if shard_id in shard_ids:
- results_file = shard_ids[shard_id]
- results_per_shard[shard_id] = self._read_shard(
- os.path.join(self.log_dir, results_file))
- else:
- results_per_shard[shard_id] = []
-
- # Compute shard stats.
- shard_stats = []
- for shard_results in results_per_shard:
- max_local_reps = self._get_max_local_reps(shard_results)
- shard_stats.append(ShardStats(
- num_local_reps_completed=len(shard_results),
- max_local_reps=max_local_reps,
- finished=ge_non_zero(len(shard_results), max_local_reps)))
-
- # Compute aggregate.
- aggregate = [
- r for shard_results in results_per_shard for r in shard_results]
-
- return aggregate, shard_stats
diff --git a/research/brain_coder/single_task/results_lib_test.py b/research/brain_coder/single_task/results_lib_test.py
deleted file mode 100644
index 6fe838d74d6a3bdea4c3b219a4d3ceea4385a97e..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/results_lib_test.py
+++ /dev/null
@@ -1,84 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for results_lib."""
-
-import contextlib
-import os
-import shutil
-import tempfile
-from six.moves import xrange
-import tensorflow as tf
-
-from single_task import results_lib # brain coder
-
-
-@contextlib.contextmanager
-def temporary_directory(suffix='', prefix='tmp', base_path=None):
- """A context manager to create a temporary directory and clean up on exit.
-
- The parameters are the same ones expected by tempfile.mkdtemp.
- The directory will be securely and atomically created.
- Everything under it will be removed when exiting the context.
-
- Args:
- suffix: optional suffix.
- prefix: options prefix.
- base_path: the base path under which to create the temporary directory.
- Yields:
- The absolute path of the new temporary directory.
- """
- temp_dir_path = tempfile.mkdtemp(suffix, prefix, base_path)
- try:
- yield temp_dir_path
- finally:
- try:
- shutil.rmtree(temp_dir_path)
- except OSError as e:
- if e.message == 'Cannot call rmtree on a symbolic link':
- # Interesting synthetic exception made up by shutil.rmtree.
- # Means we received a symlink from mkdtemp.
- # Also means must clean up the symlink instead.
- os.unlink(temp_dir_path)
- else:
- raise
-
-
-def freeze(dictionary):
- """Convert dict to hashable frozenset."""
- return frozenset(dictionary.iteritems())
-
-
-class ResultsLibTest(tf.test.TestCase):
-
- def testResults(self):
- with temporary_directory() as logdir:
- results_obj = results_lib.Results(logdir)
- self.assertEqual(results_obj.read_this_shard(), [])
- results_obj.append(
- {'foo': 1.5, 'bar': 2.5, 'baz': 0})
- results_obj.append(
- {'foo': 5.5, 'bar': -1, 'baz': 2})
- self.assertEqual(
- results_obj.read_this_shard(),
- [{'foo': 1.5, 'bar': 2.5, 'baz': 0},
- {'foo': 5.5, 'bar': -1, 'baz': 2}])
-
- def testShardedResults(self):
- with temporary_directory() as logdir:
- n = 4 # Number of shards.
- results_objs = [
- results_lib.Results(logdir, shard_id=i) for i in xrange(n)]
- for i, robj in enumerate(results_objs):
- robj.append({'foo': i, 'bar': 1 + i * 2})
- results_list, _ = results_objs[0].read_all()
-
- # Check results. Order does not matter here.
- self.assertEqual(
- set(freeze(r) for r in results_list),
- set(freeze({'foo': i, 'bar': 1 + i * 2}) for i in xrange(n)))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/single_task/run.py b/research/brain_coder/single_task/run.py
deleted file mode 100644
index 9d8f37c973dcca3bbf8e25bce3d181e5405c6167..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/run.py
+++ /dev/null
@@ -1,142 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-r"""Run training.
-
-Choose training algorithm and task(s) and follow these examples.
-
-Run synchronous policy gradient training locally:
-
-CONFIG="agent=c(algorithm='pg'),env=c(task='reverse')"
-OUT_DIR="/tmp/bf_pg_local"
-rm -rf $OUT_DIR
-bazel run -c opt single_task:run -- \
- --alsologtostderr \
- --config="$CONFIG" \
- --max_npe=0 \
- --logdir="$OUT_DIR" \
- --summary_interval=1 \
- --model_v=0
-learning/brain/tensorboard/tensorboard.sh --port 12345 --logdir "$OUT_DIR"
-
-
-Run genetic algorithm locally:
-
-CONFIG="agent=c(algorithm='ga'),env=c(task='reverse')"
-OUT_DIR="/tmp/bf_ga_local"
-rm -rf $OUT_DIR
-bazel run -c opt single_task:run -- \
- --alsologtostderr \
- --config="$CONFIG" \
- --max_npe=0 \
- --logdir="$OUT_DIR"
-
-
-Run uniform random search locally:
-
-CONFIG="agent=c(algorithm='rand'),env=c(task='reverse')"
-OUT_DIR="/tmp/bf_rand_local"
-rm -rf $OUT_DIR
-bazel run -c opt single_task:run -- \
- --alsologtostderr \
- --config="$CONFIG" \
- --max_npe=0 \
- --logdir="$OUT_DIR"
-"""
-
-from absl import app
-from absl import flags
-from absl import logging
-
-from single_task import defaults # brain coder
-from single_task import ga_train # brain coder
-from single_task import pg_train # brain coder
-
-FLAGS = flags.FLAGS
-flags.DEFINE_string('config', '', 'Configuration.')
-flags.DEFINE_string(
- 'logdir', None, 'Absolute path where to write results.')
-flags.DEFINE_integer('task_id', 0, 'ID for this worker.')
-flags.DEFINE_integer('num_workers', 1, 'How many workers there are.')
-flags.DEFINE_integer(
- 'max_npe', 0,
- 'NPE = number of programs executed. Maximum number of programs to execute '
- 'in each run. Training will complete when this threshold is reached. Set '
- 'to 0 for unlimited training.')
-flags.DEFINE_integer(
- 'num_repetitions', 1,
- 'Number of times the same experiment will be run (globally across all '
- 'workers). Each run is independent.')
-flags.DEFINE_string(
- 'log_level', 'INFO',
- 'The threshold for what messages will be logged. One of DEBUG, INFO, WARN, '
- 'ERROR, or FATAL.')
-
-
-# To register an algorithm:
-# 1) Add dependency in the BUILD file to this build rule.
-# 2) Import the algorithm's module at the top of this file.
-# 3) Add a new entry in the following dict. The key is the algorithm name
-# (used to select the algorithm in the config). The value is the module
-# defining the expected functions for training and tuning. See the docstring
-# for `get_namespace` for further details.
-ALGORITHM_REGISTRATION = {
- 'pg': pg_train,
- 'ga': ga_train,
- 'rand': ga_train,
-}
-
-
-def get_namespace(config_string):
- """Get namespace for the selected algorithm.
-
- Users who want to add additional algorithm types should modify this function.
- The algorithm's namespace should contain the following functions:
- run_training: Run the main training loop.
- define_tuner_hparam_space: Return the hparam tuning space for the algo.
- write_hparams_to_config: Helper for tuning. Write hparams chosen for tuning
- to the Config object.
- Look at pg_train.py and ga_train.py for function signatures and
- implementations.
-
- Args:
- config_string: String representation of a Config object. This will get
- parsed into a Config in order to determine what algorithm to use.
-
- Returns:
- algorithm_namespace: The module corresponding to the algorithm given in the
- config.
- config: The Config object resulting from parsing `config_string`.
-
- Raises:
- ValueError: If config.agent.algorithm is not one of the registered
- algorithms.
- """
- config = defaults.default_config_with_updates(config_string)
- if config.agent.algorithm not in ALGORITHM_REGISTRATION:
- raise ValueError('Unknown algorithm type "%s"' % (config.agent.algorithm,))
- else:
- return ALGORITHM_REGISTRATION[config.agent.algorithm], config
-
-
-def main(argv):
- del argv # Unused.
-
- logging.set_verbosity(FLAGS.log_level)
-
- flags.mark_flag_as_required('logdir')
- if FLAGS.num_workers <= 0:
- raise ValueError('num_workers flag must be greater than 0.')
- if FLAGS.task_id < 0:
- raise ValueError('task_id flag must be greater than or equal to 0.')
- if FLAGS.task_id >= FLAGS.num_workers:
- raise ValueError(
- 'task_id flag must be strictly less than num_workers flag.')
-
- ns, _ = get_namespace(FLAGS.config)
- ns.run_training(is_chief=FLAGS.task_id == 0)
-
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/brain_coder/single_task/run_eval_tasks.py b/research/brain_coder/single_task/run_eval_tasks.py
deleted file mode 100755
index eb684c344381462cd3626404b5d7fd7cf5d72b22..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/run_eval_tasks.py
+++ /dev/null
@@ -1,296 +0,0 @@
-#!/usr/bin/env python
-from __future__ import print_function
-
-r"""This script can launch any eval experiments from the paper.
-
-This is a script. Run with python, not bazel.
-
-Usage:
-./single_task/run_eval_tasks.py \
- --exp EXP --desc DESC [--tuning_tasks] [--iclr_tasks] [--task TASK] \
- [--tasks TASK1 TASK2 ...]
-
-where EXP is one of the keys in `experiments`,
-and DESC is a string description of the set of experiments (such as "v0")
-
-Set only one of these flags:
---tuning_tasks flag only runs tuning tasks.
---iclr_tasks flag only runs the tasks included in the paper.
---regression_tests flag runs tasks which function as regression tests.
---task flag manually selects a single task to run.
---tasks flag takes a custom list of tasks.
-
-Other flags:
---reps N specifies N repetitions per experiment, Default is 25.
---training_replicas R specifies that R workers will be launched to train one
- task (for neural network algorithms). These workers will update a global
- model stored on a parameter server. Defaults to 1. If R > 1, a parameter
- server will also be launched.
-
-
-Run everything:
-exps=( pg-20M pg-topk-20M topk-20M ga-20M rand-20M )
-BIN_DIR="single_task"
-for exp in "${exps[@]}"
-do
- ./$BIN_DIR/run_eval_tasks.py \
- --exp "$exp" --iclr_tasks
-done
-"""
-
-import argparse
-from collections import namedtuple
-import subprocess
-
-
-S = namedtuple('S', ['length'])
-default_length = 100
-
-
-iclr_tasks = [
- 'reverse', 'remove-char', 'count-char', 'add', 'bool-logic', 'print-hello',
- 'echo-twice', 'echo-thrice', 'copy-reverse', 'zero-cascade', 'cascade',
- 'shift-left', 'shift-right', 'riffle', 'unriffle', 'middle-char',
- 'remove-last', 'remove-last-two', 'echo-alternating', 'echo-half', 'length',
- 'echo-second-seq', 'echo-nth-seq', 'substring', 'divide-2', 'dedup']
-
-
-regression_test_tasks = ['reverse', 'test-hill-climb']
-
-
-E = namedtuple(
- 'E',
- ['name', 'method_type', 'config', 'simplify', 'batch_size', 'max_npe'])
-
-
-def make_experiment_settings(name, **kwargs):
- # Unpack experiment info from name.
- def split_last(string, char):
- i = string.rindex(char)
- return string[:i], string[i+1:]
- def si_to_int(si_string):
- return int(
- si_string.upper().replace('K', '0'*3).replace('M', '0'*6)
- .replace('G', '0'*9))
- method_type, max_npe = split_last(name, '-')
- assert method_type
- assert max_npe
- return E(
- name=name, method_type=method_type, max_npe=si_to_int(max_npe), **kwargs)
-
-
-experiments_set = {
- make_experiment_settings(
- 'pg-20M',
- config='entropy_beta=0.05,lr=0.0001,topk_loss_hparam=0.0,topk=0,'
- 'pi_loss_hparam=1.0,alpha=0.0',
- simplify=False,
- batch_size=64),
- make_experiment_settings(
- 'pg-topk-20M',
- config='entropy_beta=0.01,lr=0.0001,topk_loss_hparam=50.0,topk=10,'
- 'pi_loss_hparam=1.0,alpha=0.0',
- simplify=False,
- batch_size=64),
- make_experiment_settings(
- 'topk-20M',
- config='entropy_beta=0.01,lr=0.0001,topk_loss_hparam=200.0,topk=10,'
- 'pi_loss_hparam=0.0,alpha=0.0',
- simplify=False,
- batch_size=64),
- make_experiment_settings(
- 'topk-0ent-20M',
- config='entropy_beta=0.000,lr=0.0001,topk_loss_hparam=200.0,topk=10,'
- 'pi_loss_hparam=0.0,alpha=0.0',
- simplify=False,
- batch_size=64),
- make_experiment_settings(
- 'ga-20M',
- config='crossover_rate=0.95,mutation_rate=0.15',
- simplify=False,
- batch_size=100), # Population size.
- make_experiment_settings(
- 'rand-20M',
- config='',
- simplify=False,
- batch_size=1),
- make_experiment_settings(
- 'simpl-500M',
- config='entropy_beta=0.05,lr=0.0001,topk_loss_hparam=0.5,topk=10,'
- 'pi_loss_hparam=1.0,alpha=0.0',
- simplify=True,
- batch_size=64),
-}
-
-experiments = {e.name: e for e in experiments_set}
-
-
-# pylint: disable=redefined-outer-name
-def parse_args(extra_args=()):
- """Parse arguments and extract task and experiment info."""
- parser = argparse.ArgumentParser(description='Run all eval tasks.')
- parser.add_argument('--exp', required=True)
- parser.add_argument('--tuning_tasks', action='store_true')
- parser.add_argument('--iclr_tasks', action='store_true')
- parser.add_argument('--regression_tests', action='store_true')
- parser.add_argument('--desc', default='v0')
- parser.add_argument('--reps', default=25)
- parser.add_argument('--task')
- parser.add_argument('--tasks', nargs='+')
- for arg_string, default in extra_args:
- parser.add_argument(arg_string, default=default)
- args = parser.parse_args()
-
- print('Running experiment: %s' % (args.exp,))
- if args.desc:
- print('Extra description: "%s"' % (args.desc,))
- if args.exp not in experiments:
- raise ValueError('Experiment name is not valid')
- experiment_name = args.exp
- experiment_settings = experiments[experiment_name]
- assert experiment_settings.name == experiment_name
-
- if args.tasks:
- print('Launching tasks from args: %s' % (args.tasks,))
- tasks = {t: S(length=default_length) for t in args.tasks}
- elif args.task:
- print('Launching single task "%s"' % args.task)
- tasks = {args.task: S(length=default_length)}
- elif args.tuning_tasks:
- print('Only running tuning tasks')
- tasks = {name: S(length=default_length)
- for name in ['reverse-tune', 'remove-char-tune']}
- elif args.iclr_tasks:
- print('Running eval tasks from ICLR paper.')
- tasks = {name: S(length=default_length) for name in iclr_tasks}
- elif args.regression_tests:
- tasks = {name: S(length=default_length) for name in regression_test_tasks}
- print('Tasks: %s' % tasks.keys())
-
- print('reps = %d' % (int(args.reps),))
-
- return args, tasks, experiment_settings
-
-
-def run(command_string):
- subprocess.call(command_string, shell=True)
-
-
-if __name__ == '__main__':
- LAUNCH_TRAINING_COMMAND = 'single_task/launch_training.sh'
- COMPILE_COMMAND = 'bazel build -c opt single_task:run.par'
-
- args, tasks, experiment_settings = parse_args(
- extra_args=(('--training_replicas', 1),))
-
- if experiment_settings.method_type in (
- 'pg', 'pg-topk', 'topk', 'topk-0ent', 'simpl'):
- # Runs PG and TopK.
-
- def make_run_cmd(job_name, task, max_npe, num_reps, code_length,
- batch_size, do_simplify, custom_config_str):
- """Constructs terminal command for launching NN based algorithms.
-
- The arguments to this function will be used to create config for the
- experiment.
-
- Args:
- job_name: Name of the job to launch. Should uniquely identify this
- experiment run.
- task: Name of the coding task to solve.
- max_npe: Maximum number of programs executed. An integer.
- num_reps: Number of times to run the experiment. An integer.
- code_length: Maximum allowed length of synthesized code.
- batch_size: Minibatch size for gradient descent.
- do_simplify: Whether to run the experiment in code simplification mode.
- A bool.
- custom_config_str: Additional config for the model config string.
-
- Returns:
- The terminal command that launches the specified experiment.
- """
- config = """
- env=c(task='{0}',correct_syntax=False),
- agent=c(
- algorithm='pg',
- policy_lstm_sizes=[35,35],value_lstm_sizes=[35,35],
- grad_clip_threshold=50.0,param_init_factor=0.5,regularizer=0.0,
- softmax_tr=1.0,optimizer='rmsprop',ema_baseline_decay=0.99,
- eos_token={3},{4}),
- timestep_limit={1},batch_size={2}
- """.replace(' ', '').replace('\n', '').format(
- task, code_length, batch_size, do_simplify, custom_config_str)
- num_ps = 0 if args.training_replicas == 1 else 1
- return (
- r'{0} --job_name={1} --config="{2}" --max_npe={3} '
- '--num_repetitions={4} --num_workers={5} --num_ps={6} '
- '--stop_on_success={7}'
- .format(LAUNCH_TRAINING_COMMAND, job_name, config, max_npe, num_reps,
- args.training_replicas, num_ps, str(not do_simplify).lower()))
-
- else:
- # Runs GA and Rand.
- assert experiment_settings.method_type in ('ga', 'rand')
-
- def make_run_cmd(job_name, task, max_npe, num_reps, code_length,
- batch_size, do_simplify, custom_config_str):
- """Constructs terminal command for launching GA or uniform random search.
-
- The arguments to this function will be used to create config for the
- experiment.
-
- Args:
- job_name: Name of the job to launch. Should uniquely identify this
- experiment run.
- task: Name of the coding task to solve.
- max_npe: Maximum number of programs executed. An integer.
- num_reps: Number of times to run the experiment. An integer.
- code_length: Maximum allowed length of synthesized code.
- batch_size: Minibatch size for gradient descent.
- do_simplify: Whether to run the experiment in code simplification mode.
- A bool.
- custom_config_str: Additional config for the model config string.
-
- Returns:
- The terminal command that launches the specified experiment.
- """
- assert not do_simplify
- if custom_config_str:
- custom_config_str = ',' + custom_config_str
- config = """
- env=c(task='{0}',correct_syntax=False),
- agent=c(
- algorithm='{4}'
- {3}),
- timestep_limit={1},batch_size={2}
- """.replace(' ', '').replace('\n', '').format(
- task, code_length, batch_size, custom_config_str,
- experiment_settings.method_type)
- num_workers = num_reps # Do each rep in parallel.
- return (
- r'{0} --job_name={1} --config="{2}" --max_npe={3} '
- '--num_repetitions={4} --num_workers={5} --num_ps={6} '
- '--stop_on_success={7}'
- .format(LAUNCH_TRAINING_COMMAND, job_name, config, max_npe, num_reps,
- num_workers, 0, str(not do_simplify).lower()))
-
- print('Compiling...')
- run(COMPILE_COMMAND)
-
- print('Launching %d coding tasks...' % len(tasks))
- for task, task_settings in tasks.iteritems():
- name = 'bf_rl_iclr'
- desc = '{0}.{1}_{2}'.format(args.desc, experiment_settings.name, task)
- job_name = '{}.{}'.format(name, desc)
- print('Job name: %s' % job_name)
- reps = int(args.reps) if not experiment_settings.simplify else 1
- run_cmd = make_run_cmd(
- job_name, task, experiment_settings.max_npe, reps,
- task_settings.length, experiment_settings.batch_size,
- experiment_settings.simplify,
- experiment_settings.config)
- print('Running command:\n' + run_cmd)
- run(run_cmd)
-
- print('Done.')
-# pylint: enable=redefined-outer-name
diff --git a/research/brain_coder/single_task/test_tasks.py b/research/brain_coder/single_task/test_tasks.py
deleted file mode 100644
index fb07a12653ebad6b38dc3786e749d3e8bf2b2072..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/test_tasks.py
+++ /dev/null
@@ -1,127 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tasks that test correctness of algorithms."""
-
-from six.moves import xrange
-from common import reward as reward_lib # brain coder
-from single_task import misc # brain coder
-
-
-class BasicTaskManager(object):
- """Wraps a generic reward function."""
-
- def __init__(self, reward_fn):
- self.reward_fn = reward_fn
- self.good_reward = 1.0
-
- def _score_string(self, string):
- actions = misc.bf_string_to_tokens(string)
- reward, correct = self.reward_fn(actions)
- return misc.RewardInfo(
- episode_rewards=[0.0] * (len(string) - 1) + [reward],
- input_case=None,
- correct_output=None,
- code_output=actions,
- input_type=None,
- output_type=misc.IOType.integer,
- reason='correct' if correct else 'wrong')
-
- def rl_batch(self, batch_size):
- reward_fns = [self._score_string] * batch_size
- return reward_fns
-
-
-class Trie(object):
- """Trie for sequences."""
- EOS = ()
-
- def __init__(self):
- self.trie = {}
-
- def insert(self, sequence):
- d = self.trie
- for e in sequence:
- if e not in d:
- d[e] = {}
- d = d[e]
- d[self.EOS] = True # Terminate sequence.
-
- def prefix_match(self, sequence):
- """Return prefix of `sequence` which exists in the trie."""
- d = self.trie
- index = 0
- for i, e in enumerate(sequence + [self.EOS]):
- index = i
- if e in d:
- d = d[e]
- if e == self.EOS:
- return sequence, True
- else:
- break
- return sequence[:index], False
-
- def next_choices(self, sequence):
- d = self.trie
- for e in sequence:
- if e in d:
- d = d[e]
- else:
- raise ValueError('Sequence not a prefix: %s' % (sequence,))
- return d.keys()
-
-
-class HillClimbingTask(object):
- """Simple task that tests reward hill climbing ability.
-
- There are a set of paths (sequences of tokens) which are rewarded. The total
- reward for a path is proportional to its length, so the longest path is the
- target. Shorter paths can be dead ends.
- """
-
- def __init__(self):
- # Paths are sequences of sub-sequences. Here we form unique sub-sequences
- # out of 3 arbitrary ints. We use sub-sequences instead of single entities
- # to make the task harder by making the episodes last longer, i.e. more
- # for the agent to remember.
- a = (1, 2, 3)
- b = (4, 5, 6)
- c = (7, 8, 7)
- d = (6, 5, 4)
- e = (3, 2, 1)
- f = (8, 5, 1)
- g = (6, 4, 2)
- h = (1, 8, 3)
- self.paths = Trie()
- self.paths.insert([a, b, h])
- self.paths.insert([a, b, c, d, e, f, g, h])
- self.paths.insert([a, b, c, d, e, b, a])
- self.paths.insert([a, b, g, h])
- self.paths.insert([a, e, f, g])
- self.correct_sequence = misc.flatten([a, b, c, d, e, f, g, h])
-
- def distance_fn(a, b):
- len_diff = abs(len(a) - len(b))
- return sum(reward_lib.mod_abs_diff(ai - 1, bi - 1, 8)
- for ai, bi in zip(a, b)) + len_diff * 4 # 8 / 2 = 4
- self.distance_fn = distance_fn
-
- def __call__(self, actions):
- # Compute reward for action sequence.
- actions = [a for a in actions if a > 0]
- sequence = [tuple(actions[i: i + 3]) for i in xrange(0, len(actions), 3)]
- prefix, complete = self.paths.prefix_match(sequence)
- if complete:
- return float(len(prefix)), actions == self.correct_sequence
- if len(prefix) == len(sequence):
- return float(len(prefix)), False
- next_pred = sequence[len(prefix)]
- choices = self.paths.next_choices(prefix)
- if choices == [()]:
- return (len(prefix) - len(next_pred) / 3.0), False
- min_dist = min(self.distance_fn(c, next_pred) for c in choices)
- # +1 reward for each element in the sequence correct, plus fraction torwards
- # closest next element.
- # Maximum distance possible is num_actions * base / 2 = 3 * 8 / 2 = 12
- return (len(prefix) + (1 - min_dist / 12.0)), False
diff --git a/research/brain_coder/single_task/test_tasks_test.py b/research/brain_coder/single_task/test_tasks_test.py
deleted file mode 100644
index bc905c6936de4c686e6cac1203c65c36bd7a0b16..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/test_tasks_test.py
+++ /dev/null
@@ -1,63 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-"""Tests for test_tasks."""
-
-import numpy as np
-import tensorflow as tf
-
-from single_task import misc # brain coder
-from single_task import test_tasks # brain coder
-
-
-def get_reward(reward_fn, candidate):
- return sum(reward_fn(misc.bf_tokens_to_string(candidate)).episode_rewards)
-
-
-class TestTasksTest(tf.test.TestCase):
-
- def testHillClimbingTask(self):
- task = test_tasks.BasicTaskManager(test_tasks.HillClimbingTask())
- reward_fns = task.rl_batch(1)
- reward_fn = reward_fns[0]
- self.assertTrue(np.isclose(get_reward(reward_fn, [1, 2, 0]), 8 / 12.))
- self.assertTrue(np.isclose(get_reward(reward_fn, [1, 2, 2, 0]), 11 / 12.))
- self.assertTrue(np.isclose(get_reward(reward_fn, [1, 2, 3, 0]), 1.0))
- self.assertTrue(
- np.isclose(get_reward(reward_fn, [1, 2, 3, 4, 5, 2, 0]), 1. + 8 / 12.))
- self.assertTrue(
- np.isclose(get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 0]), 2.0))
- self.assertTrue(
- np.isclose(get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 1, 8, 3, 0]), 3.0))
- self.assertTrue(
- np.isclose(get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 7, 8, 7, 0]), 3.0))
- self.assertTrue(
- np.isclose(get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 1, 8, 3, 1, 0]),
- 3.0 - 4 / 12.))
- self.assertTrue(
- np.isclose(
- get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 1, 8, 3, 1, 1, 1, 1, 0]),
- 2.0))
- self.assertTrue(
- np.isclose(get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 7, 8, 7, 3, 0]),
- 3.0 + 1 / 12.))
- self.assertTrue(
- np.isclose(
- get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 7, 8, 7, 6, 5, 4, 3, 2, 1,
- 8, 5, 1, 6, 4, 2, 1, 8, 3, 0]),
- 8.0))
- self.assertTrue(
- np.isclose(
- get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 7, 8, 7, 6, 5, 4, 3, 2, 1,
- 8, 5, 1, 6, 4, 2, 1, 8, 3, 1, 1, 0]),
- 8.0 - 8 / 12.))
- self.assertTrue(
- np.isclose(get_reward(reward_fn, [1, 2, 3, 4, 5, 6, 7, 8, 7, 6, 5, 4, 3,
- 2, 1, 8, 5, 1, 6, 4, 2, 1, 8, 3, 1, 1,
- 1, 1, 1, 1, 1, 0]),
- 7.0))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/brain_coder/single_task/tune.py b/research/brain_coder/single_task/tune.py
deleted file mode 100644
index 3473b5e94bd3c1f737a18f0187790d5df2d7a2aa..0000000000000000000000000000000000000000
--- a/research/brain_coder/single_task/tune.py
+++ /dev/null
@@ -1,262 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-r"""Run grid search.
-
-Look at launch_tuning.sh for details on how to tune at scale.
-
-Usage example:
-Tune with one worker on the local machine.
-
-CONFIG="agent=c(algorithm='pg'),"
-CONFIG+="env=c(task_cycle=['reverse-tune', 'remove-tune'])"
-HPARAM_SPACE_TYPE="pg"
-OUT_DIR="/tmp/bf_pg_tune"
-MAX_NPE=5000000
-NUM_REPETITIONS=50
-rm -rf $OUT_DIR
-mkdir $OUT_DIR
-bazel run -c opt single_task:tune -- \
- --alsologtostderr \
- --config="$CONFIG" \
- --max_npe="$MAX_NPE" \
- --num_repetitions="$NUM_REPETITIONS" \
- --logdir="$OUT_DIR" \
- --summary_interval=1 \
- --model_v=0 \
- --hparam_space="$HPARAM_SPACE_TYPE" \
- --tuner_id=0 \
- --num_tuners=1 \
- 2>&1 >"$OUT_DIR/tuner_0.log"
-learning/brain/tensorboard/tensorboard.sh --port 12345 --logdir "$OUT_DIR"
-"""
-
-import ast
-import os
-
-from absl import app
-from absl import flags
-from absl import logging
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from single_task import defaults # brain coder
-from single_task import run as run_lib # brain coder
-
-FLAGS = flags.FLAGS
-flags.DEFINE_integer(
- 'tuner_id', 0,
- 'The unique ID for this tuning worker.')
-flags.DEFINE_integer(
- 'num_tuners', 1,
- 'How many tuners are there.')
-flags.DEFINE_string(
- 'hparam_space', 'default',
- 'String name which denotes the hparam space to tune over. This is '
- 'algorithm dependent.')
-flags.DEFINE_string(
- 'fixed_hparams', '',
- 'HParams string. Used to fix hparams during tuning.')
-flags.DEFINE_float(
- 'success_rate_objective_weight', 1.0,
- 'How much to weight success rate vs num programs seen. By default, only '
- 'success rate is optimized (this is the setting used in the paper).')
-
-
-def parse_hparams_string(hparams_str):
- hparams = {}
- for term in hparams_str.split(','):
- if not term:
- continue
- name, value = term.split('=')
- hparams[name.strip()] = ast.literal_eval(value)
- return hparams
-
-
-def int_to_multibase(n, bases):
- digits = [0] * len(bases)
- for i, b in enumerate(bases):
- n, d = divmod(n, b)
- digits[i] = d
- return digits
-
-
-def hparams_for_index(index, tuning_space):
- keys = sorted(tuning_space.keys())
- indices = int_to_multibase(index, [len(tuning_space[k]) for k in keys])
- return tf.contrib.training.HParams(
- **{k: tuning_space[k][i] for k, i in zip(keys, indices)})
-
-
-def run_tuner_loop(ns):
- """Run tuning loop for this worker."""
- is_chief = FLAGS.task_id == 0
- tuning_space = ns.define_tuner_hparam_space(
- hparam_space_type=FLAGS.hparam_space)
- fixed_hparams = parse_hparams_string(FLAGS.fixed_hparams)
- for name, value in fixed_hparams.iteritems():
- tuning_space[name] = [value]
- tuning_space_size = np.prod([len(values) for values in tuning_space.values()])
-
- num_local_trials, remainder = divmod(tuning_space_size, FLAGS.num_tuners)
- if FLAGS.tuner_id < remainder:
- num_local_trials += 1
- starting_trial_id = (
- num_local_trials * FLAGS.tuner_id + min(remainder, FLAGS.tuner_id))
-
- logging.info('tuning_space_size: %d', tuning_space_size)
- logging.info('num_local_trials: %d', num_local_trials)
- logging.info('starting_trial_id: %d', starting_trial_id)
-
- for local_trial_index in xrange(num_local_trials):
- trial_config = defaults.default_config_with_updates(FLAGS.config)
- global_trial_index = local_trial_index + starting_trial_id
- trial_name = 'trial_' + str(global_trial_index)
- trial_dir = os.path.join(FLAGS.logdir, trial_name)
- hparams = hparams_for_index(global_trial_index, tuning_space)
- ns.write_hparams_to_config(
- trial_config, hparams, hparam_space_type=FLAGS.hparam_space)
-
- results_list = ns.run_training(
- config=trial_config, tuner=None, logdir=trial_dir, is_chief=is_chief,
- trial_name=trial_name)
-
- if not is_chief:
- # Only chief worker needs to write tuning results to disk.
- continue
-
- objective, metrics = compute_tuning_objective(
- results_list, hparams, trial_name, num_trials=tuning_space_size)
- logging.info('metrics:\n%s', metrics)
- logging.info('objective: %s', objective)
- logging.info('programs_seen_fraction: %s',
- metrics['programs_seen_fraction'])
- logging.info('success_rate: %s', metrics['success_rate'])
- logging.info('success_rate_objective_weight: %s',
- FLAGS.success_rate_objective_weight)
-
- tuning_results_file = os.path.join(trial_dir, 'tuning_results.txt')
- with tf.gfile.FastGFile(tuning_results_file, 'a') as writer:
- writer.write(str(metrics) + '\n')
-
- logging.info('Trial %s complete.', trial_name)
-
-
-def compute_tuning_objective(results_list, hparams, trial_name, num_trials):
- """Compute tuning objective and metrics given results and trial information.
-
- Args:
- results_list: List of results dicts read from disk. These are written by
- workers.
- hparams: tf.contrib.training.HParams instance containing the hparams used
- in this trial (only the hparams which are being tuned).
- trial_name: Name of this trial. Used to create a trial directory.
- num_trials: Total number of trials that need to be run. This is saved in the
- metrics dict for future reference.
-
- Returns:
- objective: The objective computed for this trial. Choose the hparams for the
- trial with the largest objective value.
- metrics: Information about this trial. A dict.
- """
- found_solution = [r['found_solution'] for r in results_list]
- successful_program_counts = [
- r['npe'] for r in results_list if r['found_solution']]
-
- success_rate = sum(found_solution) / float(len(results_list))
-
- max_programs = FLAGS.max_npe # Per run.
- all_program_counts = [
- r['npe'] if r['found_solution'] else max_programs
- for r in results_list]
- programs_seen_fraction = (
- float(sum(all_program_counts))
- / (max_programs * len(all_program_counts)))
-
- # min/max/avg stats are over successful runs.
- metrics = {
- 'num_runs': len(results_list),
- 'num_succeeded': sum(found_solution),
- 'success_rate': success_rate,
- 'programs_seen_fraction': programs_seen_fraction,
- 'avg_programs': np.mean(successful_program_counts),
- 'max_possible_programs_per_run': max_programs,
- 'global_step': sum([r['num_batches'] for r in results_list]),
- 'hparams': hparams.values(),
- 'trial_name': trial_name,
- 'num_trials': num_trials}
-
- # Report stats per tasks.
- tasks = [r['task'] for r in results_list]
- for task in set(tasks):
- task_list = [r for r in results_list if r['task'] == task]
- found_solution = [r['found_solution'] for r in task_list]
- successful_rewards = [
- r['best_reward'] for r in task_list
- if r['found_solution']]
- successful_num_batches = [
- r['num_batches']
- for r in task_list if r['found_solution']]
- successful_program_counts = [
- r['npe'] for r in task_list if r['found_solution']]
- metrics_append = {
- task + '__num_runs': len(task_list),
- task + '__num_succeeded': sum(found_solution),
- task + '__success_rate': (
- sum(found_solution) / float(len(task_list)))}
- metrics.update(metrics_append)
- if any(found_solution):
- metrics_append = {
- task + '__min_reward': min(successful_rewards),
- task + '__max_reward': max(successful_rewards),
- task + '__avg_reward': np.median(successful_rewards),
- task + '__min_programs': min(successful_program_counts),
- task + '__max_programs': max(successful_program_counts),
- task + '__avg_programs': np.mean(successful_program_counts),
- task + '__min_batches': min(successful_num_batches),
- task + '__max_batches': max(successful_num_batches),
- task + '__avg_batches': np.mean(successful_num_batches)}
- metrics.update(metrics_append)
-
- # Objective will be maximized.
- # Maximize success rate, minimize num programs seen.
- # Max objective is always 1.
- weight = FLAGS.success_rate_objective_weight
- objective = (
- weight * success_rate
- + (1 - weight) * (1 - programs_seen_fraction))
- metrics['objective'] = objective
-
- return objective, metrics
-
-
-def main(argv):
- del argv
-
- logging.set_verbosity(FLAGS.log_level)
-
- if not FLAGS.logdir:
- raise ValueError('logdir flag must be provided.')
- if FLAGS.num_workers <= 0:
- raise ValueError('num_workers flag must be greater than 0.')
- if FLAGS.task_id < 0:
- raise ValueError('task_id flag must be greater than or equal to 0.')
- if FLAGS.task_id >= FLAGS.num_workers:
- raise ValueError(
- 'task_id flag must be strictly less than num_workers flag.')
- if FLAGS.num_tuners <= 0:
- raise ValueError('num_tuners flag must be greater than 0.')
- if FLAGS.tuner_id < 0:
- raise ValueError('tuner_id flag must be greater than or equal to 0.')
- if FLAGS.tuner_id >= FLAGS.num_tuners:
- raise ValueError(
- 'tuner_id flag must be strictly less than num_tuners flag.')
-
- ns, _ = run_lib.get_namespace(FLAGS.config)
- run_tuner_loop(ns)
-
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/cognitive_mapping_and_planning/.gitignore b/research/cognitive_mapping_and_planning/.gitignore
deleted file mode 100644
index cbc6a8f0271075171ffdf3c2bc5fb9c528b08fc6..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-deps
-*.pyc
-lib*.so
-lib*.so*
diff --git a/research/cognitive_mapping_and_planning/README.md b/research/cognitive_mapping_and_planning/README.md
deleted file mode 100644
index 4457bafbb4d229998a01dadc46efe41f4ba1a3e0..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/README.md
+++ /dev/null
@@ -1,127 +0,0 @@
-
-
-
-
-# Cognitive Mapping and Planning for Visual Navigation
-**Saurabh Gupta, James Davidson, Sergey Levine, Rahul Sukthankar, Jitendra Malik**
-
-**Computer Vision and Pattern Recognition (CVPR) 2017.**
-
-**[ArXiv](https://arxiv.org/abs/1702.03920),
-[Project Website](https://sites.google.com/corp/view/cognitive-mapping-and-planning/)**
-
-### Citing
-If you find this code base and models useful in your research, please consider
-citing the following paper:
- ```
- @inproceedings{gupta2017cognitive,
- title={Cognitive Mapping and Planning for Visual Navigation},
- author={Gupta, Saurabh and Davidson, James and Levine, Sergey and
- Sukthankar, Rahul and Malik, Jitendra},
- booktitle={CVPR},
- year={2017}
- }
- ```
-
-### Contents
-1. [Requirements: software](#requirements-software)
-2. [Requirements: data](#requirements-data)
-3. [Test Pre-trained Models](#test-pre-trained-models)
-4. [Train your Own Models](#train-your-own-models)
-
-### Requirements: software
-1. Python Virtual Env Setup: All code is implemented in Python but depends on a
- small number of python packages and a couple of C libraries. We recommend
- using virtual environment for installing these python packages and python
- bindings for these C libraries.
- ```Shell
- VENV_DIR=venv
- pip install virtualenv
- virtualenv $VENV_DIR
- source $VENV_DIR/bin/activate
-
- # You may need to upgrade pip for installing openv-python.
- pip install --upgrade pip
- # Install simple dependencies.
- pip install -r requirements.txt
-
- # Patch bugs in dependencies.
- sh patches/apply_patches.sh
- ```
-
-2. Install [Tensorflow](https://www.tensorflow.org/) inside this virtual
- environment. You will need to use one of the latest nightly builds
- (see instructions [here](https://github.com/tensorflow/tensorflow#installation)).
-
-3. Swiftshader: We use
- [Swiftshader](https://github.com/google/swiftshader.git), a CPU based
- renderer to render the meshes. It is possible to use other renderers,
- replace `SwiftshaderRenderer` in `render/swiftshader_renderer.py` with
- bindings to your renderer.
- ```Shell
- mkdir -p deps
- git clone --recursive https://github.com/google/swiftshader.git deps/swiftshader-src
- cd deps/swiftshader-src && git checkout 91da6b00584afd7dcaed66da88e2b617429b3950
- git submodule update
- mkdir build && cd build && cmake .. && make -j 16 libEGL libGLESv2
- cd ../../../
- cp deps/swiftshader-src/build/libEGL* libEGL.so.1
- cp deps/swiftshader-src/build/libGLESv2* libGLESv2.so.2
- ```
-
-4. PyAssimp: We use [PyAssimp](https://github.com/assimp/assimp.git) to load
- meshes. It is possible to use other libraries to load meshes, replace
- `Shape` `render/swiftshader_renderer.py` with bindings to your library for
- loading meshes.
- ```Shell
- mkdir -p deps
- git clone https://github.com/assimp/assimp.git deps/assimp-src
- cd deps/assimp-src
- git checkout 2afeddd5cb63d14bc77b53740b38a54a97d94ee8
- cmake CMakeLists.txt -G 'Unix Makefiles' && make -j 16
- cd port/PyAssimp && python setup.py install
- cd ../../../..
- cp deps/assimp-src/lib/libassimp* .
- ```
-
-5. graph-tool: We use [graph-tool](https://git.skewed.de/count0/graph-tool)
- library for graph processing.
- ```Shell
- mkdir -p deps
- # If the following git clone command fails, you can also download the source
- # from https://downloads.skewed.de/graph-tool/graph-tool-2.2.44.tar.bz2
- git clone https://git.skewed.de/count0/graph-tool deps/graph-tool-src
- cd deps/graph-tool-src && git checkout 178add3a571feb6666f4f119027705d95d2951ab
- bash autogen.sh
- ./configure --disable-cairo --disable-sparsehash --prefix=$HOME/.local
- make -j 16
- make install
- cd ../../
- ```
-
-### Requirements: data
-1. Download the Stanford 3D Indoor Spaces Dataset (S3DIS Dataset) and ImageNet
- Pre-trained models for initializing different models. Follow instructions in
- `data/README.md`
-
-### Test Pre-trained Models
-1. Download pre-trained models. See `output/README.md`.
-
-2. Test models using `scripts/script_test_pretrained_models.sh`.
-
-### Train Your Own Models
-All models were trained asynchronously with 16 workers each worker using data
-from a single floor. The default hyper-parameters correspond to this setting.
-See [distributed training with
-Tensorflow](https://www.tensorflow.org/deploy/distributed) for setting up
-distributed training. Training with a single worker is possible with the current
-code base but will require some minor changes to allow each worker to load all
-training environments.
-
-### Contact
-For questions or issues open an issue on the tensorflow/models [issues
-tracker](https://github.com/tensorflow/models/issues). Please assign issues to
-@s-gupta.
-
-### Credits
-This code was written by Saurabh Gupta (@s-gupta).
diff --git a/research/cognitive_mapping_and_planning/__init__.py b/research/cognitive_mapping_and_planning/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/cognitive_mapping_and_planning/cfgs/__init__.py b/research/cognitive_mapping_and_planning/cfgs/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/cognitive_mapping_and_planning/cfgs/config_cmp.py b/research/cognitive_mapping_and_planning/cfgs/config_cmp.py
deleted file mode 100644
index 715eee2b973cb66f816ecdb65bbcc3abdd8a9483..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/cfgs/config_cmp.py
+++ /dev/null
@@ -1,283 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import os, sys
-import numpy as np
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-import logging
-import src.utils as utils
-import cfgs.config_common as cc
-
-
-import tensorflow as tf
-
-
-rgb_resnet_v2_50_path = 'data/init_models/resnet_v2_50/model.ckpt-5136169'
-d_resnet_v2_50_path = 'data/init_models/distill_rgb_to_d_resnet_v2_50/model.ckpt-120002'
-
-def get_default_args():
- summary_args = utils.Foo(display_interval=1, test_iters=26,
- arop_full_summary_iters=14)
-
- control_args = utils.Foo(train=False, test=False,
- force_batchnorm_is_training_at_test=False,
- reset_rng_seed=False, only_eval_when_done=False,
- test_mode=None)
- return summary_args, control_args
-
-def get_default_cmp_args():
- batch_norm_param = {'center': True, 'scale': True,
- 'activation_fn':tf.nn.relu}
-
- mapper_arch_args = utils.Foo(
- dim_reduce_neurons=64,
- fc_neurons=[1024, 1024],
- fc_out_size=8,
- fc_out_neurons=64,
- encoder='resnet_v2_50',
- deconv_neurons=[64, 32, 16, 8, 4, 2],
- deconv_strides=[2, 2, 2, 2, 2, 2],
- deconv_layers_per_block=2,
- deconv_kernel_size=4,
- fc_dropout=0.5,
- combine_type='wt_avg_logits',
- batch_norm_param=batch_norm_param)
-
- readout_maps_arch_args = utils.Foo(
- num_neurons=[],
- strides=[],
- kernel_size=None,
- layers_per_block=None)
-
- arch_args = utils.Foo(
- vin_val_neurons=8, vin_action_neurons=8, vin_ks=3, vin_share_wts=False,
- pred_neurons=[64, 64], pred_batch_norm_param=batch_norm_param,
- conv_on_value_map=0, fr_neurons=16, fr_ver='v2', fr_inside_neurons=64,
- fr_stride=1, crop_remove_each=30, value_crop_size=4,
- action_sample_type='sample', action_sample_combine_type='one_or_other',
- sample_gt_prob_type='inverse_sigmoid_decay', dagger_sample_bn_false=True,
- vin_num_iters=36, isd_k=750., use_agent_loc=False, multi_scale=True,
- readout_maps=False, rom_arch=readout_maps_arch_args)
-
- return arch_args, mapper_arch_args
-
-def get_arch_vars(arch_str):
- if arch_str == '': vals = []
- else: vals = arch_str.split('_')
- ks = ['var1', 'var2', 'var3']
- ks = ks[:len(vals)]
-
- # Exp Ver.
- if len(vals) == 0: ks.append('var1'); vals.append('v0')
- # custom arch.
- if len(vals) == 1: ks.append('var2'); vals.append('')
- # map scape for projection baseline.
- if len(vals) == 2: ks.append('var3'); vals.append('fr2')
-
- assert(len(vals) == 3)
-
- vars = utils.Foo()
- for k, v in zip(ks, vals):
- setattr(vars, k, v)
-
- logging.error('arch_vars: %s', vars)
- return vars
-
-def process_arch_str(args, arch_str):
- # This function modifies args.
- args.arch, args.mapper_arch = get_default_cmp_args()
-
- arch_vars = get_arch_vars(arch_str)
-
- args.navtask.task_params.outputs.ego_maps = True
- args.navtask.task_params.outputs.ego_goal_imgs = True
- args.navtask.task_params.outputs.egomotion = True
- args.navtask.task_params.toy_problem = False
-
- if arch_vars.var1 == 'lmap':
- args = process_arch_learned_map(args, arch_vars)
-
- elif arch_vars.var1 == 'pmap':
- args = process_arch_projected_map(args, arch_vars)
-
- else:
- logging.fatal('arch_vars.var1 should be lmap or pmap, but is %s', arch_vars.var1)
- assert(False)
-
- return args
-
-def process_arch_learned_map(args, arch_vars):
- # Multiscale vision based system.
- args.navtask.task_params.input_type = 'vision'
- args.navtask.task_params.outputs.images = True
-
- if args.navtask.camera_param.modalities[0] == 'rgb':
- args.solver.pretrained_path = rgb_resnet_v2_50_path
- elif args.navtask.camera_param.modalities[0] == 'depth':
- args.solver.pretrained_path = d_resnet_v2_50_path
-
- if arch_vars.var2 == 'Ssc':
- sc = 1./args.navtask.task_params.step_size
- args.arch.vin_num_iters = 40
- args.navtask.task_params.map_scales = [sc]
- max_dist = args.navtask.task_params.max_dist * \
- args.navtask.task_params.num_goals
- args.navtask.task_params.map_crop_sizes = [2*max_dist]
-
- args.arch.fr_stride = 1
- args.arch.vin_action_neurons = 8
- args.arch.vin_val_neurons = 3
- args.arch.fr_inside_neurons = 32
-
- args.mapper_arch.pad_map_with_zeros_each = [24]
- args.mapper_arch.deconv_neurons = [64, 32, 16]
- args.mapper_arch.deconv_strides = [1, 2, 1]
-
- elif (arch_vars.var2 == 'Msc' or arch_vars.var2 == 'MscROMms' or
- arch_vars.var2 == 'MscROMss' or arch_vars.var2 == 'MscNoVin'):
- # Code for multi-scale planner.
- args.arch.vin_num_iters = 8
- args.arch.crop_remove_each = 4
- args.arch.value_crop_size = 8
-
- sc = 1./args.navtask.task_params.step_size
- max_dist = args.navtask.task_params.max_dist * \
- args.navtask.task_params.num_goals
- n_scales = np.log2(float(max_dist) / float(args.arch.vin_num_iters))
- n_scales = int(np.ceil(n_scales)+1)
-
- args.navtask.task_params.map_scales = \
- list(sc*(0.5**(np.arange(n_scales))[::-1]))
- args.navtask.task_params.map_crop_sizes = [16 for x in range(n_scales)]
-
- args.arch.fr_stride = 1
- args.arch.vin_action_neurons = 8
- args.arch.vin_val_neurons = 3
- args.arch.fr_inside_neurons = 32
-
- args.mapper_arch.pad_map_with_zeros_each = [0 for _ in range(n_scales)]
- args.mapper_arch.deconv_neurons = [64*n_scales, 32*n_scales, 16*n_scales]
- args.mapper_arch.deconv_strides = [1, 2, 1]
-
- if arch_vars.var2 == 'MscNoVin':
- # No planning version.
- args.arch.fr_stride = [1, 2, 1, 2]
- args.arch.vin_action_neurons = None
- args.arch.vin_val_neurons = 16
- args.arch.fr_inside_neurons = 32
-
- args.arch.crop_remove_each = 0
- args.arch.value_crop_size = 4
- args.arch.vin_num_iters = 0
-
- elif arch_vars.var2 == 'MscROMms' or arch_vars.var2 == 'MscROMss':
- # Code with read outs, MscROMms flattens and reads out,
- # MscROMss does not flatten and produces output at multiple scales.
- args.navtask.task_params.outputs.readout_maps = True
- args.navtask.task_params.map_resize_method = 'antialiasing'
- args.arch.readout_maps = True
-
- if arch_vars.var2 == 'MscROMms':
- args.arch.rom_arch.num_neurons = [64, 1]
- args.arch.rom_arch.kernel_size = 4
- args.arch.rom_arch.strides = [2,2]
- args.arch.rom_arch.layers_per_block = 2
-
- args.navtask.task_params.readout_maps_crop_sizes = [64]
- args.navtask.task_params.readout_maps_scales = [sc]
-
- elif arch_vars.var2 == 'MscROMss':
- args.arch.rom_arch.num_neurons = \
- [64, len(args.navtask.task_params.map_scales)]
- args.arch.rom_arch.kernel_size = 4
- args.arch.rom_arch.strides = [1,1]
- args.arch.rom_arch.layers_per_block = 1
-
- args.navtask.task_params.readout_maps_crop_sizes = \
- args.navtask.task_params.map_crop_sizes
- args.navtask.task_params.readout_maps_scales = \
- args.navtask.task_params.map_scales
-
- else:
- logging.fatal('arch_vars.var2 not one of Msc, MscROMms, MscROMss, MscNoVin.')
- assert(False)
-
- map_channels = args.mapper_arch.deconv_neurons[-1] / \
- (2*len(args.navtask.task_params.map_scales))
- args.navtask.task_params.map_channels = map_channels
-
- return args
-
-def process_arch_projected_map(args, arch_vars):
- # Single scale vision based system which does not use a mapper but instead
- # uses an analytically estimated map.
- ds = int(arch_vars.var3[2])
- args.navtask.task_params.input_type = 'analytical_counts'
- args.navtask.task_params.outputs.analytical_counts = True
-
- assert(args.navtask.task_params.modalities[0] == 'depth')
- args.navtask.camera_param.img_channels = None
-
- analytical_counts = utils.Foo(map_sizes=[512/ds],
- xy_resolution=[5.*ds],
- z_bins=[[-10, 10, 150, 200]],
- non_linearity=[arch_vars.var2])
- args.navtask.task_params.analytical_counts = analytical_counts
-
- sc = 1./ds
- args.arch.vin_num_iters = 36
- args.navtask.task_params.map_scales = [sc]
- args.navtask.task_params.map_crop_sizes = [512/ds]
-
- args.arch.fr_stride = [1,2]
- args.arch.vin_action_neurons = 8
- args.arch.vin_val_neurons = 3
- args.arch.fr_inside_neurons = 32
-
- map_channels = len(analytical_counts.z_bins[0]) + 1
- args.navtask.task_params.map_channels = map_channels
- args.solver.freeze_conv = False
-
- return args
-
-def get_args_for_config(config_name):
- args = utils.Foo()
-
- args.summary, args.control = get_default_args()
-
- exp_name, mode_str = config_name.split('+')
- arch_str, solver_str, navtask_str = exp_name.split('.')
- logging.error('config_name: %s', config_name)
- logging.error('arch_str: %s', arch_str)
- logging.error('navtask_str: %s', navtask_str)
- logging.error('solver_str: %s', solver_str)
- logging.error('mode_str: %s', mode_str)
-
- args.solver = cc.process_solver_str(solver_str)
- args.navtask = cc.process_navtask_str(navtask_str)
-
- args = process_arch_str(args, arch_str)
- args.arch.isd_k = args.solver.isd_k
-
- # Train, test, etc.
- mode, imset = mode_str.split('_')
- args = cc.adjust_args_for_mode(args, mode)
- args.navtask.building_names = args.navtask.dataset.get_split(imset)
- args.control.test_name = '{:s}_on_{:s}'.format(mode, imset)
-
- # Log the arguments
- logging.error('%s', args)
- return args
diff --git a/research/cognitive_mapping_and_planning/cfgs/config_common.py b/research/cognitive_mapping_and_planning/cfgs/config_common.py
deleted file mode 100644
index 440bf5b72f87a1eeca38e22f33b22e82de7345c0..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/cfgs/config_common.py
+++ /dev/null
@@ -1,261 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import os
-import numpy as np
-import logging
-import src.utils as utils
-import datasets.nav_env_config as nec
-from datasets import factory
-
-def adjust_args_for_mode(args, mode):
- if mode == 'train':
- args.control.train = True
-
- elif mode == 'val1':
- # Same settings as for training, to make sure nothing wonky is happening
- # there.
- args.control.test = True
- args.control.test_mode = 'val'
- args.navtask.task_params.batch_size = 32
-
- elif mode == 'val2':
- # No data augmentation, not sampling but taking the argmax action, not
- # sampling from the ground truth at all.
- args.control.test = True
- args.arch.action_sample_type = 'argmax'
- args.arch.sample_gt_prob_type = 'zero'
- args.navtask.task_params.data_augment = \
- utils.Foo(lr_flip=0, delta_angle=0, delta_xy=0, relight=False,
- relight_fast=False, structured=False)
- args.control.test_mode = 'val'
- args.navtask.task_params.batch_size = 32
-
- elif mode == 'bench':
- # Actually testing the agent in settings that are kept same between
- # different runs.
- args.navtask.task_params.batch_size = 16
- args.control.test = True
- args.arch.action_sample_type = 'argmax'
- args.arch.sample_gt_prob_type = 'zero'
- args.navtask.task_params.data_augment = \
- utils.Foo(lr_flip=0, delta_angle=0, delta_xy=0, relight=False,
- relight_fast=False, structured=False)
- args.summary.test_iters = 250
- args.control.only_eval_when_done = True
- args.control.reset_rng_seed = True
- args.control.test_mode = 'test'
- else:
- logging.fatal('Unknown mode: %s.', mode)
- assert(False)
- return args
-
-def get_solver_vars(solver_str):
- if solver_str == '': vals = [];
- else: vals = solver_str.split('_')
- ks = ['clip', 'dlw', 'long', 'typ', 'isdk', 'adam_eps', 'init_lr'];
- ks = ks[:len(vals)]
-
- # Gradient clipping or not.
- if len(vals) == 0: ks.append('clip'); vals.append('noclip');
- # data loss weight.
- if len(vals) == 1: ks.append('dlw'); vals.append('dlw20')
- # how long to train for.
- if len(vals) == 2: ks.append('long'); vals.append('nolong')
- # Adam
- if len(vals) == 3: ks.append('typ'); vals.append('adam2')
- # reg loss wt
- if len(vals) == 4: ks.append('rlw'); vals.append('rlw1')
- # isd_k
- if len(vals) == 5: ks.append('isdk'); vals.append('isdk415') # 415, inflexion at 2.5k.
- # adam eps
- if len(vals) == 6: ks.append('adam_eps'); vals.append('aeps1en8')
- # init lr
- if len(vals) == 7: ks.append('init_lr'); vals.append('lr1en3')
-
- assert(len(vals) == 8)
-
- vars = utils.Foo()
- for k, v in zip(ks, vals):
- setattr(vars, k, v)
- logging.error('solver_vars: %s', vars)
- return vars
-
-def process_solver_str(solver_str):
- solver = utils.Foo(
- seed=0, learning_rate_decay=None, clip_gradient_norm=None, max_steps=None,
- initial_learning_rate=None, momentum=None, steps_per_decay=None,
- logdir=None, sync=False, adjust_lr_sync=True, wt_decay=0.0001,
- data_loss_wt=None, reg_loss_wt=None, freeze_conv=True, num_workers=1,
- task=0, ps_tasks=0, master='local', typ=None, momentum2=None,
- adam_eps=None)
-
- # Clobber with overrides from solver str.
- solver_vars = get_solver_vars(solver_str)
-
- solver.data_loss_wt = float(solver_vars.dlw[3:].replace('x', '.'))
- solver.adam_eps = float(solver_vars.adam_eps[4:].replace('x', '.').replace('n', '-'))
- solver.initial_learning_rate = float(solver_vars.init_lr[2:].replace('x', '.').replace('n', '-'))
- solver.reg_loss_wt = float(solver_vars.rlw[3:].replace('x', '.'))
- solver.isd_k = float(solver_vars.isdk[4:].replace('x', '.'))
-
- long = solver_vars.long
- if long == 'long':
- solver.steps_per_decay = 40000
- solver.max_steps = 120000
- elif long == 'long2':
- solver.steps_per_decay = 80000
- solver.max_steps = 120000
- elif long == 'nolong' or long == 'nol':
- solver.steps_per_decay = 20000
- solver.max_steps = 60000
- else:
- logging.fatal('solver_vars.long should be long, long2, nolong or nol.')
- assert(False)
-
- clip = solver_vars.clip
- if clip == 'noclip' or clip == 'nocl':
- solver.clip_gradient_norm = 0
- elif clip[:4] == 'clip':
- solver.clip_gradient_norm = float(clip[4:].replace('x', '.'))
- else:
- logging.fatal('Unknown solver_vars.clip: %s', clip)
- assert(False)
-
- typ = solver_vars.typ
- if typ == 'adam':
- solver.typ = 'adam'
- solver.momentum = 0.9
- solver.momentum2 = 0.999
- solver.learning_rate_decay = 1.0
- elif typ == 'adam2':
- solver.typ = 'adam'
- solver.momentum = 0.9
- solver.momentum2 = 0.999
- solver.learning_rate_decay = 0.1
- elif typ == 'sgd':
- solver.typ = 'sgd'
- solver.momentum = 0.99
- solver.momentum2 = None
- solver.learning_rate_decay = 0.1
- else:
- logging.fatal('Unknown solver_vars.typ: %s', typ)
- assert(False)
-
- logging.error('solver: %s', solver)
- return solver
-
-def get_navtask_vars(navtask_str):
- if navtask_str == '': vals = []
- else: vals = navtask_str.split('_')
-
- ks_all = ['dataset_name', 'modality', 'task', 'history', 'max_dist',
- 'num_steps', 'step_size', 'n_ori', 'aux_views', 'data_aug']
- ks = ks_all[:len(vals)]
-
- # All data or not.
- if len(vals) == 0: ks.append('dataset_name'); vals.append('sbpd')
- # modality
- if len(vals) == 1: ks.append('modality'); vals.append('rgb')
- # semantic task?
- if len(vals) == 2: ks.append('task'); vals.append('r2r')
- # number of history frames.
- if len(vals) == 3: ks.append('history'); vals.append('h0')
- # max steps
- if len(vals) == 4: ks.append('max_dist'); vals.append('32')
- # num steps
- if len(vals) == 5: ks.append('num_steps'); vals.append('40')
- # step size
- if len(vals) == 6: ks.append('step_size'); vals.append('8')
- # n_ori
- if len(vals) == 7: ks.append('n_ori'); vals.append('4')
- # Auxiliary views.
- if len(vals) == 8: ks.append('aux_views'); vals.append('nv0')
- # Normal data augmentation as opposed to structured data augmentation (if set
- # to straug.
- if len(vals) == 9: ks.append('data_aug'); vals.append('straug')
-
- assert(len(vals) == 10)
- for i in range(len(ks)):
- assert(ks[i] == ks_all[i])
-
- vars = utils.Foo()
- for k, v in zip(ks, vals):
- setattr(vars, k, v)
- logging.error('navtask_vars: %s', vals)
- return vars
-
-def process_navtask_str(navtask_str):
- navtask = nec.nav_env_base_config()
-
- # Clobber with overrides from strings.
- navtask_vars = get_navtask_vars(navtask_str)
-
- navtask.task_params.n_ori = int(navtask_vars.n_ori)
- navtask.task_params.max_dist = int(navtask_vars.max_dist)
- navtask.task_params.num_steps = int(navtask_vars.num_steps)
- navtask.task_params.step_size = int(navtask_vars.step_size)
- navtask.task_params.data_augment.delta_xy = int(navtask_vars.step_size)/2.
- n_aux_views_each = int(navtask_vars.aux_views[2])
- aux_delta_thetas = np.concatenate((np.arange(n_aux_views_each) + 1,
- -1 -np.arange(n_aux_views_each)))
- aux_delta_thetas = aux_delta_thetas*np.deg2rad(navtask.camera_param.fov)
- navtask.task_params.aux_delta_thetas = aux_delta_thetas
-
- if navtask_vars.data_aug == 'aug':
- navtask.task_params.data_augment.structured = False
- elif navtask_vars.data_aug == 'straug':
- navtask.task_params.data_augment.structured = True
- else:
- logging.fatal('Unknown navtask_vars.data_aug %s.', navtask_vars.data_aug)
- assert(False)
-
- navtask.task_params.num_history_frames = int(navtask_vars.history[1:])
- navtask.task_params.n_views = 1+navtask.task_params.num_history_frames
-
- navtask.task_params.goal_channels = int(navtask_vars.n_ori)
-
- if navtask_vars.task == 'hard':
- navtask.task_params.type = 'rng_rejection_sampling_many'
- navtask.task_params.rejection_sampling_M = 2000
- navtask.task_params.min_dist = 10
- elif navtask_vars.task == 'r2r':
- navtask.task_params.type = 'room_to_room_many'
- elif navtask_vars.task == 'ST':
- # Semantic task at hand.
- navtask.task_params.goal_channels = \
- len(navtask.task_params.semantic_task.class_map_names)
- navtask.task_params.rel_goal_loc_dim = \
- len(navtask.task_params.semantic_task.class_map_names)
- navtask.task_params.type = 'to_nearest_obj_acc'
- else:
- logging.fatal('navtask_vars.task: should be hard or r2r, ST')
- assert(False)
-
- if navtask_vars.modality == 'rgb':
- navtask.camera_param.modalities = ['rgb']
- navtask.camera_param.img_channels = 3
- elif navtask_vars.modality == 'd':
- navtask.camera_param.modalities = ['depth']
- navtask.camera_param.img_channels = 2
-
- navtask.task_params.img_height = navtask.camera_param.height
- navtask.task_params.img_width = navtask.camera_param.width
- navtask.task_params.modalities = navtask.camera_param.modalities
- navtask.task_params.img_channels = navtask.camera_param.img_channels
- navtask.task_params.img_fov = navtask.camera_param.fov
-
- navtask.dataset = factory.get_dataset(navtask_vars.dataset_name)
- return navtask
diff --git a/research/cognitive_mapping_and_planning/cfgs/config_distill.py b/research/cognitive_mapping_and_planning/cfgs/config_distill.py
deleted file mode 100644
index 53be2f8a5f12ee701a53c1c354079659da6958d4..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/cfgs/config_distill.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import pprint
-import copy
-import os
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-import logging
-import src.utils as utils
-import cfgs.config_common as cc
-
-
-import tensorflow as tf
-
-rgb_resnet_v2_50_path = 'cache/resnet_v2_50_inception_preprocessed/model.ckpt-5136169'
-
-def get_default_args():
- robot = utils.Foo(radius=15, base=10, height=140, sensor_height=120,
- camera_elevation_degree=-15)
-
- camera_param = utils.Foo(width=225, height=225, z_near=0.05, z_far=20.0,
- fov=60., modalities=['rgb', 'depth'])
-
- env = utils.Foo(padding=10, resolution=5, num_point_threshold=2,
- valid_min=-10, valid_max=200, n_samples_per_face=200)
-
- data_augment = utils.Foo(lr_flip=0, delta_angle=1, delta_xy=4, relight=False,
- relight_fast=False, structured=False)
-
- task_params = utils.Foo(num_actions=4, step_size=4, num_steps=0,
- batch_size=32, room_seed=0, base_class='Building',
- task='mapping', n_ori=6, data_augment=data_augment,
- output_transform_to_global_map=False,
- output_canonical_map=False,
- output_incremental_transform=False,
- output_free_space=False, move_type='shortest_path',
- toy_problem=0)
-
- buildinger_args = utils.Foo(building_names=['area1_gates_wingA_floor1_westpart'],
- env_class=None, robot=robot,
- task_params=task_params, env=env,
- camera_param=camera_param)
-
- solver_args = utils.Foo(seed=0, learning_rate_decay=0.1,
- clip_gradient_norm=0, max_steps=120000,
- initial_learning_rate=0.001, momentum=0.99,
- steps_per_decay=40000, logdir=None, sync=False,
- adjust_lr_sync=True, wt_decay=0.0001,
- data_loss_wt=1.0, reg_loss_wt=1.0,
- num_workers=1, task=0, ps_tasks=0, master='local')
-
- summary_args = utils.Foo(display_interval=1, test_iters=100)
-
- control_args = utils.Foo(train=False, test=False,
- force_batchnorm_is_training_at_test=False)
-
- arch_args = utils.Foo(rgb_encoder='resnet_v2_50', d_encoder='resnet_v2_50')
-
- return utils.Foo(solver=solver_args,
- summary=summary_args, control=control_args, arch=arch_args,
- buildinger=buildinger_args)
-
-def get_vars(config_name):
- vars = config_name.split('_')
- if len(vars) == 1: # All data or not.
- vars.append('noall')
- if len(vars) == 2: # n_ori
- vars.append('4')
- logging.error('vars: %s', vars)
- return vars
-
-def get_args_for_config(config_name):
- args = get_default_args()
- config_name, mode = config_name.split('+')
- vars = get_vars(config_name)
-
- logging.info('config_name: %s, mode: %s', config_name, mode)
-
- args.buildinger.task_params.n_ori = int(vars[2])
- args.solver.freeze_conv = True
- args.solver.pretrained_path = rgb_resnet_v2_50_path
- args.buildinger.task_params.img_channels = 5
- args.solver.data_loss_wt = 0.00001
-
- if vars[0] == 'v0':
- None
- else:
- logging.error('config_name: %s undefined', config_name)
-
- args.buildinger.task_params.height = args.buildinger.camera_param.height
- args.buildinger.task_params.width = args.buildinger.camera_param.width
- args.buildinger.task_params.modalities = args.buildinger.camera_param.modalities
-
- if vars[1] == 'all':
- args = cc.get_args_for_mode_building_all(args, mode)
- elif vars[1] == 'noall':
- args = cc.get_args_for_mode_building(args, mode)
-
- # Log the arguments
- logging.error('%s', args)
- return args
diff --git a/research/cognitive_mapping_and_planning/cfgs/config_vision_baseline.py b/research/cognitive_mapping_and_planning/cfgs/config_vision_baseline.py
deleted file mode 100644
index 3cc64fe594ab025fbcfb41543302fa42c7fc0074..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/cfgs/config_vision_baseline.py
+++ /dev/null
@@ -1,173 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import pprint
-import os
-import numpy as np
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-import logging
-import src.utils as utils
-import cfgs.config_common as cc
-import datasets.nav_env_config as nec
-
-
-import tensorflow as tf
-
-FLAGS = flags.FLAGS
-
-get_solver_vars = cc.get_solver_vars
-get_navtask_vars = cc.get_navtask_vars
-
-
-rgb_resnet_v2_50_path = 'data/init_models/resnet_v2_50/model.ckpt-5136169'
-d_resnet_v2_50_path = 'data/init_models/distill_rgb_to_d_resnet_v2_50/model.ckpt-120002'
-
-def get_default_args():
- summary_args = utils.Foo(display_interval=1, test_iters=26,
- arop_full_summary_iters=14)
-
- control_args = utils.Foo(train=False, test=False,
- force_batchnorm_is_training_at_test=False,
- reset_rng_seed=False, only_eval_when_done=False,
- test_mode=None)
- return summary_args, control_args
-
-def get_default_baseline_args():
- batch_norm_param = {'center': True, 'scale': True,
- 'activation_fn':tf.nn.relu}
- arch_args = utils.Foo(
- pred_neurons=[], goal_embed_neurons=[], img_embed_neurons=[],
- batch_norm_param=batch_norm_param, dim_reduce_neurons=64, combine_type='',
- encoder='resnet_v2_50', action_sample_type='sample',
- action_sample_combine_type='one_or_other',
- sample_gt_prob_type='inverse_sigmoid_decay', dagger_sample_bn_false=True,
- isd_k=750., use_visit_count=False, lstm_output=False, lstm_ego=False,
- lstm_img=False, fc_dropout=0.0, embed_goal_for_state=False,
- lstm_output_init_state_from_goal=False)
- return arch_args
-
-def get_arch_vars(arch_str):
- if arch_str == '': vals = []
- else: vals = arch_str.split('_')
-
- ks = ['ver', 'lstm_dim', 'dropout']
-
- # Exp Ver
- if len(vals) == 0: vals.append('v0')
- # LSTM dimentsions
- if len(vals) == 1: vals.append('lstm2048')
- # Dropout
- if len(vals) == 2: vals.append('noDO')
-
- assert(len(vals) == 3)
-
- vars = utils.Foo()
- for k, v in zip(ks, vals):
- setattr(vars, k, v)
-
- logging.error('arch_vars: %s', vars)
- return vars
-
-def process_arch_str(args, arch_str):
- # This function modifies args.
- args.arch = get_default_baseline_args()
- arch_vars = get_arch_vars(arch_str)
-
- args.navtask.task_params.outputs.rel_goal_loc = True
- args.navtask.task_params.input_type = 'vision'
- args.navtask.task_params.outputs.images = True
-
- if args.navtask.camera_param.modalities[0] == 'rgb':
- args.solver.pretrained_path = rgb_resnet_v2_50_path
- elif args.navtask.camera_param.modalities[0] == 'depth':
- args.solver.pretrained_path = d_resnet_v2_50_path
- else:
- logging.fatal('Neither of rgb or d')
-
- if arch_vars.dropout == 'DO':
- args.arch.fc_dropout = 0.5
-
- args.tfcode = 'B'
-
- exp_ver = arch_vars.ver
- if exp_ver == 'v0':
- # Multiplicative interaction between goal loc and image features.
- args.arch.combine_type = 'multiply'
- args.arch.pred_neurons = [256, 256]
- args.arch.goal_embed_neurons = [64, 8]
- args.arch.img_embed_neurons = [1024, 512, 256*8]
-
- elif exp_ver == 'v1':
- # Additive interaction between goal and image features.
- args.arch.combine_type = 'add'
- args.arch.pred_neurons = [256, 256]
- args.arch.goal_embed_neurons = [64, 256]
- args.arch.img_embed_neurons = [1024, 512, 256]
-
- elif exp_ver == 'v2':
- # LSTM at the output on top of multiple interactions.
- args.arch.combine_type = 'multiply'
- args.arch.goal_embed_neurons = [64, 8]
- args.arch.img_embed_neurons = [1024, 512, 256*8]
- args.arch.lstm_output = True
- args.arch.lstm_output_dim = int(arch_vars.lstm_dim[4:])
- args.arch.pred_neurons = [256] # The other is inside the LSTM.
-
- elif exp_ver == 'v0blind':
- # LSTM only on the goal location.
- args.arch.combine_type = 'goalonly'
- args.arch.goal_embed_neurons = [64, 256]
- args.arch.img_embed_neurons = [2] # I dont know what it will do otherwise.
- args.arch.lstm_output = True
- args.arch.lstm_output_dim = 256
- args.arch.pred_neurons = [256] # The other is inside the LSTM.
-
- else:
- logging.fatal('exp_ver: %s undefined', exp_ver)
- assert(False)
-
- # Log the arguments
- logging.error('%s', args)
- return args
-
-def get_args_for_config(config_name):
- args = utils.Foo()
-
- args.summary, args.control = get_default_args()
-
- exp_name, mode_str = config_name.split('+')
- arch_str, solver_str, navtask_str = exp_name.split('.')
- logging.error('config_name: %s', config_name)
- logging.error('arch_str: %s', arch_str)
- logging.error('navtask_str: %s', navtask_str)
- logging.error('solver_str: %s', solver_str)
- logging.error('mode_str: %s', mode_str)
-
- args.solver = cc.process_solver_str(solver_str)
- args.navtask = cc.process_navtask_str(navtask_str)
-
- args = process_arch_str(args, arch_str)
- args.arch.isd_k = args.solver.isd_k
-
- # Train, test, etc.
- mode, imset = mode_str.split('_')
- args = cc.adjust_args_for_mode(args, mode)
- args.navtask.building_names = args.navtask.dataset.get_split(imset)
- args.control.test_name = '{:s}_on_{:s}'.format(mode, imset)
-
- # Log the arguments
- logging.error('%s', args)
- return args
diff --git a/research/cognitive_mapping_and_planning/data/.gitignore b/research/cognitive_mapping_and_planning/data/.gitignore
deleted file mode 100644
index 2b6d5e46652d14a9c0a8025dbcccfc2dd4376e4a..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/data/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-stanford_building_parser_dataset_raw
-stanford_building_parser_dataset
-init_models
diff --git a/research/cognitive_mapping_and_planning/data/README.md b/research/cognitive_mapping_and_planning/data/README.md
deleted file mode 100644
index a8928345351dac19c0e12fd33f99dd2aa600e23b..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/data/README.md
+++ /dev/null
@@ -1,33 +0,0 @@
-This directory contains the data needed for training and benchmarking various
-navigation models.
-
-1. Download the data from the [dataset website]
- (http://buildingparser.stanford.edu/dataset.html).
- 1. [Raw meshes](https://goo.gl/forms/2YSPaO2UKmn5Td5m2). We need the meshes
- which are in the noXYZ folder. Download the tar files and place them in
- the `stanford_building_parser_dataset_raw` folder. You need to download
- `area_1_noXYZ.tar`, `area_3_noXYZ.tar`, `area_5a_noXYZ.tar`,
- `area_5b_noXYZ.tar`, `area_6_noXYZ.tar` for training and
- `area_4_noXYZ.tar` for evaluation.
- 2. [Annotations](https://goo.gl/forms/4SoGp4KtH1jfRqEj2) for setting up
- tasks. We will need the file called `Stanford3dDataset_v1.2.zip`. Place
- the file in the directory `stanford_building_parser_dataset_raw`.
-
-2. Preprocess the data.
- 1. Extract meshes using `scripts/script_preprocess_meshes_S3DIS.sh`. After
- this `ls data/stanford_building_parser_dataset/mesh` should have 6
- folders `area1`, `area3`, `area4`, `area5a`, `area5b`, `area6`, with
- textures and obj files within each directory.
- 2. Extract out room information and semantics from zip file using
- `scripts/script_preprocess_annoations_S3DIS.sh`. After this there should
- be `room-dimension` and `class-maps` folder in
- `data/stanford_building_parser_dataset`. (If you find this script to
- crash because of an exception in np.loadtxt while processing
- `Area_5/office_19/Annotations/ceiling_1.txt`, there is a special
- character on line 323474, that should be removed manually.)
-
-3. Download ImageNet Pre-trained models. We used ResNet-v2-50 for representing
- images. For RGB images this is pre-trained on ImageNet. For Depth images we
- [distill](https://arxiv.org/abs/1507.00448) the RGB model to depth images
- using paired RGB-D images. Both there models are available through
- `scripts/script_download_init_models.sh`
diff --git a/research/cognitive_mapping_and_planning/datasets/__init__.py b/research/cognitive_mapping_and_planning/datasets/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/cognitive_mapping_and_planning/datasets/factory.py b/research/cognitive_mapping_and_planning/datasets/factory.py
deleted file mode 100644
index 3f7b5c0a602dbacf9619dc1c2ec98e94200428b6..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/datasets/factory.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""Wrapper for selecting the navigation environment that we want to train and
-test on.
-"""
-import numpy as np
-import os, glob
-import platform
-
-import logging
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-
-import render.swiftshader_renderer as renderer
-import src.file_utils as fu
-import src.utils as utils
-
-def get_dataset(dataset_name):
- if dataset_name == 'sbpd':
- dataset = StanfordBuildingParserDataset(dataset_name)
- else:
- logging.fatal('Not one of sbpd')
- return dataset
-
-class Loader():
- def get_data_dir():
- pass
-
- def get_meta_data(self, file_name, data_dir=None):
- if data_dir is None:
- data_dir = self.get_data_dir()
- full_file_name = os.path.join(data_dir, 'meta', file_name)
- assert(fu.exists(full_file_name)), \
- '{:s} does not exist'.format(full_file_name)
- ext = os.path.splitext(full_file_name)[1]
- if ext == '.txt':
- ls = []
- with fu.fopen(full_file_name, 'r') as f:
- for l in f:
- ls.append(l.rstrip())
- elif ext == '.pkl':
- ls = utils.load_variables(full_file_name)
- return ls
-
- def load_building(self, name, data_dir=None):
- if data_dir is None:
- data_dir = self.get_data_dir()
- out = {}
- out['name'] = name
- out['data_dir'] = data_dir
- out['room_dimension_file'] = os.path.join(data_dir, 'room-dimension',
- name+'.pkl')
- out['class_map_folder'] = os.path.join(data_dir, 'class-maps')
- return out
-
- def load_building_meshes(self, building):
- dir_name = os.path.join(building['data_dir'], 'mesh', building['name'])
- mesh_file_name = glob.glob1(dir_name, '*.obj')[0]
- mesh_file_name_full = os.path.join(dir_name, mesh_file_name)
- logging.error('Loading building from obj file: %s', mesh_file_name_full)
- shape = renderer.Shape(mesh_file_name_full, load_materials=True,
- name_prefix=building['name']+'_')
- return [shape]
-
-class StanfordBuildingParserDataset(Loader):
- def __init__(self, ver):
- self.ver = ver
- self.data_dir = None
-
- def get_data_dir(self):
- if self.data_dir is None:
- self.data_dir = 'data/stanford_building_parser_dataset/'
- return self.data_dir
-
- def get_benchmark_sets(self):
- return self._get_benchmark_sets()
-
- def get_split(self, split_name):
- if self.ver == 'sbpd':
- return self._get_split(split_name)
- else:
- logging.fatal('Unknown version.')
-
- def _get_benchmark_sets(self):
- sets = ['train1', 'val', 'test']
- return sets
-
- def _get_split(self, split_name):
- train = ['area1', 'area5a', 'area5b', 'area6']
- train1 = ['area1']
- val = ['area3']
- test = ['area4']
-
- sets = {}
- sets['train'] = train
- sets['train1'] = train1
- sets['val'] = val
- sets['test'] = test
- sets['all'] = sorted(list(set(train + val + test)))
- return sets[split_name]
diff --git a/research/cognitive_mapping_and_planning/datasets/nav_env.py b/research/cognitive_mapping_and_planning/datasets/nav_env.py
deleted file mode 100644
index 5710e26dcb113121d99400cb060104224dd91749..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/datasets/nav_env.py
+++ /dev/null
@@ -1,1465 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""Navidation Environment. Includes the following classes along with some
-helper functions.
- Building: Loads buildings, computes traversibility, exposes functionality for
- rendering images.
-
- GridWorld: Base class which implements functionality for moving an agent on a
- grid world.
-
- NavigationEnv: Base class which generates navigation problems on a grid world.
-
- VisualNavigationEnv: Builds upon NavigationEnv and Building to provide
- interface that is used externally to train the agent.
-
- MeshMapper: Class used for distilling the model, testing the mapper.
-
- BuildingMultiplexer: Wrapper class that instantiates a VisualNavigationEnv for
- each building and multiplexes between them as needed.
-"""
-
-import numpy as np
-import os
-import re
-import matplotlib.pyplot as plt
-
-import graph_tool as gt
-import graph_tool.topology
-
-from tensorflow.python.platform import gfile
-import logging
-import src.file_utils as fu
-import src.utils as utils
-import src.graph_utils as gu
-import src.map_utils as mu
-import src.depth_utils as du
-import render.swiftshader_renderer as sru
-from render.swiftshader_renderer import SwiftshaderRenderer
-import cv2
-
-label_nodes_with_class = gu.label_nodes_with_class
-label_nodes_with_class_geodesic = gu.label_nodes_with_class_geodesic
-get_distance_node_list = gu.get_distance_node_list
-convert_to_graph_tool = gu.convert_to_graph_tool
-generate_graph = gu.generate_graph
-get_hardness_distribution = gu.get_hardness_distribution
-rng_next_goal_rejection_sampling = gu.rng_next_goal_rejection_sampling
-rng_next_goal = gu.rng_next_goal
-rng_room_to_room = gu.rng_room_to_room
-rng_target_dist_field = gu.rng_target_dist_field
-
-compute_traversibility = mu.compute_traversibility
-make_map = mu.make_map
-resize_maps = mu.resize_maps
-pick_largest_cc = mu.pick_largest_cc
-get_graph_origin_loc = mu.get_graph_origin_loc
-generate_egocentric_maps = mu.generate_egocentric_maps
-generate_goal_images = mu.generate_goal_images
-get_map_to_predict = mu.get_map_to_predict
-
-bin_points = du.bin_points
-make_geocentric = du.make_geocentric
-get_point_cloud_from_z = du.get_point_cloud_from_z
-get_camera_matrix = du.get_camera_matrix
-
-def _get_semantic_maps(folder_name, building_name, map, flip):
- # Load file from the cache.
- file_name = '{:s}_{:d}_{:d}_{:d}_{:d}_{:d}_{:d}.pkl'
- file_name = file_name.format(building_name, map.size[0], map.size[1],
- map.origin[0], map.origin[1], map.resolution,
- flip)
- file_name = os.path.join(folder_name, file_name)
- logging.info('Loading semantic maps from %s.', file_name)
-
- if fu.exists(file_name):
- a = utils.load_variables(file_name)
- maps = a['maps'] #HxWx#C
- cats = a['cats']
- else:
- logging.error('file_name: %s not found.', file_name)
- maps = None
- cats = None
- return maps, cats
-
-def _select_classes(all_maps, all_cats, cats_to_use):
- inds = []
- for c in cats_to_use:
- ind = all_cats.index(c)
- inds.append(ind)
- out_maps = all_maps[:,:,inds]
- return out_maps
-
-def _get_room_dimensions(file_name, resolution, origin, flip=False):
- if fu.exists(file_name):
- a = utils.load_variables(file_name)['room_dimension']
- names = a.keys()
- dims = np.concatenate(a.values(), axis=0).reshape((-1,6))
- ind = np.argsort(names)
- dims = dims[ind,:]
- names = [names[x] for x in ind]
- if flip:
- dims_new = dims*1
- dims_new[:,1] = -dims[:,4]
- dims_new[:,4] = -dims[:,1]
- dims = dims_new*1
-
- dims = dims*100.
- dims[:,0] = dims[:,0] - origin[0]
- dims[:,1] = dims[:,1] - origin[1]
- dims[:,3] = dims[:,3] - origin[0]
- dims[:,4] = dims[:,4] - origin[1]
- dims = dims / resolution
- out = {'names': names, 'dims': dims}
- else:
- out = None
- return out
-
-def _filter_rooms(room_dims, room_regex):
- pattern = re.compile(room_regex)
- ind = []
- for i, name in enumerate(room_dims['names']):
- if pattern.match(name):
- ind.append(i)
- new_room_dims = {}
- new_room_dims['names'] = [room_dims['names'][i] for i in ind]
- new_room_dims['dims'] = room_dims['dims'][ind,:]*1
- return new_room_dims
-
-def _label_nodes_with_room_id(xyt, room_dims):
- # Label the room with the ID into things.
- node_room_id = -1*np.ones((xyt.shape[0], 1))
- dims = room_dims['dims']
- for x, name in enumerate(room_dims['names']):
- all_ = np.concatenate((xyt[:,[0]] >= dims[x,0],
- xyt[:,[0]] <= dims[x,3],
- xyt[:,[1]] >= dims[x,1],
- xyt[:,[1]] <= dims[x,4]), axis=1)
- node_room_id[np.all(all_, axis=1), 0] = x
- return node_room_id
-
-def get_path_ids(start_node_id, end_node_id, pred_map):
- id = start_node_id
- path = [id]
- while id != end_node_id:
- id = pred_map[id]
- path.append(id)
- return path
-
-def image_pre(images, modalities):
- # Assumes images are ...xHxWxC.
- # We always assume images are RGB followed by Depth.
- if 'depth' in modalities:
- d = images[...,-1][...,np.newaxis]*1.
- d[d < 0.01] = np.NaN; isnan = np.isnan(d);
- d = 100./d; d[isnan] = 0.;
- images = np.concatenate((images[...,:-1], d, isnan), axis=images.ndim-1)
- if 'rgb' in modalities:
- images[...,:3] = images[...,:3]*1. - 128
- return images
-
-def _get_relative_goal_loc(goal_loc, loc, theta):
- r = np.sqrt(np.sum(np.square(goal_loc - loc), axis=1))
- t = np.arctan2(goal_loc[:,1] - loc[:,1], goal_loc[:,0] - loc[:,0])
- t = t-theta[:,0] + np.pi/2
- return np.expand_dims(r,axis=1), np.expand_dims(t, axis=1)
-
-def _gen_perturbs(rng, batch_size, num_steps, lr_flip, delta_angle, delta_xy,
- structured):
- perturbs = []
- for i in range(batch_size):
- # Doing things one by one for each episode in this batch. This way this
- # remains replicatable even when we change the batch size.
- p = np.zeros((num_steps+1, 4))
- if lr_flip:
- # Flip the whole trajectory.
- p[:,3] = rng.rand(1)-0.5
- if delta_angle > 0:
- if structured:
- p[:,2] = (rng.rand(1)-0.5)* delta_angle
- else:
- p[:,2] = (rng.rand(p.shape[0])-0.5)* delta_angle
- if delta_xy > 0:
- if structured:
- p[:,:2] = (rng.rand(1, 2)-0.5)*delta_xy
- else:
- p[:,:2] = (rng.rand(p.shape[0], 2)-0.5)*delta_xy
- perturbs.append(p)
- return perturbs
-
-def get_multiplexer_class(args, task_number):
- assert(args.task_params.base_class == 'Building')
- logging.info('Returning BuildingMultiplexer')
- R = BuildingMultiplexer(args, task_number)
- return R
-
-class GridWorld():
- def __init__(self):
- """Class members that will be assigned by any class that actually uses this
- class."""
- self.restrict_to_largest_cc = None
- self.robot = None
- self.env = None
- self.category_list = None
- self.traversible = None
-
- def get_loc_axis(self, node, delta_theta, perturb=None):
- """Based on the node orientation returns X, and Y axis. Used to sample the
- map in egocentric coordinate frame.
- """
- if type(node) == tuple:
- node = np.array([node])
- if perturb is None:
- perturb = np.zeros((node.shape[0], 4))
- xyt = self.to_actual_xyt_vec(node)
- x = xyt[:,[0]] + perturb[:,[0]]
- y = xyt[:,[1]] + perturb[:,[1]]
- t = xyt[:,[2]] + perturb[:,[2]]
- theta = t*delta_theta
- loc = np.concatenate((x,y), axis=1)
- x_axis = np.concatenate((np.cos(theta), np.sin(theta)), axis=1)
- y_axis = np.concatenate((np.cos(theta+np.pi/2.), np.sin(theta+np.pi/2.)),
- axis=1)
- # Flip the sampled map where need be.
- y_axis[np.where(perturb[:,3] > 0)[0], :] *= -1.
- return loc, x_axis, y_axis, theta
-
- def to_actual_xyt(self, pqr):
- """Converts from node to location on the map."""
- (p, q, r) = pqr
- if self.task.n_ori == 6:
- out = (p - q * 0.5 + self.task.origin_loc[0],
- q * np.sqrt(3.) / 2. + self.task.origin_loc[1], r)
- elif self.task.n_ori == 4:
- out = (p + self.task.origin_loc[0],
- q + self.task.origin_loc[1], r)
- return out
-
- def to_actual_xyt_vec(self, pqr):
- """Converts from node array to location array on the map."""
- p = pqr[:,0][:, np.newaxis]
- q = pqr[:,1][:, np.newaxis]
- r = pqr[:,2][:, np.newaxis]
- if self.task.n_ori == 6:
- out = np.concatenate((p - q * 0.5 + self.task.origin_loc[0],
- q * np.sqrt(3.) / 2. + self.task.origin_loc[1],
- r), axis=1)
- elif self.task.n_ori == 4:
- out = np.concatenate((p + self.task.origin_loc[0],
- q + self.task.origin_loc[1],
- r), axis=1)
- return out
-
- def raw_valid_fn_vec(self, xyt):
- """Returns if the given set of nodes is valid or not."""
- height = self.traversible.shape[0]
- width = self.traversible.shape[1]
- x = np.round(xyt[:,[0]]).astype(np.int32)
- y = np.round(xyt[:,[1]]).astype(np.int32)
- is_inside = np.all(np.concatenate((x >= 0, y >= 0,
- x < width, y < height), axis=1), axis=1)
- x = np.minimum(np.maximum(x, 0), width-1)
- y = np.minimum(np.maximum(y, 0), height-1)
- ind = np.ravel_multi_index((y,x), self.traversible.shape)
- is_traversible = self.traversible.ravel()[ind]
-
- is_valid = np.all(np.concatenate((is_inside[:,np.newaxis], is_traversible),
- axis=1), axis=1)
- return is_valid
-
-
- def valid_fn_vec(self, pqr):
- """Returns if the given set of nodes is valid or not."""
- xyt = self.to_actual_xyt_vec(np.array(pqr))
- height = self.traversible.shape[0]
- width = self.traversible.shape[1]
- x = np.round(xyt[:,[0]]).astype(np.int32)
- y = np.round(xyt[:,[1]]).astype(np.int32)
- is_inside = np.all(np.concatenate((x >= 0, y >= 0,
- x < width, y < height), axis=1), axis=1)
- x = np.minimum(np.maximum(x, 0), width-1)
- y = np.minimum(np.maximum(y, 0), height-1)
- ind = np.ravel_multi_index((y,x), self.traversible.shape)
- is_traversible = self.traversible.ravel()[ind]
-
- is_valid = np.all(np.concatenate((is_inside[:,np.newaxis], is_traversible),
- axis=1), axis=1)
- return is_valid
-
- def get_feasible_actions(self, node_ids):
- """Returns the feasible set of actions from the current node."""
- a = np.zeros((len(node_ids), self.task_params.num_actions), dtype=np.int32)
- gtG = self.task.gtG
- next_node = []
- for i, c in enumerate(node_ids):
- neigh = gtG.vertex(c).out_neighbours()
- neigh_edge = gtG.vertex(c).out_edges()
- nn = {}
- for n, e in zip(neigh, neigh_edge):
- _ = gtG.ep['action'][e]
- a[i,_] = 1
- nn[_] = int(n)
- next_node.append(nn)
- return a, next_node
-
- def take_action(self, current_node_ids, action):
- """Returns the new node after taking the action action. Stays at the current
- node if the action is invalid."""
- actions, next_node_ids = self.get_feasible_actions(current_node_ids)
- new_node_ids = []
- for i, (c,a) in enumerate(zip(current_node_ids, action)):
- if actions[i,a] == 1:
- new_node_ids.append(next_node_ids[i][a])
- else:
- new_node_ids.append(c)
- return new_node_ids
-
- def set_r_obj(self, r_obj):
- """Sets the SwiftshaderRenderer object used for rendering."""
- self.r_obj = r_obj
-
-class Building(GridWorld):
- def __init__(self, building_name, robot, env,
- category_list=None, small=False, flip=False, logdir=None,
- building_loader=None):
-
- self.restrict_to_largest_cc = True
- self.robot = robot
- self.env = env
- self.logdir = logdir
-
- # Load the building meta data.
- building = building_loader.load_building(building_name)
- if small:
- building['mesh_names'] = building['mesh_names'][:5]
-
- # New code.
- shapess = building_loader.load_building_meshes(building)
- if flip:
- for shapes in shapess:
- shapes.flip_shape()
-
- vs = []
- for shapes in shapess:
- vs.append(shapes.get_vertices()[0])
- vs = np.concatenate(vs, axis=0)
- map = make_map(env.padding, env.resolution, vertex=vs, sc=100.)
- map = compute_traversibility(
- map, robot.base, robot.height, robot.radius, env.valid_min,
- env.valid_max, env.num_point_threshold, shapess=shapess, sc=100.,
- n_samples_per_face=env.n_samples_per_face)
-
- room_dims = _get_room_dimensions(building['room_dimension_file'],
- env.resolution, map.origin, flip=flip)
- class_maps, class_map_names = _get_semantic_maps(
- building['class_map_folder'], building_name, map, flip)
-
- self.class_maps = class_maps
- self.class_map_names = class_map_names
- self.building = building
- self.shapess = shapess
- self.map = map
- self.traversible = map.traversible*1
- self.building_name = building_name
- self.room_dims = room_dims
- self.flipped = flip
- self.renderer_entitiy_ids = []
-
- if self.restrict_to_largest_cc:
- self.traversible = pick_largest_cc(self.traversible)
-
- def load_building_into_scene(self):
- # Loads the scene.
- self.renderer_entitiy_ids += self.r_obj.load_shapes(self.shapess)
- # Free up memory, we dont need the mesh or the materials anymore.
- self.shapess = None
-
- def add_entity_at_nodes(self, nodes, height, shape):
- xyt = self.to_actual_xyt_vec(nodes)
- nxy = xyt[:,:2]*1.
- nxy = nxy * self.map.resolution
- nxy = nxy + self.map.origin
- Ts = np.concatenate((nxy, nxy[:,:1]), axis=1)
- Ts[:,2] = height; Ts = Ts / 100.;
-
- # Merge all the shapes into a single shape and add that shape.
- shape.replicate_shape(Ts)
- entity_ids = self.r_obj.load_shapes([shape])
- self.renderer_entitiy_ids += entity_ids
- return entity_ids
-
- def add_shapes(self, shapes):
- scene = self.r_obj.viz.scene()
- for shape in shapes:
- scene.AddShape(shape)
-
- def add_materials(self, materials):
- scene = self.r_obj.viz.scene()
- for material in materials:
- scene.AddOrUpdateMaterial(material)
-
- def set_building_visibility(self, visibility):
- self.r_obj.set_entity_visible(self.renderer_entitiy_ids, visibility)
-
- def render_nodes(self, nodes, perturb=None, aux_delta_theta=0.):
- self.set_building_visibility(True)
- if perturb is None:
- perturb = np.zeros((len(nodes), 4))
-
- imgs = []
- r = 2
- elevation_z = r * np.tan(np.deg2rad(self.robot.camera_elevation_degree))
-
- for i in range(len(nodes)):
- xyt = self.to_actual_xyt(nodes[i])
- lookat_theta = 3.0 * np.pi / 2.0 - (xyt[2]+perturb[i,2]+aux_delta_theta) * (self.task.delta_theta)
- nxy = np.array([xyt[0]+perturb[i,0], xyt[1]+perturb[i,1]]).reshape(1, -1)
- nxy = nxy * self.map.resolution
- nxy = nxy + self.map.origin
- camera_xyz = np.zeros((1, 3))
- camera_xyz[...] = [nxy[0, 0], nxy[0, 1], self.robot.sensor_height]
- camera_xyz = camera_xyz / 100.
- lookat_xyz = np.array([-r * np.sin(lookat_theta),
- -r * np.cos(lookat_theta), elevation_z])
- lookat_xyz = lookat_xyz + camera_xyz[0, :]
- self.r_obj.position_camera(camera_xyz[0, :].tolist(),
- lookat_xyz.tolist(), [0.0, 0.0, 1.0])
- img = self.r_obj.render(take_screenshot=True, output_type=0)
- img = [x for x in img if x is not None]
- img = np.concatenate(img, axis=2).astype(np.float32)
- if perturb[i,3]>0:
- img = img[:,::-1,:]
- imgs.append(img)
-
- self.set_building_visibility(False)
- return imgs
-
-
-class MeshMapper(Building):
- def __init__(self, robot, env, task_params, building_name, category_list,
- flip, logdir=None, building_loader=None):
- Building.__init__(self, building_name, robot, env, category_list,
- small=task_params.toy_problem, flip=flip, logdir=logdir,
- building_loader=building_loader)
- self.task_params = task_params
- self.task = None
- self._preprocess_for_task(self.task_params.building_seed)
-
- def _preprocess_for_task(self, seed):
- if self.task is None or self.task.seed != seed:
- rng = np.random.RandomState(seed)
- origin_loc = get_graph_origin_loc(rng, self.traversible)
- self.task = utils.Foo(seed=seed, origin_loc=origin_loc,
- n_ori=self.task_params.n_ori)
- G = generate_graph(self.valid_fn_vec,
- self.task_params.step_size, self.task.n_ori,
- (0, 0, 0))
- gtG, nodes, nodes_to_id = convert_to_graph_tool(G)
- self.task.gtG = gtG
- self.task.nodes = nodes
- self.task.delta_theta = 2.0*np.pi/(self.task.n_ori*1.)
- self.task.nodes_to_id = nodes_to_id
- logging.info('Building %s, #V=%d, #E=%d', self.building_name,
- self.task.nodes.shape[0], self.task.gtG.num_edges())
-
- if self.logdir is not None:
- write_traversible = cv2.applyColorMap(self.traversible.astype(np.uint8)*255, cv2.COLORMAP_JET)
- img_path = os.path.join(self.logdir,
- '{:s}_{:d}_graph.png'.format(self.building_name,
- seed))
- node_xyt = self.to_actual_xyt_vec(self.task.nodes)
- plt.set_cmap('jet');
- fig, ax = utils.subplot(plt, (1,1), (12,12))
- ax.plot(node_xyt[:,0], node_xyt[:,1], 'm.')
- ax.imshow(self.traversible, origin='lower');
- ax.set_axis_off(); ax.axis('equal');
- ax.set_title('{:s}, {:d}, {:d}'.format(self.building_name,
- self.task.nodes.shape[0],
- self.task.gtG.num_edges()))
- if self.room_dims is not None:
- for i, r in enumerate(self.room_dims['dims']*1):
- min_ = r[:3]*1
- max_ = r[3:]*1
- xmin, ymin, zmin = min_
- xmax, ymax, zmax = max_
-
- ax.plot([xmin, xmax, xmax, xmin, xmin],
- [ymin, ymin, ymax, ymax, ymin], 'g')
- with fu.fopen(img_path, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
- plt.close(fig)
-
-
- def _gen_rng(self, rng):
- # instances is a list of list of node_ids.
- if self.task_params.move_type == 'circle':
- _, _, _, _, paths = rng_target_dist_field(self.task_params.batch_size,
- self.task.gtG, rng, 0, 1,
- compute_path=True)
- instances_ = paths
-
- instances = []
- for instance_ in instances_:
- instance = instance_
- for i in range(self.task_params.num_steps):
- instance.append(self.take_action([instance[-1]], [1])[0])
- instances.append(instance)
-
- elif self.task_params.move_type == 'shortest_path':
- _, _, _, _, paths = rng_target_dist_field(self.task_params.batch_size,
- self.task.gtG, rng,
- self.task_params.num_steps,
- self.task_params.num_steps+1,
- compute_path=True)
- instances = paths
-
- elif self.task_params.move_type == 'circle+forward':
- _, _, _, _, paths = rng_target_dist_field(self.task_params.batch_size,
- self.task.gtG, rng, 0, 1,
- compute_path=True)
- instances_ = paths
- instances = []
- for instance_ in instances_:
- instance = instance_
- for i in range(self.task_params.n_ori-1):
- instance.append(self.take_action([instance[-1]], [1])[0])
- while len(instance) <= self.task_params.num_steps:
- while self.take_action([instance[-1]], [3])[0] == instance[-1] and len(instance) <= self.task_params.num_steps:
- instance.append(self.take_action([instance[-1]], [2])[0])
- if len(instance) <= self.task_params.num_steps:
- instance.append(self.take_action([instance[-1]], [3])[0])
- instances.append(instance)
-
- # Do random perturbation if needed.
- perturbs = _gen_perturbs(rng, self.task_params.batch_size,
- self.task_params.num_steps,
- self.task_params.data_augment.lr_flip,
- self.task_params.data_augment.delta_angle,
- self.task_params.data_augment.delta_xy,
- self.task_params.data_augment.structured)
- return instances, perturbs
-
- def worker(self, instances, perturbs):
- # Output the images and the free space.
-
- # Make the instances be all the same length.
- for i in range(len(instances)):
- for j in range(self.task_params.num_steps - len(instances[i]) + 1):
- instances[i].append(instances[i][-1])
- if perturbs[i].shape[0] < self.task_params.num_steps+1:
- p = np.zeros((self.task_params.num_steps+1, 4))
- p[:perturbs[i].shape[0], :] = perturbs[i]
- p[perturbs[i].shape[0]:, :] = perturbs[i][-1,:]
- perturbs[i] = p
-
- instances_ = []
- for instance in instances:
- instances_ = instances_ + instance
- perturbs_ = np.concatenate(perturbs, axis=0)
-
- instances_nodes = self.task.nodes[instances_,:]
- instances_nodes = [tuple(x) for x in instances_nodes]
-
- imgs_ = self.render_nodes(instances_nodes, perturbs_)
- imgs = []; next = 0;
- for instance in instances:
- img_i = []
- for _ in instance:
- img_i.append(imgs_[next])
- next = next+1
- imgs.append(img_i)
- imgs = np.array(imgs)
-
- # Render out the maps in the egocentric view for all nodes and not just the
- # last node.
- all_nodes = []
- for x in instances:
- all_nodes = all_nodes + x
- all_perturbs = np.concatenate(perturbs, axis=0)
- loc, x_axis, y_axis, theta = self.get_loc_axis(
- self.task.nodes[all_nodes, :]*1, delta_theta=self.task.delta_theta,
- perturb=all_perturbs)
- fss = None
- valids = None
- loc_on_map = None
- theta_on_map = None
- cum_fs = None
- cum_valid = None
- incremental_locs = None
- incremental_thetas = None
-
- if self.task_params.output_free_space:
- fss, valids = get_map_to_predict(loc, x_axis, y_axis,
- map=self.traversible*1.,
- map_size=self.task_params.map_size)
- fss = np.array(fss) > 0.5
- fss = np.reshape(fss, [self.task_params.batch_size,
- self.task_params.num_steps+1,
- self.task_params.map_size,
- self.task_params.map_size])
- valids = np.reshape(np.array(valids), fss.shape)
-
- if self.task_params.output_transform_to_global_map:
- # Output the transform to the global map.
- loc_on_map = np.reshape(loc*1, [self.task_params.batch_size,
- self.task_params.num_steps+1, -1])
- # Converting to location wrt to first location so that warping happens
- # properly.
- theta_on_map = np.reshape(theta*1, [self.task_params.batch_size,
- self.task_params.num_steps+1, -1])
-
- if self.task_params.output_incremental_transform:
- # Output the transform to the global map.
- incremental_locs_ = np.reshape(loc*1, [self.task_params.batch_size,
- self.task_params.num_steps+1, -1])
- incremental_locs_[:,1:,:] -= incremental_locs_[:,:-1,:]
- t0 = -np.pi/2+np.reshape(theta*1, [self.task_params.batch_size,
- self.task_params.num_steps+1, -1])
- t = t0*1
- incremental_locs = incremental_locs_*1
- incremental_locs[:,:,0] = np.sum(incremental_locs_ * np.concatenate((np.cos(t), np.sin(t)), axis=-1), axis=-1)
- incremental_locs[:,:,1] = np.sum(incremental_locs_ * np.concatenate((np.cos(t+np.pi/2), np.sin(t+np.pi/2)), axis=-1), axis=-1)
- incremental_locs[:,0,:] = incremental_locs_[:,0,:]
- # print incremental_locs_[0,:,:], incremental_locs[0,:,:], t0[0,:,:]
-
- incremental_thetas = np.reshape(theta*1, [self.task_params.batch_size,
- self.task_params.num_steps+1,
- -1])
- incremental_thetas[:,1:,:] += -incremental_thetas[:,:-1,:]
-
- if self.task_params.output_canonical_map:
- loc_ = loc[0::(self.task_params.num_steps+1), :]
- x_axis = np.zeros_like(loc_); x_axis[:,1] = 1
- y_axis = np.zeros_like(loc_); y_axis[:,0] = -1
- cum_fs, cum_valid = get_map_to_predict(loc_, x_axis, y_axis,
- map=self.traversible*1.,
- map_size=self.task_params.map_size)
- cum_fs = np.array(cum_fs) > 0.5
- cum_fs = np.reshape(cum_fs, [self.task_params.batch_size, 1,
- self.task_params.map_size,
- self.task_params.map_size])
- cum_valid = np.reshape(np.array(cum_valid), cum_fs.shape)
-
-
- inputs = {'fs_maps': fss,
- 'valid_maps': valids,
- 'imgs': imgs,
- 'loc_on_map': loc_on_map,
- 'theta_on_map': theta_on_map,
- 'cum_fs_maps': cum_fs,
- 'cum_valid_maps': cum_valid,
- 'incremental_thetas': incremental_thetas,
- 'incremental_locs': incremental_locs}
- return inputs
-
- def pre(self, inputs):
- inputs['imgs'] = image_pre(inputs['imgs'], self.task_params.modalities)
- if inputs['loc_on_map'] is not None:
- inputs['loc_on_map'] = inputs['loc_on_map'] - inputs['loc_on_map'][:,[0],:]
- if inputs['theta_on_map'] is not None:
- inputs['theta_on_map'] = np.pi/2. - inputs['theta_on_map']
- return inputs
-
-def _nav_env_reset_helper(type, rng, nodes, batch_size, gtG, max_dist,
- num_steps, num_goals, data_augment, **kwargs):
- """Generates and returns a new episode."""
- max_compute = max_dist + 4*num_steps
- if type == 'general':
- start_node_ids, end_node_ids, dist, pred_map, paths = \
- rng_target_dist_field(batch_size, gtG, rng, max_dist, max_compute,
- nodes=nodes, compute_path=False)
- target_class = None
-
- elif type == 'room_to_room_many':
- goal_node_ids = []; dists = [];
- node_room_ids = kwargs['node_room_ids']
- # Sample the first one
- start_node_ids_, end_node_ids_, dist_, _, _ = rng_room_to_room(
- batch_size, gtG, rng, max_dist, max_compute,
- node_room_ids=node_room_ids, nodes=nodes)
- start_node_ids = start_node_ids_
- goal_node_ids.append(end_node_ids_)
- dists.append(dist_)
- for n in range(num_goals-1):
- start_node_ids_, end_node_ids_, dist_, _, _ = rng_next_goal(
- goal_node_ids[n], batch_size, gtG, rng, max_dist,
- max_compute, node_room_ids=node_room_ids, nodes=nodes,
- dists_from_start_node=dists[n])
- goal_node_ids.append(end_node_ids_)
- dists.append(dist_)
- target_class = None
-
- elif type == 'rng_rejection_sampling_many':
- num_goals = num_goals
- goal_node_ids = []; dists = [];
-
- n_ori = kwargs['n_ori']
- step_size = kwargs['step_size']
- min_dist = kwargs['min_dist']
- sampling_distribution = kwargs['sampling_distribution']
- target_distribution = kwargs['target_distribution']
- rejection_sampling_M = kwargs['rejection_sampling_M']
- distribution_bins = kwargs['distribution_bins']
-
- for n in range(num_goals):
- if n == 0: input_nodes = None
- else: input_nodes = goal_node_ids[n-1]
- start_node_ids_, end_node_ids_, dist_, _, _, _, _ = rng_next_goal_rejection_sampling(
- input_nodes, batch_size, gtG, rng, max_dist, min_dist,
- max_compute, sampling_distribution, target_distribution, nodes,
- n_ori, step_size, distribution_bins, rejection_sampling_M)
- if n == 0: start_node_ids = start_node_ids_
- goal_node_ids.append(end_node_ids_)
- dists.append(dist_)
- target_class = None
-
- elif type == 'room_to_room_back':
- num_goals = num_goals
- assert(num_goals == 2), 'num_goals must be 2.'
- goal_node_ids = []; dists = [];
- node_room_ids = kwargs['node_room_ids']
- # Sample the first one.
- start_node_ids_, end_node_ids_, dist_, _, _ = rng_room_to_room(
- batch_size, gtG, rng, max_dist, max_compute,
- node_room_ids=node_room_ids, nodes=nodes)
- start_node_ids = start_node_ids_
- goal_node_ids.append(end_node_ids_)
- dists.append(dist_)
-
- # Set second goal to be starting position, and compute distance to the start node.
- goal_node_ids.append(start_node_ids)
- dist = []
- for i in range(batch_size):
- dist_ = gt.topology.shortest_distance(
- gt.GraphView(gtG, reversed=True),
- source=gtG.vertex(start_node_ids[i]), target=None)
- dist_ = np.array(dist_.get_array())
- dist.append(dist_)
- dists.append(dist)
- target_class = None
-
- elif type[:14] == 'to_nearest_obj':
- # Generate an episode by sampling one of the target classes (with
- # probability proportional to the number of nodes in the world).
- # With the sampled class sample a node that is within some distance from
- # the sampled class.
- class_nodes = kwargs['class_nodes']
- sampling = kwargs['sampling']
- dist_to_class = kwargs['dist_to_class']
-
- assert(num_goals == 1), 'Only supports a single goal.'
- ind = rng.choice(class_nodes.shape[0], size=batch_size)
- target_class = class_nodes[ind,1]
- start_node_ids = []; dists = []; goal_node_ids = [];
-
- for t in target_class:
- if sampling == 'uniform':
- max_dist = max_dist
- cnts = np.bincount(dist_to_class[t], minlength=max_dist+1)*1.
- cnts[max_dist+1:] = 0
- p_each = 1./ cnts / (max_dist+1.)
- p_each[cnts == 0] = 0
- p = p_each[dist_to_class[t]]*1.; p = p/np.sum(p)
- start_node_id = rng.choice(p.shape[0], size=1, p=p)[0]
- else:
- logging.fatal('Sampling not one of uniform.')
- start_node_ids.append(start_node_id)
- dists.append(dist_to_class[t])
- # Dummy goal node, same as the start node, so that vis is better.
- goal_node_ids.append(start_node_id)
- dists = [dists]
- goal_node_ids = [goal_node_ids]
-
- return start_node_ids, goal_node_ids, dists, target_class
-
-
-class NavigationEnv(GridWorld, Building):
- """Wrapper around GridWorld which sets up navigation tasks.
- """
- def _debug_save_hardness(self, seed):
- out_path = os.path.join(self.logdir, '{:s}_{:d}_hardness.png'.format(self.building_name, seed))
- batch_size = 4000
- rng = np.random.RandomState(0)
- start_node_ids, end_node_ids, dists, pred_maps, paths, hardnesss, gt_dists = \
- rng_next_goal_rejection_sampling(
- None, batch_size, self.task.gtG, rng, self.task_params.max_dist,
- self.task_params.min_dist, self.task_params.max_dist,
- self.task.sampling_distribution, self.task.target_distribution,
- self.task.nodes, self.task_params.n_ori, self.task_params.step_size,
- self.task.distribution_bins, self.task.rejection_sampling_M)
- bins = self.task.distribution_bins
- n_bins = self.task.n_bins
- with plt.style.context('ggplot'):
- fig, axes = utils.subplot(plt, (1,2), (10,10))
- ax = axes[0]
- _ = ax.hist(hardnesss, bins=bins, weights=np.ones_like(hardnesss)/len(hardnesss))
- ax.plot(bins[:-1]+0.5/n_bins, self.task.target_distribution, 'g')
- ax.plot(bins[:-1]+0.5/n_bins, self.task.sampling_distribution, 'b')
- ax.grid('on')
-
- ax = axes[1]
- _ = ax.hist(gt_dists, bins=np.arange(self.task_params.max_dist+1))
- ax.grid('on')
- ax.set_title('Mean: {:0.2f}, Median: {:0.2f}'.format(np.mean(gt_dists),
- np.median(gt_dists)))
- with fu.fopen(out_path, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
-
- def _debug_save_map_nodes(self, seed):
- """Saves traversible space along with nodes generated on the graph. Takes
- the seed as input."""
- img_path = os.path.join(self.logdir, '{:s}_{:d}_graph.png'.format(self.building_name, seed))
- node_xyt = self.to_actual_xyt_vec(self.task.nodes)
- plt.set_cmap('jet');
- fig, ax = utils.subplot(plt, (1,1), (12,12))
- ax.plot(node_xyt[:,0], node_xyt[:,1], 'm.')
- ax.set_axis_off(); ax.axis('equal');
-
- if self.room_dims is not None:
- for i, r in enumerate(self.room_dims['dims']*1):
- min_ = r[:3]*1
- max_ = r[3:]*1
- xmin, ymin, zmin = min_
- xmax, ymax, zmax = max_
-
- ax.plot([xmin, xmax, xmax, xmin, xmin],
- [ymin, ymin, ymax, ymax, ymin], 'g')
- ax.imshow(self.traversible, origin='lower');
- with fu.fopen(img_path, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
-
- def _debug_semantic_maps(self, seed):
- """Saves traversible space along with nodes generated on the graph. Takes
- the seed as input."""
- for i, cls in enumerate(self.task_params.semantic_task.class_map_names):
- img_path = os.path.join(self.logdir, '{:s}_flip{:d}_{:s}_graph.png'.format(self.building_name, seed, cls))
- maps = self.traversible*1.
- maps += 0.5*(self.task.class_maps_dilated[:,:,i])
- write_traversible = (maps*1.+1.)/3.0
- write_traversible = (write_traversible*255.).astype(np.uint8)[:,:,np.newaxis]
- write_traversible = write_traversible + np.zeros((1,1,3), dtype=np.uint8)
- fu.write_image(img_path, write_traversible[::-1,:,:])
-
- def _preprocess_for_task(self, seed):
- """Sets up the task field for doing navigation on the grid world."""
- if self.task is None or self.task.seed != seed:
- rng = np.random.RandomState(seed)
- origin_loc = get_graph_origin_loc(rng, self.traversible)
- self.task = utils.Foo(seed=seed, origin_loc=origin_loc,
- n_ori=self.task_params.n_ori)
- G = generate_graph(self.valid_fn_vec, self.task_params.step_size,
- self.task.n_ori, (0, 0, 0))
- gtG, nodes, nodes_to_id = convert_to_graph_tool(G)
- self.task.gtG = gtG
- self.task.nodes = nodes
- self.task.delta_theta = 2.0*np.pi/(self.task.n_ori*1.)
- self.task.nodes_to_id = nodes_to_id
-
- logging.info('Building %s, #V=%d, #E=%d', self.building_name,
- self.task.nodes.shape[0], self.task.gtG.num_edges())
- type = self.task_params.type
- if type == 'general':
- # Do nothing
- _ = None
-
- elif type == 'room_to_room_many' or type == 'room_to_room_back':
- if type == 'room_to_room_back':
- assert(self.task_params.num_goals == 2), 'num_goals must be 2.'
-
- self.room_dims = _filter_rooms(self.room_dims, self.task_params.room_regex)
- xyt = self.to_actual_xyt_vec(self.task.nodes)
- self.task.node_room_ids = _label_nodes_with_room_id(xyt, self.room_dims)
- self.task.reset_kwargs = {'node_room_ids': self.task.node_room_ids}
-
- elif type == 'rng_rejection_sampling_many':
- n_bins = 20
- rejection_sampling_M = self.task_params.rejection_sampling_M
- min_dist = self.task_params.min_dist
- bins = np.arange(n_bins+1)/(n_bins*1.)
- target_d = np.zeros(n_bins); target_d[...] = 1./n_bins;
-
- sampling_d = get_hardness_distribution(
- self.task.gtG, self.task_params.max_dist, self.task_params.min_dist,
- np.random.RandomState(0), 4000, bins, self.task.nodes,
- self.task_params.n_ori, self.task_params.step_size)
-
- self.task.reset_kwargs = {'distribution_bins': bins,
- 'target_distribution': target_d,
- 'sampling_distribution': sampling_d,
- 'rejection_sampling_M': rejection_sampling_M,
- 'n_bins': n_bins,
- 'n_ori': self.task_params.n_ori,
- 'step_size': self.task_params.step_size,
- 'min_dist': self.task_params.min_dist}
- self.task.n_bins = n_bins
- self.task.distribution_bins = bins
- self.task.target_distribution = target_d
- self.task.sampling_distribution = sampling_d
- self.task.rejection_sampling_M = rejection_sampling_M
-
- if self.logdir is not None:
- self._debug_save_hardness(seed)
-
- elif type[:14] == 'to_nearest_obj':
- self.room_dims = _filter_rooms(self.room_dims, self.task_params.room_regex)
- xyt = self.to_actual_xyt_vec(self.task.nodes)
-
- self.class_maps = _select_classes(self.class_maps,
- self.class_map_names,
- self.task_params.semantic_task.class_map_names)*1
- self.class_map_names = self.task_params.semantic_task.class_map_names
- nodes_xyt = self.to_actual_xyt_vec(np.array(self.task.nodes))
-
- tt = utils.Timer(); tt.tic();
- if self.task_params.type == 'to_nearest_obj_acc':
- self.task.class_maps_dilated, self.task.node_class_label = label_nodes_with_class_geodesic(
- nodes_xyt, self.class_maps,
- self.task_params.semantic_task.pix_distance+8, self.map.traversible,
- ff_cost=1., fo_cost=1., oo_cost=4., connectivity=8.)
-
- dists = []
- for i in range(len(self.class_map_names)):
- class_nodes_ = np.where(self.task.node_class_label[:,i])[0]
- dists.append(get_distance_node_list(gtG, source_nodes=class_nodes_, direction='to'))
- self.task.dist_to_class = dists
- a_, b_ = np.where(self.task.node_class_label)
- self.task.class_nodes = np.concatenate((a_[:,np.newaxis], b_[:,np.newaxis]), axis=1)
-
- if self.logdir is not None:
- self._debug_semantic_maps(seed)
-
- self.task.reset_kwargs = {'sampling': self.task_params.semantic_task.sampling,
- 'class_nodes': self.task.class_nodes,
- 'dist_to_class': self.task.dist_to_class}
-
- if self.logdir is not None:
- self._debug_save_map_nodes(seed)
-
- def reset(self, rngs):
- rng = rngs[0]; rng_perturb = rngs[1];
- nodes = self.task.nodes
- tp = self.task_params
-
- start_node_ids, goal_node_ids, dists, target_class = \
- _nav_env_reset_helper(tp.type, rng, self.task.nodes, tp.batch_size,
- self.task.gtG, tp.max_dist, tp.num_steps,
- tp.num_goals, tp.data_augment,
- **(self.task.reset_kwargs))
-
- start_nodes = [tuple(nodes[_,:]) for _ in start_node_ids]
- goal_nodes = [[tuple(nodes[_,:]) for _ in __] for __ in goal_node_ids]
- data_augment = tp.data_augment
- perturbs = _gen_perturbs(rng_perturb, tp.batch_size,
- (tp.num_steps+1)*tp.num_goals,
- data_augment.lr_flip, data_augment.delta_angle,
- data_augment.delta_xy, data_augment.structured)
- perturbs = np.array(perturbs) # batch x steps x 4
- end_perturbs = perturbs[:,-(tp.num_goals):,:]*1 # fixed perturb for the goal.
- perturbs = perturbs[:,:-(tp.num_goals),:]*1
-
- history = -np.ones((tp.batch_size, tp.num_steps*tp.num_goals), dtype=np.int32)
- self.episode = utils.Foo(
- start_nodes=start_nodes, start_node_ids=start_node_ids,
- goal_nodes=goal_nodes, goal_node_ids=goal_node_ids, dist_to_goal=dists,
- perturbs=perturbs, goal_perturbs=end_perturbs, history=history,
- target_class=target_class, history_frames=[])
- return start_node_ids
-
- def take_action(self, current_node_ids, action, step_number):
- """In addition to returning the action, also returns the reward that the
- agent receives."""
- goal_number = step_number / self.task_params.num_steps
- new_node_ids = GridWorld.take_action(self, current_node_ids, action)
- rewards = []
- for i, n in enumerate(new_node_ids):
- reward = 0
- if n == self.episode.goal_node_ids[goal_number][i]:
- reward = self.task_params.reward_at_goal
- reward = reward - self.task_params.reward_time_penalty
- rewards.append(reward)
- return new_node_ids, rewards
-
-
- def get_optimal_action(self, current_node_ids, step_number):
- """Returns the optimal action from the current node."""
- goal_number = step_number / self.task_params.num_steps
- gtG = self.task.gtG
- a = np.zeros((len(current_node_ids), self.task_params.num_actions), dtype=np.int32)
- d_dict = self.episode.dist_to_goal[goal_number]
- for i, c in enumerate(current_node_ids):
- neigh = gtG.vertex(c).out_neighbours()
- neigh_edge = gtG.vertex(c).out_edges()
- ds = np.array([d_dict[i][int(x)] for x in neigh])
- ds_min = np.min(ds)
- for i_, e in enumerate(neigh_edge):
- if ds[i_] == ds_min:
- _ = gtG.ep['action'][e]
- a[i, _] = 1
- return a
-
- def get_targets(self, current_node_ids, step_number):
- """Returns the target actions from the current node."""
- action = self.get_optimal_action(current_node_ids, step_number)
- action = np.expand_dims(action, axis=1)
- return vars(utils.Foo(action=action))
-
- def get_targets_name(self):
- """Returns the list of names of the targets."""
- return ['action']
-
- def cleanup(self):
- self.episode = None
-
-class VisualNavigationEnv(NavigationEnv):
- """Class for doing visual navigation in environments. Functions for computing
- features on states, etc.
- """
- def __init__(self, robot, env, task_params, category_list=None,
- building_name=None, flip=False, logdir=None,
- building_loader=None, r_obj=None):
- tt = utils.Timer()
- tt.tic()
- Building.__init__(self, building_name, robot, env, category_list,
- small=task_params.toy_problem, flip=flip, logdir=logdir,
- building_loader=building_loader)
-
- self.set_r_obj(r_obj)
- self.task_params = task_params
- self.task = None
- self.episode = None
- self._preprocess_for_task(self.task_params.building_seed)
- if hasattr(self.task_params, 'map_scales'):
- self.task.scaled_maps = resize_maps(
- self.traversible.astype(np.float32)*1, self.task_params.map_scales,
- self.task_params.map_resize_method)
- else:
- logging.fatal('VisualNavigationEnv does not support scale_f anymore.')
- self.task.readout_maps_scaled = resize_maps(
- self.traversible.astype(np.float32)*1,
- self.task_params.readout_maps_scales,
- self.task_params.map_resize_method)
- tt.toc(log_at=1, log_str='VisualNavigationEnv __init__: ')
-
- def get_weight(self):
- return self.task.nodes.shape[0]
-
- def get_common_data(self):
- goal_nodes = self.episode.goal_nodes
- start_nodes = self.episode.start_nodes
- perturbs = self.episode.perturbs
- goal_perturbs = self.episode.goal_perturbs
- target_class = self.episode.target_class
-
- goal_locs = []; rel_goal_locs = [];
- for i in range(len(goal_nodes)):
- end_nodes = goal_nodes[i]
- goal_loc, _, _, goal_theta = self.get_loc_axis(
- np.array(end_nodes), delta_theta=self.task.delta_theta,
- perturb=goal_perturbs[:,i,:])
-
- # Compute the relative location to all goals from the starting location.
- loc, _, _, theta = self.get_loc_axis(np.array(start_nodes),
- delta_theta=self.task.delta_theta,
- perturb=perturbs[:,0,:])
- r_goal, t_goal = _get_relative_goal_loc(goal_loc*1., loc, theta)
- rel_goal_loc = np.concatenate((r_goal*np.cos(t_goal), r_goal*np.sin(t_goal),
- np.cos(goal_theta-theta),
- np.sin(goal_theta-theta)), axis=1)
- rel_goal_locs.append(np.expand_dims(rel_goal_loc, axis=1))
- goal_locs.append(np.expand_dims(goal_loc, axis=1))
-
- map = self.traversible*1.
- maps = np.repeat(np.expand_dims(np.expand_dims(map, axis=0), axis=0),
- self.task_params.batch_size, axis=0)*1
- if self.task_params.type[:14] == 'to_nearest_obj':
- for i in range(self.task_params.batch_size):
- maps[i,0,:,:] += 0.5*(self.task.class_maps_dilated[:,:,target_class[i]])
-
- rel_goal_locs = np.concatenate(rel_goal_locs, axis=1)
- goal_locs = np.concatenate(goal_locs, axis=1)
- maps = np.expand_dims(maps, axis=-1)
-
- if self.task_params.type[:14] == 'to_nearest_obj':
- rel_goal_locs = np.zeros((self.task_params.batch_size, 1,
- len(self.task_params.semantic_task.class_map_names)),
- dtype=np.float32)
- goal_locs = np.zeros((self.task_params.batch_size, 1, 2),
- dtype=np.float32)
- for i in range(self.task_params.batch_size):
- t = target_class[i]
- rel_goal_locs[i,0,t] = 1.
- goal_locs[i,0,0] = t
- goal_locs[i,0,1] = np.NaN
-
- return vars(utils.Foo(orig_maps=maps, goal_loc=goal_locs,
- rel_goal_loc_at_start=rel_goal_locs))
-
- def pre_common_data(self, inputs):
- return inputs
-
-
- def get_features(self, current_node_ids, step_number):
- task_params = self.task_params
- goal_number = step_number / self.task_params.num_steps
- end_nodes = self.task.nodes[self.episode.goal_node_ids[goal_number],:]*1
- current_nodes = self.task.nodes[current_node_ids,:]*1
- end_perturbs = self.episode.goal_perturbs[:,goal_number,:][:,np.newaxis,:]
- perturbs = self.episode.perturbs
- target_class = self.episode.target_class
-
- # Append to history.
- self.episode.history[:,step_number] = np.array(current_node_ids)
-
- # Render out the images from current node.
- outs = {}
-
- if self.task_params.outputs.images:
- imgs_all = []
- imgs = self.render_nodes([tuple(x) for x in current_nodes],
- perturb=perturbs[:,step_number,:])
- imgs_all.append(imgs)
- aux_delta_thetas = self.task_params.aux_delta_thetas
- for i in range(len(aux_delta_thetas)):
- imgs = self.render_nodes([tuple(x) for x in current_nodes],
- perturb=perturbs[:,step_number,:],
- aux_delta_theta=aux_delta_thetas[i])
- imgs_all.append(imgs)
- imgs_all = np.array(imgs_all) # A x B x H x W x C
- imgs_all = np.transpose(imgs_all, axes=[1,0,2,3,4])
- imgs_all = np.expand_dims(imgs_all, axis=1) # B x N x A x H x W x C
- if task_params.num_history_frames > 0:
- if step_number == 0:
- # Append the same frame 4 times
- for i in range(task_params.num_history_frames+1):
- self.episode.history_frames.insert(0, imgs_all*1.)
- self.episode.history_frames.insert(0, imgs_all)
- self.episode.history_frames.pop()
- imgs_all_with_history = np.concatenate(self.episode.history_frames, axis=2)
- else:
- imgs_all_with_history = imgs_all
- outs['imgs'] = imgs_all_with_history # B x N x A x H x W x C
-
- if self.task_params.outputs.node_ids:
- outs['node_ids'] = np.array(current_node_ids).reshape((-1,1,1))
- outs['perturbs'] = np.expand_dims(perturbs[:,step_number, :]*1., axis=1)
-
- if self.task_params.outputs.analytical_counts:
- assert(self.task_params.modalities == ['depth'])
- d = image_pre(outs['imgs']*1., self.task_params.modalities)
- cm = get_camera_matrix(self.task_params.img_width,
- self.task_params.img_height,
- self.task_params.img_fov)
- XYZ = get_point_cloud_from_z(100./d[...,0], cm)
- XYZ = make_geocentric(XYZ*100., self.robot.sensor_height,
- self.robot.camera_elevation_degree)
- for i in range(len(self.task_params.analytical_counts.map_sizes)):
- non_linearity = self.task_params.analytical_counts.non_linearity[i]
- count, isvalid = bin_points(XYZ*1.,
- map_size=self.task_params.analytical_counts.map_sizes[i],
- xy_resolution=self.task_params.analytical_counts.xy_resolution[i],
- z_bins=self.task_params.analytical_counts.z_bins[i])
- assert(count.shape[2] == 1), 'only works for n_views equal to 1.'
- count = count[:,:,0,:,:,:]
- isvalid = isvalid[:,:,0,:,:,:]
- if non_linearity == 'none':
- None
- elif non_linearity == 'min10':
- count = np.minimum(count, 10.)
- elif non_linearity == 'sqrt':
- count = np.sqrt(count)
- else:
- logging.fatal('Undefined non_linearity.')
- outs['analytical_counts_{:d}'.format(i)] = count
-
- # Compute the goal location in the cordinate frame of the robot.
- if self.task_params.outputs.rel_goal_loc:
- if self.task_params.type[:14] != 'to_nearest_obj':
- loc, _, _, theta = self.get_loc_axis(current_nodes,
- delta_theta=self.task.delta_theta,
- perturb=perturbs[:,step_number,:])
- goal_loc, _, _, goal_theta = self.get_loc_axis(end_nodes,
- delta_theta=self.task.delta_theta,
- perturb=end_perturbs[:,0,:])
- r_goal, t_goal = _get_relative_goal_loc(goal_loc, loc, theta)
-
- rel_goal_loc = np.concatenate((r_goal*np.cos(t_goal), r_goal*np.sin(t_goal),
- np.cos(goal_theta-theta),
- np.sin(goal_theta-theta)), axis=1)
- outs['rel_goal_loc'] = np.expand_dims(rel_goal_loc, axis=1)
- elif self.task_params.type[:14] == 'to_nearest_obj':
- rel_goal_loc = np.zeros((self.task_params.batch_size, 1,
- len(self.task_params.semantic_task.class_map_names)),
- dtype=np.float32)
- for i in range(self.task_params.batch_size):
- t = target_class[i]
- rel_goal_loc[i,0,t] = 1.
- outs['rel_goal_loc'] = rel_goal_loc
-
- # Location on map to plot the trajectory during validation.
- if self.task_params.outputs.loc_on_map:
- loc, x_axis, y_axis, theta = self.get_loc_axis(current_nodes,
- delta_theta=self.task.delta_theta,
- perturb=perturbs[:,step_number,:])
- outs['loc_on_map'] = np.expand_dims(loc, axis=1)
-
- # Compute gt_dist to goal
- if self.task_params.outputs.gt_dist_to_goal:
- gt_dist_to_goal = np.zeros((len(current_node_ids), 1), dtype=np.float32)
- for i, n in enumerate(current_node_ids):
- gt_dist_to_goal[i,0] = self.episode.dist_to_goal[goal_number][i][n]
- outs['gt_dist_to_goal'] = np.expand_dims(gt_dist_to_goal, axis=1)
-
- # Free space in front of you, map and goal as images.
- if self.task_params.outputs.ego_maps:
- loc, x_axis, y_axis, theta = self.get_loc_axis(current_nodes,
- delta_theta=self.task.delta_theta,
- perturb=perturbs[:,step_number,:])
- maps = generate_egocentric_maps(self.task.scaled_maps,
- self.task_params.map_scales,
- self.task_params.map_crop_sizes, loc,
- x_axis, y_axis, theta)
-
- for i in range(len(self.task_params.map_scales)):
- outs['ego_maps_{:d}'.format(i)] = \
- np.expand_dims(np.expand_dims(maps[i], axis=1), axis=-1)
-
- if self.task_params.outputs.readout_maps:
- loc, x_axis, y_axis, theta = self.get_loc_axis(current_nodes,
- delta_theta=self.task.delta_theta,
- perturb=perturbs[:,step_number,:])
- maps = generate_egocentric_maps(self.task.readout_maps_scaled,
- self.task_params.readout_maps_scales,
- self.task_params.readout_maps_crop_sizes,
- loc, x_axis, y_axis, theta)
- for i in range(len(self.task_params.readout_maps_scales)):
- outs['readout_maps_{:d}'.format(i)] = \
- np.expand_dims(np.expand_dims(maps[i], axis=1), axis=-1)
-
- # Images for the goal.
- if self.task_params.outputs.ego_goal_imgs:
- if self.task_params.type[:14] != 'to_nearest_obj':
- loc, x_axis, y_axis, theta = self.get_loc_axis(current_nodes,
- delta_theta=self.task.delta_theta,
- perturb=perturbs[:,step_number,:])
- goal_loc, _, _, _ = self.get_loc_axis(end_nodes,
- delta_theta=self.task.delta_theta,
- perturb=end_perturbs[:,0,:])
- rel_goal_orientation = np.mod(
- np.int32(current_nodes[:,2:] - end_nodes[:,2:]), self.task_params.n_ori)
- goal_dist, goal_theta = _get_relative_goal_loc(goal_loc, loc, theta)
- goals = generate_goal_images(self.task_params.map_scales,
- self.task_params.map_crop_sizes,
- self.task_params.n_ori, goal_dist,
- goal_theta, rel_goal_orientation)
- for i in range(len(self.task_params.map_scales)):
- outs['ego_goal_imgs_{:d}'.format(i)] = np.expand_dims(goals[i], axis=1)
-
- elif self.task_params.type[:14] == 'to_nearest_obj':
- for i in range(len(self.task_params.map_scales)):
- num_classes = len(self.task_params.semantic_task.class_map_names)
- outs['ego_goal_imgs_{:d}'.format(i)] = np.zeros((self.task_params.batch_size, 1,
- self.task_params.map_crop_sizes[i],
- self.task_params.map_crop_sizes[i],
- self.task_params.goal_channels))
- for i in range(self.task_params.batch_size):
- t = target_class[i]
- for j in range(len(self.task_params.map_scales)):
- outs['ego_goal_imgs_{:d}'.format(j)][i,:,:,:,t] = 1.
-
- # Incremental locs and theta (for map warping), always in the original scale
- # of the map, the subequent steps in the tf code scale appropriately.
- # Scaling is done by just multiplying incremental_locs appropriately.
- if self.task_params.outputs.egomotion:
- if step_number == 0:
- # Zero Ego Motion
- incremental_locs = np.zeros((self.task_params.batch_size, 1, 2), dtype=np.float32)
- incremental_thetas = np.zeros((self.task_params.batch_size, 1, 1), dtype=np.float32)
- else:
- previous_nodes = self.task.nodes[self.episode.history[:,step_number-1], :]*1
- loc, _, _, theta = self.get_loc_axis(current_nodes,
- delta_theta=self.task.delta_theta,
- perturb=perturbs[:,step_number,:])
- previous_loc, _, _, previous_theta = self.get_loc_axis(
- previous_nodes, delta_theta=self.task.delta_theta,
- perturb=perturbs[:,step_number-1,:])
-
- incremental_locs_ = np.reshape(loc-previous_loc, [self.task_params.batch_size, 1, -1])
-
- t = -np.pi/2+np.reshape(theta*1, [self.task_params.batch_size, 1, -1])
- incremental_locs = incremental_locs_*1
- incremental_locs[:,:,0] = np.sum(incremental_locs_ *
- np.concatenate((np.cos(t), np.sin(t)),
- axis=-1), axis=-1)
- incremental_locs[:,:,1] = np.sum(incremental_locs_ *
- np.concatenate((np.cos(t+np.pi/2),
- np.sin(t+np.pi/2)),
- axis=-1), axis=-1)
- incremental_thetas = np.reshape(theta-previous_theta,
- [self.task_params.batch_size, 1, -1])
- outs['incremental_locs'] = incremental_locs
- outs['incremental_thetas'] = incremental_thetas
-
- if self.task_params.outputs.visit_count:
- # Output the visit count for this state, how many times has the current
- # state been visited, and how far in the history was the last visit
- # (except this one)
- visit_count = np.zeros((self.task_params.batch_size, 1), dtype=np.int32)
- last_visit = -np.ones((self.task_params.batch_size, 1), dtype=np.int32)
- if step_number >= 1:
- h = self.episode.history[:,:(step_number)]
- visit_count[:,0] = np.sum(h == np.array(current_node_ids).reshape([-1,1]),
- axis=1)
- last_visit[:,0] = np.argmax(h[:,::-1] == np.array(current_node_ids).reshape([-1,1]),
- axis=1) + 1
- last_visit[visit_count == 0] = -1 # -1 if not visited.
- outs['visit_count'] = np.expand_dims(visit_count, axis=1)
- outs['last_visit'] = np.expand_dims(last_visit, axis=1)
- return outs
-
- def get_features_name(self):
- f = []
- if self.task_params.outputs.images:
- f.append('imgs')
- if self.task_params.outputs.rel_goal_loc:
- f.append('rel_goal_loc')
- if self.task_params.outputs.loc_on_map:
- f.append('loc_on_map')
- if self.task_params.outputs.gt_dist_to_goal:
- f.append('gt_dist_to_goal')
- if self.task_params.outputs.ego_maps:
- for i in range(len(self.task_params.map_scales)):
- f.append('ego_maps_{:d}'.format(i))
- if self.task_params.outputs.readout_maps:
- for i in range(len(self.task_params.readout_maps_scales)):
- f.append('readout_maps_{:d}'.format(i))
- if self.task_params.outputs.ego_goal_imgs:
- for i in range(len(self.task_params.map_scales)):
- f.append('ego_goal_imgs_{:d}'.format(i))
- if self.task_params.outputs.egomotion:
- f.append('incremental_locs')
- f.append('incremental_thetas')
- if self.task_params.outputs.visit_count:
- f.append('visit_count')
- f.append('last_visit')
- if self.task_params.outputs.analytical_counts:
- for i in range(len(self.task_params.analytical_counts.map_sizes)):
- f.append('analytical_counts_{:d}'.format(i))
- if self.task_params.outputs.node_ids:
- f.append('node_ids')
- f.append('perturbs')
- return f
-
- def pre_features(self, inputs):
- if self.task_params.outputs.images:
- inputs['imgs'] = image_pre(inputs['imgs'], self.task_params.modalities)
- return inputs
-
-class BuildingMultiplexer():
- def __init__(self, args, task_number):
- params = vars(args)
- for k in params.keys():
- setattr(self, k, params[k])
- self.task_number = task_number
- self._pick_data(task_number)
- logging.info('Env Class: %s.', self.env_class)
- if self.task_params.task == 'planning':
- self._setup_planner()
- elif self.task_params.task == 'mapping':
- self._setup_mapper()
- elif self.task_params.task == 'map+plan':
- self._setup_mapper()
- else:
- logging.error('Undefined task: %s'.format(self.task_params.task))
-
- def _pick_data(self, task_number):
- logging.error('Input Building Names: %s', self.building_names)
- self.flip = [np.mod(task_number / len(self.building_names), 2) == 1]
- id = np.mod(task_number, len(self.building_names))
- self.building_names = [self.building_names[id]]
- self.task_params.building_seed = task_number
- logging.error('BuildingMultiplexer: Picked Building Name: %s', self.building_names)
- self.building_names = self.building_names[0].split('+')
- self.flip = [self.flip[0] for _ in self.building_names]
- logging.error('BuildingMultiplexer: Picked Building Name: %s', self.building_names)
- logging.error('BuildingMultiplexer: Flipping Buildings: %s', self.flip)
- logging.error('BuildingMultiplexer: Set building_seed: %d', self.task_params.building_seed)
- self.num_buildings = len(self.building_names)
- logging.error('BuildingMultiplexer: Num buildings: %d', self.num_buildings)
-
- def _setup_planner(self):
- # Load building env class.
- self.buildings = []
- for i, building_name in enumerate(self.building_names):
- b = self.env_class(robot=self.robot, env=self.env,
- task_params=self.task_params,
- building_name=building_name, flip=self.flip[i],
- logdir=self.logdir, building_loader=self.dataset)
- self.buildings.append(b)
-
- def _setup_mapper(self):
- # Set up the renderer.
- cp = self.camera_param
- rgb_shader, d_shader = sru.get_shaders(cp.modalities)
- r_obj = SwiftshaderRenderer()
- r_obj.init_display(width=cp.width, height=cp.height, fov=cp.fov,
- z_near=cp.z_near, z_far=cp.z_far, rgb_shader=rgb_shader,
- d_shader=d_shader)
- self.r_obj = r_obj
- r_obj.clear_scene()
-
- # Load building env class.
- self.buildings = []
- wt = []
- for i, building_name in enumerate(self.building_names):
- b = self.env_class(robot=self.robot, env=self.env,
- task_params=self.task_params,
- building_name=building_name, flip=self.flip[i],
- logdir=self.logdir, building_loader=self.dataset,
- r_obj=r_obj)
- wt.append(b.get_weight())
- b.load_building_into_scene()
- b.set_building_visibility(False)
- self.buildings.append(b)
- wt = np.array(wt).astype(np.float32)
- wt = wt / np.sum(wt+0.0001)
- self.building_sampling_weights = wt
-
- def sample_building(self, rng):
- if self.num_buildings == 1:
- building_id = rng.choice(range(len(self.building_names)))
- else:
- building_id = rng.choice(self.num_buildings,
- p=self.building_sampling_weights)
- b = self.buildings[building_id]
- instances = b._gen_rng(rng)
- self._building_id = building_id
- return self.buildings[building_id], instances
-
- def sample_env(self, rngs):
- rng = rngs[0];
- if self.num_buildings == 1:
- building_id = rng.choice(range(len(self.building_names)))
- else:
- building_id = rng.choice(self.num_buildings,
- p=self.building_sampling_weights)
- return self.buildings[building_id]
-
- def pre(self, inputs):
- return self.buildings[self._building_id].pre(inputs)
-
- def __del__(self):
- self.r_obj.clear_scene()
- logging.error('Clearing scene.')
diff --git a/research/cognitive_mapping_and_planning/datasets/nav_env_config.py b/research/cognitive_mapping_and_planning/datasets/nav_env_config.py
deleted file mode 100644
index 3d71c5767c4dc0ed9f05cce5c1790f11ede3778a..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/datasets/nav_env_config.py
+++ /dev/null
@@ -1,127 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Configs for stanford navigation environment.
-
-Base config for stanford navigation enviornment.
-"""
-import numpy as np
-import src.utils as utils
-import datasets.nav_env as nav_env
-
-def nav_env_base_config():
- """Returns the base config for stanford navigation environment.
-
- Returns:
- Base config for stanford navigation environment.
- """
- robot = utils.Foo(radius=15,
- base=10,
- height=140,
- sensor_height=120,
- camera_elevation_degree=-15)
-
- env = utils.Foo(padding=10,
- resolution=5,
- num_point_threshold=2,
- valid_min=-10,
- valid_max=200,
- n_samples_per_face=200)
-
- camera_param = utils.Foo(width=225,
- height=225,
- z_near=0.05,
- z_far=20.0,
- fov=60.,
- modalities=['rgb'],
- img_channels=3)
-
- data_augment = utils.Foo(lr_flip=0,
- delta_angle=0.5,
- delta_xy=4,
- relight=True,
- relight_fast=False,
- structured=False) # if True, uses the same perturb for the whole episode.
-
- outputs = utils.Foo(images=True,
- rel_goal_loc=False,
- loc_on_map=True,
- gt_dist_to_goal=True,
- ego_maps=False,
- ego_goal_imgs=False,
- egomotion=False,
- visit_count=False,
- analytical_counts=False,
- node_ids=True,
- readout_maps=False)
-
- # class_map_names=['board', 'chair', 'door', 'sofa', 'table']
- class_map_names = ['chair', 'door', 'table']
- semantic_task = utils.Foo(class_map_names=class_map_names, pix_distance=16,
- sampling='uniform')
-
- # time per iteration for cmp is 0.82 seconds per episode with 3.4s overhead per batch.
- task_params = utils.Foo(max_dist=32,
- step_size=8,
- num_steps=40,
- num_actions=4,
- batch_size=4,
- building_seed=0,
- num_goals=1,
- img_height=None,
- img_width=None,
- img_channels=None,
- modalities=None,
- outputs=outputs,
- map_scales=[1.],
- map_crop_sizes=[64],
- rel_goal_loc_dim=4,
- base_class='Building',
- task='map+plan',
- n_ori=4,
- type='room_to_room_many',
- data_augment=data_augment,
- room_regex='^((?!hallway).)*$',
- toy_problem=False,
- map_channels=1,
- gt_coverage=False,
- input_type='maps',
- full_information=False,
- aux_delta_thetas=[],
- semantic_task=semantic_task,
- num_history_frames=0,
- node_ids_dim=1,
- perturbs_dim=4,
- map_resize_method='linear_noantialiasing',
- readout_maps_channels=1,
- readout_maps_scales=[],
- readout_maps_crop_sizes=[],
- n_views=1,
- reward_time_penalty=0.1,
- reward_at_goal=1.,
- discount_factor=0.99,
- rejection_sampling_M=100,
- min_dist=None)
-
- navtask_args = utils.Foo(
- building_names=['area1_gates_wingA_floor1_westpart'],
- env_class=nav_env.VisualNavigationEnv,
- robot=robot,
- task_params=task_params,
- env=env,
- camera_param=camera_param,
- cache_rooms=True)
- return navtask_args
-
diff --git a/research/cognitive_mapping_and_planning/matplotlibrc b/research/cognitive_mapping_and_planning/matplotlibrc
deleted file mode 100644
index ed5097572ae68680d0c9afdf510968e1c3d175d4..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/matplotlibrc
+++ /dev/null
@@ -1 +0,0 @@
-backend : agg
diff --git a/research/cognitive_mapping_and_planning/output/.gitignore b/research/cognitive_mapping_and_planning/output/.gitignore
deleted file mode 100644
index a767cafbbd864d0baf76530294598e4c2be60a24..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/output/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*
diff --git a/research/cognitive_mapping_and_planning/output/README.md b/research/cognitive_mapping_and_planning/output/README.md
deleted file mode 100644
index 7518c3874390da7e2aa65a89ccdec035ca7610e8..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/output/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-### Pre-Trained Models
-
-We provide the following pre-trained models:
-
-Config Name | Checkpoint | Mean Dist. | 50%ile Dist. | 75%ile Dist. | Success %age |
-:-: | :-: | :-: | :-: | :-: | :-: |
-cmp.lmap_Msc.clip5.sbpd_d_r2r | [ckpt](http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/cmp.lmap_Msc.clip5.sbpd_d_r2r.tar) | 4.79 | 0 | 1 | 78.9 |
-cmp.lmap_Msc.clip5.sbpd_rgb_r2r | [ckpt](http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/cmp.lmap_Msc.clip5.sbpd_rgb_r2r.tar) | 7.74 | 0 | 14 | 62.4 |
-cmp.lmap_Msc.clip5.sbpd_d_ST | [ckpt](http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/cmp.lmap_Msc.clip5.sbpd_d_ST.tar) | 10.67 | 9 | 19 | 39.7 |
-cmp.lmap_Msc.clip5.sbpd_rgb_ST | [ckpt](http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/cmp.lmap_Msc.clip5.sbpd_rgb_ST.tar) | 11.27 | 10 | 19 | 35.6 |
-cmp.lmap_Msc.clip5.sbpd_d_r2r_h0_64_80 | [ckpt](http:////download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/cmp.lmap_Msc.clip5.sbpd_d_r2r_h0_64_80.tar) | 11.6 | 0 | 19 | 66.9 |
-bl.v2.noclip.sbpd_d_r2r | [ckpt](http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/bl.v2.noclip.sbpd_d_r2r.tar) | 5.90 | 0 | 6 | 71.2 |
-bl.v2.noclip.sbpd_rgb_r2r | [ckpt](http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/bl.v2.noclip.sbpd_rgb_r2r.tar) | 10.21 | 1 | 21 | 53.4 |
-bl.v2.noclip.sbpd_d_ST | [ckpt](http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/bl.v2.noclip.sbpd_d_ST.tar) | 13.29 | 14 | 23 | 28.0 |
-bl.v2.noclip.sbpd_rgb_ST | [ckpt](http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/bl.v2.noclip.sbpd_rgb_ST.tar) | 13.37 | 13 | 20 | 24.2 |
-bl.v2.noclip.sbpd_d_r2r_h0_64_80 | [ckpt](http:////download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/bl.v2.noclip.sbpd_d_r2r_h0_64_80.tar) | 15.30 | 0 | 29 | 57.9 |
diff --git a/research/cognitive_mapping_and_planning/patches/GLES2_2_0.py.patch b/research/cognitive_mapping_and_planning/patches/GLES2_2_0.py.patch
deleted file mode 100644
index de1be442d5b9fff44862d37b9329e32face2b663..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/patches/GLES2_2_0.py.patch
+++ /dev/null
@@ -1,14 +0,0 @@
-10c10
-< from OpenGL import platform, constant, arrays
----
-> from OpenGL import platform, constant, arrays, contextdata
-249a250
-> from OpenGL._bytes import _NULL_8_BYTE
-399c400
-< array = ArrayDatatype.asArray( pointer, type )
----
-> array = arrays.ArrayDatatype.asArray( pointer, type )
-405c406
-< ArrayDatatype.voidDataPointer( array )
----
-> arrays.ArrayDatatype.voidDataPointer( array )
diff --git a/research/cognitive_mapping_and_planning/patches/apply_patches.sh b/research/cognitive_mapping_and_planning/patches/apply_patches.sh
deleted file mode 100644
index 4a786058258decdfb381eff25684183d92788ebe..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/patches/apply_patches.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-echo $VIRTUAL_ENV
-patch $VIRTUAL_ENV/local/lib/python2.7/site-packages/OpenGL/GLES2/VERSION/GLES2_2_0.py patches/GLES2_2_0.py.patch
-patch $VIRTUAL_ENV/local/lib/python2.7/site-packages/OpenGL/platform/ctypesloader.py patches/ctypesloader.py.patch
diff --git a/research/cognitive_mapping_and_planning/patches/ctypesloader.py.patch b/research/cognitive_mapping_and_planning/patches/ctypesloader.py.patch
deleted file mode 100644
index 27dd43b18010dc5fdcd605b9a5d470abaa19151f..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/patches/ctypesloader.py.patch
+++ /dev/null
@@ -1,15 +0,0 @@
-45c45,46
-< return dllType( name, mode )
----
-> print './' + name
-> return dllType( './' + name, mode )
-47,48c48,53
-< err.args += (name,fullName)
-< raise
----
-> try:
-> print name
-> return dllType( name, mode )
-> except:
-> err.args += (name,fullName)
-> raise
diff --git a/research/cognitive_mapping_and_planning/render/__init__.py b/research/cognitive_mapping_and_planning/render/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/cognitive_mapping_and_planning/render/depth_rgb_encoded.fp b/research/cognitive_mapping_and_planning/render/depth_rgb_encoded.fp
deleted file mode 100644
index 23e93d27f585e93896799f177888e9c50fa03eed..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/render/depth_rgb_encoded.fp
+++ /dev/null
@@ -1,30 +0,0 @@
-// This shader computes per-pixel depth (-z coordinate in the camera space, or
-// orthogonal distance to the camera plane). The result is multiplied by the
-// `kFixedPointFraction` constant and is encoded to RGB channels as an integer
-// (R being the least significant byte).
-
-#ifdef GL_ES
-#ifdef GL_FRAGMENT_PRECISION_HIGH
-precision highp float;
-#else
-precision mediump float;
-#endif
-#endif
-
-const float kFixedPointFraction = 1000.0;
-
-varying float vDepth;
-
-void main(void) {
- float d = vDepth;
-
- // Encode the depth to RGB.
- d *= (kFixedPointFraction / 255.0);
- gl_FragColor.r = mod(d, 1.0);
- d = (d - gl_FragColor.r) / 255.0;
- gl_FragColor.g = mod(d, 1.0);
- d = (d - gl_FragColor.g) / 255.0;
- gl_FragColor.b = mod(d, 1.0);
-
- gl_FragColor.a = 1.0;
-}
diff --git a/research/cognitive_mapping_and_planning/render/depth_rgb_encoded.vp b/research/cognitive_mapping_and_planning/render/depth_rgb_encoded.vp
deleted file mode 100644
index 2db74f14aa7f253b8f544ec1ab519129f13426a0..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/render/depth_rgb_encoded.vp
+++ /dev/null
@@ -1,15 +0,0 @@
-uniform mat4 uViewMatrix;
-uniform mat4 uProjectionMatrix;
-
-attribute vec3 aPosition;
-
-varying float vDepth;
-
-void main(void) {
- vec4 worldPosition = vec4(aPosition, 1.0);
- vec4 viewPosition = uViewMatrix * worldPosition;
- gl_Position = uProjectionMatrix * viewPosition;
-
- // Orthogonal depth is simply -z in the camera space.
- vDepth = -viewPosition.z;
-}
diff --git a/research/cognitive_mapping_and_planning/render/rgb_flat_color.fp b/research/cognitive_mapping_and_planning/render/rgb_flat_color.fp
deleted file mode 100644
index c8c24d76103793d9cfa9166517177cb332d1a92c..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/render/rgb_flat_color.fp
+++ /dev/null
@@ -1,11 +0,0 @@
-precision highp float;
-varying vec4 vColor;
-varying vec2 vTextureCoord;
-
-uniform sampler2D uTexture;
-
-void main(void) {
- vec4 color = vColor;
- color = texture2D(uTexture, vTextureCoord);
- gl_FragColor = color;
-}
diff --git a/research/cognitive_mapping_and_planning/render/rgb_flat_color.vp b/research/cognitive_mapping_and_planning/render/rgb_flat_color.vp
deleted file mode 100644
index ebc79173405f7449921fd40f778fe3695aab5ea8..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/render/rgb_flat_color.vp
+++ /dev/null
@@ -1,18 +0,0 @@
-uniform mat4 uViewMatrix;
-uniform mat4 uProjectionMatrix;
-uniform vec4 uColor;
-
-attribute vec4 aColor;
-attribute vec3 aPosition;
-attribute vec2 aTextureCoord;
-
-varying vec4 vColor;
-varying vec2 vTextureCoord;
-
-void main(void) {
- vec4 worldPosition = vec4(aPosition, 1.0);
- gl_Position = uProjectionMatrix * (uViewMatrix * worldPosition);
-
- vColor = aColor * uColor;
- vTextureCoord = aTextureCoord;
-}
diff --git a/research/cognitive_mapping_and_planning/render/swiftshader_renderer.py b/research/cognitive_mapping_and_planning/render/swiftshader_renderer.py
deleted file mode 100644
index 74b1be72c11a2877231a66886d02babfd4793ce8..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/render/swiftshader_renderer.py
+++ /dev/null
@@ -1,427 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""Implements loading and rendering of meshes. Contains 2 classes:
- Shape: Class that exposes high level functions for loading and manipulating
- shapes. This currently is bound to assimp
- (https://github.com/assimp/assimp). If you want to interface to a different
- library, reimplement this class with bindings to your mesh loading library.
-
- SwiftshaderRenderer: Class that renders Shapes. Currently this uses python
- bindings to OpenGL (EGL), bindings to an alternate renderer may be implemented
- here.
-"""
-
-import numpy as np, os
-import cv2, ctypes, logging, os, numpy as np
-import pyassimp as assimp
-from OpenGL.GLES2 import *
-from OpenGL.EGL import *
-import src.rotation_utils as ru
-
-__version__ = 'swiftshader_renderer'
-
-def get_shaders(modalities):
- rgb_shader = 'rgb_flat_color' if 'rgb' in modalities else None
- d_shader = 'depth_rgb_encoded' if 'depth' in modalities else None
- return rgb_shader, d_shader
-
-def sample_points_on_faces(vs, fs, rng, n_samples_per_face):
- idx = np.repeat(np.arange(fs.shape[0]), n_samples_per_face)
-
- r = rng.rand(idx.size, 2)
- r1 = r[:,:1]; r2 = r[:,1:]; sqrt_r1 = np.sqrt(r1);
-
- v1 = vs[fs[idx, 0], :]; v2 = vs[fs[idx, 1], :]; v3 = vs[fs[idx, 2], :];
- pts = (1-sqrt_r1)*v1 + sqrt_r1*(1-r2)*v2 + sqrt_r1*r2*v3
-
- v1 = vs[fs[:,0], :]; v2 = vs[fs[:, 1], :]; v3 = vs[fs[:, 2], :];
- ar = 0.5*np.sqrt(np.sum(np.cross(v1-v3, v2-v3)**2, 1))
-
- return pts, ar, idx
-
-class Shape():
- def get_pyassimp_load_options(self):
- load_flags = assimp.postprocess.aiProcess_Triangulate;
- load_flags = load_flags | assimp.postprocess.aiProcess_SortByPType;
- load_flags = load_flags | assimp.postprocess.aiProcess_OptimizeMeshes;
- load_flags = load_flags | assimp.postprocess.aiProcess_RemoveRedundantMaterials;
- load_flags = load_flags | assimp.postprocess.aiProcess_FindDegenerates;
- load_flags = load_flags | assimp.postprocess.aiProcess_GenSmoothNormals;
- load_flags = load_flags | assimp.postprocess.aiProcess_JoinIdenticalVertices;
- load_flags = load_flags | assimp.postprocess.aiProcess_ImproveCacheLocality;
- load_flags = load_flags | assimp.postprocess.aiProcess_GenUVCoords;
- load_flags = load_flags | assimp.postprocess.aiProcess_FindInvalidData;
- return load_flags
-
- def __init__(self, obj_file, material_file=None, load_materials=True,
- name_prefix='', name_suffix=''):
- if material_file is not None:
- logging.error('Ignoring material file input, reading them off obj file.')
- load_flags = self.get_pyassimp_load_options()
- scene = assimp.load(obj_file, processing=load_flags)
- filter_ind = self._filter_triangles(scene.meshes)
- self.meshes = [scene.meshes[i] for i in filter_ind]
- for m in self.meshes:
- m.name = name_prefix + m.name + name_suffix
-
- dir_name = os.path.dirname(obj_file)
- # Load materials
- materials = None
- if load_materials:
- materials = []
- for m in self.meshes:
- file_name = os.path.join(dir_name, m.material.properties[('file', 1)])
- assert(os.path.exists(file_name)), \
- 'Texture file {:s} foes not exist.'.format(file_name)
- img_rgb = cv2.imread(file_name)[::-1,:,::-1]
- if img_rgb.shape[0] != img_rgb.shape[1]:
- logging.warn('Texture image not square.')
- sz = np.maximum(img_rgb.shape[0], img_rgb.shape[1])
- sz = int(np.power(2., np.ceil(np.log2(sz))))
- img_rgb = cv2.resize(img_rgb, (sz,sz), interpolation=cv2.INTER_LINEAR)
- else:
- sz = img_rgb.shape[0]
- sz_ = int(np.power(2., np.ceil(np.log2(sz))))
- if sz != sz_:
- logging.warn('Texture image not square of power of 2 size. ' +
- 'Changing size from %d to %d.', sz, sz_)
- sz = sz_
- img_rgb = cv2.resize(img_rgb, (sz,sz), interpolation=cv2.INTER_LINEAR)
- materials.append(img_rgb)
- self.scene = scene
- self.materials = materials
-
- def _filter_triangles(self, meshes):
- select = []
- for i in range(len(meshes)):
- if meshes[i].primitivetypes == 4:
- select.append(i)
- return select
-
- def flip_shape(self):
- for m in self.meshes:
- m.vertices[:,1] = -m.vertices[:,1]
- bb = m.faces*1
- bb[:,1] = m.faces[:,2]
- bb[:,2] = m.faces[:,1]
- m.faces = bb
- # m.vertices[:,[0,1]] = m.vertices[:,[1,0]]
-
- def get_vertices(self):
- vs = []
- for m in self.meshes:
- vs.append(m.vertices)
- vss = np.concatenate(vs, axis=0)
- return vss, vs
-
- def get_faces(self):
- vs = []
- for m in self.meshes:
- v = m.faces
- vs.append(v)
- return vs
-
- def get_number_of_meshes(self):
- return len(self.meshes)
-
- def scale(self, sx=1., sy=1., sz=1.):
- pass
-
- def sample_points_on_face_of_shape(self, i, n_samples_per_face, sc):
- v = self.meshes[i].vertices*sc
- f = self.meshes[i].faces
- p, face_areas, face_idx = sample_points_on_faces(
- v, f, np.random.RandomState(0), n_samples_per_face)
- return p, face_areas, face_idx
-
- def __del__(self):
- scene = self.scene
- assimp.release(scene)
-
-class SwiftshaderRenderer():
- def __init__(self):
- self.entities = {}
-
- def init_display(self, width, height, fov, z_near, z_far, rgb_shader,
- d_shader):
- self.init_renderer_egl(width, height)
- dir_path = os.path.dirname(os.path.realpath(__file__))
- if d_shader is not None and rgb_shader is not None:
- logging.fatal('Does not support setting both rgb_shader and d_shader.')
-
- if d_shader is not None:
- assert rgb_shader is None
- shader = d_shader
- self.modality = 'depth'
-
- if rgb_shader is not None:
- assert d_shader is None
- shader = rgb_shader
- self.modality = 'rgb'
-
- self.create_shaders(os.path.join(dir_path, shader+'.vp'),
- os.path.join(dir_path, shader + '.fp'))
- aspect = width*1./(height*1.)
- self.set_camera(fov, z_near, z_far, aspect)
-
- def init_renderer_egl(self, width, height):
- major,minor = ctypes.c_long(),ctypes.c_long()
- logging.info('init_renderer_egl: EGL_DEFAULT_DISPLAY: %s', EGL_DEFAULT_DISPLAY)
-
- egl_display = eglGetDisplay(EGL_DEFAULT_DISPLAY)
- logging.info('init_renderer_egl: egl_display: %s', egl_display)
-
- eglInitialize(egl_display, major, minor)
- logging.info('init_renderer_egl: EGL_OPENGL_API, EGL_OPENGL_ES_API: %s, %s',
- EGL_OPENGL_API, EGL_OPENGL_ES_API)
- eglBindAPI(EGL_OPENGL_ES_API)
-
- num_configs = ctypes.c_long()
- configs = (EGLConfig*1)()
- local_attributes = [EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8,
- EGL_DEPTH_SIZE, 16, EGL_SURFACE_TYPE, EGL_PBUFFER_BIT,
- EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, EGL_NONE,]
- logging.error('init_renderer_egl: local attributes: %s', local_attributes)
- local_attributes = arrays.GLintArray.asArray(local_attributes)
- success = eglChooseConfig(egl_display, local_attributes, configs, 1, num_configs)
- logging.error('init_renderer_egl: eglChooseConfig success, num_configs: %d, %d', success, num_configs.value)
- egl_config = configs[0]
-
-
- context_attributes = [EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE]
- context_attributes = arrays.GLintArray.asArray(context_attributes)
- egl_context = eglCreateContext(egl_display, egl_config, EGL_NO_CONTEXT, context_attributes)
-
- buffer_attributes = [EGL_WIDTH, width, EGL_HEIGHT, height, EGL_NONE]
- buffer_attributes = arrays.GLintArray.asArray(buffer_attributes)
- egl_surface = eglCreatePbufferSurface(egl_display, egl_config, buffer_attributes)
-
-
- eglMakeCurrent(egl_display, egl_surface, egl_surface, egl_context)
- logging.error("init_renderer_egl: egl_display: %s egl_surface: %s, egl_config: %s", egl_display, egl_surface, egl_context)
-
- glViewport(0, 0, width, height);
-
- self.egl_display = egl_display
- self.egl_surface = egl_surface
- self.egl_config = egl_config
- self.egl_mapping = {}
- self.render_timer = None
- self.load_timer = None
- self.height = height
- self.width = width
-
- def create_shaders(self, v_shader_file, f_shader_file):
- v_shader = glCreateShader(GL_VERTEX_SHADER)
- with open(v_shader_file, 'r') as f:
- ls = ''
- for l in f:
- ls = ls + l
- glShaderSource(v_shader, ls)
- glCompileShader(v_shader);
- assert(glGetShaderiv(v_shader, GL_COMPILE_STATUS) == 1)
-
- f_shader = glCreateShader(GL_FRAGMENT_SHADER)
- with open(f_shader_file, 'r') as f:
- ls = ''
- for l in f:
- ls = ls + l
- glShaderSource(f_shader, ls)
- glCompileShader(f_shader);
- assert(glGetShaderiv(f_shader, GL_COMPILE_STATUS) == 1)
-
- egl_program = glCreateProgram();
- assert(egl_program)
- glAttachShader(egl_program, v_shader)
- glAttachShader(egl_program, f_shader)
- glLinkProgram(egl_program);
- assert(glGetProgramiv(egl_program, GL_LINK_STATUS) == 1)
- glUseProgram(egl_program)
-
- glBindAttribLocation(egl_program, 0, "aPosition")
- glBindAttribLocation(egl_program, 1, "aColor")
- glBindAttribLocation(egl_program, 2, "aTextureCoord")
-
- self.egl_program = egl_program
- self.egl_mapping['vertexs'] = 0
- self.egl_mapping['vertexs_color'] = 1
- self.egl_mapping['vertexs_tc'] = 2
-
- glClearColor(0.0, 0.0, 0.0, 1.0);
- # glEnable(GL_CULL_FACE); glCullFace(GL_BACK);
- glEnable(GL_DEPTH_TEST);
-
- glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
-
- def set_camera(self, fov_vertical, z_near, z_far, aspect):
- width = 2*np.tan(np.deg2rad(fov_vertical)/2.0)*z_near*aspect;
- height = 2*np.tan(np.deg2rad(fov_vertical)/2.0)*z_near;
- egl_program = self.egl_program
- c = np.eye(4, dtype=np.float32)
- c[3,3] = 0
- c[3,2] = -1
- c[2,2] = -(z_near+z_far)/(z_far-z_near)
- c[2,3] = -2.0*(z_near*z_far)/(z_far-z_near)
- c[0,0] = 2.0*z_near/width
- c[1,1] = 2.0*z_near/height
- c = c.T
-
- projection_matrix_o = glGetUniformLocation(egl_program, 'uProjectionMatrix')
- projection_matrix = np.eye(4, dtype=np.float32)
- projection_matrix[...] = c
- projection_matrix = np.reshape(projection_matrix, (-1))
- glUniformMatrix4fv(projection_matrix_o, 1, GL_FALSE, projection_matrix)
-
-
- def load_default_object(self):
- v = np.array([[0.0, 0.5, 0.0, 1.0, 1.0, 0.0, 1.0],
- [-0.5, -0.5, 0.0, 1.0, 0.0, 1.0, 1.0],
- [0.5, -0.5, 0.0, 1.0, 1.0, 1.0, 1.0]], dtype=np.float32)
- v = np.concatenate((v,v+0.1), axis=0)
- v = np.ascontiguousarray(v, dtype=np.float32)
-
- vbo = glGenBuffers(1)
- glBindBuffer (GL_ARRAY_BUFFER, vbo)
- glBufferData (GL_ARRAY_BUFFER, v.dtype.itemsize*v.size, v, GL_STATIC_DRAW)
- glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 28, ctypes.c_void_p(0))
- glVertexAttribPointer(1, 4, GL_FLOAT, GL_FALSE, 28, ctypes.c_void_p(12))
- glEnableVertexAttribArray(0);
- glEnableVertexAttribArray(1);
-
- self.num_to_render = 6;
-
- def _actual_render(self):
- for entity_id, entity in self.entities.iteritems():
- if entity['visible']:
- vbo = entity['vbo']
- tbo = entity['tbo']
- num = entity['num']
-
- glBindBuffer(GL_ARRAY_BUFFER, vbo)
- glVertexAttribPointer(self.egl_mapping['vertexs'], 3, GL_FLOAT, GL_FALSE,
- 20, ctypes.c_void_p(0))
- glVertexAttribPointer(self.egl_mapping['vertexs_tc'], 2, GL_FLOAT,
- GL_FALSE, 20, ctypes.c_void_p(12))
- glEnableVertexAttribArray(self.egl_mapping['vertexs']);
- glEnableVertexAttribArray(self.egl_mapping['vertexs_tc']);
-
- glBindTexture(GL_TEXTURE_2D, tbo)
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
- glDrawArrays(GL_TRIANGLES, 0, num)
-
- def render(self, take_screenshot=False, output_type=0):
- # self.render_timer.tic()
- self._actual_render()
- # self.render_timer.toc(log_at=1000, log_str='render timer', type='time')
-
- np_rgb_img = None
- np_d_img = None
- c = 1000.
- if take_screenshot:
- if self.modality == 'rgb':
- screenshot_rgba = np.zeros((self.height, self.width, 4), dtype=np.uint8)
- glReadPixels(0, 0, self.width, self.height, GL_RGBA, GL_UNSIGNED_BYTE, screenshot_rgba)
- np_rgb_img = screenshot_rgba[::-1,:,:3];
-
- if self.modality == 'depth':
- screenshot_d = np.zeros((self.height, self.width, 4), dtype=np.uint8)
- glReadPixels(0, 0, self.width, self.height, GL_RGBA, GL_UNSIGNED_BYTE, screenshot_d)
- np_d_img = screenshot_d[::-1,:,:3];
- np_d_img = np_d_img[:,:,2]*(255.*255./c) + np_d_img[:,:,1]*(255./c) + np_d_img[:,:,0]*(1./c)
- np_d_img = np_d_img.astype(np.float32)
- np_d_img[np_d_img == 0] = np.NaN
- np_d_img = np_d_img[:,:,np.newaxis]
-
- glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
- return np_rgb_img, np_d_img
-
- def _load_mesh_into_gl(self, mesh, material):
- vvt = np.concatenate((mesh.vertices, mesh.texturecoords[0,:,:2]), axis=1)
- vvt = np.ascontiguousarray(vvt[mesh.faces.reshape((-1)),:], dtype=np.float32)
- num = vvt.shape[0]
- vvt = np.reshape(vvt, (-1))
-
- vbo = glGenBuffers(1)
- glBindBuffer(GL_ARRAY_BUFFER, vbo)
- glBufferData(GL_ARRAY_BUFFER, vvt.dtype.itemsize*vvt.size, vvt, GL_STATIC_DRAW)
-
- tbo = glGenTextures(1)
- glBindTexture(GL_TEXTURE_2D, tbo)
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, material.shape[1],
- material.shape[0], 0, GL_RGB, GL_UNSIGNED_BYTE,
- np.reshape(material, (-1)))
- return num, vbo, tbo
-
- def load_shapes(self, shapes):
- entities = self.entities
- entity_ids = []
- for i, shape in enumerate(shapes):
- for j in range(len(shape.meshes)):
- name = shape.meshes[j].name
- assert name not in entities, '{:s} entity already exists.'.format(name)
- num, vbo, tbo = self._load_mesh_into_gl(shape.meshes[j], shape.materials[j])
- entities[name] = {'num': num, 'vbo': vbo, 'tbo': tbo, 'visible': False}
- entity_ids.append(name)
- return entity_ids
-
- def set_entity_visible(self, entity_ids, visibility):
- for entity_id in entity_ids:
- self.entities[entity_id]['visible'] = visibility
-
- def position_camera(self, camera_xyz, lookat_xyz, up):
- camera_xyz = np.array(camera_xyz)
- lookat_xyz = np.array(lookat_xyz)
- up = np.array(up)
- lookat_to = lookat_xyz - camera_xyz
- lookat_from = np.array([0, 1., 0.])
- up_from = np.array([0, 0., 1.])
- up_to = up * 1.
- # np.set_printoptions(precision=2, suppress=True)
- # print up_from, lookat_from, up_to, lookat_to
- r = ru.rotate_camera_to_point_at(up_from, lookat_from, up_to, lookat_to)
- R = np.eye(4, dtype=np.float32)
- R[:3,:3] = r
-
- t = np.eye(4, dtype=np.float32)
- t[:3,3] = -camera_xyz
-
- view_matrix = np.dot(R.T, t)
- flip_yz = np.eye(4, dtype=np.float32)
- flip_yz[1,1] = 0; flip_yz[2,2] = 0; flip_yz[1,2] = 1; flip_yz[2,1] = -1;
- view_matrix = np.dot(flip_yz, view_matrix)
- view_matrix = view_matrix.T
- # print np.concatenate((R, t, view_matrix), axis=1)
- view_matrix = np.reshape(view_matrix, (-1))
- view_matrix_o = glGetUniformLocation(self.egl_program, 'uViewMatrix')
- glUniformMatrix4fv(view_matrix_o, 1, GL_FALSE, view_matrix)
- return None, None #camera_xyz, q
-
- def clear_scene(self):
- keys = self.entities.keys()
- for entity_id in keys:
- entity = self.entities.pop(entity_id, None)
- vbo = entity['vbo']
- tbo = entity['tbo']
- num = entity['num']
- glDeleteBuffers(1, [vbo])
- glDeleteTextures(1, [tbo])
-
- def __del__(self):
- self.clear_scene()
- eglMakeCurrent(self.egl_display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT)
- eglDestroySurface(self.egl_display, self.egl_surface)
- eglTerminate(self.egl_display)
diff --git a/research/cognitive_mapping_and_planning/requirements.txt b/research/cognitive_mapping_and_planning/requirements.txt
deleted file mode 100644
index 306c807a6c9fd9404afa1c05108e5e835e84edc6..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/requirements.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-numpy
-pillow
-PyOpenGL
-PyOpenGL-accelerate
-six
-networkx
-scikit-image
-scipy
-opencv-python
diff --git a/research/cognitive_mapping_and_planning/scripts/__init__.py b/research/cognitive_mapping_and_planning/scripts/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/cognitive_mapping_and_planning/scripts/script_distill.py b/research/cognitive_mapping_and_planning/scripts/script_distill.py
deleted file mode 100644
index 010c690412ed28011146ab44109dc099d02324e7..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_distill.py
+++ /dev/null
@@ -1,177 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r""" Script to setup the grid moving agent.
-
-blaze build --define=ION_GFX_OGLES20=1 -c opt --copt=-mavx --config=cuda_clang \
- learning/brain/public/tensorflow_std_server{,_gpu} \
- experimental/users/saurabhgupta/navigation/cmp/scripts/script_distill.par \
- experimental/users/saurabhgupta/navigation/cmp/scripts/script_distill
-
-
-./blaze-bin/experimental/users/saurabhgupta/navigation/cmp/scripts/script_distill \
- --logdir=/cns/iq-d/home/saurabhgupta/output/stanford-distill/local/v0/ \
- --config_name 'v0+train' --gfs_user robot-intelligence-gpu
-
-"""
-import sys, os, numpy as np
-import copy
-import argparse, pprint
-import time
-import cProfile
-
-
-import tensorflow as tf
-from tensorflow.contrib import slim
-from tensorflow.python.framework import ops
-from tensorflow.contrib.framework.python.ops import variables
-
-import logging
-from tensorflow.python.platform import gfile
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-from cfgs import config_distill
-from tfcode import tf_utils
-import src.utils as utils
-import src.file_utils as fu
-import tfcode.distillation as distill
-import datasets.nav_env as nav_env
-
-FLAGS = flags.FLAGS
-
-flags.DEFINE_string('master', 'local',
- 'The name of the TensorFlow master to use.')
-flags.DEFINE_integer('ps_tasks', 0, 'The number of parameter servers. If the '
- 'value is 0, then the parameters are handled locally by '
- 'the worker.')
-flags.DEFINE_integer('task', 0, 'The Task ID. This value is used when training '
- 'with multiple workers to identify each worker.')
-
-flags.DEFINE_integer('num_workers', 1, '')
-
-flags.DEFINE_string('config_name', '', '')
-
-flags.DEFINE_string('logdir', '', '')
-
-def main(_):
- args = config_distill.get_args_for_config(FLAGS.config_name)
- args.logdir = FLAGS.logdir
- args.solver.num_workers = FLAGS.num_workers
- args.solver.task = FLAGS.task
- args.solver.ps_tasks = FLAGS.ps_tasks
- args.solver.master = FLAGS.master
-
- args.buildinger.env_class = nav_env.MeshMapper
- fu.makedirs(args.logdir)
- args.buildinger.logdir = args.logdir
- R = nav_env.get_multiplexor_class(args.buildinger, args.solver.task)
-
- if False:
- pr = cProfile.Profile()
- pr.enable()
- rng = np.random.RandomState(0)
- for i in range(1):
- b, instances_perturbs = R.sample_building(rng)
- inputs = b.worker(*(instances_perturbs))
- for j in range(inputs['imgs'].shape[0]):
- p = os.path.join('tmp', '{:d}.png'.format(j))
- img = inputs['imgs'][j,0,:,:,:3]*1
- img = (img).astype(np.uint8)
- fu.write_image(p, img)
- print(inputs['imgs'].shape)
- inputs = R.pre(inputs)
- pr.disable()
- pr.print_stats(2)
-
- if args.control.train:
- if not gfile.Exists(args.logdir):
- gfile.MakeDirs(args.logdir)
-
- m = utils.Foo()
- m.tf_graph = tf.Graph()
-
- config = tf.ConfigProto()
- config.device_count['GPU'] = 1
- config.gpu_options.allow_growth = True
- config.gpu_options.per_process_gpu_memory_fraction = 0.8
-
- with m.tf_graph.as_default():
- with tf.device(tf.train.replica_device_setter(args.solver.ps_tasks)):
- m = distill.setup_to_run(m, args, is_training=True,
- batch_norm_is_training=True)
-
- train_step_kwargs = distill.setup_train_step_kwargs_mesh(
- m, R, os.path.join(args.logdir, 'train'),
- rng_seed=args.solver.task, is_chief=args.solver.task==0, iters=1,
- train_display_interval=args.summary.display_interval)
-
- final_loss = slim.learning.train(
- train_op=m.train_op,
- logdir=args.logdir,
- master=args.solver.master,
- is_chief=args.solver.task == 0,
- number_of_steps=args.solver.max_steps,
- train_step_fn=tf_utils.train_step_custom,
- train_step_kwargs=train_step_kwargs,
- global_step=m.global_step_op,
- init_op=m.init_op,
- init_fn=m.init_fn,
- sync_optimizer=m.sync_optimizer,
- saver=m.saver_op,
- summary_op=None, session_config=config)
-
- if args.control.test:
- m = utils.Foo()
- m.tf_graph = tf.Graph()
- checkpoint_dir = os.path.join(format(args.logdir))
- with m.tf_graph.as_default():
- m = distill.setup_to_run(m, args, is_training=False,
- batch_norm_is_training=args.control.force_batchnorm_is_training_at_test)
-
- train_step_kwargs = distill.setup_train_step_kwargs_mesh(
- m, R, os.path.join(args.logdir, args.control.test_name),
- rng_seed=args.solver.task+1, is_chief=args.solver.task==0,
- iters=args.summary.test_iters, train_display_interval=None)
-
- sv = slim.learning.supervisor.Supervisor(
- graph=ops.get_default_graph(), logdir=None, init_op=m.init_op,
- summary_op=None, summary_writer=None, global_step=None, saver=m.saver_op)
-
- last_checkpoint = None
- while True:
- last_checkpoint = slim.evaluation.wait_for_new_checkpoint(checkpoint_dir, last_checkpoint)
- checkpoint_iter = int(os.path.basename(last_checkpoint).split('-')[1])
- start = time.time()
- logging.info('Starting evaluation at %s using checkpoint %s.',
- time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime()),
- last_checkpoint)
-
- config = tf.ConfigProto()
- config.device_count['GPU'] = 1
- config.gpu_options.allow_growth = True
- config.gpu_options.per_process_gpu_memory_fraction = 0.8
-
- with sv.managed_session(args.solver.master,config=config,
- start_standard_services=False) as sess:
- sess.run(m.init_op)
- sv.saver.restore(sess, last_checkpoint)
- sv.start_queue_runners(sess)
- vals, _ = tf_utils.train_step_custom(
- sess, None, m.global_step_op, train_step_kwargs, mode='val')
- if checkpoint_iter >= args.solver.max_steps:
- break
-
-if __name__ == '__main__':
- app.run()
diff --git a/research/cognitive_mapping_and_planning/scripts/script_download_init_models.sh b/research/cognitive_mapping_and_planning/scripts/script_download_init_models.sh
deleted file mode 100644
index 1900bd0b03566d29dac8a8de5f4fce623be98a92..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_download_init_models.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-# Script to download models to initialize the RGB and D models for training.We
-# use ResNet-v2-50 for both modalities.
-
-mkdir -p data/init_models
-cd data/init_models
-
-# RGB Models are initialized by pre-training on ImageNet.
-mkdir -p resnet_v2_50
-RGB_URL="http://download.tensorflow.org/models/resnet_v2_50_2017_04_14.tar.gz"
-wget $RGB_URL
-tar -xf resnet_v2_50_2017_04_14.tar.gz -C resnet_v2_50
-
-# Depth models are initialized by distilling the RGB model to D images using
-# Cross-Modal Distillation (https://arxiv.org/abs/1507.00448).
-mkdir -p distill_rgb_to_d_resnet_v2_50
-D_URL="http://download.tensorflow.org/models/cognitive_mapping_and_planning/2017_04_16/distill_rgb_to_d_resnet_v2_50.tar"
-wget $D_URL
-tar -xf distill_rgb_to_d_resnet_v2_50.tar -C distill_rgb_to_d_resnet_v2_50
diff --git a/research/cognitive_mapping_and_planning/scripts/script_env_vis.py b/research/cognitive_mapping_and_planning/scripts/script_env_vis.py
deleted file mode 100644
index 3690ff484fea9344db6fbe20ac54731200f0c84e..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_env_vis.py
+++ /dev/null
@@ -1,186 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A simple python function to walk in the enviornments that we have created.
-PYTHONPATH='.' PYOPENGL_PLATFORM=egl python scripts/script_env_vis.py \
- --dataset_name sbpd --building_name area3
-"""
-import sys
-import numpy as np
-import matplotlib
-matplotlib.use('TkAgg')
-from PIL import ImageTk, Image
-import Tkinter as tk
-import logging
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-
-import datasets.nav_env_config as nec
-import datasets.nav_env as nav_env
-import cv2
-from datasets import factory
-import render.swiftshader_renderer as renderer
-
-SwiftshaderRenderer = renderer.SwiftshaderRenderer
-VisualNavigationEnv = nav_env.VisualNavigationEnv
-
-FLAGS = flags.FLAGS
-flags.DEFINE_string('dataset_name', 'sbpd', 'Name of the dataset.')
-flags.DEFINE_float('fov', 60., 'Field of view')
-flags.DEFINE_integer('image_size', 512, 'Size of the image.')
-flags.DEFINE_string('building_name', '', 'Name of the building.')
-
-def get_args():
- navtask = nec.nav_env_base_config()
- navtask.task_params.type = 'rng_rejection_sampling_many'
- navtask.task_params.rejection_sampling_M = 2000
- navtask.task_params.min_dist = 10
- sz = FLAGS.image_size
- navtask.camera_param.fov = FLAGS.fov
- navtask.camera_param.height = sz
- navtask.camera_param.width = sz
- navtask.task_params.img_height = sz
- navtask.task_params.img_width = sz
-
- # navtask.task_params.semantic_task.class_map_names = ['chair', 'door', 'table']
- # navtask.task_params.type = 'to_nearest_obj_acc'
-
- logging.info('navtask: %s', navtask)
- return navtask
-
-def load_building(dataset_name, building_name):
- dataset = factory.get_dataset(dataset_name)
-
- navtask = get_args()
- cp = navtask.camera_param
- rgb_shader, d_shader = renderer.get_shaders(cp.modalities)
- r_obj = SwiftshaderRenderer()
- r_obj.init_display(width=cp.width, height=cp.height,
- fov=cp.fov, z_near=cp.z_near, z_far=cp.z_far,
- rgb_shader=rgb_shader, d_shader=d_shader)
- r_obj.clear_scene()
- b = VisualNavigationEnv(robot=navtask.robot, env=navtask.env,
- task_params=navtask.task_params,
- building_name=building_name, flip=False,
- logdir=None, building_loader=dataset,
- r_obj=r_obj)
- b.load_building_into_scene()
- b.set_building_visibility(False)
- return b
-
-def walk_through(b):
- # init agent at a random location in the environment.
- init_env_state = b.reset([np.random.RandomState(0), np.random.RandomState(0)])
-
- global current_node
- rng = np.random.RandomState(0)
- current_node = rng.choice(b.task.nodes.shape[0])
-
- root = tk.Tk()
- image = b.render_nodes(b.task.nodes[[current_node],:])[0]
- print(image.shape)
- image = image.astype(np.uint8)
- im = Image.fromarray(image)
- im = ImageTk.PhotoImage(im)
- panel = tk.Label(root, image=im)
-
- map_size = b.traversible.shape
- sc = np.max(map_size)/256.
- loc = np.array([[map_size[1]/2., map_size[0]/2.]])
- x_axis = np.zeros_like(loc); x_axis[:,1] = sc
- y_axis = np.zeros_like(loc); y_axis[:,0] = -sc
- cum_fs, cum_valid = nav_env.get_map_to_predict(loc, x_axis, y_axis,
- map=b.traversible*1.,
- map_size=256)
- cum_fs = cum_fs[0]
- cum_fs = cv2.applyColorMap((cum_fs*255).astype(np.uint8), cv2.COLORMAP_JET)
- im = Image.fromarray(cum_fs)
- im = ImageTk.PhotoImage(im)
- panel_overhead = tk.Label(root, image=im)
-
- def refresh():
- global current_node
- image = b.render_nodes(b.task.nodes[[current_node],:])[0]
- image = image.astype(np.uint8)
- im = Image.fromarray(image)
- im = ImageTk.PhotoImage(im)
- panel.configure(image=im)
- panel.image = im
-
- def left_key(event):
- global current_node
- current_node = b.take_action([current_node], [2], 1)[0][0]
- refresh()
-
- def up_key(event):
- global current_node
- current_node = b.take_action([current_node], [3], 1)[0][0]
- refresh()
-
- def right_key(event):
- global current_node
- current_node = b.take_action([current_node], [1], 1)[0][0]
- refresh()
-
- def quit(event):
- root.destroy()
-
- panel_overhead.grid(row=4, column=5, rowspan=1, columnspan=1,
- sticky=tk.W+tk.E+tk.N+tk.S)
- panel.bind('', left_key)
- panel.bind('', up_key)
- panel.bind('', right_key)
- panel.bind('q', quit)
- panel.focus_set()
- panel.grid(row=0, column=0, rowspan=5, columnspan=5,
- sticky=tk.W+tk.E+tk.N+tk.S)
- root.mainloop()
-
-def simple_window():
- root = tk.Tk()
-
- image = np.zeros((128, 128, 3), dtype=np.uint8)
- image[32:96, 32:96, 0] = 255
- im = Image.fromarray(image)
- im = ImageTk.PhotoImage(im)
-
- image = np.zeros((128, 128, 3), dtype=np.uint8)
- image[32:96, 32:96, 1] = 255
- im2 = Image.fromarray(image)
- im2 = ImageTk.PhotoImage(im2)
-
- panel = tk.Label(root, image=im)
-
- def left_key(event):
- panel.configure(image=im2)
- panel.image = im2
-
- def quit(event):
- sys.exit()
-
- panel.bind('', left_key)
- panel.bind('', left_key)
- panel.bind('', left_key)
- panel.bind('q', quit)
- panel.focus_set()
- panel.pack(side = "bottom", fill = "both", expand = "yes")
- root.mainloop()
-
-def main(_):
- b = load_building(FLAGS.dataset_name, FLAGS.building_name)
- walk_through(b)
-
-if __name__ == '__main__':
- app.run()
diff --git a/research/cognitive_mapping_and_planning/scripts/script_nav_agent_release.py b/research/cognitive_mapping_and_planning/scripts/script_nav_agent_release.py
deleted file mode 100644
index dab2819a6fcf100cb2e385e45b7aa694c4c5f033..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_nav_agent_release.py
+++ /dev/null
@@ -1,253 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r""" Script to train and test the grid navigation agent.
-Usage:
- 1. Testing a model.
- CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 \
- PYTHONPATH='.' PYOPENGL_PLATFORM=egl python scripts/script_nav_agent_release.py \
- --config_name cmp.lmap_Msc.clip5.sbpd_d_r2r+bench_test \
- --logdir output/cmp.lmap_Msc.clip5.sbpd_d_r2r
-
- 2. Training a model (locally).
- CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 \
- PYTHONPATH='.' PYOPENGL_PLATFORM=egl python scripts/script_nav_agent_release.py \
- --config_name cmp.lmap_Msc.clip5.sbpd_d_r2r+train_train \
- --logdir output/cmp.lmap_Msc.clip5.sbpd_d_r2r_
-
- 3. Training a model (distributed).
- # See https://www.tensorflow.org/deploy/distributed on how to setup distributed
- # training.
- CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 \
- PYTHONPATH='.' PYOPENGL_PLATFORM=egl python scripts/script_nav_agent_release.py \
- --config_name cmp.lmap_Msc.clip5.sbpd_d_r2r+train_train \
- --logdir output/cmp.lmap_Msc.clip5.sbpd_d_r2r_ \
- --ps_tasks $num_ps --master $master_name --task $worker_id
-"""
-
-import sys, os, numpy as np
-import copy
-import argparse, pprint
-import time
-import cProfile
-import platform
-
-
-import tensorflow as tf
-from tensorflow.contrib import slim
-from tensorflow.python.framework import ops
-from tensorflow.contrib.framework.python.ops import variables
-
-import logging
-from tensorflow.python.platform import gfile
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-from cfgs import config_cmp
-from cfgs import config_vision_baseline
-import datasets.nav_env as nav_env
-import src.file_utils as fu
-import src.utils as utils
-import tfcode.cmp as cmp
-from tfcode import tf_utils
-from tfcode import vision_baseline_lstm
-
-FLAGS = flags.FLAGS
-
-flags.DEFINE_string('master', '',
- 'The address of the tensorflow master')
-flags.DEFINE_integer('ps_tasks', 0, 'The number of parameter servers. If the '
- 'value is 0, then the parameters are handled locally by '
- 'the worker.')
-flags.DEFINE_integer('task', 0, 'The Task ID. This value is used when training '
- 'with multiple workers to identify each worker.')
-
-flags.DEFINE_integer('num_workers', 1, '')
-
-flags.DEFINE_string('config_name', '', '')
-
-flags.DEFINE_string('logdir', '', '')
-
-flags.DEFINE_integer('solver_seed', 0, '')
-
-flags.DEFINE_integer('delay_start_iters', 20, '')
-
-logging.basicConfig(level=logging.INFO)
-
-def main(_):
- _launcher(FLAGS.config_name, FLAGS.logdir)
-
-def _launcher(config_name, logdir):
- args = _setup_args(config_name, logdir)
-
- fu.makedirs(args.logdir)
-
- if args.control.train:
- _train(args)
-
- if args.control.test:
- _test(args)
-
-def get_args_for_config(config_name):
- configs = config_name.split('.')
- type = configs[0]
- config_name = '.'.join(configs[1:])
- if type == 'cmp':
- args = config_cmp.get_args_for_config(config_name)
- args.setup_to_run = cmp.setup_to_run
- args.setup_train_step_kwargs = cmp.setup_train_step_kwargs
-
- elif type == 'bl':
- args = config_vision_baseline.get_args_for_config(config_name)
- args.setup_to_run = vision_baseline_lstm.setup_to_run
- args.setup_train_step_kwargs = vision_baseline_lstm.setup_train_step_kwargs
-
- else:
- logging.fatal('Unknown type: {:s}'.format(type))
- return args
-
-def _setup_args(config_name, logdir):
- args = get_args_for_config(config_name)
- args.solver.num_workers = FLAGS.num_workers
- args.solver.task = FLAGS.task
- args.solver.ps_tasks = FLAGS.ps_tasks
- args.solver.master = FLAGS.master
- args.solver.seed = FLAGS.solver_seed
- args.logdir = logdir
- args.navtask.logdir = None
- return args
-
-def _train(args):
- container_name = ""
-
- R = lambda: nav_env.get_multiplexer_class(args.navtask, args.solver.task)
- m = utils.Foo()
- m.tf_graph = tf.Graph()
-
- config = tf.ConfigProto()
- config.device_count['GPU'] = 1
-
- with m.tf_graph.as_default():
- with tf.device(tf.train.replica_device_setter(args.solver.ps_tasks,
- merge_devices=True)):
- with tf.container(container_name):
- m = args.setup_to_run(m, args, is_training=True,
- batch_norm_is_training=True, summary_mode='train')
-
- train_step_kwargs = args.setup_train_step_kwargs(
- m, R(), os.path.join(args.logdir, 'train'), rng_seed=args.solver.task,
- is_chief=args.solver.task==0,
- num_steps=args.navtask.task_params.num_steps*args.navtask.task_params.num_goals, iters=1,
- train_display_interval=args.summary.display_interval,
- dagger_sample_bn_false=args.arch.dagger_sample_bn_false)
-
- delay_start = (args.solver.task*(args.solver.task+1))/2 * FLAGS.delay_start_iters
- logging.error('delaying start for task %d by %d steps.',
- args.solver.task, delay_start)
-
- additional_args = {}
- final_loss = slim.learning.train(
- train_op=m.train_op,
- logdir=args.logdir,
- master=args.solver.master,
- is_chief=args.solver.task == 0,
- number_of_steps=args.solver.max_steps,
- train_step_fn=tf_utils.train_step_custom_online_sampling,
- train_step_kwargs=train_step_kwargs,
- global_step=m.global_step_op,
- init_op=m.init_op,
- init_fn=m.init_fn,
- sync_optimizer=m.sync_optimizer,
- saver=m.saver_op,
- startup_delay_steps=delay_start,
- summary_op=None, session_config=config, **additional_args)
-
-def _test(args):
- args.solver.master = ''
- container_name = ""
- checkpoint_dir = os.path.join(format(args.logdir))
- logging.error('Checkpoint_dir: %s', args.logdir)
-
- config = tf.ConfigProto();
- config.device_count['GPU'] = 1;
-
- m = utils.Foo()
- m.tf_graph = tf.Graph()
-
- rng_data_seed = 0; rng_action_seed = 0;
- R = lambda: nav_env.get_multiplexer_class(args.navtask, rng_data_seed)
- with m.tf_graph.as_default():
- with tf.container(container_name):
- m = args.setup_to_run(
- m, args, is_training=False,
- batch_norm_is_training=args.control.force_batchnorm_is_training_at_test,
- summary_mode=args.control.test_mode)
- train_step_kwargs = args.setup_train_step_kwargs(
- m, R(), os.path.join(args.logdir, args.control.test_name),
- rng_seed=rng_data_seed, is_chief=True,
- num_steps=args.navtask.task_params.num_steps*args.navtask.task_params.num_goals,
- iters=args.summary.test_iters, train_display_interval=None,
- dagger_sample_bn_false=args.arch.dagger_sample_bn_false)
-
- saver = slim.learning.tf_saver.Saver(variables.get_variables_to_restore())
-
- sv = slim.learning.supervisor.Supervisor(
- graph=ops.get_default_graph(), logdir=None, init_op=m.init_op,
- summary_op=None, summary_writer=None, global_step=None, saver=m.saver_op)
-
- last_checkpoint = None
- reported = False
- while True:
- last_checkpoint_ = None
- while last_checkpoint_ is None:
- last_checkpoint_ = slim.evaluation.wait_for_new_checkpoint(
- checkpoint_dir, last_checkpoint, seconds_to_sleep=10, timeout=60)
- if last_checkpoint_ is None: break
-
- last_checkpoint = last_checkpoint_
- checkpoint_iter = int(os.path.basename(last_checkpoint).split('-')[1])
-
- logging.info('Starting evaluation at %s using checkpoint %s.',
- time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime()),
- last_checkpoint)
-
- if (args.control.only_eval_when_done == False or
- checkpoint_iter >= args.solver.max_steps):
- start = time.time()
- logging.info('Starting evaluation at %s using checkpoint %s.',
- time.strftime('%Y-%m-%d-%H:%M:%S', time.localtime()),
- last_checkpoint)
-
- with sv.managed_session(args.solver.master, config=config,
- start_standard_services=False) as sess:
- sess.run(m.init_op)
- sv.saver.restore(sess, last_checkpoint)
- sv.start_queue_runners(sess)
- if args.control.reset_rng_seed:
- train_step_kwargs['rng_data'] = [np.random.RandomState(rng_data_seed),
- np.random.RandomState(rng_data_seed)]
- train_step_kwargs['rng_action'] = np.random.RandomState(rng_action_seed)
- vals, _ = tf_utils.train_step_custom_online_sampling(
- sess, None, m.global_step_op, train_step_kwargs,
- mode=args.control.test_mode)
- should_stop = False
-
- if checkpoint_iter >= args.solver.max_steps:
- should_stop = True
-
- if should_stop:
- break
-
-if __name__ == '__main__':
- app.run()
diff --git a/research/cognitive_mapping_and_planning/scripts/script_plot_trajectory.py b/research/cognitive_mapping_and_planning/scripts/script_plot_trajectory.py
deleted file mode 100644
index 08273a83b512fa3100f7df6e20d41d666b037aad..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_plot_trajectory.py
+++ /dev/null
@@ -1,339 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""
-Code for plotting trajectories in the top view, and also plot first person views
-from saved trajectories. Does not run the network but only loads the mesh data
-to plot the view points.
- CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64
- PYTHONPATH='.' PYOPENGL_PLATFORM=egl python scripts/script_plot_trajectory.py \
- --first_person --num_steps 40 \
- --config_name cmp.lmap_Msc.clip5.sbpd_d_r2r \
- --imset test --alsologtostderr --base_dir output --out_dir vis
-
-"""
-import os, sys, numpy as np, copy
-import matplotlib
-matplotlib.use("Agg")
-import matplotlib.pyplot as plt
-import matplotlib.animation as animation
-from matplotlib.gridspec import GridSpec
-
-import tensorflow as tf
-from tensorflow.contrib import slim
-import cv2
-import logging
-from tensorflow.python.platform import gfile
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-
-from datasets import nav_env
-import scripts.script_nav_agent_release as sna
-import src.file_utils as fu
-from src import graph_utils
-from src import utils
-FLAGS = flags.FLAGS
-
-flags.DEFINE_string('out_dir', 'vis', 'Directory where to store the output')
-flags.DEFINE_string('type', '', 'Optional type.')
-flags.DEFINE_bool('first_person', False, 'Visualize the first person view.')
-flags.DEFINE_bool('top_view', False, 'Visualize the trajectory in the top view.')
-flags.DEFINE_integer('num_steps', 40, 'Number of steps to run the model for.')
-flags.DEFINE_string('imset', 'test', '')
-flags.DEFINE_string('base_dir', 'output', 'Cache directory.')
-
-def _get_suffix_str():
- return ''
-
-
-def _load_trajectory():
- base_dir = FLAGS.base_dir
- config_name = FLAGS.config_name+_get_suffix_str()
-
- dir_name = os.path.join(base_dir, FLAGS.type, config_name)
- logging.info('Waiting for snapshot in directory %s.', dir_name)
- last_checkpoint = slim.evaluation.wait_for_new_checkpoint(dir_name, None)
- checkpoint_iter = int(os.path.basename(last_checkpoint).split('-')[1])
-
- # Load the distances.
- a = utils.load_variables(os.path.join(dir_name, 'bench_on_'+FLAGS.imset,
- 'all_locs_at_t_{:d}.pkl'.format(checkpoint_iter)))
- return a
-
-def _compute_hardness():
- # Load the stanford data to compute the hardness.
- if FLAGS.type == '':
- args = sna.get_args_for_config(FLAGS.config_name+'+bench_'+FLAGS.imset)
- else:
- args = sna.get_args_for_config(FLAGS.type+'.'+FLAGS.config_name+'+bench_'+FLAGS.imset)
-
- args.navtask.logdir = None
- R = lambda: nav_env.get_multiplexer_class(args.navtask, 0)
- R = R()
-
- rng_data = [np.random.RandomState(0), np.random.RandomState(0)]
-
- # Sample a room.
- h_dists = []
- gt_dists = []
- for i in range(250):
- e = R.sample_env(rng_data)
- nodes = e.task.nodes
-
- # Initialize the agent.
- init_env_state = e.reset(rng_data)
-
- gt_dist_to_goal = [e.episode.dist_to_goal[0][j][s]
- for j, s in enumerate(e.episode.start_node_ids)]
-
- for j in range(args.navtask.task_params.batch_size):
- start_node_id = e.episode.start_node_ids[j]
- end_node_id =e.episode.goal_node_ids[0][j]
- h_dist = graph_utils.heuristic_fn_vec(
- nodes[[start_node_id],:], nodes[[end_node_id], :],
- n_ori=args.navtask.task_params.n_ori,
- step_size=args.navtask.task_params.step_size)[0][0]
- gt_dist = e.episode.dist_to_goal[0][j][start_node_id]
- h_dists.append(h_dist)
- gt_dists.append(gt_dist)
-
- h_dists = np.array(h_dists)
- gt_dists = np.array(gt_dists)
- e = R.sample_env([np.random.RandomState(0), np.random.RandomState(0)])
- input = e.get_common_data()
- orig_maps = input['orig_maps'][0,0,:,:,0]
- return h_dists, gt_dists, orig_maps
-
-def plot_trajectory_first_person(dt, orig_maps, out_dir):
- out_dir = os.path.join(out_dir, FLAGS.config_name+_get_suffix_str(),
- FLAGS.imset)
- fu.makedirs(out_dir)
-
- # Load the model so that we can render.
- plt.set_cmap('gray')
- samples_per_action = 8; wait_at_action = 0;
-
- Writer = animation.writers['mencoder']
- writer = Writer(fps=3*(samples_per_action+wait_at_action),
- metadata=dict(artist='anonymous'), bitrate=1800)
-
- args = sna.get_args_for_config(FLAGS.config_name + '+bench_'+FLAGS.imset)
- args.navtask.logdir = None
- navtask_ = copy.deepcopy(args.navtask)
- navtask_.camera_param.modalities = ['rgb']
- navtask_.task_params.modalities = ['rgb']
- sz = 512
- navtask_.camera_param.height = sz
- navtask_.camera_param.width = sz
- navtask_.task_params.img_height = sz
- navtask_.task_params.img_width = sz
- R = lambda: nav_env.get_multiplexer_class(navtask_, 0)
- R = R()
- b = R.buildings[0]
-
- f = [0 for _ in range(wait_at_action)] + \
- [float(_)/samples_per_action for _ in range(samples_per_action)];
-
- # Generate things for it to render.
- inds_to_do = []
- inds_to_do += [1, 4, 10] #1291, 1268, 1273, 1289, 1302, 1426, 1413, 1449, 1399, 1390]
-
- for i in inds_to_do:
- fig = plt.figure(figsize=(10,8))
- gs = GridSpec(3,4)
- gs.update(wspace=0.05, hspace=0.05, left=0.0, top=0.97, right=1.0, bottom=0.)
- ax = fig.add_subplot(gs[:,:-1])
- ax1 = fig.add_subplot(gs[0,-1])
- ax2 = fig.add_subplot(gs[1,-1])
- ax3 = fig.add_subplot(gs[2,-1])
- axes = [ax, ax1, ax2, ax3]
- # ax = fig.add_subplot(gs[:,:])
- # axes = [ax]
- for ax in axes:
- ax.set_axis_off()
-
- node_ids = dt['all_node_ids'][i, :, 0]*1
- # Prune so that last node is not repeated more than 3 times?
- if np.all(node_ids[-4:] == node_ids[-1]):
- while node_ids[-4] == node_ids[-1]:
- node_ids = node_ids[:-1]
- num_steps = np.minimum(FLAGS.num_steps, len(node_ids))
-
- xyt = b.to_actual_xyt_vec(b.task.nodes[node_ids])
- xyt_diff = xyt[1:,:] - xyt[:-1:,:]
- xyt_diff[:,2] = np.mod(xyt_diff[:,2], 4)
- ind = np.where(xyt_diff[:,2] == 3)[0]
- xyt_diff[ind, 2] = -1
- xyt_diff = np.expand_dims(xyt_diff, axis=1)
- to_cat = [xyt_diff*_ for _ in f]
- perturbs_all = np.concatenate(to_cat, axis=1)
- perturbs_all = np.concatenate([perturbs_all, np.zeros_like(perturbs_all[:,:,:1])], axis=2)
- node_ids_all = np.expand_dims(node_ids, axis=1)*1
- node_ids_all = np.concatenate([node_ids_all for _ in f], axis=1)
- node_ids_all = np.reshape(node_ids_all[:-1,:], -1)
- perturbs_all = np.reshape(perturbs_all, [-1, 4])
- imgs = b.render_nodes(b.task.nodes[node_ids_all,:], perturb=perturbs_all)
-
- # Get action at each node.
- actions = []
- _, action_to_nodes = b.get_feasible_actions(node_ids)
- for j in range(num_steps-1):
- action_to_node = action_to_nodes[j]
- node_to_action = dict(zip(action_to_node.values(), action_to_node.keys()))
- actions.append(node_to_action[node_ids[j+1]])
-
- def init_fn():
- return fig,
- gt_dist_to_goal = []
-
- # Render trajectories.
- def worker(j):
- # Plot the image.
- step_number = j/(samples_per_action + wait_at_action)
- img = imgs[j]; ax = axes[0]; ax.clear(); ax.set_axis_off();
- img = img.astype(np.uint8); ax.imshow(img);
- tt = ax.set_title(
- "First Person View\n" +
- "Top corners show diagnostics (distance, agents' action) not input to agent.",
- fontsize=12)
- plt.setp(tt, color='white')
-
- # Distance to goal.
- t = 'Dist to Goal:\n{:2d} steps'.format(int(dt['all_d_at_t'][i, step_number]))
- t = ax.text(0.01, 0.99, t,
- horizontalalignment='left',
- verticalalignment='top',
- fontsize=20, color='red',
- transform=ax.transAxes, alpha=1.0)
- t.set_bbox(dict(color='white', alpha=0.85, pad=-0.1))
-
- # Action to take.
- action_latex = ['$\odot$ ', '$\curvearrowright$ ', '$\curvearrowleft$ ', r'$\Uparrow$ ']
- t = ax.text(0.99, 0.99, action_latex[actions[step_number]],
- horizontalalignment='right',
- verticalalignment='top',
- fontsize=40, color='green',
- transform=ax.transAxes, alpha=1.0)
- t.set_bbox(dict(color='white', alpha=0.85, pad=-0.1))
-
-
- # Plot the map top view.
- ax = axes[-1]
- if j == 0:
- # Plot the map
- locs = dt['all_locs'][i,:num_steps,:]
- goal_loc = dt['all_goal_locs'][i,:,:]
- xymin = np.minimum(np.min(goal_loc, axis=0), np.min(locs, axis=0))
- xymax = np.maximum(np.max(goal_loc, axis=0), np.max(locs, axis=0))
- xy1 = (xymax+xymin)/2. - 0.7*np.maximum(np.max(xymax-xymin), 24)
- xy2 = (xymax+xymin)/2. + 0.7*np.maximum(np.max(xymax-xymin), 24)
-
- ax.set_axis_on()
- ax.patch.set_facecolor((0.333, 0.333, 0.333))
- ax.set_xticks([]); ax.set_yticks([]);
- ax.imshow(orig_maps, origin='lower', vmin=-1.0, vmax=2.0)
- ax.plot(goal_loc[:,0], goal_loc[:,1], 'g*', markersize=12)
-
- locs = dt['all_locs'][i,:1,:]
- ax.plot(locs[:,0], locs[:,1], 'b.', markersize=12)
-
- ax.set_xlim([xy1[0], xy2[0]])
- ax.set_ylim([xy1[1], xy2[1]])
-
- locs = dt['all_locs'][i,step_number,:]
- locs = np.expand_dims(locs, axis=0)
- ax.plot(locs[:,0], locs[:,1], 'r.', alpha=1.0, linewidth=0, markersize=4)
- tt = ax.set_title('Trajectory in topview', fontsize=14)
- plt.setp(tt, color='white')
- return fig,
-
- line_ani = animation.FuncAnimation(fig, worker,
- (num_steps-1)*(wait_at_action+samples_per_action),
- interval=500, blit=True, init_func=init_fn)
- tmp_file_name = 'tmp.mp4'
- line_ani.save(tmp_file_name, writer=writer, savefig_kwargs={'facecolor':'black'})
- out_file_name = os.path.join(out_dir, 'vis_{:04d}.mp4'.format(i))
- print(out_file_name)
-
- if fu.exists(out_file_name):
- gfile.Remove(out_file_name)
- gfile.Copy(tmp_file_name, out_file_name)
- gfile.Remove(tmp_file_name)
- plt.close(fig)
-
-def plot_trajectory(dt, hardness, orig_maps, out_dir):
- out_dir = os.path.join(out_dir, FLAGS.config_name+_get_suffix_str(),
- FLAGS.imset)
- fu.makedirs(out_dir)
- out_file = os.path.join(out_dir, 'all_locs_at_t.pkl')
- dt['hardness'] = hardness
- utils.save_variables(out_file, dt.values(), dt.keys(), overwrite=True)
-
- #Plot trajectories onto the maps
- plt.set_cmap('gray')
- for i in range(4000):
- goal_loc = dt['all_goal_locs'][i, :, :]
- locs = np.concatenate((dt['all_locs'][i,:,:],
- dt['all_locs'][i,:,:]), axis=0)
- xymin = np.minimum(np.min(goal_loc, axis=0), np.min(locs, axis=0))
- xymax = np.maximum(np.max(goal_loc, axis=0), np.max(locs, axis=0))
- xy1 = (xymax+xymin)/2. - 1.*np.maximum(np.max(xymax-xymin), 24)
- xy2 = (xymax+xymin)/2. + 1.*np.maximum(np.max(xymax-xymin), 24)
-
- fig, ax = utils.tight_imshow_figure(plt, figsize=(6,6))
- ax.set_axis_on()
- ax.patch.set_facecolor((0.333, 0.333, 0.333))
- ax.set_xticks([])
- ax.set_yticks([])
-
- all_locs = dt['all_locs'][i,:,:]*1
- uniq = np.where(np.any(all_locs[1:,:] != all_locs[:-1,:], axis=1))[0]+1
- uniq = np.sort(uniq).tolist()
- uniq.insert(0,0)
- uniq = np.array(uniq)
- all_locs = all_locs[uniq, :]
-
- ax.plot(dt['all_locs'][i, 0, 0],
- dt['all_locs'][i, 0, 1], 'b.', markersize=24)
- ax.plot(dt['all_goal_locs'][i, 0, 0],
- dt['all_goal_locs'][i, 0, 1], 'g*', markersize=19)
- ax.plot(all_locs[:,0], all_locs[:,1], 'r', alpha=0.4, linewidth=2)
- ax.scatter(all_locs[:,0], all_locs[:,1],
- c=5+np.arange(all_locs.shape[0])*1./all_locs.shape[0],
- cmap='Reds', s=30, linewidth=0)
- ax.imshow(orig_maps, origin='lower', vmin=-1.0, vmax=2.0, aspect='equal')
- ax.set_xlim([xy1[0], xy2[0]])
- ax.set_ylim([xy1[1], xy2[1]])
-
- file_name = os.path.join(out_dir, 'trajectory_{:04d}.png'.format(i))
- print(file_name)
- with fu.fopen(file_name, 'w') as f:
- plt.savefig(f)
- plt.close(fig)
-
-
-def main(_):
- a = _load_trajectory()
- h_dists, gt_dists, orig_maps = _compute_hardness()
- hardness = 1.-h_dists*1./ gt_dists
-
- if FLAGS.top_view:
- plot_trajectory(a, hardness, orig_maps, out_dir=FLAGS.out_dir)
-
- if FLAGS.first_person:
- plot_trajectory_first_person(a, orig_maps, out_dir=FLAGS.out_dir)
-
-if __name__ == '__main__':
- app.run()
diff --git a/research/cognitive_mapping_and_planning/scripts/script_preprocess_annoations_S3DIS.py b/research/cognitive_mapping_and_planning/scripts/script_preprocess_annoations_S3DIS.py
deleted file mode 100644
index 58f32d121acf4c638625079907b02161e808af68..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_preprocess_annoations_S3DIS.py
+++ /dev/null
@@ -1,197 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import os
-import glob
-import numpy as np
-import logging
-import cPickle
-from datasets import nav_env
-from datasets import factory
-from src import utils
-from src import map_utils as mu
-
-logging.basicConfig(level=logging.INFO)
-DATA_DIR = 'data/stanford_building_parser_dataset_raw/'
-
-mkdir_if_missing = utils.mkdir_if_missing
-save_variables = utils.save_variables
-
-def _get_semantic_maps(building_name, transform, map_, flip, cats):
- rooms = get_room_in_building(building_name)
- maps = []
- for cat in cats:
- maps.append(np.zeros((map_.size[1], map_.size[0])))
-
- for r in rooms:
- room = load_room(building_name, r, category_list=cats)
- classes = room['class_id']
- for i, cat in enumerate(cats):
- c_ind = cats.index(cat)
- ind = [_ for _, c in enumerate(classes) if c == c_ind]
- if len(ind) > 0:
- vs = [room['vertexs'][x]*1 for x in ind]
- vs = np.concatenate(vs, axis=0)
- if transform:
- vs = np.array([vs[:,1], vs[:,0], vs[:,2]]).T
- vs[:,0] = -vs[:,0]
- vs[:,1] += 4.20
- vs[:,0] += 6.20
- vs = vs*100.
- if flip:
- vs[:,1] = -vs[:,1]
- maps[i] = maps[i] + \
- mu._project_to_map(map_, vs, ignore_points_outside_map=True)
- return maps
-
-def _map_building_name(building_name):
- b = int(building_name.split('_')[0][4])
- out_name = 'Area_{:d}'.format(b)
- if b == 5:
- if int(building_name.split('_')[0][5]) == 1:
- transform = True
- else:
- transform = False
- else:
- transform = False
- return out_name, transform
-
-def get_categories():
- cats = ['beam', 'board', 'bookcase', 'ceiling', 'chair', 'clutter', 'column',
- 'door', 'floor', 'sofa', 'table', 'wall', 'window']
- return cats
-
-def _write_map_files(b_in, b_out, transform):
- cats = get_categories()
-
- env = utils.Foo(padding=10, resolution=5, num_point_threshold=2,
- valid_min=-10, valid_max=200, n_samples_per_face=200)
- robot = utils.Foo(radius=15, base=10, height=140, sensor_height=120,
- camera_elevation_degree=-15)
-
- building_loader = factory.get_dataset('sbpd')
- for flip in [False, True]:
- b = nav_env.Building(b_out, robot, env, flip=flip,
- building_loader=building_loader)
- logging.info("building_in: %s, building_out: %s, transform: %d", b_in,
- b_out, transform)
- maps = _get_semantic_maps(b_in, transform, b.map, flip, cats)
- maps = np.transpose(np.array(maps), axes=[1,2,0])
-
- # Load file from the cache.
- file_name = '{:s}_{:d}_{:d}_{:d}_{:d}_{:d}_{:d}.pkl'
- file_name = file_name.format(b.building_name, b.map.size[0], b.map.size[1],
- b.map.origin[0], b.map.origin[1],
- b.map.resolution, flip)
- out_file = os.path.join(DATA_DIR, 'processing', 'class-maps', file_name)
- logging.info('Writing semantic maps to %s.', out_file)
- save_variables(out_file, [maps, cats], ['maps', 'cats'], overwrite=True)
-
-def _transform_area5b(room_dimension):
- for a in room_dimension.keys():
- r = room_dimension[a]*1
- r[[0,1,3,4]] = r[[1,0,4,3]]
- r[[0,3]] = -r[[3,0]]
- r[[1,4]] += 4.20
- r[[0,3]] += 6.20
- room_dimension[a] = r
- return room_dimension
-
-def collect_room(building_name, room_name):
- room_dir = os.path.join(DATA_DIR, 'Stanford3dDataset_v1.2', building_name,
- room_name, 'Annotations')
- files = glob.glob1(room_dir, '*.txt')
- files = sorted(files, key=lambda s: s.lower())
- vertexs = []; colors = [];
- for f in files:
- file_name = os.path.join(room_dir, f)
- logging.info(' %s', file_name)
- a = np.loadtxt(file_name)
- vertex = a[:,:3]*1.
- color = a[:,3:]*1
- color = color.astype(np.uint8)
- vertexs.append(vertex)
- colors.append(color)
- files = [f.split('.')[0] for f in files]
- out = {'vertexs': vertexs, 'colors': colors, 'names': files}
- return out
-
-def load_room(building_name, room_name, category_list=None):
- room = collect_room(building_name, room_name)
- room['building_name'] = building_name
- room['room_name'] = room_name
- instance_id = range(len(room['names']))
- room['instance_id'] = instance_id
- if category_list is not None:
- name = [r.split('_')[0] for r in room['names']]
- class_id = []
- for n in name:
- if n in category_list:
- class_id.append(category_list.index(n))
- else:
- class_id.append(len(category_list))
- room['class_id'] = class_id
- room['category_list'] = category_list
- return room
-
-def get_room_in_building(building_name):
- building_dir = os.path.join(DATA_DIR, 'Stanford3dDataset_v1.2', building_name)
- rn = os.listdir(building_dir)
- rn = [x for x in rn if os.path.isdir(os.path.join(building_dir, x))]
- rn = sorted(rn, key=lambda s: s.lower())
- return rn
-
-def write_room_dimensions(b_in, b_out, transform):
- rooms = get_room_in_building(b_in)
- room_dimension = {}
- for r in rooms:
- room = load_room(b_in, r, category_list=None)
- vertex = np.concatenate(room['vertexs'], axis=0)
- room_dimension[r] = np.concatenate((np.min(vertex, axis=0), np.max(vertex, axis=0)), axis=0)
- if transform == 1:
- room_dimension = _transform_area5b(room_dimension)
-
- out_file = os.path.join(DATA_DIR, 'processing', 'room-dimension', b_out+'.pkl')
- save_variables(out_file, [room_dimension], ['room_dimension'], overwrite=True)
-
-def write_room_dimensions_all(I):
- mkdir_if_missing(os.path.join(DATA_DIR, 'processing', 'room-dimension'))
- bs_in = ['Area_1', 'Area_2', 'Area_3', 'Area_4', 'Area_5', 'Area_5', 'Area_6']
- bs_out = ['area1', 'area2', 'area3', 'area4', 'area5a', 'area5b', 'area6']
- transforms = [0, 0, 0, 0, 0, 1, 0]
-
- for i in I:
- b_in = bs_in[i]
- b_out = bs_out[i]
- t = transforms[i]
- write_room_dimensions(b_in, b_out, t)
-
-def write_class_maps_all(I):
- mkdir_if_missing(os.path.join(DATA_DIR, 'processing', 'class-maps'))
- bs_in = ['Area_1', 'Area_2', 'Area_3', 'Area_4', 'Area_5', 'Area_5', 'Area_6']
- bs_out = ['area1', 'area2', 'area3', 'area4', 'area5a', 'area5b', 'area6']
- transforms = [0, 0, 0, 0, 0, 1, 0]
-
- for i in I:
- b_in = bs_in[i]
- b_out = bs_out[i]
- t = transforms[i]
- _write_map_files(b_in, b_out, t)
-
-
-if __name__ == '__main__':
- write_room_dimensions_all([0, 2, 3, 4, 5, 6])
- write_class_maps_all([0, 2, 3, 4, 5, 6])
-
diff --git a/research/cognitive_mapping_and_planning/scripts/script_preprocess_annoations_S3DIS.sh b/research/cognitive_mapping_and_planning/scripts/script_preprocess_annoations_S3DIS.sh
deleted file mode 100644
index 1384fabe69259ccc514a14d62aee358d1909bffb..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_preprocess_annoations_S3DIS.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-cd data/stanford_building_parser_dataset_raw
-unzip Stanford3dDataset_v1.2.zip
-cd ../../
-PYOPENGL_PLATFORM=egl PYTHONPATH='.' python scripts/script_preprocess_annoations_S3DIS.py
-
-mv data/stanford_building_parser_dataset_raw/processing/room-dimension data/stanford_building_parser_dataset/.
-mv data/stanford_building_parser_dataset_raw/processing/class-maps data/stanford_building_parser_dataset/.
-
-echo "You may now delete data/stanford_building_parser_dataset_raw if needed."
diff --git a/research/cognitive_mapping_and_planning/scripts/script_preprocess_meshes_S3DIS.sh b/research/cognitive_mapping_and_planning/scripts/script_preprocess_meshes_S3DIS.sh
deleted file mode 100644
index 557a4dde611d42e71d71dd1589abf96f55e6eec6..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_preprocess_meshes_S3DIS.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-mkdir -p data/stanford_building_parser_dataset
-mkdir -p data/stanford_building_parser_dataset/mesh
-cd data/stanford_building_parser_dataset_raw
-
-# Untar the files and extract the meshes.
-for t in "1" "3" "4" "5a" "5b" "6"; do
- tar -xf area_"$t"_noXYZ.tar area_$t/3d/rgb_textures
- mv area_$t/3d/rgb_textures ../stanford_building_parser_dataset/mesh/area$t
- rmdir area_$t/3d
- rmdir area_$t
-done
-
-cd ../../
-
-# Preprocess meshes to remove the group and chunk information.
-cd data/stanford_building_parser_dataset/
-for t in "1" "3" "4" "5a" "5b" "6"; do
- obj_name=`ls mesh/area$t/*.obj`
- cp $obj_name "$obj_name".bck
- cat $obj_name.bck | grep -v '^g' | grep -v '^o' > $obj_name
-done
-cd ../../
diff --git a/research/cognitive_mapping_and_planning/scripts/script_test_pretrained_models.sh b/research/cognitive_mapping_and_planning/scripts/script_test_pretrained_models.sh
deleted file mode 100644
index a4299fff5346afb53783a61de5c3e84f102a6304..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/scripts/script_test_pretrained_models.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Test CMP models.
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name cmp.lmap_Msc.clip5.sbpd_d_r2r+bench_test \
- --logdir output/cmp.lmap_Msc.clip5.sbpd_d_r2r
-
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name cmp.lmap_Msc.clip5.sbpd_rgb_r2r+bench_test \
- --logdir output/cmp.lmap_Msc.clip5.sbpd_rgb_r2r
-
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name cmp.lmap_Msc.clip5.sbpd_d_ST+bench_test \
- --logdir output/cmp.lmap_Msc.clip5.sbpd_d_ST
-
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name cmp.lmap_Msc.clip5.sbpd_rgb_ST+bench_test \
- --logdir output/cmp.lmap_Msc.clip5.sbpd_rgb_ST
-
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name cmp.lmap_Msc.clip5.sbpd_d_r2r_h0_64_80+bench_test \
- --logdir output/cmp.lmap_Msc.clip5.sbpd_d_r2r_h0_64_80
-
-# Test LSTM baseline models.
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name bl.v2.noclip.sbpd_d_r2r+bench_test \
- --logdir output/bl.v2.noclip.sbpd_d_r2r
-
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name bl.v2.noclip.sbpd_rgb_r2r+bench_test \
- --logdir output/bl.v2.noclip.sbpd_rgb_r2r
-
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name bl.v2.noclip.sbpd_d_ST+bench_test \
- --logdir output/bl.v2.noclip.sbpd_d_ST
-
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name bl.v2.noclip.sbpd_rgb_ST+bench_test \
- --logdir output/bl.v2.noclip.sbpd_rgb_ST
-
-CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
- python scripts/script_nav_agent_release.py --config_name bl.v2.noclip.sbpd_d_r2r_h0_64_80+bench_test \
- --logdir output/bl.v2.noclip.sbpd_d_r2r_h0_64_80
-
-# Visualize test trajectories in top view.
-# CUDA_VISIBLE_DEVICES=0 LD_LIBRARY_PATH=/opt/cuda-8.0/lib64:/opt/cudnnv51/lib64 PYTHONPATH='.' PYOPENGL_PLATFORM=egl \
-# python scripts/script_plot_trajectory.py \
-# --first_person --num_steps 40 \
-# --config_name cmp.lmap_Msc.clip5.sbpd_d_r2r \
-# --imset test --alsologtostderr
diff --git a/research/cognitive_mapping_and_planning/src/__init__.py b/research/cognitive_mapping_and_planning/src/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/cognitive_mapping_and_planning/src/depth_utils.py b/research/cognitive_mapping_and_planning/src/depth_utils.py
deleted file mode 100644
index 35f14fc7c37fffb2a408decede11e378867a2834..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/src/depth_utils.py
+++ /dev/null
@@ -1,96 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities for processing depth images.
-"""
-import numpy as np
-import src.rotation_utils as ru
-import src.utils as utils
-
-def get_camera_matrix(width, height, fov):
- """Returns a camera matrix from image size and fov."""
- xc = (width-1.) / 2.
- zc = (height-1.) / 2.
- f = (width / 2.) / np.tan(np.deg2rad(fov / 2.))
- camera_matrix = utils.Foo(xc=xc, zc=zc, f=f)
- return camera_matrix
-
-def get_point_cloud_from_z(Y, camera_matrix):
- """Projects the depth image Y into a 3D point cloud.
- Inputs:
- Y is ...xHxW
- camera_matrix
- Outputs:
- X is positive going right
- Y is positive into the image
- Z is positive up in the image
- XYZ is ...xHxWx3
- """
- x, z = np.meshgrid(np.arange(Y.shape[-1]),
- np.arange(Y.shape[-2]-1, -1, -1))
- for i in range(Y.ndim-2):
- x = np.expand_dims(x, axis=0)
- z = np.expand_dims(z, axis=0)
- X = (x-camera_matrix.xc) * Y / camera_matrix.f
- Z = (z-camera_matrix.zc) * Y / camera_matrix.f
- XYZ = np.concatenate((X[...,np.newaxis], Y[...,np.newaxis],
- Z[...,np.newaxis]), axis=X.ndim)
- return XYZ
-
-def make_geocentric(XYZ, sensor_height, camera_elevation_degree):
- """Transforms the point cloud into geocentric coordinate frame.
- Input:
- XYZ : ...x3
- sensor_height : height of the sensor
- camera_elevation_degree : camera elevation to rectify.
- Output:
- XYZ : ...x3
- """
- R = ru.get_r_matrix([1.,0.,0.], angle=np.deg2rad(camera_elevation_degree))
- XYZ = np.matmul(XYZ.reshape(-1,3), R.T).reshape(XYZ.shape)
- XYZ[...,2] = XYZ[...,2] + sensor_height
- return XYZ
-
-def bin_points(XYZ_cms, map_size, z_bins, xy_resolution):
- """Bins points into xy-z bins
- XYZ_cms is ... x H x W x3
- Outputs is ... x map_size x map_size x (len(z_bins)+1)
- """
- sh = XYZ_cms.shape
- XYZ_cms = XYZ_cms.reshape([-1, sh[-3], sh[-2], sh[-1]])
- n_z_bins = len(z_bins)+1
- map_center = (map_size-1.)/2.
- counts = []
- isvalids = []
- for XYZ_cm in XYZ_cms:
- isnotnan = np.logical_not(np.isnan(XYZ_cm[:,:,0]))
- X_bin = np.round(XYZ_cm[:,:,0] / xy_resolution + map_center).astype(np.int32)
- Y_bin = np.round(XYZ_cm[:,:,1] / xy_resolution + map_center).astype(np.int32)
- Z_bin = np.digitize(XYZ_cm[:,:,2], bins=z_bins).astype(np.int32)
-
- isvalid = np.array([X_bin >= 0, X_bin < map_size, Y_bin >= 0, Y_bin < map_size,
- Z_bin >= 0, Z_bin < n_z_bins, isnotnan])
- isvalid = np.all(isvalid, axis=0)
-
- ind = (Y_bin * map_size + X_bin) * n_z_bins + Z_bin
- ind[np.logical_not(isvalid)] = 0
- count = np.bincount(ind.ravel(), isvalid.ravel().astype(np.int32),
- minlength=map_size*map_size*n_z_bins)
- count = np.reshape(count, [map_size, map_size, n_z_bins])
- counts.append(count)
- isvalids.append(isvalid)
- counts = np.array(counts).reshape(list(sh[:-3]) + [map_size, map_size, n_z_bins])
- isvalids = np.array(isvalids).reshape(list(sh[:-3]) + [sh[-3], sh[-2], 1])
- return counts, isvalids
diff --git a/research/cognitive_mapping_and_planning/src/file_utils.py b/research/cognitive_mapping_and_planning/src/file_utils.py
deleted file mode 100644
index b386236ca6e04c9fa1e452b6ad3e70c6ab9bb88a..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/src/file_utils.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities for manipulating files.
-"""
-import os
-import numpy as np
-import PIL
-from tensorflow.python.platform import gfile
-import cv2
-
-exists = lambda path: gfile.Exists(path)
-fopen = lambda path, mode: gfile.Open(path, mode)
-makedirs = lambda path: gfile.MakeDirs(path)
-listdir = lambda path: gfile.ListDir(path)
-copyfile = lambda a, b, o: gfile.Copy(a,b,o)
-
-def write_image(image_path, rgb):
- ext = os.path.splitext(image_path)[1]
- with gfile.GFile(image_path, 'w') as f:
- img_str = cv2.imencode(ext, rgb[:,:,::-1])[1].tostring()
- f.write(img_str)
-
-def read_image(image_path, type='rgb'):
- with fopen(image_path, 'r') as f:
- I = PIL.Image.open(f)
- II = np.array(I)
- if type == 'rgb':
- II = II[:,:,:3]
- return II
diff --git a/research/cognitive_mapping_and_planning/src/graph_utils.py b/research/cognitive_mapping_and_planning/src/graph_utils.py
deleted file mode 100644
index cd99fd22a2f630438f31eecd7fbfece2c6008ead..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/src/graph_utils.py
+++ /dev/null
@@ -1,552 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Various function to manipulate graphs for computing distances.
-"""
-import skimage.morphology
-import numpy as np
-import networkx as nx
-import itertools
-import logging
-from datasets.nav_env import get_path_ids
-import graph_tool as gt
-import graph_tool.topology
-import graph_tool.generation
-import src.utils as utils
-
-# Compute shortest path from all nodes to or from all source nodes
-def get_distance_node_list(gtG, source_nodes, direction, weights=None):
- gtG_ = gt.Graph(gtG)
- v = gtG_.add_vertex()
-
- if weights is not None:
- weights = gtG_.edge_properties[weights]
-
- for s in source_nodes:
- e = gtG_.add_edge(s, int(v))
- if weights is not None:
- weights[e] = 0.
-
- if direction == 'to':
- dist = gt.topology.shortest_distance(
- gt.GraphView(gtG_, reversed=True), source=gtG_.vertex(int(v)),
- target=None, weights=weights)
- elif direction == 'from':
- dist = gt.topology.shortest_distance(
- gt.GraphView(gtG_, reversed=False), source=gtG_.vertex(int(v)),
- target=None, weights=weights)
- dist = np.array(dist.get_array())
- dist = dist[:-1]
- if weights is None:
- dist = dist-1
- return dist
-
-# Functions for semantically labelling nodes in the traversal graph.
-def generate_lattice(sz_x, sz_y):
- """Generates a lattice with sz_x vertices along x and sz_y vertices along y
- direction Each of these vertices is step_size distance apart. Origin is at
- (0,0). """
- g = gt.generation.lattice([sz_x, sz_y])
- x, y = np.meshgrid(np.arange(sz_x), np.arange(sz_y))
- x = np.reshape(x, [-1,1]); y = np.reshape(y, [-1,1]);
- nodes = np.concatenate((x,y), axis=1)
- return g, nodes
-
-def add_diagonal_edges(g, nodes, sz_x, sz_y, edge_len):
- offset = [sz_x+1, sz_x-1]
- for o in offset:
- s = np.arange(nodes.shape[0]-o-1)
- t = s + o
- ind = np.all(np.abs(nodes[s,:] - nodes[t,:]) == np.array([[1,1]]), axis=1)
- s = s[ind][:,np.newaxis]
- t = t[ind][:,np.newaxis]
- st = np.concatenate((s,t), axis=1)
- for i in range(st.shape[0]):
- e = g.add_edge(st[i,0], st[i,1], add_missing=False)
- g.ep['wts'][e] = edge_len
-
-def convert_traversible_to_graph(traversible, ff_cost=1., fo_cost=1.,
- oo_cost=1., connectivity=4):
- assert(connectivity == 4 or connectivity == 8)
-
- sz_x = traversible.shape[1]
- sz_y = traversible.shape[0]
- g, nodes = generate_lattice(sz_x, sz_y)
-
- # Assign costs.
- edge_wts = g.new_edge_property('float')
- g.edge_properties['wts'] = edge_wts
- wts = np.ones(g.num_edges(), dtype=np.float32)
- edge_wts.get_array()[:] = wts
-
- if connectivity == 8:
- add_diagonal_edges(g, nodes, sz_x, sz_y, np.sqrt(2.))
-
- se = np.array([[int(e.source()), int(e.target())] for e in g.edges()])
- s_xy = nodes[se[:,0]]
- t_xy = nodes[se[:,1]]
- s_t = np.ravel_multi_index((s_xy[:,1], s_xy[:,0]), traversible.shape)
- t_t = np.ravel_multi_index((t_xy[:,1], t_xy[:,0]), traversible.shape)
- s_t = traversible.ravel()[s_t]
- t_t = traversible.ravel()[t_t]
-
- wts = np.zeros(g.num_edges(), dtype=np.float32)
- wts[np.logical_and(s_t == True, t_t == True)] = ff_cost
- wts[np.logical_and(s_t == False, t_t == False)] = oo_cost
- wts[np.logical_xor(s_t, t_t)] = fo_cost
-
- edge_wts = g.edge_properties['wts']
- for i, e in enumerate(g.edges()):
- edge_wts[e] = edge_wts[e] * wts[i]
- # d = edge_wts.get_array()*1.
- # edge_wts.get_array()[:] = d*wts
- return g, nodes
-
-def label_nodes_with_class(nodes_xyt, class_maps, pix):
- """
- Returns:
- class_maps__: one-hot class_map for each class.
- node_class_label: one-hot class_map for each class, nodes_xyt.shape[0] x n_classes
- """
- # Assign each pixel to a node.
- selem = skimage.morphology.disk(pix)
- class_maps_ = class_maps*1.
- for i in range(class_maps.shape[2]):
- class_maps_[:,:,i] = skimage.morphology.dilation(class_maps[:,:,i]*1, selem)
- class_maps__ = np.argmax(class_maps_, axis=2)
- class_maps__[np.max(class_maps_, axis=2) == 0] = -1
-
- # For each node pick out the label from this class map.
- x = np.round(nodes_xyt[:,[0]]).astype(np.int32)
- y = np.round(nodes_xyt[:,[1]]).astype(np.int32)
- ind = np.ravel_multi_index((y,x), class_maps__.shape)
- node_class_label = class_maps__.ravel()[ind][:,0]
-
- # Convert to one hot versions.
- class_maps_one_hot = np.zeros(class_maps.shape, dtype=np.bool)
- node_class_label_one_hot = np.zeros((node_class_label.shape[0], class_maps.shape[2]), dtype=np.bool)
- for i in range(class_maps.shape[2]):
- class_maps_one_hot[:,:,i] = class_maps__ == i
- node_class_label_one_hot[:,i] = node_class_label == i
- return class_maps_one_hot, node_class_label_one_hot
-
-def label_nodes_with_class_geodesic(nodes_xyt, class_maps, pix, traversible,
- ff_cost=1., fo_cost=1., oo_cost=1.,
- connectivity=4):
- """Labels nodes in nodes_xyt with class labels using geodesic distance as
- defined by traversible from class_maps.
- Inputs:
- nodes_xyt
- class_maps: counts for each class.
- pix: distance threshold to consider close enough to target.
- traversible: binary map of whether traversible or not.
- Output:
- labels: For each node in nodes_xyt returns a label of the class or -1 is
- unlabelled.
- """
- g, nodes = convert_traversible_to_graph(traversible, ff_cost=ff_cost,
- fo_cost=fo_cost, oo_cost=oo_cost,
- connectivity=connectivity)
-
- class_dist = np.zeros_like(class_maps*1.)
- n_classes = class_maps.shape[2]
- if False:
- # Assign each pixel to a class based on number of points.
- selem = skimage.morphology.disk(pix)
- class_maps_ = class_maps*1.
- class_maps__ = np.argmax(class_maps_, axis=2)
- class_maps__[np.max(class_maps_, axis=2) == 0] = -1
-
- # Label nodes with classes.
- for i in range(n_classes):
- # class_node_ids = np.where(class_maps__.ravel() == i)[0]
- class_node_ids = np.where(class_maps[:,:,i].ravel() > 0)[0]
- dist_i = get_distance_node_list(g, class_node_ids, 'to', weights='wts')
- class_dist[:,:,i] = np.reshape(dist_i, class_dist[:,:,i].shape)
- class_map_geodesic = (class_dist <= pix)
- class_map_geodesic = np.reshape(class_map_geodesic, [-1, n_classes])
-
- # For each node pick out the label from this class map.
- x = np.round(nodes_xyt[:,[0]]).astype(np.int32)
- y = np.round(nodes_xyt[:,[1]]).astype(np.int32)
- ind = np.ravel_multi_index((y,x), class_dist[:,:,0].shape)
- node_class_label = class_map_geodesic[ind[:,0],:]
- class_map_geodesic = class_dist <= pix
- return class_map_geodesic, node_class_label
-
-def _get_next_nodes_undirected(n, sc, n_ori):
- nodes_to_add = []
- nodes_to_validate = []
- (p, q, r) = n
- nodes_to_add.append((n, (p, q, r), 0))
- if n_ori == 4:
- for _ in [1, 2, 3, 4]:
- if _ == 1:
- v = (p - sc, q, r)
- elif _ == 2:
- v = (p + sc, q, r)
- elif _ == 3:
- v = (p, q - sc, r)
- elif _ == 4:
- v = (p, q + sc, r)
- nodes_to_validate.append((n, v, _))
- return nodes_to_add, nodes_to_validate
-
-def _get_next_nodes(n, sc, n_ori):
- nodes_to_add = []
- nodes_to_validate = []
- (p, q, r) = n
- for r_, a_ in zip([-1, 0, 1], [1, 0, 2]):
- nodes_to_add.append((n, (p, q, np.mod(r+r_, n_ori)), a_))
-
- if n_ori == 6:
- if r == 0:
- v = (p + sc, q, r)
- elif r == 1:
- v = (p + sc, q + sc, r)
- elif r == 2:
- v = (p, q + sc, r)
- elif r == 3:
- v = (p - sc, q, r)
- elif r == 4:
- v = (p - sc, q - sc, r)
- elif r == 5:
- v = (p, q - sc, r)
- elif n_ori == 4:
- if r == 0:
- v = (p + sc, q, r)
- elif r == 1:
- v = (p, q + sc, r)
- elif r == 2:
- v = (p - sc, q, r)
- elif r == 3:
- v = (p, q - sc, r)
- nodes_to_validate.append((n,v,3))
-
- return nodes_to_add, nodes_to_validate
-
-def generate_graph(valid_fn_vec=None, sc=1., n_ori=6,
- starting_location=(0, 0, 0), vis=False, directed=True):
- timer = utils.Timer()
- timer.tic()
- if directed: G = nx.DiGraph(directed=True)
- else: G = nx.Graph()
- G.add_node(starting_location)
- new_nodes = G.nodes()
- while len(new_nodes) != 0:
- nodes_to_add = []
- nodes_to_validate = []
- for n in new_nodes:
- if directed:
- na, nv = _get_next_nodes(n, sc, n_ori)
- else:
- na, nv = _get_next_nodes_undirected(n, sc, n_ori)
- nodes_to_add = nodes_to_add + na
- if valid_fn_vec is not None:
- nodes_to_validate = nodes_to_validate + nv
- else:
- node_to_add = nodes_to_add + nv
-
- # Validate nodes.
- vs = [_[1] for _ in nodes_to_validate]
- valids = valid_fn_vec(vs)
-
- for nva, valid in zip(nodes_to_validate, valids):
- if valid:
- nodes_to_add.append(nva)
-
- new_nodes = []
- for n,v,a in nodes_to_add:
- if not G.has_node(v):
- new_nodes.append(v)
- G.add_edge(n, v, action=a)
-
- timer.toc(average=True, log_at=1, log_str='src.graph_utils.generate_graph')
- return (G)
-
-def vis_G(G, ax, vertex_color='r', edge_color='b', r=None):
- if edge_color is not None:
- for e in G.edges():
- XYT = zip(*e)
- x = XYT[-3]
- y = XYT[-2]
- t = XYT[-1]
- if r is None or t[0] == r:
- ax.plot(x, y, edge_color)
- if vertex_color is not None:
- XYT = zip(*G.nodes())
- x = XYT[-3]
- y = XYT[-2]
- t = XYT[-1]
- ax.plot(x, y, vertex_color + '.')
-
-def convert_to_graph_tool(G):
- timer = utils.Timer()
- timer.tic()
- gtG = gt.Graph(directed=G.is_directed())
- gtG.ep['action'] = gtG.new_edge_property('int')
-
- nodes_list = G.nodes()
- nodes_array = np.array(nodes_list)
-
- nodes_id = np.zeros((nodes_array.shape[0],), dtype=np.int64)
-
- for i in range(nodes_array.shape[0]):
- v = gtG.add_vertex()
- nodes_id[i] = int(v)
-
- # d = {key: value for (key, value) in zip(nodes_list, nodes_id)}
- d = dict(itertools.izip(nodes_list, nodes_id))
-
- for src, dst, data in G.edges_iter(data=True):
- e = gtG.add_edge(d[src], d[dst])
- gtG.ep['action'][e] = data['action']
- nodes_to_id = d
- timer.toc(average=True, log_at=1, log_str='src.graph_utils.convert_to_graph_tool')
- return gtG, nodes_array, nodes_to_id
-
-
-def _rejection_sampling(rng, sampling_d, target_d, bins, hardness, M):
- bin_ind = np.digitize(hardness, bins)-1
- i = 0
- ratio = target_d[bin_ind] / (M*sampling_d[bin_ind])
- while i < ratio.size and rng.rand() > ratio[i]:
- i = i+1
- return i
-
-def heuristic_fn_vec(n1, n2, n_ori, step_size):
- # n1 is a vector and n2 is a single point.
- dx = (n1[:,0] - n2[0,0])/step_size
- dy = (n1[:,1] - n2[0,1])/step_size
- dt = n1[:,2] - n2[0,2]
- dt = np.mod(dt, n_ori)
- dt = np.minimum(dt, n_ori-dt)
-
- if n_ori == 6:
- if dx*dy > 0:
- d = np.maximum(np.abs(dx), np.abs(dy))
- else:
- d = np.abs(dy-dx)
- elif n_ori == 4:
- d = np.abs(dx) + np.abs(dy)
-
- return (d + dt).reshape((-1,1))
-
-def get_hardness_distribution(gtG, max_dist, min_dist, rng, trials, bins, nodes,
- n_ori, step_size):
- heuristic_fn = lambda node_ids, node_id: \
- heuristic_fn_vec(nodes[node_ids, :], nodes[[node_id], :], n_ori, step_size)
- num_nodes = gtG.num_vertices()
- gt_dists = []; h_dists = [];
- for i in range(trials):
- end_node_id = rng.choice(num_nodes)
- gt_dist = gt.topology.shortest_distance(gt.GraphView(gtG, reversed=True),
- source=gtG.vertex(end_node_id),
- target=None, max_dist=max_dist)
- gt_dist = np.array(gt_dist.get_array())
- ind = np.where(np.logical_and(gt_dist <= max_dist, gt_dist >= min_dist))[0]
- gt_dist = gt_dist[ind]
- h_dist = heuristic_fn(ind, end_node_id)[:,0]
- gt_dists.append(gt_dist)
- h_dists.append(h_dist)
- gt_dists = np.concatenate(gt_dists)
- h_dists = np.concatenate(h_dists)
- hardness = 1. - h_dists*1./gt_dists
- hist, _ = np.histogram(hardness, bins)
- hist = hist.astype(np.float64)
- hist = hist / np.sum(hist)
- return hist
-
-def rng_next_goal_rejection_sampling(start_node_ids, batch_size, gtG, rng,
- max_dist, min_dist, max_dist_to_compute,
- sampling_d, target_d,
- nodes, n_ori, step_size, bins, M):
- sample_start_nodes = start_node_ids is None
- dists = []; pred_maps = []; end_node_ids = []; start_node_ids_ = [];
- hardnesss = []; gt_dists = [];
- num_nodes = gtG.num_vertices()
- for i in range(batch_size):
- done = False
- while not done:
- if sample_start_nodes:
- start_node_id = rng.choice(num_nodes)
- else:
- start_node_id = start_node_ids[i]
-
- gt_dist = gt.topology.shortest_distance(
- gt.GraphView(gtG, reversed=False), source=start_node_id, target=None,
- max_dist=max_dist)
- gt_dist = np.array(gt_dist.get_array())
- ind = np.where(np.logical_and(gt_dist <= max_dist, gt_dist >= min_dist))[0]
- ind = rng.permutation(ind)
- gt_dist = gt_dist[ind]*1.
- h_dist = heuristic_fn_vec(nodes[ind, :], nodes[[start_node_id], :],
- n_ori, step_size)[:,0]
- hardness = 1. - h_dist / gt_dist
- sampled_ind = _rejection_sampling(rng, sampling_d, target_d, bins,
- hardness, M)
- if sampled_ind < ind.size:
- # print sampled_ind
- end_node_id = ind[sampled_ind]
- hardness = hardness[sampled_ind]
- gt_dist = gt_dist[sampled_ind]
- done = True
-
- # Compute distance from end node to all nodes, to return.
- dist, pred_map = gt.topology.shortest_distance(
- gt.GraphView(gtG, reversed=True), source=end_node_id, target=None,
- max_dist=max_dist_to_compute, pred_map=True)
- dist = np.array(dist.get_array())
- pred_map = np.array(pred_map.get_array())
-
- hardnesss.append(hardness); dists.append(dist); pred_maps.append(pred_map);
- start_node_ids_.append(start_node_id); end_node_ids.append(end_node_id);
- gt_dists.append(gt_dist);
- paths = None
- return start_node_ids_, end_node_ids, dists, pred_maps, paths, hardnesss, gt_dists
-
-
-def rng_next_goal(start_node_ids, batch_size, gtG, rng, max_dist,
- max_dist_to_compute, node_room_ids, nodes=None,
- compute_path=False, dists_from_start_node=None):
- # Compute the distance field from the starting location, and then pick a
- # destination in another room if possible otherwise anywhere outside this
- # room.
- dists = []; pred_maps = []; paths = []; end_node_ids = [];
- for i in range(batch_size):
- room_id = node_room_ids[start_node_ids[i]]
- # Compute distances.
- if dists_from_start_node == None:
- dist, pred_map = gt.topology.shortest_distance(
- gt.GraphView(gtG, reversed=False), source=gtG.vertex(start_node_ids[i]),
- target=None, max_dist=max_dist_to_compute, pred_map=True)
- dist = np.array(dist.get_array())
- else:
- dist = dists_from_start_node[i]
-
- # Randomly sample nodes which are within max_dist.
- near_ids = dist <= max_dist
- near_ids = near_ids[:, np.newaxis]
- # Check to see if there is a non-negative node which is close enough.
- non_same_room_ids = node_room_ids != room_id
- non_hallway_ids = node_room_ids != -1
- good1_ids = np.logical_and(near_ids, np.logical_and(non_same_room_ids, non_hallway_ids))
- good2_ids = np.logical_and(near_ids, non_hallway_ids)
- good3_ids = near_ids
- if np.any(good1_ids):
- end_node_id = rng.choice(np.where(good1_ids)[0])
- elif np.any(good2_ids):
- end_node_id = rng.choice(np.where(good2_ids)[0])
- elif np.any(good3_ids):
- end_node_id = rng.choice(np.where(good3_ids)[0])
- else:
- logging.error('Did not find any good nodes.')
-
- # Compute distance to this new goal for doing distance queries.
- dist, pred_map = gt.topology.shortest_distance(
- gt.GraphView(gtG, reversed=True), source=gtG.vertex(end_node_id),
- target=None, max_dist=max_dist_to_compute, pred_map=True)
- dist = np.array(dist.get_array())
- pred_map = np.array(pred_map.get_array())
-
- dists.append(dist)
- pred_maps.append(pred_map)
- end_node_ids.append(end_node_id)
-
- path = None
- if compute_path:
- path = get_path_ids(start_node_ids[i], end_node_ids[i], pred_map)
- paths.append(path)
-
- return start_node_ids, end_node_ids, dists, pred_maps, paths
-
-
-def rng_room_to_room(batch_size, gtG, rng, max_dist, max_dist_to_compute,
- node_room_ids, nodes=None, compute_path=False):
- # Sample one of the rooms, compute the distance field. Pick a destination in
- # another room if possible otherwise anywhere outside this room.
- dists = []; pred_maps = []; paths = []; start_node_ids = []; end_node_ids = [];
- room_ids = np.unique(node_room_ids[node_room_ids[:,0] >= 0, 0])
- for i in range(batch_size):
- room_id = rng.choice(room_ids)
- end_node_id = rng.choice(np.where(node_room_ids[:,0] == room_id)[0])
- end_node_ids.append(end_node_id)
-
- # Compute distances.
- dist, pred_map = gt.topology.shortest_distance(
- gt.GraphView(gtG, reversed=True), source=gtG.vertex(end_node_id),
- target=None, max_dist=max_dist_to_compute, pred_map=True)
- dist = np.array(dist.get_array())
- pred_map = np.array(pred_map.get_array())
- dists.append(dist)
- pred_maps.append(pred_map)
-
- # Randomly sample nodes which are within max_dist.
- near_ids = dist <= max_dist
- near_ids = near_ids[:, np.newaxis]
-
- # Check to see if there is a non-negative node which is close enough.
- non_same_room_ids = node_room_ids != room_id
- non_hallway_ids = node_room_ids != -1
- good1_ids = np.logical_and(near_ids, np.logical_and(non_same_room_ids, non_hallway_ids))
- good2_ids = np.logical_and(near_ids, non_hallway_ids)
- good3_ids = near_ids
- if np.any(good1_ids):
- start_node_id = rng.choice(np.where(good1_ids)[0])
- elif np.any(good2_ids):
- start_node_id = rng.choice(np.where(good2_ids)[0])
- elif np.any(good3_ids):
- start_node_id = rng.choice(np.where(good3_ids)[0])
- else:
- logging.error('Did not find any good nodes.')
-
- start_node_ids.append(start_node_id)
-
- path = None
- if compute_path:
- path = get_path_ids(start_node_ids[i], end_node_ids[i], pred_map)
- paths.append(path)
-
- return start_node_ids, end_node_ids, dists, pred_maps, paths
-
-
-def rng_target_dist_field(batch_size, gtG, rng, max_dist, max_dist_to_compute,
- nodes=None, compute_path=False):
- # Sample a single node, compute distance to all nodes less than max_dist,
- # sample nodes which are a particular distance away.
- dists = []; pred_maps = []; paths = []; start_node_ids = []
- end_node_ids = rng.choice(gtG.num_vertices(), size=(batch_size,),
- replace=False).tolist()
-
- for i in range(batch_size):
- dist, pred_map = gt.topology.shortest_distance(
- gt.GraphView(gtG, reversed=True), source=gtG.vertex(end_node_ids[i]),
- target=None, max_dist=max_dist_to_compute, pred_map=True)
- dist = np.array(dist.get_array())
- pred_map = np.array(pred_map.get_array())
- dists.append(dist)
- pred_maps.append(pred_map)
-
- # Randomly sample nodes which are withing max_dist
- near_ids = np.where(dist <= max_dist)[0]
- start_node_id = rng.choice(near_ids, size=(1,), replace=False)[0]
- start_node_ids.append(start_node_id)
-
- path = None
- if compute_path:
- path = get_path_ids(start_node_ids[i], end_node_ids[i], pred_map)
- paths.append(path)
-
- return start_node_ids, end_node_ids, dists, pred_maps, paths
diff --git a/research/cognitive_mapping_and_planning/src/map_utils.py b/research/cognitive_mapping_and_planning/src/map_utils.py
deleted file mode 100644
index 6756131a9eac161e7633ef089ed573e324f859e1..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/src/map_utils.py
+++ /dev/null
@@ -1,245 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Various function to compute the ground truth map for training etc.
-"""
-import copy
-import skimage.morphology
-import logging
-import numpy as np
-import scipy.ndimage
-import matplotlib.pyplot as plt
-import PIL
-
-import src.utils as utils
-import cv2
-
-def _get_xy_bounding_box(vertex, padding):
- """Returns the xy bounding box of the environment."""
- min_ = np.floor(np.min(vertex[:, :2], axis=0) - padding).astype(np.int)
- max_ = np.ceil(np.max(vertex[:, :2], axis=0) + padding).astype(np.int)
- return min_, max_
-
-def _project_to_map(map, vertex, wt=None, ignore_points_outside_map=False):
- """Projects points to map, returns how many points are present at each
- location."""
- num_points = np.zeros((map.size[1], map.size[0]))
- vertex_ = vertex[:, :2] - map.origin
- vertex_ = np.round(vertex_ / map.resolution).astype(np.int)
- if ignore_points_outside_map:
- good_ind = np.all(np.array([vertex_[:,1] >= 0, vertex_[:,1] < map.size[1],
- vertex_[:,0] >= 0, vertex_[:,0] < map.size[0]]),
- axis=0)
- vertex_ = vertex_[good_ind, :]
- if wt is not None:
- wt = wt[good_ind, :]
- if wt is None:
- np.add.at(num_points, (vertex_[:, 1], vertex_[:, 0]), 1)
- else:
- assert(wt.shape[0] == vertex.shape[0]), \
- 'number of weights should be same as vertices.'
- np.add.at(num_points, (vertex_[:, 1], vertex_[:, 0]), wt)
- return num_points
-
-def make_map(padding, resolution, vertex=None, sc=1.):
- """Returns a map structure."""
- min_, max_ = _get_xy_bounding_box(vertex*sc, padding=padding)
- sz = np.ceil((max_ - min_ + 1) / resolution).astype(np.int32)
- max_ = min_ + sz * resolution - 1
- map = utils.Foo(origin=min_, size=sz, max=max_, resolution=resolution,
- padding=padding)
- return map
-
-def _fill_holes(img, thresh):
- """Fills holes less than thresh area (assumes 4 connectivity when computing
- hole area."""
- l, n = scipy.ndimage.label(np.logical_not(img))
- img_ = img == True
- cnts = np.bincount(l.reshape(-1))
- for i, cnt in enumerate(cnts):
- if cnt < thresh:
- l[l == i] = -1
- img_[l == -1] = True
- return img_
-
-def compute_traversibility(map, robot_base, robot_height, robot_radius,
- valid_min, valid_max, num_point_threshold, shapess,
- sc=100., n_samples_per_face=200):
- """Returns a bit map with pixels that are traversible or not as long as the
- robot center is inside this volume we are good colisions can be detected by
- doing a line search on things, or walking from current location to final
- location in the bitmap, or doing bwlabel on the traversibility map."""
-
- tt = utils.Timer()
- tt.tic()
- num_obstcale_points = np.zeros((map.size[1], map.size[0]))
- num_points = np.zeros((map.size[1], map.size[0]))
-
- for i, shapes in enumerate(shapess):
- for j in range(shapes.get_number_of_meshes()):
- p, face_areas, face_idx = shapes.sample_points_on_face_of_shape(
- j, n_samples_per_face, sc)
- wt = face_areas[face_idx]/n_samples_per_face
-
- ind = np.all(np.concatenate(
- (p[:, [2]] > robot_base,
- p[:, [2]] < robot_base + robot_height), axis=1),axis=1)
- num_obstcale_points += _project_to_map(map, p[ind, :], wt[ind])
-
- ind = np.all(np.concatenate(
- (p[:, [2]] > valid_min,
- p[:, [2]] < valid_max), axis=1),axis=1)
- num_points += _project_to_map(map, p[ind, :], wt[ind])
-
- selem = skimage.morphology.disk(robot_radius / map.resolution)
- obstacle_free = skimage.morphology.binary_dilation(
- _fill_holes(num_obstcale_points > num_point_threshold, 20), selem) != True
- valid_space = _fill_holes(num_points > num_point_threshold, 20)
- traversible = np.all(np.concatenate((obstacle_free[...,np.newaxis],
- valid_space[...,np.newaxis]), axis=2),
- axis=2)
- # plt.imshow(np.concatenate((obstacle_free, valid_space, traversible), axis=1))
- # plt.show()
-
- map_out = copy.deepcopy(map)
- map_out.num_obstcale_points = num_obstcale_points
- map_out.num_points = num_points
- map_out.traversible = traversible
- map_out.obstacle_free = obstacle_free
- map_out.valid_space = valid_space
- tt.toc(log_at=1, log_str='src.map_utils.compute_traversibility: ')
- return map_out
-
-
-def resize_maps(map, map_scales, resize_method):
- scaled_maps = []
- for i, sc in enumerate(map_scales):
- if resize_method == 'antialiasing':
- # Resize using open cv so that we can compute the size.
- # Use PIL resize to use anti aliasing feature.
- map_ = cv2.resize(map*1, None, None, fx=sc, fy=sc, interpolation=cv2.INTER_LINEAR)
- w = map_.shape[1]; h = map_.shape[0]
-
- map_img = PIL.Image.fromarray((map*255).astype(np.uint8))
- map__img = map_img.resize((w,h), PIL.Image.ANTIALIAS)
- map_ = np.asarray(map__img).astype(np.float32)
- map_ = map_/255.
- map_ = np.minimum(map_, 1.0)
- map_ = np.maximum(map_, 0.0)
- elif resize_method == 'linear_noantialiasing':
- map_ = cv2.resize(map*1, None, None, fx=sc, fy=sc, interpolation=cv2.INTER_LINEAR)
- else:
- logging.error('Unknown resizing method')
- scaled_maps.append(map_)
- return scaled_maps
-
-
-def pick_largest_cc(traversible):
- out = scipy.ndimage.label(traversible)[0]
- cnt = np.bincount(out.reshape(-1))[1:]
- return out == np.argmax(cnt) + 1
-
-def get_graph_origin_loc(rng, traversible):
- """Erode the traversibility mask so that we get points in the bulk of the
- graph, and not end up with a situation where the graph is localized in the
- corner of a cramped room. Output Locs is in the coordinate frame of the
- map."""
-
- aa = pick_largest_cc(skimage.morphology.binary_erosion(traversible == True,
- selem=np.ones((15,15))))
- y, x = np.where(aa > 0)
- ind = rng.choice(y.size)
- locs = np.array([x[ind], y[ind]])
- locs = locs + rng.rand(*(locs.shape)) - 0.5
- return locs
-
-
-def generate_egocentric_maps(scaled_maps, map_scales, map_crop_sizes, loc,
- x_axis, y_axis, theta):
- maps = []
- for i, (map_, sc, map_crop_size) in enumerate(zip(scaled_maps, map_scales, map_crop_sizes)):
- maps_i = np.array(get_map_to_predict(loc*sc, x_axis, y_axis, map_,
- map_crop_size,
- interpolation=cv2.INTER_LINEAR)[0])
- maps_i[np.isnan(maps_i)] = 0
- maps.append(maps_i)
- return maps
-
-def generate_goal_images(map_scales, map_crop_sizes, n_ori, goal_dist,
- goal_theta, rel_goal_orientation):
- goal_dist = goal_dist[:,0]
- goal_theta = goal_theta[:,0]
- rel_goal_orientation = rel_goal_orientation[:,0]
-
- goals = [];
- # Generate the map images.
- for i, (sc, map_crop_size) in enumerate(zip(map_scales, map_crop_sizes)):
- goal_i = np.zeros((goal_dist.shape[0], map_crop_size, map_crop_size, n_ori),
- dtype=np.float32)
- x = goal_dist*np.cos(goal_theta)*sc + (map_crop_size-1.)/2.
- y = goal_dist*np.sin(goal_theta)*sc + (map_crop_size-1.)/2.
-
- for j in range(goal_dist.shape[0]):
- gc = rel_goal_orientation[j]
- x0 = np.floor(x[j]).astype(np.int32); x1 = x0 + 1;
- y0 = np.floor(y[j]).astype(np.int32); y1 = y0 + 1;
- if x0 >= 0 and x0 <= map_crop_size-1:
- if y0 >= 0 and y0 <= map_crop_size-1:
- goal_i[j, y0, x0, gc] = (x1-x[j])*(y1-y[j])
- if y1 >= 0 and y1 <= map_crop_size-1:
- goal_i[j, y1, x0, gc] = (x1-x[j])*(y[j]-y0)
-
- if x1 >= 0 and x1 <= map_crop_size-1:
- if y0 >= 0 and y0 <= map_crop_size-1:
- goal_i[j, y0, x1, gc] = (x[j]-x0)*(y1-y[j])
- if y1 >= 0 and y1 <= map_crop_size-1:
- goal_i[j, y1, x1, gc] = (x[j]-x0)*(y[j]-y0)
-
- goals.append(goal_i)
- return goals
-
-def get_map_to_predict(src_locs, src_x_axiss, src_y_axiss, map, map_size,
- interpolation=cv2.INTER_LINEAR):
- fss = []
- valids = []
-
- center = (map_size-1.0)/2.0
- dst_theta = np.pi/2.0
- dst_loc = np.array([center, center])
- dst_x_axis = np.array([np.cos(dst_theta), np.sin(dst_theta)])
- dst_y_axis = np.array([np.cos(dst_theta+np.pi/2), np.sin(dst_theta+np.pi/2)])
-
- def compute_points(center, x_axis, y_axis):
- points = np.zeros((3,2),dtype=np.float32)
- points[0,:] = center
- points[1,:] = center + x_axis
- points[2,:] = center + y_axis
- return points
-
- dst_points = compute_points(dst_loc, dst_x_axis, dst_y_axis)
- for i in range(src_locs.shape[0]):
- src_loc = src_locs[i,:]
- src_x_axis = src_x_axiss[i,:]
- src_y_axis = src_y_axiss[i,:]
- src_points = compute_points(src_loc, src_x_axis, src_y_axis)
- M = cv2.getAffineTransform(src_points, dst_points)
-
- fs = cv2.warpAffine(map, M, (map_size, map_size), None, flags=interpolation,
- borderValue=np.NaN)
- valid = np.invert(np.isnan(fs))
- valids.append(valid)
- fss.append(fs)
- return fss, valids
-
diff --git a/research/cognitive_mapping_and_planning/src/rotation_utils.py b/research/cognitive_mapping_and_planning/src/rotation_utils.py
deleted file mode 100644
index 8d6d4f3cbdb1f808d210dce8b22fa3ba831d45a9..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/src/rotation_utils.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities for generating and applying rotation matrices.
-"""
-import numpy as np
-
-ANGLE_EPS = 0.001
-
-
-def normalize(v):
- return v / np.linalg.norm(v)
-
-
-def get_r_matrix(ax_, angle):
- ax = normalize(ax_)
- if np.abs(angle) > ANGLE_EPS:
- S_hat = np.array(
- [[0.0, -ax[2], ax[1]], [ax[2], 0.0, -ax[0]], [-ax[1], ax[0], 0.0]],
- dtype=np.float32)
- R = np.eye(3) + np.sin(angle)*S_hat + \
- (1-np.cos(angle))*(np.linalg.matrix_power(S_hat, 2))
- else:
- R = np.eye(3)
- return R
-
-
-def r_between(v_from_, v_to_):
- v_from = normalize(v_from_)
- v_to = normalize(v_to_)
- ax = normalize(np.cross(v_from, v_to))
- angle = np.arccos(np.dot(v_from, v_to))
- return get_r_matrix(ax, angle)
-
-
-def rotate_camera_to_point_at(up_from, lookat_from, up_to, lookat_to):
- inputs = [up_from, lookat_from, up_to, lookat_to]
- for i in range(4):
- inputs[i] = normalize(np.array(inputs[i]).reshape((-1,)))
- up_from, lookat_from, up_to, lookat_to = inputs
- r1 = r_between(lookat_from, lookat_to)
-
- new_x = np.dot(r1, np.array([1, 0, 0]).reshape((-1, 1))).reshape((-1))
- to_x = normalize(np.cross(lookat_to, up_to))
- angle = np.arccos(np.dot(new_x, to_x))
- if angle > ANGLE_EPS:
- if angle < np.pi - ANGLE_EPS:
- ax = normalize(np.cross(new_x, to_x))
- flip = np.dot(lookat_to, ax)
- if flip > 0:
- r2 = get_r_matrix(lookat_to, angle)
- elif flip < 0:
- r2 = get_r_matrix(lookat_to, -1. * angle)
- else:
- # Angle of rotation is too close to 180 degrees, direction of rotation
- # does not matter.
- r2 = get_r_matrix(lookat_to, angle)
- else:
- r2 = np.eye(3)
- return np.dot(r2, r1)
-
diff --git a/research/cognitive_mapping_and_planning/src/utils.py b/research/cognitive_mapping_and_planning/src/utils.py
deleted file mode 100644
index a1b9e44260b7c7884855761f56ac60d6f508c2fb..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/src/utils.py
+++ /dev/null
@@ -1,168 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""Generaly Utilities.
-"""
-
-import numpy as np, cPickle, os, time
-from six.moves import xrange
-import src.file_utils as fu
-import logging
-
-class Timer():
- def __init__(self):
- self.calls = 0.
- self.start_time = 0.
- self.time_per_call = 0.
- self.total_time = 0.
- self.last_log_time = 0.
-
- def tic(self):
- self.start_time = time.time()
-
- def toc(self, average=True, log_at=-1, log_str='', type='calls'):
- if self.start_time == 0:
- logging.error('Timer not started by calling tic().')
- t = time.time()
- diff = time.time() - self.start_time
- self.total_time += diff
- self.calls += 1.
- self.time_per_call = self.total_time/self.calls
-
- if type == 'calls' and log_at > 0 and np.mod(self.calls, log_at) == 0:
- _ = []
- logging.info('%s: %f seconds.', log_str, self.time_per_call)
- elif type == 'time' and log_at > 0 and t - self.last_log_time >= log_at:
- _ = []
- logging.info('%s: %f seconds.', log_str, self.time_per_call)
- self.last_log_time = t
-
- if average:
- return self.time_per_call
- else:
- return diff
-
-class Foo(object):
- def __init__(self, **kwargs):
- self.__dict__.update(kwargs)
- def __str__(self):
- str_ = ''
- for v in vars(self).keys():
- a = getattr(self, v)
- if True: #isinstance(v, object):
- str__ = str(a)
- str__ = str__.replace('\n', '\n ')
- else:
- str__ = str(a)
- str_ += '{:s}: {:s}'.format(v, str__)
- str_ += '\n'
- return str_
-
-
-def dict_equal(dict1, dict2):
- assert(set(dict1.keys()) == set(dict2.keys())), "Sets of keys between 2 dictionaries are different."
- for k in dict1.keys():
- assert(type(dict1[k]) == type(dict2[k])), "Type of key '{:s}' if different.".format(k)
- if type(dict1[k]) == np.ndarray:
- assert(dict1[k].dtype == dict2[k].dtype), "Numpy Type of key '{:s}' if different.".format(k)
- assert(np.allclose(dict1[k], dict2[k])), "Value for key '{:s}' do not match.".format(k)
- else:
- assert(dict1[k] == dict2[k]), "Value for key '{:s}' do not match.".format(k)
- return True
-
-def subplot(plt, Y_X, sz_y_sz_x = (10, 10)):
- Y,X = Y_X
- sz_y, sz_x = sz_y_sz_x
- plt.rcParams['figure.figsize'] = (X*sz_x, Y*sz_y)
- fig, axes = plt.subplots(Y, X)
- plt.subplots_adjust(wspace=0.1, hspace=0.1)
- return fig, axes
-
-def tic_toc_print(interval, string):
- global tic_toc_print_time_old
- if 'tic_toc_print_time_old' not in globals():
- tic_toc_print_time_old = time.time()
- print(string)
- else:
- new_time = time.time()
- if new_time - tic_toc_print_time_old > interval:
- tic_toc_print_time_old = new_time;
- print(string)
-
-def mkdir_if_missing(output_dir):
- if not fu.exists(output_dir):
- fu.makedirs(output_dir)
-
-def save_variables(pickle_file_name, var, info, overwrite = False):
- if fu.exists(pickle_file_name) and overwrite == False:
- raise Exception('{:s} exists and over write is false.'.format(pickle_file_name))
- # Construct the dictionary
- assert(type(var) == list); assert(type(info) == list);
- d = {}
- for i in xrange(len(var)):
- d[info[i]] = var[i]
- with fu.fopen(pickle_file_name, 'w') as f:
- cPickle.dump(d, f, cPickle.HIGHEST_PROTOCOL)
-
-def load_variables(pickle_file_name):
- if fu.exists(pickle_file_name):
- with fu.fopen(pickle_file_name, 'r') as f:
- d = cPickle.load(f)
- return d
- else:
- raise Exception('{:s} does not exists.'.format(pickle_file_name))
-
-def voc_ap(rec, prec):
- rec = rec.reshape((-1,1))
- prec = prec.reshape((-1,1))
- z = np.zeros((1,1))
- o = np.ones((1,1))
- mrec = np.vstack((z, rec, o))
- mpre = np.vstack((z, prec, z))
- for i in range(len(mpre)-2, -1, -1):
- mpre[i] = max(mpre[i], mpre[i+1])
-
- I = np.where(mrec[1:] != mrec[0:-1])[0]+1;
- ap = 0;
- for i in I:
- ap = ap + (mrec[i] - mrec[i-1])*mpre[i];
- return ap
-
-def tight_imshow_figure(plt, figsize=None):
- fig = plt.figure(figsize=figsize)
- ax = plt.Axes(fig, [0,0,1,1])
- ax.set_axis_off()
- fig.add_axes(ax)
- return fig, ax
-
-def calc_pr(gt, out, wt=None):
- if wt is None:
- wt = np.ones((gt.size,1))
-
- gt = gt.astype(np.float64).reshape((-1,1))
- wt = wt.astype(np.float64).reshape((-1,1))
- out = out.astype(np.float64).reshape((-1,1))
-
- gt = gt*wt
- tog = np.concatenate([gt, wt, out], axis=1)*1.
- ind = np.argsort(tog[:,2], axis=0)[::-1]
- tog = tog[ind,:]
- cumsumsortgt = np.cumsum(tog[:,0])
- cumsumsortwt = np.cumsum(tog[:,1])
- prec = cumsumsortgt / cumsumsortwt
- rec = cumsumsortgt / np.sum(tog[:,0])
-
- ap = voc_ap(rec, prec)
- return ap, rec, prec
diff --git a/research/cognitive_mapping_and_planning/tfcode/__init__.py b/research/cognitive_mapping_and_planning/tfcode/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/cognitive_mapping_and_planning/tfcode/cmp.py b/research/cognitive_mapping_and_planning/tfcode/cmp.py
deleted file mode 100644
index 228ef90fddcd9ff41b26795544d93a1f18466158..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/tfcode/cmp.py
+++ /dev/null
@@ -1,553 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Code for setting up the network for CMP.
-
-Sets up the mapper and the planner.
-"""
-
-import sys, os, numpy as np
-import matplotlib.pyplot as plt
-import copy
-import argparse, pprint
-import time
-
-
-import tensorflow as tf
-
-from tensorflow.contrib import slim
-from tensorflow.contrib.slim import arg_scope
-
-import logging
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-from src import utils
-import src.file_utils as fu
-import tfcode.nav_utils as nu
-import tfcode.cmp_utils as cu
-import tfcode.cmp_summary as cmp_s
-from tfcode import tf_utils
-
-value_iteration_network = cu.value_iteration_network
-rotate_preds = cu.rotate_preds
-deconv = cu.deconv
-get_visual_frustum = cu.get_visual_frustum
-fr_v2 = cu.fr_v2
-
-setup_train_step_kwargs = nu.default_train_step_kwargs
-compute_losses_multi_or = nu.compute_losses_multi_or
-
-get_repr_from_image = nu.get_repr_from_image
-
-_save_d_at_t = nu.save_d_at_t
-_save_all = nu.save_all
-_eval_ap = nu.eval_ap
-_eval_dist = nu.eval_dist
-_plot_trajectories = nu.plot_trajectories
-
-_vis_readout_maps = cmp_s._vis_readout_maps
-_vis = cmp_s._vis
-_summary_vis = cmp_s._summary_vis
-_summary_readout_maps = cmp_s._summary_readout_maps
-_add_summaries = cmp_s._add_summaries
-
-def _inputs(problem):
- # Set up inputs.
- with tf.name_scope('inputs'):
- inputs = []
- inputs.append(('orig_maps', tf.float32,
- (problem.batch_size, 1, None, None, 1)))
- inputs.append(('goal_loc', tf.float32,
- (problem.batch_size, problem.num_goals, 2)))
- common_input_data, _ = tf_utils.setup_inputs(inputs)
-
- inputs = []
- if problem.input_type == 'vision':
- # Multiple images from an array of cameras.
- inputs.append(('imgs', tf.float32,
- (problem.batch_size, None, len(problem.aux_delta_thetas)+1,
- problem.img_height, problem.img_width,
- problem.img_channels)))
- elif problem.input_type == 'analytical_counts':
- for i in range(len(problem.map_crop_sizes)):
- inputs.append(('analytical_counts_{:d}'.format(i), tf.float32,
- (problem.batch_size, None, problem.map_crop_sizes[i],
- problem.map_crop_sizes[i], problem.map_channels)))
-
- if problem.outputs.readout_maps:
- for i in range(len(problem.readout_maps_crop_sizes)):
- inputs.append(('readout_maps_{:d}'.format(i), tf.float32,
- (problem.batch_size, None,
- problem.readout_maps_crop_sizes[i],
- problem.readout_maps_crop_sizes[i],
- problem.readout_maps_channels)))
-
- for i in range(len(problem.map_crop_sizes)):
- inputs.append(('ego_goal_imgs_{:d}'.format(i), tf.float32,
- (problem.batch_size, None, problem.map_crop_sizes[i],
- problem.map_crop_sizes[i], problem.goal_channels)))
- for s in ['sum_num', 'sum_denom', 'max_denom']:
- inputs.append(('running_'+s+'_{:d}'.format(i), tf.float32,
- (problem.batch_size, 1, problem.map_crop_sizes[i],
- problem.map_crop_sizes[i], problem.map_channels)))
-
- inputs.append(('incremental_locs', tf.float32,
- (problem.batch_size, None, 2)))
- inputs.append(('incremental_thetas', tf.float32,
- (problem.batch_size, None, 1)))
- inputs.append(('step_number', tf.int32, (1, None, 1)))
- inputs.append(('node_ids', tf.int32, (problem.batch_size, None,
- problem.node_ids_dim)))
- inputs.append(('perturbs', tf.float32, (problem.batch_size, None,
- problem.perturbs_dim)))
-
- # For plotting result plots
- inputs.append(('loc_on_map', tf.float32, (problem.batch_size, None, 2)))
- inputs.append(('gt_dist_to_goal', tf.float32, (problem.batch_size, None, 1)))
-
- step_input_data, _ = tf_utils.setup_inputs(inputs)
-
- inputs = []
- inputs.append(('action', tf.int32, (problem.batch_size, None, problem.num_actions)))
- train_data, _ = tf_utils.setup_inputs(inputs)
- train_data.update(step_input_data)
- train_data.update(common_input_data)
- return common_input_data, step_input_data, train_data
-
-def readout_general(multi_scale_belief, num_neurons, strides, layers_per_block,
- kernel_size, batch_norm_is_training_op, wt_decay):
- multi_scale_belief = tf.stop_gradient(multi_scale_belief)
- with tf.variable_scope('readout_maps_deconv'):
- x, outs = deconv(multi_scale_belief, batch_norm_is_training_op,
- wt_decay=wt_decay, neurons=num_neurons, strides=strides,
- layers_per_block=layers_per_block, kernel_size=kernel_size,
- conv_fn=slim.conv2d_transpose, offset=0,
- name='readout_maps_deconv')
- probs = tf.sigmoid(x)
- return x, probs
-
-
-def running_combine(fss_logits, confs_probs, incremental_locs,
- incremental_thetas, previous_sum_num, previous_sum_denom,
- previous_max_denom, map_size, num_steps):
- # fss_logits is B x N x H x W x C
- # confs_logits is B x N x H x W x C
- # incremental_locs is B x N x 2
- # incremental_thetas is B x N x 1
- # previous_sum_num etc is B x 1 x H x W x C
-
- with tf.name_scope('combine_{:d}'.format(num_steps)):
- running_sum_nums_ = []; running_sum_denoms_ = [];
- running_max_denoms_ = [];
-
- fss_logits_ = tf.unstack(fss_logits, axis=1, num=num_steps)
- confs_probs_ = tf.unstack(confs_probs, axis=1, num=num_steps)
- incremental_locs_ = tf.unstack(incremental_locs, axis=1, num=num_steps)
- incremental_thetas_ = tf.unstack(incremental_thetas, axis=1, num=num_steps)
- running_sum_num = tf.unstack(previous_sum_num, axis=1, num=1)[0]
- running_sum_denom = tf.unstack(previous_sum_denom, axis=1, num=1)[0]
- running_max_denom = tf.unstack(previous_max_denom, axis=1, num=1)[0]
-
- for i in range(num_steps):
- # Rotate the previous running_num and running_denom
- running_sum_num, running_sum_denom, running_max_denom = rotate_preds(
- incremental_locs_[i], incremental_thetas_[i], map_size,
- [running_sum_num, running_sum_denom, running_max_denom],
- output_valid_mask=False)[0]
- # print i, num_steps, running_sum_num.get_shape().as_list()
- running_sum_num = running_sum_num + fss_logits_[i] * confs_probs_[i]
- running_sum_denom = running_sum_denom + confs_probs_[i]
- running_max_denom = tf.maximum(running_max_denom, confs_probs_[i])
- running_sum_nums_.append(running_sum_num)
- running_sum_denoms_.append(running_sum_denom)
- running_max_denoms_.append(running_max_denom)
-
- running_sum_nums = tf.stack(running_sum_nums_, axis=1)
- running_sum_denoms = tf.stack(running_sum_denoms_, axis=1)
- running_max_denoms = tf.stack(running_max_denoms_, axis=1)
- return running_sum_nums, running_sum_denoms, running_max_denoms
-
-def get_map_from_images(imgs, mapper_arch, task_params, freeze_conv, wt_decay,
- is_training, batch_norm_is_training_op, num_maps,
- split_maps=True):
- # Hit image with a resnet.
- n_views = len(task_params.aux_delta_thetas) + 1
- out = utils.Foo()
-
- images_reshaped = tf.reshape(imgs,
- shape=[-1, task_params.img_height,
- task_params.img_width,
- task_params.img_channels], name='re_image')
-
- x, out.vars_to_restore = get_repr_from_image(
- images_reshaped, task_params.modalities, task_params.data_augment,
- mapper_arch.encoder, freeze_conv, wt_decay, is_training)
-
- # Reshape into nice things so that these can be accumulated over time steps
- # for faster backprop.
- sh_before = x.get_shape().as_list()
- out.encoder_output = tf.reshape(x, shape=[task_params.batch_size, -1, n_views] + sh_before[1:])
- x = tf.reshape(out.encoder_output, shape=[-1] + sh_before[1:])
-
- # Add a layer to reduce dimensions for a fc layer.
- if mapper_arch.dim_reduce_neurons > 0:
- ks = 1; neurons = mapper_arch.dim_reduce_neurons;
- init_var = np.sqrt(2.0/(ks**2)/neurons)
- batch_norm_param = mapper_arch.batch_norm_param
- batch_norm_param['is_training'] = batch_norm_is_training_op
- out.conv_feat = slim.conv2d(x, neurons, kernel_size=ks, stride=1,
- normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_param,
- padding='SAME', scope='dim_reduce',
- weights_regularizer=slim.l2_regularizer(wt_decay),
- weights_initializer=tf.random_normal_initializer(stddev=init_var))
- reshape_conv_feat = slim.flatten(out.conv_feat)
- sh = reshape_conv_feat.get_shape().as_list()
- out.reshape_conv_feat = tf.reshape(reshape_conv_feat, shape=[-1, sh[1]*n_views])
-
- with tf.variable_scope('fc'):
- # Fully connected layers to compute the representation in top-view space.
- fc_batch_norm_param = {'center': True, 'scale': True,
- 'activation_fn':tf.nn.relu,
- 'is_training': batch_norm_is_training_op}
- f = out.reshape_conv_feat
- out_neurons = (mapper_arch.fc_out_size**2)*mapper_arch.fc_out_neurons
- neurons = mapper_arch.fc_neurons + [out_neurons]
- f, _ = tf_utils.fc_network(f, neurons=neurons, wt_decay=wt_decay,
- name='fc', offset=0,
- batch_norm_param=fc_batch_norm_param,
- is_training=is_training,
- dropout_ratio=mapper_arch.fc_dropout)
- f = tf.reshape(f, shape=[-1, mapper_arch.fc_out_size,
- mapper_arch.fc_out_size,
- mapper_arch.fc_out_neurons], name='re_fc')
-
- # Use pool5 to predict the free space map via deconv layers.
- with tf.variable_scope('deconv'):
- x, outs = deconv(f, batch_norm_is_training_op, wt_decay=wt_decay,
- neurons=mapper_arch.deconv_neurons,
- strides=mapper_arch.deconv_strides,
- layers_per_block=mapper_arch.deconv_layers_per_block,
- kernel_size=mapper_arch.deconv_kernel_size,
- conv_fn=slim.conv2d_transpose, offset=0, name='deconv')
-
- # Reshape x the right way.
- sh = x.get_shape().as_list()
- x = tf.reshape(x, shape=[task_params.batch_size, -1] + sh[1:])
- out.deconv_output = x
-
- # Separate out the map and the confidence predictions, pass the confidence
- # through a sigmoid.
- if split_maps:
- with tf.name_scope('split'):
- out_all = tf.split(value=x, axis=4, num_or_size_splits=2*num_maps)
- out.fss_logits = out_all[:num_maps]
- out.confs_logits = out_all[num_maps:]
- with tf.name_scope('sigmoid'):
- out.confs_probs = [tf.nn.sigmoid(x) for x in out.confs_logits]
- return out
-
-def setup_to_run(m, args, is_training, batch_norm_is_training, summary_mode):
- assert(args.arch.multi_scale), 'removed support for old single scale code.'
- # Set up the model.
- tf.set_random_seed(args.solver.seed)
- task_params = args.navtask.task_params
-
- batch_norm_is_training_op = \
- tf.placeholder_with_default(batch_norm_is_training, shape=[],
- name='batch_norm_is_training_op')
-
- # Setup the inputs
- m.input_tensors = {}
- m.train_ops = {}
- m.input_tensors['common'], m.input_tensors['step'], m.input_tensors['train'] = \
- _inputs(task_params)
-
- m.init_fn = None
-
- if task_params.input_type == 'vision':
- m.vision_ops = get_map_from_images(
- m.input_tensors['step']['imgs'], args.mapper_arch,
- task_params, args.solver.freeze_conv,
- args.solver.wt_decay, is_training, batch_norm_is_training_op,
- num_maps=len(task_params.map_crop_sizes))
-
- # Load variables from snapshot if needed.
- if args.solver.pretrained_path is not None:
- m.init_fn = slim.assign_from_checkpoint_fn(args.solver.pretrained_path,
- m.vision_ops.vars_to_restore)
-
- # Set up caching of vision features if needed.
- if args.solver.freeze_conv:
- m.train_ops['step_data_cache'] = [m.vision_ops.encoder_output]
- else:
- m.train_ops['step_data_cache'] = []
-
- # Set up blobs that are needed for the computation in rest of the graph.
- m.ego_map_ops = m.vision_ops.fss_logits
- m.coverage_ops = m.vision_ops.confs_probs
-
- # Zero pad these to make them same size as what the planner expects.
- for i in range(len(m.ego_map_ops)):
- if args.mapper_arch.pad_map_with_zeros_each[i] > 0:
- paddings = np.zeros((5,2), dtype=np.int32)
- paddings[2:4,:] = args.mapper_arch.pad_map_with_zeros_each[i]
- paddings_op = tf.constant(paddings, dtype=tf.int32)
- m.ego_map_ops[i] = tf.pad(m.ego_map_ops[i], paddings=paddings_op)
- m.coverage_ops[i] = tf.pad(m.coverage_ops[i], paddings=paddings_op)
-
- elif task_params.input_type == 'analytical_counts':
- m.ego_map_ops = []; m.coverage_ops = []
- for i in range(len(task_params.map_crop_sizes)):
- ego_map_op = m.input_tensors['step']['analytical_counts_{:d}'.format(i)]
- coverage_op = tf.cast(tf.greater_equal(
- tf.reduce_max(ego_map_op, reduction_indices=[4],
- keep_dims=True), 1), tf.float32)
- coverage_op = tf.ones_like(ego_map_op) * coverage_op
- m.ego_map_ops.append(ego_map_op)
- m.coverage_ops.append(coverage_op)
- m.train_ops['step_data_cache'] = []
-
- num_steps = task_params.num_steps
- num_goals = task_params.num_goals
-
- map_crop_size_ops = []
- for map_crop_size in task_params.map_crop_sizes:
- map_crop_size_ops.append(tf.constant(map_crop_size, dtype=tf.int32, shape=(2,)))
-
- with tf.name_scope('check_size'):
- is_single_step = tf.equal(tf.unstack(tf.shape(m.ego_map_ops[0]), num=5)[1], 1)
-
- fr_ops = []; value_ops = [];
- fr_intermediate_ops = []; value_intermediate_ops = [];
- crop_value_ops = [];
- resize_crop_value_ops = [];
- confs = []; occupancys = [];
-
- previous_value_op = None
- updated_state = []; state_names = [];
-
- for i in range(len(task_params.map_crop_sizes)):
- map_crop_size = task_params.map_crop_sizes[i]
- with tf.variable_scope('scale_{:d}'.format(i)):
- # Accumulate the map.
- fn = lambda ns: running_combine(
- m.ego_map_ops[i],
- m.coverage_ops[i],
- m.input_tensors['step']['incremental_locs'] * task_params.map_scales[i],
- m.input_tensors['step']['incremental_thetas'],
- m.input_tensors['step']['running_sum_num_{:d}'.format(i)],
- m.input_tensors['step']['running_sum_denom_{:d}'.format(i)],
- m.input_tensors['step']['running_max_denom_{:d}'.format(i)],
- map_crop_size, ns)
-
- running_sum_num, running_sum_denom, running_max_denom = \
- tf.cond(is_single_step, lambda: fn(1), lambda: fn(num_steps*num_goals))
- updated_state += [running_sum_num, running_sum_denom, running_max_denom]
- state_names += ['running_sum_num_{:d}'.format(i),
- 'running_sum_denom_{:d}'.format(i),
- 'running_max_denom_{:d}'.format(i)]
-
- # Concat the accumulated map and goal
- occupancy = running_sum_num / tf.maximum(running_sum_denom, 0.001)
- conf = running_max_denom
- # print occupancy.get_shape().as_list()
-
- # Concat occupancy, how much occupied and goal.
- with tf.name_scope('concat'):
- sh = [-1, map_crop_size, map_crop_size, task_params.map_channels]
- occupancy = tf.reshape(occupancy, shape=sh)
- conf = tf.reshape(conf, shape=sh)
-
- sh = [-1, map_crop_size, map_crop_size, task_params.goal_channels]
- goal = tf.reshape(m.input_tensors['step']['ego_goal_imgs_{:d}'.format(i)], shape=sh)
- to_concat = [occupancy, conf, goal]
-
- if previous_value_op is not None:
- to_concat.append(previous_value_op)
-
- x = tf.concat(to_concat, 3)
-
- # Pass the map, previous rewards and the goal through a few convolutional
- # layers to get fR.
- fr_op, fr_intermediate_op = fr_v2(
- x, output_neurons=args.arch.fr_neurons,
- inside_neurons=args.arch.fr_inside_neurons,
- is_training=batch_norm_is_training_op, name='fr',
- wt_decay=args.solver.wt_decay, stride=args.arch.fr_stride)
-
- # Do Value Iteration on the fR
- if args.arch.vin_num_iters > 0:
- value_op, value_intermediate_op = value_iteration_network(
- fr_op, num_iters=args.arch.vin_num_iters,
- val_neurons=args.arch.vin_val_neurons,
- action_neurons=args.arch.vin_action_neurons,
- kernel_size=args.arch.vin_ks, share_wts=args.arch.vin_share_wts,
- name='vin', wt_decay=args.solver.wt_decay)
- else:
- value_op = fr_op
- value_intermediate_op = []
-
- # Crop out and upsample the previous value map.
- remove = args.arch.crop_remove_each
- if remove > 0:
- crop_value_op = value_op[:, remove:-remove, remove:-remove,:]
- else:
- crop_value_op = value_op
- crop_value_op = tf.reshape(crop_value_op, shape=[-1, args.arch.value_crop_size,
- args.arch.value_crop_size,
- args.arch.vin_val_neurons])
- if i < len(task_params.map_crop_sizes)-1:
- # Reshape it to shape of the next scale.
- previous_value_op = tf.image.resize_bilinear(crop_value_op,
- map_crop_size_ops[i+1],
- align_corners=True)
- resize_crop_value_ops.append(previous_value_op)
-
- occupancys.append(occupancy)
- confs.append(conf)
- value_ops.append(value_op)
- crop_value_ops.append(crop_value_op)
- fr_ops.append(fr_op)
- fr_intermediate_ops.append(fr_intermediate_op)
-
- m.value_ops = value_ops
- m.value_intermediate_ops = value_intermediate_ops
- m.fr_ops = fr_ops
- m.fr_intermediate_ops = fr_intermediate_ops
- m.final_value_op = crop_value_op
- m.crop_value_ops = crop_value_ops
- m.resize_crop_value_ops = resize_crop_value_ops
- m.confs = confs
- m.occupancys = occupancys
-
- sh = [-1, args.arch.vin_val_neurons*((args.arch.value_crop_size)**2)]
- m.value_features_op = tf.reshape(m.final_value_op, sh, name='reshape_value_op')
-
- # Determine what action to take.
- with tf.variable_scope('action_pred'):
- batch_norm_param = args.arch.pred_batch_norm_param
- if batch_norm_param is not None:
- batch_norm_param['is_training'] = batch_norm_is_training_op
- m.action_logits_op, _ = tf_utils.fc_network(
- m.value_features_op, neurons=args.arch.pred_neurons,
- wt_decay=args.solver.wt_decay, name='pred', offset=0,
- num_pred=task_params.num_actions,
- batch_norm_param=batch_norm_param)
- m.action_prob_op = tf.nn.softmax(m.action_logits_op)
-
- init_state = tf.constant(0., dtype=tf.float32, shape=[
- task_params.batch_size, 1, map_crop_size, map_crop_size,
- task_params.map_channels])
-
- m.train_ops['state_names'] = state_names
- m.train_ops['updated_state'] = updated_state
- m.train_ops['init_state'] = [init_state for _ in updated_state]
-
- m.train_ops['step'] = m.action_prob_op
- m.train_ops['common'] = [m.input_tensors['common']['orig_maps'],
- m.input_tensors['common']['goal_loc']]
- m.train_ops['batch_norm_is_training_op'] = batch_norm_is_training_op
- m.loss_ops = []; m.loss_ops_names = [];
-
- if args.arch.readout_maps:
- with tf.name_scope('readout_maps'):
- all_occupancys = tf.concat(m.occupancys + m.confs, 3)
- readout_maps, probs = readout_general(
- all_occupancys, num_neurons=args.arch.rom_arch.num_neurons,
- strides=args.arch.rom_arch.strides,
- layers_per_block=args.arch.rom_arch.layers_per_block,
- kernel_size=args.arch.rom_arch.kernel_size,
- batch_norm_is_training_op=batch_norm_is_training_op,
- wt_decay=args.solver.wt_decay)
-
- gt_ego_maps = [m.input_tensors['step']['readout_maps_{:d}'.format(i)]
- for i in range(len(task_params.readout_maps_crop_sizes))]
- m.readout_maps_gt = tf.concat(gt_ego_maps, 4)
- gt_shape = tf.shape(m.readout_maps_gt)
- m.readout_maps_logits = tf.reshape(readout_maps, gt_shape)
- m.readout_maps_probs = tf.reshape(probs, gt_shape)
-
- # Add a loss op
- m.readout_maps_loss_op = tf.losses.sigmoid_cross_entropy(
- tf.reshape(m.readout_maps_gt, [-1, len(task_params.readout_maps_crop_sizes)]),
- tf.reshape(readout_maps, [-1, len(task_params.readout_maps_crop_sizes)]),
- scope='loss')
- m.readout_maps_loss_op = 10.*m.readout_maps_loss_op
-
- ewma_decay = 0.99 if is_training else 0.0
- weight = tf.ones_like(m.input_tensors['train']['action'], dtype=tf.float32,
- name='weight')
- m.reg_loss_op, m.data_loss_op, m.total_loss_op, m.acc_ops = \
- compute_losses_multi_or(m.action_logits_op,
- m.input_tensors['train']['action'], weights=weight,
- num_actions=task_params.num_actions,
- data_loss_wt=args.solver.data_loss_wt,
- reg_loss_wt=args.solver.reg_loss_wt,
- ewma_decay=ewma_decay)
-
- if args.arch.readout_maps:
- m.total_loss_op = m.total_loss_op + m.readout_maps_loss_op
- m.loss_ops += [m.readout_maps_loss_op]
- m.loss_ops_names += ['readout_maps_loss']
-
- m.loss_ops += [m.reg_loss_op, m.data_loss_op, m.total_loss_op]
- m.loss_ops_names += ['reg_loss', 'data_loss', 'total_loss']
-
- if args.solver.freeze_conv:
- vars_to_optimize = list(set(tf.trainable_variables()) -
- set(m.vision_ops.vars_to_restore))
- else:
- vars_to_optimize = None
-
- m.lr_op, m.global_step_op, m.train_op, m.should_stop_op, m.optimizer, \
- m.sync_optimizer = tf_utils.setup_training(
- m.total_loss_op,
- args.solver.initial_learning_rate,
- args.solver.steps_per_decay,
- args.solver.learning_rate_decay,
- args.solver.momentum,
- args.solver.max_steps,
- args.solver.sync,
- args.solver.adjust_lr_sync,
- args.solver.num_workers,
- args.solver.task,
- vars_to_optimize=vars_to_optimize,
- clip_gradient_norm=args.solver.clip_gradient_norm,
- typ=args.solver.typ, momentum2=args.solver.momentum2,
- adam_eps=args.solver.adam_eps)
-
- if args.arch.sample_gt_prob_type == 'inverse_sigmoid_decay':
- m.sample_gt_prob_op = tf_utils.inverse_sigmoid_decay(args.arch.isd_k,
- m.global_step_op)
- elif args.arch.sample_gt_prob_type == 'zero':
- m.sample_gt_prob_op = tf.constant(-1.0, dtype=tf.float32)
-
- elif args.arch.sample_gt_prob_type.split('_')[0] == 'step':
- step = int(args.arch.sample_gt_prob_type.split('_')[1])
- m.sample_gt_prob_op = tf_utils.step_gt_prob(
- step, m.input_tensors['step']['step_number'][0,0,0])
-
- m.sample_action_type = args.arch.action_sample_type
- m.sample_action_combine_type = args.arch.action_sample_combine_type
-
- m.summary_ops = {
- summary_mode: _add_summaries(m, args, summary_mode,
- args.summary.arop_full_summary_iters)}
-
- m.init_op = tf.group(tf.global_variables_initializer(),
- tf.local_variables_initializer())
- m.saver_op = tf.train.Saver(keep_checkpoint_every_n_hours=4,
- write_version=tf.train.SaverDef.V2)
- return m
diff --git a/research/cognitive_mapping_and_planning/tfcode/cmp_summary.py b/research/cognitive_mapping_and_planning/tfcode/cmp_summary.py
deleted file mode 100644
index 55313bfbd52a9e079e1de5093ae1882a9bf1d858..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/tfcode/cmp_summary.py
+++ /dev/null
@@ -1,213 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Code for setting up summaries for CMP.
-"""
-
-import sys, os, numpy as np
-import matplotlib.pyplot as plt
-
-
-import tensorflow as tf
-
-from tensorflow.contrib import slim
-from tensorflow.contrib.slim import arg_scope
-
-import logging
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-from src import utils
-import src.file_utils as fu
-import tfcode.nav_utils as nu
-
-def _vis_readout_maps(outputs, global_step, output_dir, metric_summary, N):
- # outputs is [gt_map, pred_map]:
- if N >= 0:
- outputs = outputs[:N]
- N = len(outputs)
-
- plt.set_cmap('jet')
- fig, axes = utils.subplot(plt, (N, outputs[0][0].shape[4]*2), (5,5))
- axes = axes.ravel()[::-1].tolist()
- for i in range(N):
- gt_map, pred_map = outputs[i]
- for j in [0]:
- for k in range(gt_map.shape[4]):
- # Display something like the midpoint of the trajectory.
- id = np.int(gt_map.shape[1]/2)
-
- ax = axes.pop();
- ax.imshow(gt_map[j,id,:,:,k], origin='lower', interpolation='none',
- vmin=0., vmax=1.)
- ax.set_axis_off();
- if i == 0: ax.set_title('gt_map')
-
- ax = axes.pop();
- ax.imshow(pred_map[j,id,:,:,k], origin='lower', interpolation='none',
- vmin=0., vmax=1.)
- ax.set_axis_off();
- if i == 0: ax.set_title('pred_map')
-
- file_name = os.path.join(output_dir, 'readout_map_{:d}.png'.format(global_step))
- with fu.fopen(file_name, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
- plt.close(fig)
-
-def _vis(outputs, global_step, output_dir, metric_summary, N):
- # Plot the value map, goal for various maps to see what if the model is
- # learning anything useful.
- #
- # outputs is [values, goals, maps, occupancy, conf].
- #
- if N >= 0:
- outputs = outputs[:N]
- N = len(outputs)
-
- plt.set_cmap('jet')
- fig, axes = utils.subplot(plt, (N, outputs[0][0].shape[4]*5), (5,5))
- axes = axes.ravel()[::-1].tolist()
- for i in range(N):
- values, goals, maps, occupancy, conf = outputs[i]
- for j in [0]:
- for k in range(values.shape[4]):
- # Display something like the midpoint of the trajectory.
- id = np.int(values.shape[1]/2)
-
- ax = axes.pop();
- ax.imshow(goals[j,id,:,:,k], origin='lower', interpolation='none')
- ax.set_axis_off();
- if i == 0: ax.set_title('goal')
-
- ax = axes.pop();
- ax.imshow(occupancy[j,id,:,:,k], origin='lower', interpolation='none')
- ax.set_axis_off();
- if i == 0: ax.set_title('occupancy')
-
- ax = axes.pop();
- ax.imshow(conf[j,id,:,:,k], origin='lower', interpolation='none',
- vmin=0., vmax=1.)
- ax.set_axis_off();
- if i == 0: ax.set_title('conf')
-
- ax = axes.pop();
- ax.imshow(values[j,id,:,:,k], origin='lower', interpolation='none')
- ax.set_axis_off();
- if i == 0: ax.set_title('value')
-
- ax = axes.pop();
- ax.imshow(maps[j,id,:,:,k], origin='lower', interpolation='none')
- ax.set_axis_off();
- if i == 0: ax.set_title('incr map')
-
- file_name = os.path.join(output_dir, 'value_vis_{:d}.png'.format(global_step))
- with fu.fopen(file_name, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
- plt.close(fig)
-
-def _summary_vis(m, batch_size, num_steps, arop_full_summary_iters):
- arop = []; arop_summary_iters = []; arop_eval_fns = [];
- vis_value_ops = []; vis_goal_ops = []; vis_map_ops = [];
- vis_occupancy_ops = []; vis_conf_ops = [];
- for i, val_op in enumerate(m.value_ops):
- vis_value_op = tf.reduce_mean(tf.abs(val_op), axis=3, keep_dims=True)
- vis_value_ops.append(vis_value_op)
-
- vis_occupancy_op = tf.reduce_mean(tf.abs(m.occupancys[i]), 3, True)
- vis_occupancy_ops.append(vis_occupancy_op)
-
- vis_conf_op = tf.reduce_max(tf.abs(m.confs[i]), axis=3, keep_dims=True)
- vis_conf_ops.append(vis_conf_op)
-
- ego_goal_imgs_i_op = m.input_tensors['step']['ego_goal_imgs_{:d}'.format(i)]
- vis_goal_op = tf.reduce_max(ego_goal_imgs_i_op, 4, True)
- vis_goal_ops.append(vis_goal_op)
-
- vis_map_op = tf.reduce_mean(tf.abs(m.ego_map_ops[i]), 4, True)
- vis_map_ops.append(vis_map_op)
-
- vis_goal_ops = tf.concat(vis_goal_ops, 4)
- vis_map_ops = tf.concat(vis_map_ops, 4)
- vis_value_ops = tf.concat(vis_value_ops, 3)
- vis_occupancy_ops = tf.concat(vis_occupancy_ops, 3)
- vis_conf_ops = tf.concat(vis_conf_ops, 3)
-
- sh = tf.unstack(tf.shape(vis_value_ops))[1:]
- vis_value_ops = tf.reshape(vis_value_ops, shape=[batch_size, -1] + sh)
-
- sh = tf.unstack(tf.shape(vis_conf_ops))[1:]
- vis_conf_ops = tf.reshape(vis_conf_ops, shape=[batch_size, -1] + sh)
-
- sh = tf.unstack(tf.shape(vis_occupancy_ops))[1:]
- vis_occupancy_ops = tf.reshape(vis_occupancy_ops, shape=[batch_size,-1] + sh)
-
- # Save memory, only return time steps that need to be visualized, factor of
- # 32 CPU memory saving.
- id = np.int(num_steps/2)
- vis_goal_ops = tf.expand_dims(vis_goal_ops[:,id,:,:,:], axis=1)
- vis_map_ops = tf.expand_dims(vis_map_ops[:,id,:,:,:], axis=1)
- vis_value_ops = tf.expand_dims(vis_value_ops[:,id,:,:,:], axis=1)
- vis_conf_ops = tf.expand_dims(vis_conf_ops[:,id,:,:,:], axis=1)
- vis_occupancy_ops = tf.expand_dims(vis_occupancy_ops[:,id,:,:,:], axis=1)
-
- arop += [[vis_value_ops, vis_goal_ops, vis_map_ops, vis_occupancy_ops,
- vis_conf_ops]]
- arop_summary_iters += [arop_full_summary_iters]
- arop_eval_fns += [_vis]
- return arop, arop_summary_iters, arop_eval_fns
-
-def _summary_readout_maps(m, num_steps, arop_full_summary_iters):
- arop = []; arop_summary_iters = []; arop_eval_fns = [];
- id = np.int(num_steps-1)
- vis_readout_maps_gt = m.readout_maps_gt
- vis_readout_maps_prob = tf.reshape(m.readout_maps_probs,
- shape=tf.shape(vis_readout_maps_gt))
- vis_readout_maps_gt = tf.expand_dims(vis_readout_maps_gt[:,id,:,:,:], 1)
- vis_readout_maps_prob = tf.expand_dims(vis_readout_maps_prob[:,id,:,:,:], 1)
- arop += [[vis_readout_maps_gt, vis_readout_maps_prob]]
- arop_summary_iters += [arop_full_summary_iters]
- arop_eval_fns += [_vis_readout_maps]
- return arop, arop_summary_iters, arop_eval_fns
-
-def _add_summaries(m, args, summary_mode, arop_full_summary_iters):
- task_params = args.navtask.task_params
-
- summarize_ops = [m.lr_op, m.global_step_op, m.sample_gt_prob_op] + \
- m.loss_ops + m.acc_ops
- summarize_names = ['lr', 'global_step', 'sample_gt_prob_op'] + \
- m.loss_ops_names + ['acc_{:d}'.format(i) for i in range(len(m.acc_ops))]
- to_aggregate = [0, 0, 0] + [1]*len(m.loss_ops_names) + [1]*len(m.acc_ops)
-
- scope_name = 'summary'
- with tf.name_scope(scope_name):
- s_ops = nu.add_default_summaries(summary_mode, arop_full_summary_iters,
- summarize_ops, summarize_names,
- to_aggregate, m.action_prob_op,
- m.input_tensors, scope_name=scope_name)
- if summary_mode == 'val':
- arop, arop_summary_iters, arop_eval_fns = _summary_vis(
- m, task_params.batch_size, task_params.num_steps,
- arop_full_summary_iters)
- s_ops.additional_return_ops += arop
- s_ops.arop_summary_iters += arop_summary_iters
- s_ops.arop_eval_fns += arop_eval_fns
-
- if args.arch.readout_maps:
- arop, arop_summary_iters, arop_eval_fns = _summary_readout_maps(
- m, task_params.num_steps, arop_full_summary_iters)
- s_ops.additional_return_ops += arop
- s_ops.arop_summary_iters += arop_summary_iters
- s_ops.arop_eval_fns += arop_eval_fns
-
- return s_ops
diff --git a/research/cognitive_mapping_and_planning/tfcode/cmp_utils.py b/research/cognitive_mapping_and_planning/tfcode/cmp_utils.py
deleted file mode 100644
index 6d87c697b4b29128c8b8a42caac27aeb4d657ec6..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/tfcode/cmp_utils.py
+++ /dev/null
@@ -1,164 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utility functions for setting up the CMP graph.
-"""
-
-import os, numpy as np
-import matplotlib.pyplot as plt
-
-
-import tensorflow as tf
-
-from tensorflow.contrib import slim
-from tensorflow.contrib.slim import arg_scope
-import logging
-from src import utils
-import src.file_utils as fu
-from tfcode import tf_utils
-
-resnet_v2 = tf_utils.resnet_v2
-custom_residual_block = tf_utils.custom_residual_block
-
-def value_iteration_network(
- fr, num_iters, val_neurons, action_neurons, kernel_size, share_wts=False,
- name='vin', wt_decay=0.0001, activation_fn=None, shape_aware=False):
- """
- Constructs a Value Iteration Network, convolutions and max pooling across
- channels.
- Input:
- fr: NxWxHxC
- val_neurons: Number of channels for maintaining the value.
- action_neurons: Computes action_neurons * val_neurons at each iteration to
- max pool over.
- Output:
- value image: NxHxWx(val_neurons)
- """
- init_var = np.sqrt(2.0/(kernel_size**2)/(val_neurons*action_neurons))
- vals = []
- with tf.variable_scope(name) as varscope:
- if shape_aware == False:
- fr_shape = tf.unstack(tf.shape(fr))
- val_shape = tf.stack(fr_shape[:-1] + [val_neurons])
- val = tf.zeros(val_shape, name='val_init')
- else:
- val = tf.expand_dims(tf.zeros_like(fr[:,:,:,0]), dim=-1) * \
- tf.constant(0., dtype=tf.float32, shape=[1,1,1,val_neurons])
- val_shape = tf.shape(val)
- vals.append(val)
- for i in range(num_iters):
- if share_wts:
- # The first Value Iteration maybe special, so it can have its own
- # paramterss.
- scope = 'conv'
- if i == 0: scope = 'conv_0'
- if i > 1: varscope.reuse_variables()
- else:
- scope = 'conv_{:d}'.format(i)
- val = slim.conv2d(tf.concat([val, fr], 3, name='concat_{:d}'.format(i)),
- num_outputs=action_neurons*val_neurons,
- kernel_size=kernel_size, stride=1, activation_fn=activation_fn,
- scope=scope, normalizer_fn=None,
- weights_regularizer=slim.l2_regularizer(wt_decay),
- weights_initializer=tf.random_normal_initializer(stddev=init_var),
- biases_initializer=tf.zeros_initializer())
- val = tf.reshape(val, [-1, action_neurons*val_neurons, 1, 1],
- name='re_{:d}'.format(i))
- val = slim.max_pool2d(val, kernel_size=[action_neurons,1],
- stride=[action_neurons,1], padding='VALID',
- scope='val_{:d}'.format(i))
- val = tf.reshape(val, val_shape, name='unre_{:d}'.format(i))
- vals.append(val)
- return val, vals
-
-
-def rotate_preds(loc_on_map, relative_theta, map_size, preds,
- output_valid_mask):
- with tf.name_scope('rotate'):
- flow_op = tf_utils.get_flow(loc_on_map, relative_theta, map_size=map_size)
- if type(preds) != list:
- rotated_preds, valid_mask_warps = tf_utils.dense_resample(preds, flow_op,
- output_valid_mask)
- else:
- rotated_preds = [] ;valid_mask_warps = []
- for pred in preds:
- rotated_pred, valid_mask_warp = tf_utils.dense_resample(pred, flow_op,
- output_valid_mask)
- rotated_preds.append(rotated_pred)
- valid_mask_warps.append(valid_mask_warp)
- return rotated_preds, valid_mask_warps
-
-def get_visual_frustum(map_size, shape_like, expand_dims=[0,0]):
- with tf.name_scope('visual_frustum'):
- l = np.tril(np.ones(map_size)) ;l = l + l[:,::-1]
- l = (l == 2).astype(np.float32)
- for e in expand_dims:
- l = np.expand_dims(l, axis=e)
- confs_probs = tf.constant(l, dtype=tf.float32)
- confs_probs = tf.ones_like(shape_like, dtype=tf.float32) * confs_probs
- return confs_probs
-
-def deconv(x, is_training, wt_decay, neurons, strides, layers_per_block,
- kernel_size, conv_fn, name, offset=0):
- """Generates a up sampling network with residual connections.
- """
- batch_norm_param = {'center': True, 'scale': True,
- 'activation_fn': tf.nn.relu,
- 'is_training': is_training}
- outs = []
- for i, (neuron, stride) in enumerate(zip(neurons, strides)):
- for s in range(layers_per_block):
- scope = '{:s}_{:d}_{:d}'.format(name, i+1+offset,s+1)
- x = custom_residual_block(x, neuron, kernel_size, stride, scope,
- is_training, wt_decay, use_residual=True,
- residual_stride_conv=True, conv_fn=conv_fn,
- batch_norm_param=batch_norm_param)
- stride = 1
- outs.append((x,True))
- return x, outs
-
-def fr_v2(x, output_neurons, inside_neurons, is_training, name='fr',
- wt_decay=0.0001, stride=1, updates_collections=tf.GraphKeys.UPDATE_OPS):
- """Performs fusion of information between the map and the reward map.
- Inputs
- x: NxHxWxC1
-
- Outputs
- fr map: NxHxWx(output_neurons)
- """
- if type(stride) != list:
- stride = [stride]
- with slim.arg_scope(resnet_v2.resnet_utils.resnet_arg_scope(
- is_training=is_training, weight_decay=wt_decay)):
- with slim.arg_scope([slim.batch_norm], updates_collections=updates_collections) as arg_sc:
- # Change the updates_collections for the conv normalizer_params to None
- for i in range(len(arg_sc.keys())):
- if 'convolution' in arg_sc.keys()[i]:
- arg_sc.values()[i]['normalizer_params']['updates_collections'] = updates_collections
- with slim.arg_scope(arg_sc):
- bottleneck = resnet_v2.bottleneck
- blocks = []
- for i, s in enumerate(stride):
- b = resnet_v2.resnet_utils.Block(
- 'block{:d}'.format(i + 1), bottleneck, [{
- 'depth': output_neurons,
- 'depth_bottleneck': inside_neurons,
- 'stride': stride[i]
- }])
- blocks.append(b)
- x, outs = resnet_v2.resnet_v2(x, blocks, num_classes=None, global_pool=False,
- output_stride=None, include_root_block=False,
- reuse=False, scope=name)
- return x, outs
diff --git a/research/cognitive_mapping_and_planning/tfcode/nav_utils.py b/research/cognitive_mapping_and_planning/tfcode/nav_utils.py
deleted file mode 100644
index 2f764f33df91a80f6539dcbae1e0fa7093becd29..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/tfcode/nav_utils.py
+++ /dev/null
@@ -1,435 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Various losses for training navigation agents.
-
-Defines various loss functions for navigation agents,
-compute_losses_multi_or.
-"""
-
-import os, numpy as np
-import matplotlib.pyplot as plt
-
-
-import tensorflow as tf
-
-from tensorflow.contrib import slim
-from tensorflow.contrib.slim import arg_scope
-from tensorflow.contrib.slim.nets import resnet_v2
-from tensorflow.python.training import moving_averages
-import logging
-from src import utils
-import src.file_utils as fu
-from tfcode import tf_utils
-
-
-def compute_losses_multi_or(logits, actions_one_hot, weights=None,
- num_actions=-1, data_loss_wt=1., reg_loss_wt=1.,
- ewma_decay=0.99, reg_loss_op=None):
- assert(num_actions > 0), 'num_actions must be specified and must be > 0.'
-
- with tf.name_scope('loss'):
- if weights is None:
- weight = tf.ones_like(actions_one_hot, dtype=tf.float32, name='weight')
-
- actions_one_hot = tf.cast(tf.reshape(actions_one_hot, [-1, num_actions],
- 're_actions_one_hot'), tf.float32)
- weights = tf.reduce_sum(tf.reshape(weights, [-1, num_actions], 're_weight'),
- reduction_indices=1)
- total = tf.reduce_sum(weights)
-
- action_prob = tf.nn.softmax(logits)
- action_prob = tf.reduce_sum(tf.multiply(action_prob, actions_one_hot),
- reduction_indices=1)
- example_loss = -tf.log(tf.maximum(tf.constant(1e-4), action_prob))
-
- data_loss_op = tf.reduce_sum(example_loss * weights) / total
- if reg_loss_op is None:
- if reg_loss_wt > 0:
- reg_loss_op = tf.add_n(tf.losses.get_regularization_losses())
- else:
- reg_loss_op = tf.constant(0.)
-
- if reg_loss_wt > 0:
- total_loss_op = data_loss_wt*data_loss_op + reg_loss_wt*reg_loss_op
- else:
- total_loss_op = data_loss_wt*data_loss_op
-
- is_correct = tf.cast(tf.greater(action_prob, 0.5, name='pred_class'), tf.float32)
- acc_op = tf.reduce_sum(is_correct*weights) / total
-
- ewma_acc_op = moving_averages.weighted_moving_average(
- acc_op, ewma_decay, weight=total, name='ewma_acc')
-
- acc_ops = [ewma_acc_op]
-
- return reg_loss_op, data_loss_op, total_loss_op, acc_ops
-
-
-def get_repr_from_image(images_reshaped, modalities, data_augment, encoder,
- freeze_conv, wt_decay, is_training):
- # Pass image through lots of convolutional layers, to obtain pool5
- if modalities == ['rgb']:
- with tf.name_scope('pre_rgb'):
- x = (images_reshaped + 128.) / 255. # Convert to brightness between 0 and 1.
- if data_augment.relight and is_training:
- x = tf_utils.distort_image(x, fast_mode=data_augment.relight_fast)
- x = (x-0.5)*2.0
- scope_name = encoder
- elif modalities == ['depth']:
- with tf.name_scope('pre_d'):
- d_image = images_reshaped
- x = 2*(d_image[...,0] - 80.0)/100.0
- y = d_image[...,1]
- d_image = tf.concat([tf.expand_dims(x, -1), tf.expand_dims(y, -1)], 3)
- x = d_image
- scope_name = 'd_'+encoder
-
- resnet_is_training = is_training and (not freeze_conv)
- with slim.arg_scope(resnet_v2.resnet_utils.resnet_arg_scope(resnet_is_training)):
- fn = getattr(tf_utils, encoder)
- x, end_points = fn(x, num_classes=None, global_pool=False,
- output_stride=None, reuse=None,
- scope=scope_name)
- vars_ = slim.get_variables_to_restore()
-
- conv_feat = x
- return conv_feat, vars_
-
-def default_train_step_kwargs(m, obj, logdir, rng_seed, is_chief, num_steps,
- iters, train_display_interval,
- dagger_sample_bn_false):
- train_step_kwargs = {}
- train_step_kwargs['obj'] = obj
- train_step_kwargs['m'] = m
-
- # rng_data has 2 independent rngs, one for sampling episodes and one for
- # sampling perturbs (so that we can make results reproducible.
- train_step_kwargs['rng_data'] = [np.random.RandomState(rng_seed),
- np.random.RandomState(rng_seed)]
- train_step_kwargs['rng_action'] = np.random.RandomState(rng_seed)
- if is_chief:
- train_step_kwargs['writer'] = tf.summary.FileWriter(logdir) #, m.tf_graph)
- else:
- train_step_kwargs['writer'] = None
- train_step_kwargs['iters'] = iters
- train_step_kwargs['train_display_interval'] = train_display_interval
- train_step_kwargs['num_steps'] = num_steps
- train_step_kwargs['logdir'] = logdir
- train_step_kwargs['dagger_sample_bn_false'] = dagger_sample_bn_false
- return train_step_kwargs
-
-# Utilities for visualizing and analysing validation output.
-def save_d_at_t(outputs, global_step, output_dir, metric_summary, N):
- """Save distance to goal at all time steps.
-
- Args:
- outputs : [gt_dist_to_goal].
- global_step : number of iterations.
- output_dir : output directory.
- metric_summary : to append scalars to summary.
- N : number of outputs to process.
-
- """
- d_at_t = np.concatenate(map(lambda x: x[0][:,:,0]*1, outputs), axis=0)
- fig, axes = utils.subplot(plt, (1,1), (5,5))
- axes.plot(np.arange(d_at_t.shape[1]), np.mean(d_at_t, axis=0), 'r.')
- axes.set_xlabel('time step')
- axes.set_ylabel('dist to next goal')
- axes.grid('on')
- file_name = os.path.join(output_dir, 'dist_at_t_{:d}.png'.format(global_step))
- with fu.fopen(file_name, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
- file_name = os.path.join(output_dir, 'dist_at_t_{:d}.pkl'.format(global_step))
- utils.save_variables(file_name, [d_at_t], ['d_at_t'], overwrite=True)
- plt.close(fig)
- return None
-
-def save_all(outputs, global_step, output_dir, metric_summary, N):
- """Save numerous statistics.
-
- Args:
- outputs : [locs, goal_loc, gt_dist_to_goal, node_ids, perturbs]
- global_step : number of iterations.
- output_dir : output directory.
- metric_summary : to append scalars to summary.
- N : number of outputs to process.
- """
- all_locs = np.concatenate(map(lambda x: x[0], outputs), axis=0)
- all_goal_locs = np.concatenate(map(lambda x: x[1], outputs), axis=0)
- all_d_at_t = np.concatenate(map(lambda x: x[2][:,:,0]*1, outputs), axis=0)
- all_node_ids = np.concatenate(map(lambda x: x[3], outputs), axis=0)
- all_perturbs = np.concatenate(map(lambda x: x[4], outputs), axis=0)
-
- file_name = os.path.join(output_dir, 'all_locs_at_t_{:d}.pkl'.format(global_step))
- vars = [all_locs, all_goal_locs, all_d_at_t, all_node_ids, all_perturbs]
- var_names = ['all_locs', 'all_goal_locs', 'all_d_at_t', 'all_node_ids', 'all_perturbs']
- utils.save_variables(file_name, vars, var_names, overwrite=True)
- return None
-
-def eval_ap(outputs, global_step, output_dir, metric_summary, N, num_classes=4):
- """Processes the collected outputs to compute AP for action prediction.
-
- Args:
- outputs : [logits, labels]
- global_step : global_step.
- output_dir : where to store results.
- metric_summary : summary object to add summaries to.
- N : number of outputs to process.
- num_classes : number of classes to compute AP over, and to reshape tensors.
- """
- if N >= 0:
- outputs = outputs[:N]
- logits = np.concatenate(map(lambda x: x[0], outputs), axis=0).reshape((-1, num_classes))
- labels = np.concatenate(map(lambda x: x[1], outputs), axis=0).reshape((-1, num_classes))
- aps = []
- for i in range(logits.shape[1]):
- ap, rec, prec = utils.calc_pr(labels[:,i], logits[:,i])
- ap = ap[0]
- tf_utils.add_value_to_summary(metric_summary, 'aps/ap_{:d}: '.format(i), ap)
- aps.append(ap)
- return aps
-
-def eval_dist(outputs, global_step, output_dir, metric_summary, N):
- """Processes the collected outputs during validation to
- 1. Plot the distance over time curve.
- 2. Compute mean and median distances.
- 3. Plots histogram of end distances.
-
- Args:
- outputs : [locs, goal_loc, gt_dist_to_goal].
- global_step : global_step.
- output_dir : where to store results.
- metric_summary : summary object to add summaries to.
- N : number of outputs to process.
- """
- SUCCESS_THRESH = 3
- if N >= 0:
- outputs = outputs[:N]
-
- # Plot distance at time t.
- d_at_t = []
- for i in range(len(outputs)):
- locs, goal_loc, gt_dist_to_goal = outputs[i]
- d_at_t.append(gt_dist_to_goal[:,:,0]*1)
-
- # Plot the distance.
- fig, axes = utils.subplot(plt, (1,1), (5,5))
- d_at_t = np.concatenate(d_at_t, axis=0)
- axes.plot(np.arange(d_at_t.shape[1]), np.mean(d_at_t, axis=0), 'r.')
- axes.set_xlabel('time step')
- axes.set_ylabel('dist to next goal')
- axes.grid('on')
- file_name = os.path.join(output_dir, 'dist_at_t_{:d}.png'.format(global_step))
- with fu.fopen(file_name, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
- file_name = os.path.join(output_dir, 'dist_at_t_{:d}.pkl'.format(global_step))
- utils.save_variables(file_name, [d_at_t], ['d_at_t'], overwrite=True)
- plt.close(fig)
-
- # Plot the trajectories and the init_distance and final distance.
- d_inits = []
- d_ends = []
- for i in range(len(outputs)):
- locs, goal_loc, gt_dist_to_goal = outputs[i]
- d_inits.append(gt_dist_to_goal[:,0,0]*1)
- d_ends.append(gt_dist_to_goal[:,-1,0]*1)
-
- # Plot the distance.
- fig, axes = utils.subplot(plt, (1,1), (5,5))
- d_inits = np.concatenate(d_inits, axis=0)
- d_ends = np.concatenate(d_ends, axis=0)
- axes.plot(d_inits+np.random.rand(*(d_inits.shape))-0.5,
- d_ends+np.random.rand(*(d_ends.shape))-0.5, '.', mec='red', mew=1.0)
- axes.set_xlabel('init dist'); axes.set_ylabel('final dist');
- axes.grid('on'); axes.axis('equal');
- title_str = 'mean: {:0.1f}, 50: {:0.1f}, 75: {:0.2f}, s: {:0.1f}'
- title_str = title_str.format(
- np.mean(d_ends), np.median(d_ends), np.percentile(d_ends, q=75),
- 100*(np.mean(d_ends <= SUCCESS_THRESH)))
- axes.set_title(title_str)
- file_name = os.path.join(output_dir, 'dist_{:d}.png'.format(global_step))
- with fu.fopen(file_name, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
-
- file_name = os.path.join(output_dir, 'dist_{:d}.pkl'.format(global_step))
- utils.save_variables(file_name, [d_inits, d_ends], ['d_inits', 'd_ends'],
- overwrite=True)
- plt.close(fig)
-
- # Plot the histogram of the end_distance.
- with plt.style.context('seaborn-white'):
- d_ends_ = np.sort(d_ends)
- d_inits_ = np.sort(d_inits)
- leg = [];
- fig, ax = utils.subplot(plt, (1,1), (5,5))
- ax.grid('on')
- ax.set_xlabel('Distance from goal'); ax.xaxis.label.set_fontsize(16);
- ax.set_ylabel('Fraction of data'); ax.yaxis.label.set_fontsize(16);
- ax.plot(d_ends_, np.arange(d_ends_.size)*1./d_ends_.size, 'r')
- ax.plot(d_inits_, np.arange(d_inits_.size)*1./d_inits_.size, 'k')
- leg.append('Final'); leg.append('Init');
- ax.legend(leg, fontsize='x-large');
- ax.set_axis_on()
- title_str = 'mean: {:0.1f}, 50: {:0.1f}, 75: {:0.2f}, s: {:0.1f}'
- title_str = title_str.format(
- np.mean(d_ends), np.median(d_ends), np.percentile(d_ends, q=75),
- 100*(np.mean(d_ends <= SUCCESS_THRESH)))
- ax.set_title(title_str)
- file_name = os.path.join(output_dir, 'dist_hist_{:d}.png'.format(global_step))
- with fu.fopen(file_name, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
-
- # Log distance metrics.
- tf_utils.add_value_to_summary(metric_summary, 'dists/success_init: ',
- 100*(np.mean(d_inits <= SUCCESS_THRESH)))
- tf_utils.add_value_to_summary(metric_summary, 'dists/success_end: ',
- 100*(np.mean(d_ends <= SUCCESS_THRESH)))
- tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (75): ',
- np.percentile(d_inits, q=75))
- tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (75): ',
- np.percentile(d_ends, q=75))
- tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (median): ',
- np.median(d_inits))
- tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (median): ',
- np.median(d_ends))
- tf_utils.add_value_to_summary(metric_summary, 'dists/dist_init (mean): ',
- np.mean(d_inits))
- tf_utils.add_value_to_summary(metric_summary, 'dists/dist_end (mean): ',
- np.mean(d_ends))
- return np.median(d_inits), np.median(d_ends), np.mean(d_inits), np.mean(d_ends), \
- np.percentile(d_inits, q=75), np.percentile(d_ends, q=75), \
- 100*(np.mean(d_inits) <= SUCCESS_THRESH), 100*(np.mean(d_ends) <= SUCCESS_THRESH)
-
-def plot_trajectories(outputs, global_step, output_dir, metric_summary, N):
- """Processes the collected outputs during validation to plot the trajectories
- in the top view.
-
- Args:
- outputs : [locs, orig_maps, goal_loc].
- global_step : global_step.
- output_dir : where to store results.
- metric_summary : summary object to add summaries to.
- N : number of outputs to process.
- """
- if N >= 0:
- outputs = outputs[:N]
- N = len(outputs)
-
- plt.set_cmap('gray')
- fig, axes = utils.subplot(plt, (N, outputs[0][1].shape[0]), (5,5))
- axes = axes.ravel()[::-1].tolist()
- for i in range(N):
- locs, orig_maps, goal_loc = outputs[i]
- is_semantic = np.isnan(goal_loc[0,0,1])
- for j in range(orig_maps.shape[0]):
- ax = axes.pop();
- ax.plot(locs[j,0,0], locs[j,0,1], 'ys')
- # Plot one by one, so that they come in different colors.
- for k in range(goal_loc.shape[1]):
- if not is_semantic:
- ax.plot(goal_loc[j,k,0], goal_loc[j,k,1], 's')
- if False:
- ax.plot(locs[j,:,0], locs[j,:,1], 'r.', ms=3)
- ax.imshow(orig_maps[j,0,:,:,0], origin='lower')
- ax.set_axis_off();
- else:
- ax.scatter(locs[j,:,0], locs[j,:,1], c=np.arange(locs.shape[1]),
- cmap='jet', s=10, lw=0)
- ax.imshow(orig_maps[j,0,:,:,0], origin='lower', vmin=-1.0, vmax=2.0)
- if not is_semantic:
- xymin = np.minimum(np.min(goal_loc[j,:,:], axis=0), np.min(locs[j,:,:], axis=0))
- xymax = np.maximum(np.max(goal_loc[j,:,:], axis=0), np.max(locs[j,:,:], axis=0))
- else:
- xymin = np.min(locs[j,:,:], axis=0)
- xymax = np.max(locs[j,:,:], axis=0)
- xy1 = (xymax+xymin)/2. - np.maximum(np.max(xymax-xymin), 12)
- xy2 = (xymax+xymin)/2. + np.maximum(np.max(xymax-xymin), 12)
- ax.set_xlim([xy1[0], xy2[0]])
- ax.set_ylim([xy1[1], xy2[1]])
- ax.set_axis_off()
- file_name = os.path.join(output_dir, 'trajectory_{:d}.png'.format(global_step))
- with fu.fopen(file_name, 'w') as f:
- fig.savefig(f, bbox_inches='tight', transparent=True, pad_inches=0)
- plt.close(fig)
- return None
-
-def add_default_summaries(mode, arop_full_summary_iters, summarize_ops,
- summarize_names, to_aggregate, action_prob_op,
- input_tensors, scope_name):
- assert(mode == 'train' or mode == 'val' or mode == 'test'), \
- 'add_default_summaries mode is neither train or val or test.'
-
- s_ops = tf_utils.get_default_summary_ops()
-
- if mode == 'train':
- s_ops.summary_ops, s_ops.print_summary_ops, additional_return_ops, \
- arop_summary_iters, arop_eval_fns = tf_utils.simple_summaries(
- summarize_ops, summarize_names, mode, to_aggregate=False,
- scope_name=scope_name)
- s_ops.additional_return_ops += additional_return_ops
- s_ops.arop_summary_iters += arop_summary_iters
- s_ops.arop_eval_fns += arop_eval_fns
- elif mode == 'val':
- s_ops.summary_ops, s_ops.print_summary_ops, additional_return_ops, \
- arop_summary_iters, arop_eval_fns = tf_utils.simple_summaries(
- summarize_ops, summarize_names, mode, to_aggregate=to_aggregate,
- scope_name=scope_name)
- s_ops.additional_return_ops += additional_return_ops
- s_ops.arop_summary_iters += arop_summary_iters
- s_ops.arop_eval_fns += arop_eval_fns
-
- elif mode == 'test':
- s_ops.summary_ops, s_ops.print_summary_ops, additional_return_ops, \
- arop_summary_iters, arop_eval_fns = tf_utils.simple_summaries(
- [], [], mode, to_aggregate=[], scope_name=scope_name)
- s_ops.additional_return_ops += additional_return_ops
- s_ops.arop_summary_iters += arop_summary_iters
- s_ops.arop_eval_fns += arop_eval_fns
-
-
- if mode == 'val':
- arop = s_ops.additional_return_ops
- arop += [[action_prob_op, input_tensors['train']['action']]]
- arop += [[input_tensors['step']['loc_on_map'],
- input_tensors['common']['goal_loc'],
- input_tensors['step']['gt_dist_to_goal']]]
- arop += [[input_tensors['step']['loc_on_map'],
- input_tensors['common']['orig_maps'],
- input_tensors['common']['goal_loc']]]
- s_ops.arop_summary_iters += [-1, arop_full_summary_iters,
- arop_full_summary_iters]
- s_ops.arop_eval_fns += [eval_ap, eval_dist, plot_trajectories]
-
- elif mode == 'test':
- arop = s_ops.additional_return_ops
- arop += [[input_tensors['step']['loc_on_map'],
- input_tensors['common']['goal_loc'],
- input_tensors['step']['gt_dist_to_goal']]]
- arop += [[input_tensors['step']['gt_dist_to_goal']]]
- arop += [[input_tensors['step']['loc_on_map'],
- input_tensors['common']['goal_loc'],
- input_tensors['step']['gt_dist_to_goal'],
- input_tensors['step']['node_ids'],
- input_tensors['step']['perturbs']]]
- arop += [[input_tensors['step']['loc_on_map'],
- input_tensors['common']['orig_maps'],
- input_tensors['common']['goal_loc']]]
- s_ops.arop_summary_iters += [-1, -1, -1, arop_full_summary_iters]
- s_ops.arop_eval_fns += [eval_dist, save_d_at_t, save_all,
- plot_trajectories]
- return s_ops
-
-
diff --git a/research/cognitive_mapping_and_planning/tfcode/tf_utils.py b/research/cognitive_mapping_and_planning/tfcode/tf_utils.py
deleted file mode 100644
index 5f96d8ff5ce7473f0ec49096abcbac274e6c4fcc..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/tfcode/tf_utils.py
+++ /dev/null
@@ -1,840 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import numpy as np
-import sys
-import tensorflow as tf
-import src.utils as utils
-import logging
-from tensorflow.contrib import slim
-from tensorflow.contrib.metrics.python.ops import confusion_matrix_ops
-from tensorflow.contrib.slim import arg_scope
-from tensorflow.contrib.slim.nets import resnet_v2
-from tensorflow.python.framework import dtypes
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import check_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import variable_scope
-sys.path.insert(0, '../slim')
-from preprocessing import inception_preprocessing as ip
-
-resnet_v2_50 = resnet_v2.resnet_v2_50
-
-
-def custom_residual_block(x, neurons, kernel_size, stride, name, is_training,
- wt_decay=0.0001, use_residual=True,
- residual_stride_conv=True, conv_fn=slim.conv2d,
- batch_norm_param=None):
-
- # batch norm x and relu
- init_var = np.sqrt(2.0/(kernel_size**2)/neurons)
- with arg_scope([conv_fn],
- weights_regularizer=slim.l2_regularizer(wt_decay),
- weights_initializer=tf.random_normal_initializer(stddev=init_var),
- biases_initializer=tf.zeros_initializer()):
-
- if batch_norm_param is None:
- batch_norm_param = {'center': True, 'scale': False,
- 'activation_fn':tf.nn.relu,
- 'is_training': is_training}
-
- y = slim.batch_norm(x, scope=name+'_bn', **batch_norm_param)
-
- y = conv_fn(y, num_outputs=neurons, kernel_size=kernel_size, stride=stride,
- activation_fn=None, scope=name+'_1',
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_param)
-
- y = conv_fn(y, num_outputs=neurons, kernel_size=kernel_size,
- stride=1, activation_fn=None, scope=name+'_2')
-
- if use_residual:
- if stride != 1 or x.get_shape().as_list()[-1] != neurons:
- batch_norm_param_ = dict(batch_norm_param)
- batch_norm_param_['activation_fn'] = None
- x = conv_fn(x, num_outputs=neurons, kernel_size=1,
- stride=stride if residual_stride_conv else 1,
- activation_fn=None, scope=name+'_0_1x1',
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_param_)
- if not residual_stride_conv:
- x = slim.avg_pool2d(x, 1, stride=stride, scope=name+'_0_avg')
-
- y = tf.add(x, y, name=name+'_add')
-
- return y
-
-def step_gt_prob(step, step_number_op):
- # Change samping probability from 1 to -1 at step steps.
- with tf.name_scope('step_gt_prob'):
- out = tf.cond(tf.less(step_number_op, step),
- lambda: tf.constant(1.), lambda: tf.constant(-1.))
- return out
-
-def inverse_sigmoid_decay(k, global_step_op):
- with tf.name_scope('inverse_sigmoid_decay'):
- k = tf.constant(k, dtype=tf.float32)
- tmp = k*tf.exp(-tf.cast(global_step_op, tf.float32)/k)
- tmp = tmp / (1. + tmp)
- return tmp
-
-def dense_resample(im, flow_im, output_valid_mask, name='dense_resample'):
- """ Resample reward at particular locations.
- Args:
- im: ...xHxWxC matrix to sample from.
- flow_im: ...xHxWx2 matrix, samples the image using absolute offsets as given
- by the flow_im.
- """
- with tf.name_scope(name):
- valid_mask = None
-
- x, y = tf.unstack(flow_im, axis=-1)
- x = tf.cast(tf.reshape(x, [-1]), tf.float32)
- y = tf.cast(tf.reshape(y, [-1]), tf.float32)
-
- # constants
- shape = tf.unstack(tf.shape(im))
- channels = shape[-1]
- width = shape[-2]
- height = shape[-3]
- num_batch = tf.cast(tf.reduce_prod(tf.stack(shape[:-3])), 'int32')
- zero = tf.constant(0, dtype=tf.int32)
-
- # Round up and down.
- x0 = tf.cast(tf.floor(x), 'int32'); x1 = x0 + 1;
- y0 = tf.cast(tf.floor(y), 'int32'); y1 = y0 + 1;
-
- if output_valid_mask:
- valid_mask = tf.logical_and(
- tf.logical_and(tf.less_equal(x, tf.cast(width, tf.float32)-1.), tf.greater_equal(x, 0.)),
- tf.logical_and(tf.less_equal(y, tf.cast(height, tf.float32)-1.), tf.greater_equal(y, 0.)))
- valid_mask = tf.reshape(valid_mask, shape=shape[:-1] + [1])
-
- x0 = tf.clip_by_value(x0, zero, width-1)
- x1 = tf.clip_by_value(x1, zero, width-1)
- y0 = tf.clip_by_value(y0, zero, height-1)
- y1 = tf.clip_by_value(y1, zero, height-1)
-
- dim2 = width; dim1 = width * height;
-
- # Create base index
- base = tf.reshape(tf.range(num_batch) * dim1, shape=[-1,1])
- base = tf.reshape(tf.tile(base, [1, height*width]), shape=[-1])
-
- base_y0 = base + y0 * dim2
- base_y1 = base + y1 * dim2
- idx_a = base_y0 + x0
- idx_b = base_y1 + x0
- idx_c = base_y0 + x1
- idx_d = base_y1 + x1
-
- # use indices to lookup pixels in the flat image and restore channels dim
- sh = tf.stack([tf.constant(-1,dtype=tf.int32), channels])
- im_flat = tf.cast(tf.reshape(im, sh), dtype=tf.float32)
- pixel_a = tf.gather(im_flat, idx_a)
- pixel_b = tf.gather(im_flat, idx_b)
- pixel_c = tf.gather(im_flat, idx_c)
- pixel_d = tf.gather(im_flat, idx_d)
-
- # and finally calculate interpolated values
- x1_f = tf.to_float(x1)
- y1_f = tf.to_float(y1)
-
- wa = tf.expand_dims(((x1_f - x) * (y1_f - y)), 1)
- wb = tf.expand_dims((x1_f - x) * (1.0 - (y1_f - y)), 1)
- wc = tf.expand_dims(((1.0 - (x1_f - x)) * (y1_f - y)), 1)
- wd = tf.expand_dims(((1.0 - (x1_f - x)) * (1.0 - (y1_f - y))), 1)
-
- output = tf.add_n([wa * pixel_a, wb * pixel_b, wc * pixel_c, wd * pixel_d])
- output = tf.reshape(output, shape=tf.shape(im))
- return output, valid_mask
-
-def get_flow(t, theta, map_size, name_scope='gen_flow'):
- """
- Rotates the map by theta and translates the rotated map by t.
-
- Assume that the robot rotates by an angle theta and then moves forward by
- translation t. This function returns the flow field field. For every pixel in
- the new image it tells us which pixel in the original image it came from:
- NewI(x, y) = OldI(flow_x(x,y), flow_y(x,y)).
-
- Assume there is a point p in the original image. Robot rotates by R and moves
- forward by t. p1 = Rt*p; p2 = p1 - t; (the world moves in opposite direction.
- So, p2 = Rt*p - t, thus p2 came from R*(p2+t), which is what this function
- calculates.
-
- t: ... x 2 (translation for B batches of N motions each).
- theta: ... x 1 (rotation for B batches of N motions each).
-
- Output: ... x map_size x map_size x 2
- """
-
- with tf.name_scope(name_scope):
- tx, ty = tf.unstack(tf.reshape(t, shape=[-1, 1, 1, 1, 2]), axis=4)
- theta = tf.reshape(theta, shape=[-1, 1, 1, 1])
- c = tf.constant((map_size-1.)/2., dtype=tf.float32)
-
- x, y = np.meshgrid(np.arange(map_size), np.arange(map_size))
- x = tf.constant(x[np.newaxis, :, :, np.newaxis], dtype=tf.float32, name='x',
- shape=[1, map_size, map_size, 1])
- y = tf.constant(y[np.newaxis, :, :, np.newaxis], dtype=tf.float32, name='y',
- shape=[1,map_size, map_size, 1])
-
- x = x-(-tx+c)
- y = y-(-ty+c)
-
- sin_theta = tf.sin(theta)
- cos_theta = tf.cos(theta)
- xr = cos_theta*x - sin_theta*y
- yr = sin_theta*x + cos_theta*y
-
- xr = xr + c
- yr = yr + c
-
- flow = tf.stack([xr, yr], axis=-1)
- sh = tf.unstack(tf.shape(t), axis=0)
- sh = tf.stack(sh[:-1]+[tf.constant(_, dtype=tf.int32) for _ in [map_size, map_size, 2]])
- flow = tf.reshape(flow, shape=sh)
- return flow
-
-def distort_image(im, fast_mode=False):
- # All images in the same batch are transformed the same way, but over
- # iterations you see different distortions.
- # im should be float with values between 0 and 1.
- im_ = tf.reshape(im, shape=(-1,1,3))
- im_ = ip.apply_with_random_selector(
- im_, lambda x, ordering: ip.distort_color(x, ordering, fast_mode),
- num_cases=4)
- im_ = tf.reshape(im_, tf.shape(im))
- return im_
-
-def fc_network(x, neurons, wt_decay, name, num_pred=None, offset=0,
- batch_norm_param=None, dropout_ratio=0.0, is_training=None):
- if dropout_ratio > 0:
- assert(is_training is not None), \
- 'is_training needs to be defined when trainnig with dropout.'
-
- repr = []
- for i, neuron in enumerate(neurons):
- init_var = np.sqrt(2.0/neuron)
- if batch_norm_param is not None:
- x = slim.fully_connected(x, neuron, activation_fn=None,
- weights_initializer=tf.random_normal_initializer(stddev=init_var),
- weights_regularizer=slim.l2_regularizer(wt_decay),
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_param,
- biases_initializer=tf.zeros_initializer(),
- scope='{:s}_{:d}'.format(name, offset+i))
- else:
- x = slim.fully_connected(x, neuron, activation_fn=tf.nn.relu,
- weights_initializer=tf.random_normal_initializer(stddev=init_var),
- weights_regularizer=slim.l2_regularizer(wt_decay),
- biases_initializer=tf.zeros_initializer(),
- scope='{:s}_{:d}'.format(name, offset+i))
- if dropout_ratio > 0:
- x = slim.dropout(x, keep_prob=1-dropout_ratio, is_training=is_training,
- scope='{:s}_{:d}'.format('dropout_'+name, offset+i))
- repr.append(x)
-
- if num_pred is not None:
- init_var = np.sqrt(2.0/num_pred)
- x = slim.fully_connected(x, num_pred,
- weights_regularizer=slim.l2_regularizer(wt_decay),
- weights_initializer=tf.random_normal_initializer(stddev=init_var),
- biases_initializer=tf.zeros_initializer(),
- activation_fn=None,
- scope='{:s}_pred'.format(name))
- return x, repr
-
-def concat_state_x_list(f, names):
- af = {}
- for i, k in enumerate(names):
- af[k] = np.concatenate([x[i] for x in f], axis=1)
- return af
-
-def concat_state_x(f, names):
- af = {}
- for k in names:
- af[k] = np.concatenate([x[k] for x in f], axis=1)
- # af[k] = np.swapaxes(af[k], 0, 1)
- return af
-
-def sample_action(rng, action_probs, optimal_action, sample_gt_prob,
- type='sample', combine_type='one_or_other'):
- optimal_action_ = optimal_action/np.sum(optimal_action+0., 1, keepdims=True)
- action_probs_ = action_probs/np.sum(action_probs+0.001, 1, keepdims=True)
- batch_size = action_probs_.shape[0]
-
- action = np.zeros((batch_size), dtype=np.int32)
- action_sample_wt = np.zeros((batch_size), dtype=np.float32)
- if combine_type == 'add':
- sample_gt_prob_ = np.minimum(np.maximum(sample_gt_prob, 0.), 1.)
-
- for i in range(batch_size):
- if combine_type == 'one_or_other':
- sample_gt = rng.rand() < sample_gt_prob
- if sample_gt: distr_ = optimal_action_[i,:]*1.
- else: distr_ = action_probs_[i,:]*1.
- elif combine_type == 'add':
- distr_ = optimal_action_[i,:]*sample_gt_prob_ + \
- (1.-sample_gt_prob_)*action_probs_[i,:]
- distr_ = distr_ / np.sum(distr_)
-
- if type == 'sample':
- action[i] = np.argmax(rng.multinomial(1, distr_, size=1))
- elif type == 'argmax':
- action[i] = np.argmax(distr_)
- action_sample_wt[i] = action_probs_[i, action[i]] / distr_[action[i]]
- return action, action_sample_wt
-
-def train_step_custom_online_sampling(sess, train_op, global_step,
- train_step_kwargs, mode='train'):
- m = train_step_kwargs['m']
- obj = train_step_kwargs['obj']
- rng_data = train_step_kwargs['rng_data']
- rng_action = train_step_kwargs['rng_action']
- writer = train_step_kwargs['writer']
- iters = train_step_kwargs['iters']
- num_steps = train_step_kwargs['num_steps']
- logdir = train_step_kwargs['logdir']
- dagger_sample_bn_false = train_step_kwargs['dagger_sample_bn_false']
- train_display_interval = train_step_kwargs['train_display_interval']
- if 'outputs' not in m.train_ops:
- m.train_ops['outputs'] = []
-
- s_ops = m.summary_ops[mode]
- val_additional_ops = []
-
- # Print all variables here.
- if False:
- v = tf.get_collection(tf.GraphKeys.VARIABLES)
- v_op = [_.value() for _ in v]
- v_op_value = sess.run(v_op)
-
- filter = lambda x, y: 'Adam' in x.name
- # filter = lambda x, y: np.is_any_nan(y)
- ind = [i for i, (_, __) in enumerate(zip(v, v_op_value)) if filter(_, __)]
- v = [v[i] for i in ind]
- v_op_value = [v_op_value[i] for i in ind]
-
- for i in range(len(v)):
- logging.info('XXXX: variable: %30s, is_any_nan: %5s, norm: %f.',
- v[i].name, np.any(np.isnan(v_op_value[i])),
- np.linalg.norm(v_op_value[i]))
-
- tt = utils.Timer()
- for i in range(iters):
- tt.tic()
- # Sample a room.
- e = obj.sample_env(rng_data)
-
- # Initialize the agent.
- init_env_state = e.reset(rng_data)
-
- # Get and process the common data.
- input = e.get_common_data()
- input = e.pre_common_data(input)
- feed_dict = prepare_feed_dict(m.input_tensors['common'], input)
- if dagger_sample_bn_false:
- feed_dict[m.train_ops['batch_norm_is_training_op']] = False
- common_data = sess.run(m.train_ops['common'], feed_dict=feed_dict)
-
- states = []
- state_features = []
- state_targets = []
- net_state_to_input = []
- step_data_cache = []
- executed_actions = []
- rewards = []
- action_sample_wts = []
- states.append(init_env_state)
-
- net_state = sess.run(m.train_ops['init_state'], feed_dict=feed_dict)
- net_state = dict(zip(m.train_ops['state_names'], net_state))
- net_state_to_input.append(net_state)
- for j in range(num_steps):
- f = e.get_features(states[j], j)
- f = e.pre_features(f)
- f.update(net_state)
- f['step_number'] = np.ones((1,1,1), dtype=np.int32)*j
- state_features.append(f)
-
- feed_dict = prepare_feed_dict(m.input_tensors['step'], state_features[-1])
- optimal_action = e.get_optimal_action(states[j], j)
- for x, v in zip(m.train_ops['common'], common_data):
- feed_dict[x] = v
- if dagger_sample_bn_false:
- feed_dict[m.train_ops['batch_norm_is_training_op']] = False
- outs = sess.run([m.train_ops['step'], m.sample_gt_prob_op,
- m.train_ops['step_data_cache'],
- m.train_ops['updated_state'],
- m.train_ops['outputs']], feed_dict=feed_dict)
- action_probs = outs[0]
- sample_gt_prob = outs[1]
- step_data_cache.append(dict(zip(m.train_ops['step_data_cache'], outs[2])))
- net_state = outs[3]
- if hasattr(e, 'update_state'):
- outputs = outs[4]
- outputs = dict(zip(m.train_ops['output_names'], outputs))
- e.update_state(outputs, j)
- state_targets.append(e.get_targets(states[j], j))
-
- if j < num_steps-1:
- # Sample from action_probs and optimal action.
- action, action_sample_wt = sample_action(
- rng_action, action_probs, optimal_action, sample_gt_prob,
- m.sample_action_type, m.sample_action_combine_type)
- next_state, reward = e.take_action(states[j], action, j)
- executed_actions.append(action)
- states.append(next_state)
- rewards.append(reward)
- action_sample_wts.append(action_sample_wt)
- net_state = dict(zip(m.train_ops['state_names'], net_state))
- net_state_to_input.append(net_state)
-
- # Concatenate things together for training.
- rewards = np.array(rewards).T
- action_sample_wts = np.array(action_sample_wts).T
- executed_actions = np.array(executed_actions).T
- all_state_targets = concat_state_x(state_targets, e.get_targets_name())
- all_state_features = concat_state_x(state_features,
- e.get_features_name()+['step_number'])
- # all_state_net = concat_state_x(net_state_to_input,
- # m.train_ops['state_names'])
- all_step_data_cache = concat_state_x(step_data_cache,
- m.train_ops['step_data_cache'])
-
- dict_train = dict(input)
- dict_train.update(all_state_features)
- dict_train.update(all_state_targets)
- # dict_train.update(all_state_net)
- dict_train.update(net_state_to_input[0])
- dict_train.update(all_step_data_cache)
- dict_train.update({'rewards': rewards,
- 'action_sample_wts': action_sample_wts,
- 'executed_actions': executed_actions})
- feed_dict = prepare_feed_dict(m.input_tensors['train'], dict_train)
- for x in m.train_ops['step_data_cache']:
- feed_dict[x] = all_step_data_cache[x]
- if mode == 'train':
- n_step = sess.run(global_step)
-
- if np.mod(n_step, train_display_interval) == 0:
- total_loss, np_global_step, summary, print_summary = sess.run(
- [train_op, global_step, s_ops.summary_ops, s_ops.print_summary_ops],
- feed_dict=feed_dict)
- logging.error("")
- else:
- total_loss, np_global_step, summary = sess.run(
- [train_op, global_step, s_ops.summary_ops], feed_dict=feed_dict)
-
- if writer is not None and summary is not None:
- writer.add_summary(summary, np_global_step)
-
- should_stop = sess.run(m.should_stop_op)
-
- if mode != 'train':
- arop = [[] for j in range(len(s_ops.additional_return_ops))]
- for j in range(len(s_ops.additional_return_ops)):
- if s_ops.arop_summary_iters[j] < 0 or i < s_ops.arop_summary_iters[j]:
- arop[j] = s_ops.additional_return_ops[j]
- val = sess.run(arop, feed_dict=feed_dict)
- val_additional_ops.append(val)
- tt.toc(log_at=60, log_str='val timer {:d} / {:d}: '.format(i, iters),
- type='time')
-
- if mode != 'train':
- # Write the default val summaries.
- summary, print_summary, np_global_step = sess.run(
- [s_ops.summary_ops, s_ops.print_summary_ops, global_step])
- if writer is not None and summary is not None:
- writer.add_summary(summary, np_global_step)
-
- # write custom validation ops
- val_summarys = []
- val_additional_ops = zip(*val_additional_ops)
- if len(s_ops.arop_eval_fns) > 0:
- val_metric_summary = tf.summary.Summary()
- for i in range(len(s_ops.arop_eval_fns)):
- val_summary = None
- if s_ops.arop_eval_fns[i] is not None:
- val_summary = s_ops.arop_eval_fns[i](val_additional_ops[i],
- np_global_step, logdir,
- val_metric_summary,
- s_ops.arop_summary_iters[i])
- val_summarys.append(val_summary)
- if writer is not None:
- writer.add_summary(val_metric_summary, np_global_step)
-
- # Return the additional val_ops
- total_loss = (val_additional_ops, val_summarys)
- should_stop = None
-
- return total_loss, should_stop
-
-def train_step_custom_v2(sess, train_op, global_step, train_step_kwargs,
- mode='train'):
- m = train_step_kwargs['m']
- obj = train_step_kwargs['obj']
- rng = train_step_kwargs['rng']
- writer = train_step_kwargs['writer']
- iters = train_step_kwargs['iters']
- logdir = train_step_kwargs['logdir']
- train_display_interval = train_step_kwargs['train_display_interval']
-
- s_ops = m.summary_ops[mode]
- val_additional_ops = []
-
- # Print all variables here.
- if False:
- v = tf.get_collection(tf.GraphKeys.VARIABLES)
- v_op = [_.value() for _ in v]
- v_op_value = sess.run(v_op)
-
- filter = lambda x, y: 'Adam' in x.name
- # filter = lambda x, y: np.is_any_nan(y)
- ind = [i for i, (_, __) in enumerate(zip(v, v_op_value)) if filter(_, __)]
- v = [v[i] for i in ind]
- v_op_value = [v_op_value[i] for i in ind]
-
- for i in range(len(v)):
- logging.info('XXXX: variable: %30s, is_any_nan: %5s, norm: %f.',
- v[i].name, np.any(np.isnan(v_op_value[i])),
- np.linalg.norm(v_op_value[i]))
-
- tt = utils.Timer()
- for i in range(iters):
- tt.tic()
- e = obj.sample_env(rng)
- rngs = e.gen_rng(rng)
- input_data = e.gen_data(*rngs)
- input_data = e.pre_data(input_data)
- feed_dict = prepare_feed_dict(m.input_tensors, input_data)
-
- if mode == 'train':
- n_step = sess.run(global_step)
-
- if np.mod(n_step, train_display_interval) == 0:
- total_loss, np_global_step, summary, print_summary = sess.run(
- [train_op, global_step, s_ops.summary_ops, s_ops.print_summary_ops],
- feed_dict=feed_dict)
- else:
- total_loss, np_global_step, summary = sess.run(
- [train_op, global_step, s_ops.summary_ops],
- feed_dict=feed_dict)
-
- if writer is not None and summary is not None:
- writer.add_summary(summary, np_global_step)
-
- should_stop = sess.run(m.should_stop_op)
-
- if mode != 'train':
- arop = [[] for j in range(len(s_ops.additional_return_ops))]
- for j in range(len(s_ops.additional_return_ops)):
- if s_ops.arop_summary_iters[j] < 0 or i < s_ops.arop_summary_iters[j]:
- arop[j] = s_ops.additional_return_ops[j]
- val = sess.run(arop, feed_dict=feed_dict)
- val_additional_ops.append(val)
- tt.toc(log_at=60, log_str='val timer {:d} / {:d}: '.format(i, iters),
- type='time')
-
- if mode != 'train':
- # Write the default val summaries.
- summary, print_summary, np_global_step = sess.run(
- [s_ops.summary_ops, s_ops.print_summary_ops, global_step])
- if writer is not None and summary is not None:
- writer.add_summary(summary, np_global_step)
-
- # write custom validation ops
- val_summarys = []
- val_additional_ops = zip(*val_additional_ops)
- if len(s_ops.arop_eval_fns) > 0:
- val_metric_summary = tf.summary.Summary()
- for i in range(len(s_ops.arop_eval_fns)):
- val_summary = None
- if s_ops.arop_eval_fns[i] is not None:
- val_summary = s_ops.arop_eval_fns[i](val_additional_ops[i],
- np_global_step, logdir,
- val_metric_summary,
- s_ops.arop_summary_iters[i])
- val_summarys.append(val_summary)
- if writer is not None:
- writer.add_summary(val_metric_summary, np_global_step)
-
- # Return the additional val_ops
- total_loss = (val_additional_ops, val_summarys)
- should_stop = None
-
- return total_loss, should_stop
-
-def train_step_custom(sess, train_op, global_step, train_step_kwargs,
- mode='train'):
- m = train_step_kwargs['m']
- params = train_step_kwargs['params']
- rng = train_step_kwargs['rng']
- writer = train_step_kwargs['writer']
- iters = train_step_kwargs['iters']
- gen_rng = train_step_kwargs['gen_rng']
- logdir = train_step_kwargs['logdir']
- gen_data = train_step_kwargs['gen_data']
- pre_data = train_step_kwargs['pre_data']
- train_display_interval = train_step_kwargs['train_display_interval']
-
- val_additional_ops = []
- # Print all variables here.
- if False:
- v = tf.get_collection(tf.GraphKeys.VARIABLES)
- for _ in v:
- val = sess.run(_.value())
- logging.info('variable: %30s, is_any_nan: %5s, norm: %f.', _.name,
- np.any(np.isnan(val)), np.linalg.norm(val))
-
- for i in range(iters):
- rngs = gen_rng(params, rng)
- input_data = gen_data(params, *rngs)
- input_data = pre_data(params, input_data)
- feed_dict = prepare_feed_dict(m.input_tensors, input_data)
-
- if mode == 'train':
- n_step = sess.run(global_step)
-
- if np.mod(n_step, train_display_interval) == 0:
- total_loss, np_global_step, summary, print_summary = sess.run(
- [train_op, global_step, m.summary_op[mode], m.print_summary_op[mode]],
- feed_dict=feed_dict)
- else:
- total_loss, np_global_step, summary = sess.run(
- [train_op, global_step, m.summary_op[mode]],
- feed_dict=feed_dict)
-
- if writer is not None:
- writer.add_summary(summary, np_global_step)
-
- should_stop = sess.run(m.should_stop_op)
-
- if mode == 'val':
- val = sess.run(m.agg_update_op[mode] + m.additional_return_op[mode],
- feed_dict=feed_dict)
- val_additional_ops.append(val[len(m.agg_update_op[mode]):])
-
- if mode == 'val':
- summary, print_summary, np_global_step = sess.run(
- [m.summary_op[mode], m.print_summary_op[mode], global_step])
- if writer is not None:
- writer.add_summary(summary, np_global_step)
- sess.run([m.agg_reset_op[mode]])
-
- # write custom validation ops
- if m.eval_metrics_fn[mode] is not None:
- val_metric_summary = m.eval_metrics_fn[mode](val_additional_ops,
- np_global_step, logdir)
- if writer is not None:
- writer.add_summary(val_metric_summary, np_global_step)
-
- total_loss = val_additional_ops
- should_stop = None
-
- return total_loss, should_stop
-
-def setup_training(loss_op, initial_learning_rate, steps_per_decay,
- learning_rate_decay, momentum, max_steps,
- sync=False, adjust_lr_sync=True,
- num_workers=1, replica_id=0, vars_to_optimize=None,
- clip_gradient_norm=0, typ=None, momentum2=0.999,
- adam_eps=1e-8):
- if sync and adjust_lr_sync:
- initial_learning_rate = initial_learning_rate * num_workers
- max_steps = np.int(max_steps / num_workers)
- steps_per_decay = np.int(steps_per_decay / num_workers)
-
- global_step_op = slim.get_or_create_global_step()
- lr_op = tf.train.exponential_decay(initial_learning_rate,
- global_step_op, steps_per_decay, learning_rate_decay, staircase=True)
- if typ == 'sgd':
- optimizer = tf.train.MomentumOptimizer(lr_op, momentum)
- elif typ == 'adam':
- optimizer = tf.train.AdamOptimizer(learning_rate=lr_op, beta1=momentum,
- beta2=momentum2, epsilon=adam_eps)
-
- if sync:
-
- sync_optimizer = tf.train.SyncReplicasOptimizer(optimizer,
- replicas_to_aggregate=num_workers,
- replica_id=replica_id,
- total_num_replicas=num_workers)
- train_op = slim.learning.create_train_op(loss_op, sync_optimizer,
- variables_to_train=vars_to_optimize,
- clip_gradient_norm=clip_gradient_norm)
- else:
- sync_optimizer = None
- train_op = slim.learning.create_train_op(loss_op, optimizer,
- variables_to_train=vars_to_optimize,
- clip_gradient_norm=clip_gradient_norm)
- should_stop_op = tf.greater_equal(global_step_op, max_steps)
- return lr_op, global_step_op, train_op, should_stop_op, optimizer, sync_optimizer
-
-def add_value_to_summary(metric_summary, tag, val, log=True, tag_str=None):
- """Adds a scalar summary to the summary object. Optionally also logs to
- logging."""
- new_value = metric_summary.value.add();
- new_value.tag = tag
- new_value.simple_value = val
- if log:
- if tag_str is None:
- tag_str = tag + '%f'
- logging.info(tag_str, val)
-
-def add_scalar_summary_op(tensor, name=None,
- summary_key='summaries', print_summary_key='print_summaries', prefix=''):
- collections = []
- op = tf.summary.scalar(name, tensor, collections=collections)
- if summary_key != print_summary_key:
- tf.add_to_collection(summary_key, op)
-
- op = tf.Print(op, [tensor], ' {:-<25s}: '.format(name) + prefix)
- tf.add_to_collection(print_summary_key, op)
- return op
-
-def setup_inputs(inputs):
- input_tensors = {}
- input_shapes = {}
- for (name, typ, sz) in inputs:
- _ = tf.placeholder(typ, shape=sz, name=name)
- input_tensors[name] = _
- input_shapes[name] = sz
- return input_tensors, input_shapes
-
-def prepare_feed_dict(input_tensors, inputs):
- feed_dict = {}
- for n in input_tensors.keys():
- feed_dict[input_tensors[n]] = inputs[n].astype(input_tensors[n].dtype.as_numpy_dtype)
- return feed_dict
-
-def simple_add_summaries(summarize_ops, summarize_names,
- summary_key='summaries',
- print_summary_key='print_summaries', prefix=''):
- for op, name, in zip(summarize_ops, summarize_names):
- add_scalar_summary_op(op, name, summary_key, print_summary_key, prefix)
-
- summary_op = tf.summary.merge_all(summary_key)
- print_summary_op = tf.summary.merge_all(print_summary_key)
- return summary_op, print_summary_op
-
-def add_summary_ops(m, summarize_ops, summarize_names, to_aggregate=None,
- summary_key='summaries',
- print_summary_key='print_summaries', prefix=''):
- if type(to_aggregate) != list:
- to_aggregate = [to_aggregate for _ in summarize_ops]
-
- # set up aggregating metrics
- if np.any(to_aggregate):
- agg_ops = []
- for op, name, to_agg in zip(summarize_ops, summarize_names, to_aggregate):
- if to_agg:
- # agg_ops.append(slim.metrics.streaming_mean(op, return_reset_op=True))
- agg_ops.append(tf.contrib.metrics.streaming_mean(op))
- # agg_ops.append(tf.contrib.metrics.streaming_mean(op, return_reset_op=True))
- else:
- agg_ops.append([None, None, None])
-
- # agg_values_op, agg_update_op, agg_reset_op = zip(*agg_ops)
- # agg_update_op = [x for x in agg_update_op if x is not None]
- # agg_reset_op = [x for x in agg_reset_op if x is not None]
- agg_values_op, agg_update_op = zip(*agg_ops)
- agg_update_op = [x for x in agg_update_op if x is not None]
- agg_reset_op = [tf.no_op()]
- else:
- agg_values_op = [None for _ in to_aggregate]
- agg_update_op = [tf.no_op()]
- agg_reset_op = [tf.no_op()]
-
- for op, name, to_agg, agg_op in zip(summarize_ops, summarize_names, to_aggregate, agg_values_op):
- if to_agg:
- add_scalar_summary_op(agg_op, name, summary_key, print_summary_key, prefix)
- else:
- add_scalar_summary_op(op, name, summary_key, print_summary_key, prefix)
-
- summary_op = tf.summary.merge_all(summary_key)
- print_summary_op = tf.summary.merge_all(print_summary_key)
- return summary_op, print_summary_op, agg_update_op, agg_reset_op
-
-
-
-def accum_val_ops(outputs, names, global_step, output_dir, metric_summary, N):
- """Processes the collected outputs to compute AP for action prediction.
-
- Args:
- outputs : List of scalar ops to summarize.
- names : Name of the scalar ops.
- global_step : global_step.
- output_dir : where to store results.
- metric_summary : summary object to add summaries to.
- N : number of outputs to process.
- """
- outs = []
- if N >= 0:
- outputs = outputs[:N]
- for i in range(len(outputs[0])):
- scalar = np.array(map(lambda x: x[i], outputs))
- assert(scalar.ndim == 1)
- add_value_to_summary(metric_summary, names[i], np.mean(scalar),
- tag_str='{:>27s}: [{:s}]: %f'.format(names[i], ''))
- outs.append(np.mean(scalar))
- return outs
-
-def get_default_summary_ops():
- return utils.Foo(summary_ops=None, print_summary_ops=None,
- additional_return_ops=[], arop_summary_iters=[],
- arop_eval_fns=[])
-
-
-def simple_summaries(summarize_ops, summarize_names, mode, to_aggregate=False,
- scope_name='summary'):
-
- if type(to_aggregate) != list:
- to_aggregate = [to_aggregate for _ in summarize_ops]
-
- summary_key = '{:s}_summaries'.format(mode)
- print_summary_key = '{:s}_print_summaries'.format(mode)
- prefix=' [{:s}]: '.format(mode)
-
- # Default ops for things that dont need to be aggregated.
- if not np.all(to_aggregate):
- for op, name, to_agg in zip(summarize_ops, summarize_names, to_aggregate):
- if not to_agg:
- add_scalar_summary_op(op, name, summary_key, print_summary_key, prefix)
- summary_ops = tf.summary.merge_all(summary_key)
- print_summary_ops = tf.summary.merge_all(print_summary_key)
- else:
- summary_ops = tf.no_op()
- print_summary_ops = tf.no_op()
-
- # Default ops for things that dont need to be aggregated.
- if np.any(to_aggregate):
- additional_return_ops = [[summarize_ops[i]
- for i, x in enumerate(to_aggregate )if x]]
- arop_summary_iters = [-1]
- s_names = ['{:s}/{:s}'.format(scope_name, summarize_names[i])
- for i, x in enumerate(to_aggregate) if x]
- fn = lambda outputs, global_step, output_dir, metric_summary, N: \
- accum_val_ops(outputs, s_names, global_step, output_dir, metric_summary,
- N)
- arop_eval_fns = [fn]
- else:
- additional_return_ops = []
- arop_summary_iters = []
- arop_eval_fns = []
- return summary_ops, print_summary_ops, additional_return_ops, \
- arop_summary_iters, arop_eval_fns
diff --git a/research/cognitive_mapping_and_planning/tfcode/vision_baseline_lstm.py b/research/cognitive_mapping_and_planning/tfcode/vision_baseline_lstm.py
deleted file mode 100644
index ccf3ab23b06b71ed2a6d300b9a7d2a67a396c52e..0000000000000000000000000000000000000000
--- a/research/cognitive_mapping_and_planning/tfcode/vision_baseline_lstm.py
+++ /dev/null
@@ -1,533 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import numpy as np
-
-
-import tensorflow as tf
-
-from tensorflow.contrib import slim
-
-import logging
-from tensorflow.python.platform import app
-from tensorflow.python.platform import flags
-from src import utils
-import src.file_utils as fu
-import tfcode.nav_utils as nu
-from tfcode import tf_utils
-
-setup_train_step_kwargs = nu.default_train_step_kwargs
-compute_losses_multi_or = nu.compute_losses_multi_or
-get_repr_from_image = nu.get_repr_from_image
-
-_save_d_at_t = nu.save_d_at_t
-_save_all = nu.save_all
-_eval_ap = nu.eval_ap
-_eval_dist = nu.eval_dist
-_plot_trajectories = nu.plot_trajectories
-
-def lstm_online(cell_fn, num_steps, inputs, state, varscope):
- # inputs is B x num_steps x C, C channels.
- # state is 2 tuple with B x 1 x C1, B x 1 x C2
- # Output state is always B x 1 x C
- inputs = tf.unstack(inputs, axis=1, num=num_steps)
- state = tf.unstack(state, axis=1, num=1)[0]
- outputs = []
-
- if num_steps > 1:
- varscope.reuse_variables()
-
- for s in range(num_steps):
- output, state = cell_fn(inputs[s], state)
- outputs.append(output)
- outputs = tf.stack(outputs, axis=1)
- state = tf.stack([state], axis=1)
- return outputs, state
-
-def _inputs(problem, lstm_states, lstm_state_dims):
- # Set up inputs.
- with tf.name_scope('inputs'):
- n_views = problem.n_views
-
- inputs = []
- inputs.append(('orig_maps', tf.float32,
- (problem.batch_size, 1, None, None, 1)))
- inputs.append(('goal_loc', tf.float32,
- (problem.batch_size, problem.num_goals, 2)))
-
- # For initing LSTM.
- inputs.append(('rel_goal_loc_at_start', tf.float32,
- (problem.batch_size, problem.num_goals,
- problem.rel_goal_loc_dim)))
- common_input_data, _ = tf_utils.setup_inputs(inputs)
-
- inputs = []
- inputs.append(('imgs', tf.float32, (problem.batch_size, None, n_views,
- problem.img_height, problem.img_width,
- problem.img_channels)))
- # Goal location as a tuple of delta location and delta theta.
- inputs.append(('rel_goal_loc', tf.float32, (problem.batch_size, None,
- problem.rel_goal_loc_dim)))
- if problem.outputs.visit_count:
- inputs.append(('visit_count', tf.int32, (problem.batch_size, None, 1)))
- inputs.append(('last_visit', tf.int32, (problem.batch_size, None, 1)))
-
- for i, (state, dim) in enumerate(zip(lstm_states, lstm_state_dims)):
- inputs.append((state, tf.float32, (problem.batch_size, 1, dim)))
-
- if problem.outputs.egomotion:
- inputs.append(('incremental_locs', tf.float32,
- (problem.batch_size, None, 2)))
- inputs.append(('incremental_thetas', tf.float32,
- (problem.batch_size, None, 1)))
-
- inputs.append(('step_number', tf.int32, (1, None, 1)))
- inputs.append(('node_ids', tf.int32, (problem.batch_size, None,
- problem.node_ids_dim)))
- inputs.append(('perturbs', tf.float32, (problem.batch_size, None,
- problem.perturbs_dim)))
-
- # For plotting result plots
- inputs.append(('loc_on_map', tf.float32, (problem.batch_size, None, 2)))
- inputs.append(('gt_dist_to_goal', tf.float32, (problem.batch_size, None, 1)))
- step_input_data, _ = tf_utils.setup_inputs(inputs)
-
- inputs = []
- inputs.append(('executed_actions', tf.int32, (problem.batch_size, None)))
- inputs.append(('rewards', tf.float32, (problem.batch_size, None)))
- inputs.append(('action_sample_wts', tf.float32, (problem.batch_size, None)))
- inputs.append(('action', tf.int32, (problem.batch_size, None,
- problem.num_actions)))
- train_data, _ = tf_utils.setup_inputs(inputs)
- train_data.update(step_input_data)
- train_data.update(common_input_data)
- return common_input_data, step_input_data, train_data
-
-
-def _add_summaries(m, summary_mode, arop_full_summary_iters):
- summarize_ops = [m.lr_op, m.global_step_op, m.sample_gt_prob_op,
- m.total_loss_op, m.data_loss_op, m.reg_loss_op] + m.acc_ops
- summarize_names = ['lr', 'global_step', 'sample_gt_prob_op', 'total_loss',
- 'data_loss', 'reg_loss'] + \
- ['acc_{:d}'.format(i) for i in range(len(m.acc_ops))]
- to_aggregate = [0, 0, 0, 1, 1, 1] + [1]*len(m.acc_ops)
-
- scope_name = 'summary'
- with tf.name_scope(scope_name):
- s_ops = nu.add_default_summaries(summary_mode, arop_full_summary_iters,
- summarize_ops, summarize_names,
- to_aggregate, m.action_prob_op,
- m.input_tensors, scope_name=scope_name)
- m.summary_ops = {summary_mode: s_ops}
-
-def visit_count_fc(visit_count, last_visit, embed_neurons, wt_decay, fc_dropout):
- with tf.variable_scope('embed_visit_count'):
- visit_count = tf.reshape(visit_count, shape=[-1])
- last_visit = tf.reshape(last_visit, shape=[-1])
-
- visit_count = tf.clip_by_value(visit_count, clip_value_min=-1,
- clip_value_max=15)
- last_visit = tf.clip_by_value(last_visit, clip_value_min=-1,
- clip_value_max=15)
- visit_count = tf.one_hot(visit_count, depth=16, axis=1, dtype=tf.float32,
- on_value=10., off_value=0.)
- last_visit = tf.one_hot(last_visit, depth=16, axis=1, dtype=tf.float32,
- on_value=10., off_value=0.)
- f = tf.concat([visit_count, last_visit], 1)
- x, _ = tf_utils.fc_network(
- f, neurons=embed_neurons, wt_decay=wt_decay, name='visit_count_embed',
- offset=0, batch_norm_param=None, dropout_ratio=fc_dropout,
- is_training=is_training)
- return x
-
-def lstm_setup(name, x, batch_size, is_single_step, lstm_dim, lstm_out,
- num_steps, state_input_op):
- # returns state_name, state_init_op, updated_state_op, out_op
- with tf.name_scope('reshape_'+name):
- sh = x.get_shape().as_list()
- x = tf.reshape(x, shape=[batch_size, -1, sh[-1]])
-
- with tf.variable_scope(name) as varscope:
- cell = tf.contrib.rnn.LSTMCell(
- num_units=lstm_dim, forget_bias=1.0, state_is_tuple=False,
- num_proj=lstm_out, use_peepholes=True,
- initializer=tf.random_uniform_initializer(-0.01, 0.01, seed=0),
- cell_clip=None, proj_clip=None)
-
- sh = [batch_size, 1, lstm_dim+lstm_out]
- state_init_op = tf.constant(0., dtype=tf.float32, shape=sh)
-
- fn = lambda ns: lstm_online(cell, ns, x, state_input_op, varscope)
- out_op, updated_state_op = tf.cond(is_single_step, lambda: fn(1), lambda:
- fn(num_steps))
-
- return name, state_init_op, updated_state_op, out_op
-
-def combine_setup(name, combine_type, embed_img, embed_goal, num_img_neuorons=None,
- num_goal_neurons=None):
- with tf.name_scope(name + '_' + combine_type):
- if combine_type == 'add':
- # Simple concat features from goal and image
- out = embed_img + embed_goal
-
- elif combine_type == 'multiply':
- # Multiply things together
- re_embed_img = tf.reshape(
- embed_img, shape=[-1, num_img_neuorons / num_goal_neurons,
- num_goal_neurons])
- re_embed_goal = tf.reshape(embed_goal, shape=[-1, num_goal_neurons, 1])
- x = tf.matmul(re_embed_img, re_embed_goal, transpose_a=False, transpose_b=False)
- out = slim.flatten(x)
- elif combine_type == 'none' or combine_type == 'imgonly':
- out = embed_img
- elif combine_type == 'goalonly':
- out = embed_goal
- else:
- logging.fatal('Undefined combine_type: %s', combine_type)
- return out
-
-
-def preprocess_egomotion(locs, thetas):
- with tf.name_scope('pre_ego'):
- pre_ego = tf.concat([locs, tf.sin(thetas), tf.cos(thetas)], 2)
- sh = pre_ego.get_shape().as_list()
- pre_ego = tf.reshape(pre_ego, [-1, sh[-1]])
- return pre_ego
-
-def setup_to_run(m, args, is_training, batch_norm_is_training, summary_mode):
- # Set up the model.
- tf.set_random_seed(args.solver.seed)
- task_params = args.navtask.task_params
- num_steps = task_params.num_steps
- num_goals = task_params.num_goals
- num_actions = task_params.num_actions
- num_actions_ = num_actions
-
- n_views = task_params.n_views
-
- batch_norm_is_training_op = \
- tf.placeholder_with_default(batch_norm_is_training, shape=[],
- name='batch_norm_is_training_op')
- # Setup the inputs
- m.input_tensors = {}
- lstm_states = []; lstm_state_dims = [];
- state_names = []; updated_state_ops = []; init_state_ops = [];
- if args.arch.lstm_output:
- lstm_states += ['lstm_output']
- lstm_state_dims += [args.arch.lstm_output_dim+task_params.num_actions]
- if args.arch.lstm_ego:
- lstm_states += ['lstm_ego']
- lstm_state_dims += [args.arch.lstm_ego_dim + args.arch.lstm_ego_out]
- lstm_states += ['lstm_img']
- lstm_state_dims += [args.arch.lstm_img_dim + args.arch.lstm_img_out]
- elif args.arch.lstm_img:
- # An LSTM only on the image
- lstm_states += ['lstm_img']
- lstm_state_dims += [args.arch.lstm_img_dim + args.arch.lstm_img_out]
- else:
- # No LSTMs involved here.
- None
-
- m.input_tensors['common'], m.input_tensors['step'], m.input_tensors['train'] = \
- _inputs(task_params, lstm_states, lstm_state_dims)
-
- with tf.name_scope('check_size'):
- is_single_step = tf.equal(tf.unstack(tf.shape(m.input_tensors['step']['imgs']),
- num=6)[1], 1)
-
- images_reshaped = tf.reshape(m.input_tensors['step']['imgs'],
- shape=[-1, task_params.img_height, task_params.img_width,
- task_params.img_channels], name='re_image')
-
- rel_goal_loc_reshaped = tf.reshape(m.input_tensors['step']['rel_goal_loc'],
- shape=[-1, task_params.rel_goal_loc_dim], name='re_rel_goal_loc')
-
- x, vars_ = get_repr_from_image(
- images_reshaped, task_params.modalities, task_params.data_augment,
- args.arch.encoder, args.solver.freeze_conv, args.solver.wt_decay,
- is_training)
-
- # Reshape into nice things so that these can be accumulated over time steps
- # for faster backprop.
- sh_before = x.get_shape().as_list()
- m.encoder_output = tf.reshape(
- x, shape=[task_params.batch_size, -1, n_views] + sh_before[1:])
- x = tf.reshape(m.encoder_output, shape=[-1] + sh_before[1:])
-
- # Add a layer to reduce dimensions for a fc layer.
- if args.arch.dim_reduce_neurons > 0:
- ks = 1; neurons = args.arch.dim_reduce_neurons;
- init_var = np.sqrt(2.0/(ks**2)/neurons)
- batch_norm_param = args.arch.batch_norm_param
- batch_norm_param['is_training'] = batch_norm_is_training_op
- m.conv_feat = slim.conv2d(
- x, neurons, kernel_size=ks, stride=1, normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_param, padding='SAME', scope='dim_reduce',
- weights_regularizer=slim.l2_regularizer(args.solver.wt_decay),
- weights_initializer=tf.random_normal_initializer(stddev=init_var))
- reshape_conv_feat = slim.flatten(m.conv_feat)
- sh = reshape_conv_feat.get_shape().as_list()
- m.reshape_conv_feat = tf.reshape(reshape_conv_feat,
- shape=[-1, sh[1]*n_views])
-
- # Restore these from a checkpoint.
- if args.solver.pretrained_path is not None:
- m.init_fn = slim.assign_from_checkpoint_fn(args.solver.pretrained_path,
- vars_)
- else:
- m.init_fn = None
-
- # Hit the goal_location with a bunch of fully connected layers, to embed it
- # into some space.
- with tf.variable_scope('embed_goal'):
- batch_norm_param = args.arch.batch_norm_param
- batch_norm_param['is_training'] = batch_norm_is_training_op
- m.embed_goal, _ = tf_utils.fc_network(
- rel_goal_loc_reshaped, neurons=args.arch.goal_embed_neurons,
- wt_decay=args.solver.wt_decay, name='goal_embed', offset=0,
- batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout,
- is_training=is_training)
-
- if args.arch.embed_goal_for_state:
- with tf.variable_scope('embed_goal_for_state'):
- batch_norm_param = args.arch.batch_norm_param
- batch_norm_param['is_training'] = batch_norm_is_training_op
- m.embed_goal_for_state, _ = tf_utils.fc_network(
- m.input_tensors['common']['rel_goal_loc_at_start'][:,0,:],
- neurons=args.arch.goal_embed_neurons, wt_decay=args.solver.wt_decay,
- name='goal_embed', offset=0, batch_norm_param=batch_norm_param,
- dropout_ratio=args.arch.fc_dropout, is_training=is_training)
-
- # Hit the goal_location with a bunch of fully connected layers, to embed it
- # into some space.
- with tf.variable_scope('embed_img'):
- batch_norm_param = args.arch.batch_norm_param
- batch_norm_param['is_training'] = batch_norm_is_training_op
- m.embed_img, _ = tf_utils.fc_network(
- m.reshape_conv_feat, neurons=args.arch.img_embed_neurons,
- wt_decay=args.solver.wt_decay, name='img_embed', offset=0,
- batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout,
- is_training=is_training)
-
- # For lstm_ego, and lstm_image, embed the ego motion, accumulate it into an
- # LSTM, combine with image features and accumulate those in an LSTM. Finally
- # combine what you get from the image LSTM with the goal to output an action.
- if args.arch.lstm_ego:
- ego_reshaped = preprocess_egomotion(m.input_tensors['step']['incremental_locs'],
- m.input_tensors['step']['incremental_thetas'])
- with tf.variable_scope('embed_ego'):
- batch_norm_param = args.arch.batch_norm_param
- batch_norm_param['is_training'] = batch_norm_is_training_op
- m.embed_ego, _ = tf_utils.fc_network(
- ego_reshaped, neurons=args.arch.ego_embed_neurons,
- wt_decay=args.solver.wt_decay, name='ego_embed', offset=0,
- batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout,
- is_training=is_training)
-
- state_name, state_init_op, updated_state_op, out_op = lstm_setup(
- 'lstm_ego', m.embed_ego, task_params.batch_size, is_single_step,
- args.arch.lstm_ego_dim, args.arch.lstm_ego_out, num_steps*num_goals,
- m.input_tensors['step']['lstm_ego'])
- state_names += [state_name]
- init_state_ops += [state_init_op]
- updated_state_ops += [updated_state_op]
-
- # Combine the output with the vision features.
- m.img_ego_op = combine_setup('img_ego', args.arch.combine_type_ego,
- m.embed_img, out_op,
- args.arch.img_embed_neurons[-1],
- args.arch.lstm_ego_out)
-
- # LSTM on these vision features.
- state_name, state_init_op, updated_state_op, out_op = lstm_setup(
- 'lstm_img', m.img_ego_op, task_params.batch_size, is_single_step,
- args.arch.lstm_img_dim, args.arch.lstm_img_out, num_steps*num_goals,
- m.input_tensors['step']['lstm_img'])
- state_names += [state_name]
- init_state_ops += [state_init_op]
- updated_state_ops += [updated_state_op]
-
- m.img_for_goal = out_op
- num_img_for_goal_neurons = args.arch.lstm_img_out
-
- elif args.arch.lstm_img:
- # LSTM on just the image features.
- state_name, state_init_op, updated_state_op, out_op = lstm_setup(
- 'lstm_img', m.embed_img, task_params.batch_size, is_single_step,
- args.arch.lstm_img_dim, args.arch.lstm_img_out, num_steps*num_goals,
- m.input_tensors['step']['lstm_img'])
- state_names += [state_name]
- init_state_ops += [state_init_op]
- updated_state_ops += [updated_state_op]
- m.img_for_goal = out_op
- num_img_for_goal_neurons = args.arch.lstm_img_out
-
- else:
- m.img_for_goal = m.embed_img
- num_img_for_goal_neurons = args.arch.img_embed_neurons[-1]
-
-
- if args.arch.use_visit_count:
- m.embed_visit_count = visit_count_fc(
- m.input_tensors['step']['visit_count'],
- m.input_tensors['step']['last_visit'], args.arch.goal_embed_neurons,
- args.solver.wt_decay, args.arch.fc_dropout, is_training=is_training)
- m.embed_goal = m.embed_goal + m.embed_visit_count
-
- m.combined_f = combine_setup('img_goal', args.arch.combine_type,
- m.img_for_goal, m.embed_goal,
- num_img_for_goal_neurons,
- args.arch.goal_embed_neurons[-1])
-
- # LSTM on the combined representation.
- if args.arch.lstm_output:
- name = 'lstm_output'
- # A few fully connected layers here.
- with tf.variable_scope('action_pred'):
- batch_norm_param = args.arch.batch_norm_param
- batch_norm_param['is_training'] = batch_norm_is_training_op
- x, _ = tf_utils.fc_network(
- m.combined_f, neurons=args.arch.pred_neurons,
- wt_decay=args.solver.wt_decay, name='pred', offset=0,
- batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout)
-
- if args.arch.lstm_output_init_state_from_goal:
- # Use the goal embedding to initialize the LSTM state.
- # UGLY CLUGGY HACK: if this is doing computation for a single time step
- # then this will not involve back prop, so we can use the state input from
- # the feed dict, otherwise we compute the state representation from the
- # goal and feed that in. Necessary for using goal location to generate the
- # state representation.
- m.embed_goal_for_state = tf.expand_dims(m.embed_goal_for_state, dim=1)
- state_op = tf.cond(is_single_step, lambda: m.input_tensors['step'][name],
- lambda: m.embed_goal_for_state)
- state_name, state_init_op, updated_state_op, out_op = lstm_setup(
- name, x, task_params.batch_size, is_single_step,
- args.arch.lstm_output_dim,
- num_actions_,
- num_steps*num_goals, state_op)
- init_state_ops += [m.embed_goal_for_state]
- else:
- state_op = m.input_tensors['step'][name]
- state_name, state_init_op, updated_state_op, out_op = lstm_setup(
- name, x, task_params.batch_size, is_single_step,
- args.arch.lstm_output_dim,
- num_actions_, num_steps*num_goals, state_op)
- init_state_ops += [state_init_op]
-
- state_names += [state_name]
- updated_state_ops += [updated_state_op]
-
- out_op = tf.reshape(out_op, shape=[-1, num_actions_])
- if num_actions_ > num_actions:
- m.action_logits_op = out_op[:,:num_actions]
- m.baseline_op = out_op[:,num_actions:]
- else:
- m.action_logits_op = out_op
- m.baseline_op = None
- m.action_prob_op = tf.nn.softmax(m.action_logits_op)
-
- else:
- # A few fully connected layers here.
- with tf.variable_scope('action_pred'):
- batch_norm_param = args.arch.batch_norm_param
- batch_norm_param['is_training'] = batch_norm_is_training_op
- out_op, _ = tf_utils.fc_network(
- m.combined_f, neurons=args.arch.pred_neurons,
- wt_decay=args.solver.wt_decay, name='pred', offset=0,
- num_pred=num_actions_,
- batch_norm_param=batch_norm_param,
- dropout_ratio=args.arch.fc_dropout, is_training=is_training)
- if num_actions_ > num_actions:
- m.action_logits_op = out_op[:,:num_actions]
- m.baseline_op = out_op[:,num_actions:]
- else:
- m.action_logits_op = out_op
- m.baseline_op = None
- m.action_prob_op = tf.nn.softmax(m.action_logits_op)
-
- m.train_ops = {}
- m.train_ops['step'] = m.action_prob_op
- m.train_ops['common'] = [m.input_tensors['common']['orig_maps'],
- m.input_tensors['common']['goal_loc'],
- m.input_tensors['common']['rel_goal_loc_at_start']]
- m.train_ops['state_names'] = state_names
- m.train_ops['init_state'] = init_state_ops
- m.train_ops['updated_state'] = updated_state_ops
- m.train_ops['batch_norm_is_training_op'] = batch_norm_is_training_op
-
- # Flat list of ops which cache the step data.
- m.train_ops['step_data_cache'] = [tf.no_op()]
-
- if args.solver.freeze_conv:
- m.train_ops['step_data_cache'] = [m.encoder_output]
- else:
- m.train_ops['step_data_cache'] = []
-
- ewma_decay = 0.99 if is_training else 0.0
- weight = tf.ones_like(m.input_tensors['train']['action'], dtype=tf.float32,
- name='weight')
-
- m.reg_loss_op, m.data_loss_op, m.total_loss_op, m.acc_ops = \
- compute_losses_multi_or(
- m.action_logits_op, m.input_tensors['train']['action'],
- weights=weight, num_actions=num_actions,
- data_loss_wt=args.solver.data_loss_wt,
- reg_loss_wt=args.solver.reg_loss_wt, ewma_decay=ewma_decay)
-
-
- if args.solver.freeze_conv:
- vars_to_optimize = list(set(tf.trainable_variables()) - set(vars_))
- else:
- vars_to_optimize = None
-
- m.lr_op, m.global_step_op, m.train_op, m.should_stop_op, m.optimizer, \
- m.sync_optimizer = tf_utils.setup_training(
- m.total_loss_op,
- args.solver.initial_learning_rate,
- args.solver.steps_per_decay,
- args.solver.learning_rate_decay,
- args.solver.momentum,
- args.solver.max_steps,
- args.solver.sync,
- args.solver.adjust_lr_sync,
- args.solver.num_workers,
- args.solver.task,
- vars_to_optimize=vars_to_optimize,
- clip_gradient_norm=args.solver.clip_gradient_norm,
- typ=args.solver.typ, momentum2=args.solver.momentum2,
- adam_eps=args.solver.adam_eps)
-
-
- if args.arch.sample_gt_prob_type == 'inverse_sigmoid_decay':
- m.sample_gt_prob_op = tf_utils.inverse_sigmoid_decay(args.arch.isd_k,
- m.global_step_op)
- elif args.arch.sample_gt_prob_type == 'zero':
- m.sample_gt_prob_op = tf.constant(-1.0, dtype=tf.float32)
- elif args.arch.sample_gt_prob_type.split('_')[0] == 'step':
- step = int(args.arch.sample_gt_prob_type.split('_')[1])
- m.sample_gt_prob_op = tf_utils.step_gt_prob(
- step, m.input_tensors['step']['step_number'][0,0,0])
-
- m.sample_action_type = args.arch.action_sample_type
- m.sample_action_combine_type = args.arch.action_sample_combine_type
- _add_summaries(m, summary_mode, args.summary.arop_full_summary_iters)
-
- m.init_op = tf.group(tf.global_variables_initializer(),
- tf.local_variables_initializer())
- m.saver_op = tf.train.Saver(keep_checkpoint_every_n_hours=4,
- write_version=tf.train.SaverDef.V2)
-
- return m
diff --git a/research/compression/README.md b/research/compression/README.md
deleted file mode 100644
index 7f431b5eac6805fbecc276783cef2bc6c62068e5..0000000000000000000000000000000000000000
--- a/research/compression/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
-
-# Compression with Neural Networks
-
-This is a [TensorFlow](http://www.tensorflow.org/) model repo containing
-research on compression with neural networks. This repo currently contains
-code for the following papers:
-
-[Full Resolution Image Compression with Recurrent Neural Networks](https://arxiv.org/abs/1608.05148)
-
-## Organization
-[Image Encoder](image_encoder/): Encoding and decoding images into their binary representation.
-
-[Entropy Coder](entropy_coder/): Lossless compression of the binary representation.
-
-## Contact Info
-Model repository maintained by Nick Johnston ([nmjohn](https://github.com/nmjohn)).
diff --git a/research/compression/entropy_coder/README.md b/research/compression/entropy_coder/README.md
deleted file mode 100644
index 59e889990aab71e12ed13122c9b5a796a048402a..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/README.md
+++ /dev/null
@@ -1,109 +0,0 @@
-# Neural net based entropy coding
-
-This is a [TensorFlow](http://www.tensorflow.org/) model for additional
-lossless compression of bitstreams generated by neural net based image
-encoders as described in
-[https://arxiv.org/abs/1703.10114](https://arxiv.org/abs/1703.10114).
-
-To be more specific, the entropy coder aims at compressing further binary
-codes which have a 3D tensor structure with:
-
-* the first two dimensions of the tensors corresponding to the height and
-the width of the binary codes,
-* the last dimension being the depth of the codes. The last dimension can be
-sliced into N groups of K, where each additional group is used by the image
-decoder to add more details to the reconstructed image.
-
-The code in this directory only contains the underlying code probability model
-but does not perform the actual compression using arithmetic coding.
-The code probability model is enough to compute the theoretical compression
-ratio.
-
-
-## Prerequisites
-The only software requirements for running the encoder and decoder is having
-Tensorflow installed.
-
-You will also need to add the top level source directory of the entropy coder
-to your `PYTHONPATH`, for example:
-
-`export PYTHONPATH=${PYTHONPATH}:/tmp/models/compression`
-
-
-## Training the entropy coder
-
-### Synthetic dataset
-If you do not have a training dataset, there is a simple code generative model
-that you can use to generate a dataset and play with the entropy coder.
-The generative model is located under dataset/gen\_synthetic\_dataset.py. Note
-that this simple generative model is not going to give good results on real
-images as it is not supposed to be close to the statistics of the binary
-representation of encoded images. Consider it as a toy dataset, no more, no
-less.
-
-To generate a synthetic dataset with 20000 samples:
-
-`mkdir -p /tmp/dataset`
-
-`python ./dataset/gen_synthetic_dataset.py --dataset_dir=/tmp/dataset/
---count=20000`
-
-Note that the generator has not been optimized at all, generating the synthetic
-dataset is currently pretty slow.
-
-### Training
-
-If you just want to play with the entropy coder trainer, here is the command
-line that can be used to train the entropy coder on the synthetic dataset:
-
-`mkdir -p /tmp/entropy_coder_train`
-
-`python ./core/entropy_coder_train.py --task=0
---train_dir=/tmp/entropy_coder_train/
---model=progressive
---model_config=./configs/synthetic/model_config.json
---train_config=./configs/synthetic/train_config.json
---input_config=./configs/synthetic/input_config.json
-`
-
-Training is configured using 3 files formatted using JSON:
-
-* One file is used to configure the underlying entropy coder model.
- Currently, only the *progressive* model is supported.
- This model takes 2 mandatory parameters and an optional one:
- * `layer_depth`: the number of bits per layer (a.k.a. iteration).
- Background: the image decoder takes each layer to add more detail
- to the image.
- * `layer_count`: the maximum number of layers that should be supported
- by the model. This should be equal or greater than the maximum number
- of layers in the input binary codes.
- * `coded_layer_count`: This can be used to consider only partial codes,
- keeping only the first `coded_layer_count` layers and ignoring the
- remaining layers. If left empty, the binary codes are left unchanged.
-* One file to configure the training, including the learning rate, ...
- The meaning of the parameters are pretty straightforward. Note that this
- file is only used during training and is not needed during inference.
-* One file to specify the input dataset to use during training.
- The dataset is formatted using tf.RecordIO.
-
-
-## Inference: file size after entropy coding.
-
-### Using a synthetic sample
-
-Here is the command line to generate a single synthetic sample formatted
-in the same way as what is provided by the image encoder:
-
-`python ./dataset/gen_synthetic_single.py
---sample_filename=/tmp/dataset/sample_0000.npz`
-
-To actually compute the additional compression ratio using the entropy coder
-trained in the previous step:
-
-`python ./core/entropy_coder_single.py
---model=progressive
---model_config=./configs/synthetic/model_config.json
---input_codes=/tmp/dataset/sample_0000.npz
---checkpoint=/tmp/entropy_coder_train/model.ckpt-209078`
-
-where the checkpoint number should be adjusted accordingly.
diff --git a/research/compression/entropy_coder/__init__.py b/research/compression/entropy_coder/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/compression/entropy_coder/all_models/__init__.py b/research/compression/entropy_coder/all_models/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/compression/entropy_coder/all_models/all_models.py b/research/compression/entropy_coder/all_models/all_models.py
deleted file mode 100644
index e376dac737667a348065eec622920b0a81ed1ac9..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/all_models/all_models.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Import and register all the entropy coder models."""
-
-# pylint: disable=unused-import
-from entropy_coder.progressive import progressive
diff --git a/research/compression/entropy_coder/all_models/all_models_test.py b/research/compression/entropy_coder/all_models/all_models_test.py
deleted file mode 100644
index b8aff504a0a00d579d1b2768164b78b6c095b235..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/all_models/all_models_test.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Basic test of all registered models."""
-
-import tensorflow as tf
-
-# pylint: disable=unused-import
-import all_models
-# pylint: enable=unused-import
-from entropy_coder.model import model_factory
-
-
-class AllModelsTest(tf.test.TestCase):
-
- def testBuildModelForTraining(self):
- factory = model_factory.GetModelRegistry()
- model_names = factory.GetAvailableModels()
-
- for m in model_names:
- tf.reset_default_graph()
-
- global_step = tf.Variable(tf.zeros([], dtype=tf.int64),
- trainable=False,
- name='global_step')
-
- optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
-
- batch_size = 3
- height = 40
- width = 20
- depth = 5
- binary_codes = tf.placeholder(dtype=tf.float32,
- shape=[batch_size, height, width, depth])
-
- # Create a model with the default configuration.
- print('Creating model: {}'.format(m))
- model = factory.CreateModel(m)
- model.Initialize(global_step,
- optimizer,
- model.GetConfigStringForUnitTest())
- self.assertTrue(model.loss is None, 'model: {}'.format(m))
- self.assertTrue(model.train_op is None, 'model: {}'.format(m))
- self.assertTrue(model.average_code_length is None, 'model: {}'.format(m))
-
- # Build the Tensorflow graph corresponding to the model.
- model.BuildGraph(binary_codes)
- self.assertTrue(model.loss is not None, 'model: {}'.format(m))
- self.assertTrue(model.average_code_length is not None,
- 'model: {}'.format(m))
- if model.train_op is None:
- print('Model {} is not trainable'.format(m))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/compression/entropy_coder/configs/gru_prime3/model_config.json b/research/compression/entropy_coder/configs/gru_prime3/model_config.json
deleted file mode 100644
index cf63a4c454df5c47c732c5eaeea481b2aa714665..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/configs/gru_prime3/model_config.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "layer_count": 16,
- "layer_depth": 32
-}
diff --git a/research/compression/entropy_coder/configs/synthetic/input_config.json b/research/compression/entropy_coder/configs/synthetic/input_config.json
deleted file mode 100644
index 18455e65120cd45cb04106ed8b6b2d6641e1d49a..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/configs/synthetic/input_config.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "data": "/tmp/dataset/synthetic_dataset",
- "unique_code_size": true
-}
diff --git a/research/compression/entropy_coder/configs/synthetic/model_config.json b/research/compression/entropy_coder/configs/synthetic/model_config.json
deleted file mode 100644
index c6f1f3e11547a75c05019e24c59a7fc6d2a29e3b..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/configs/synthetic/model_config.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "layer_depth": 2,
- "layer_count": 8
-}
diff --git a/research/compression/entropy_coder/configs/synthetic/train_config.json b/research/compression/entropy_coder/configs/synthetic/train_config.json
deleted file mode 100644
index 79e4909fd3f93df983d79890e25b7b61ba14aa40..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/configs/synthetic/train_config.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "batch_size": 4,
- "learning_rate": 0.1,
- "decay_rate": 0.9,
- "samples_per_decay": 20000
-}
diff --git a/research/compression/entropy_coder/core/code_loader.py b/research/compression/entropy_coder/core/code_loader.py
deleted file mode 100644
index 603ab724afb0e6c4e94db9c121d7799eaf30fa02..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/core/code_loader.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Load binary codes stored as tf.Example in a TFRecord table."""
-
-import tensorflow as tf
-
-
-def ReadFirstCode(dataset):
- """Read the first example from a binary code RecordIO table."""
- for record in tf.python_io.tf_record_iterator(dataset):
- tf_example = tf.train.Example()
- tf_example.ParseFromString(record)
- break
- return tf_example
-
-
-def LoadBinaryCode(input_config, batch_size):
- """Load a batch of binary codes from a tf.Example dataset.
-
- Args:
- input_config: An InputConfig proto containing the input configuration.
- batch_size: Output batch size of examples.
-
- Returns:
- A batched tensor of binary codes.
- """
- data = input_config.data
-
- # TODO: Possibly use multiple files (instead of just one).
- file_list = [data]
- filename_queue = tf.train.string_input_producer(file_list,
- capacity=4)
- reader = tf.TFRecordReader()
- _, values = reader.read(filename_queue)
-
- serialized_example = tf.reshape(values, shape=[1])
- serialized_features = {
- 'code_shape': tf.FixedLenFeature([3],
- dtype=tf.int64),
- 'code': tf.VarLenFeature(tf.float32),
- }
- example = tf.parse_example(serialized_example, serialized_features)
-
- # 3D shape: height x width x binary_code_depth
- z = example['code_shape']
- code_shape = tf.reshape(tf.cast(z, tf.int32), [3])
- # Un-flatten the binary codes.
- code = tf.reshape(tf.sparse_tensor_to_dense(example['code']), code_shape)
-
- queue_size = 10
- queue = tf.PaddingFIFOQueue(
- queue_size + 3 * batch_size,
- dtypes=[code.dtype],
- shapes=[[None, None, None]])
- enqueue_op = queue.enqueue([code])
- dequeue_code = queue.dequeue_many(batch_size)
- queue_runner = tf.train.queue_runner.QueueRunner(queue, [enqueue_op])
- tf.add_to_collection(tf.GraphKeys.QUEUE_RUNNERS, queue_runner)
-
- return dequeue_code
diff --git a/research/compression/entropy_coder/core/config_helper.py b/research/compression/entropy_coder/core/config_helper.py
deleted file mode 100644
index a7d949e329b93f33d330d1ba494f71ae1704fa3f..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/core/config_helper.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Helper functions used in both train and inference."""
-
-import json
-import os.path
-
-import tensorflow as tf
-
-
-def GetConfigString(config_file):
- config_string = ''
- if config_file is not None:
- config_string = open(config_file).read()
- return config_string
-
-
-class InputConfig(object):
-
- def __init__(self, config_string):
- config = json.loads(config_string)
- self.data = config["data"]
- self.unique_code_size = config["unique_code_size"]
-
-
-class TrainConfig(object):
-
- def __init__(self, config_string):
- config = json.loads(config_string)
- self.batch_size = config["batch_size"]
- self.learning_rate = config["learning_rate"]
- self.decay_rate = config["decay_rate"]
- self.samples_per_decay = config["samples_per_decay"]
-
-
-def SaveConfig(directory, filename, config_string):
- path = os.path.join(directory, filename)
- with tf.gfile.Open(path, mode='w') as f:
- f.write(config_string)
diff --git a/research/compression/entropy_coder/core/entropy_coder_single.py b/research/compression/entropy_coder/core/entropy_coder_single.py
deleted file mode 100644
index 8a61b488b6bdd11e1cff4a2da672129240eb7240..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/core/entropy_coder_single.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Compute the additional compression ratio after entropy coding."""
-
-import io
-import os
-
-import numpy as np
-import tensorflow as tf
-
-import config_helper
-
-# pylint: disable=unused-import
-from entropy_coder.all_models import all_models
-# pylint: enable=unused-import
-from entropy_coder.model import model_factory
-
-
-# Checkpoint used to restore the model parameters.
-tf.app.flags.DEFINE_string('checkpoint', None,
- """Model checkpoint.""")
-
-# Model selection and configuration.
-tf.app.flags.DEFINE_string('model', None, """Underlying encoder model.""")
-tf.app.flags.DEFINE_string('model_config', None,
- """Model config protobuf given as text file.""")
-
-# File holding the binary codes.
-tf.flags.DEFINE_string('input_codes', None, 'Location of binary code file.')
-
-FLAGS = tf.flags.FLAGS
-
-
-def main(_):
- if (FLAGS.input_codes is None or FLAGS.model is None):
- print ('\nUsage: python entropy_coder_single.py --model=progressive '
- '--model_config=model_config.json'
- '--iteration=15\n\n')
- return
-
- #if FLAGS.iteration < -1 or FLAGS.iteration > 15:
- # print ('\n--iteration must be between 0 and 15 inclusive, or -1 to infer '
- # 'from file.\n')
- # return
- #iteration = FLAGS.iteration
-
- if not tf.gfile.Exists(FLAGS.input_codes):
- print('\nInput codes not found.\n')
- return
-
- with tf.gfile.FastGFile(FLAGS.input_codes, 'rb') as code_file:
- contents = code_file.read()
- loaded_codes = np.load(io.BytesIO(contents))
- assert ['codes', 'shape'] not in loaded_codes.files
- loaded_shape = loaded_codes['shape']
- loaded_array = loaded_codes['codes']
-
- # Unpack and recover code shapes.
- unpacked_codes = np.reshape(np.unpackbits(loaded_array)
- [:np.prod(loaded_shape)],
- loaded_shape)
-
- numpy_int_codes = unpacked_codes.transpose([1, 2, 3, 0, 4])
- numpy_int_codes = numpy_int_codes.reshape([numpy_int_codes.shape[0],
- numpy_int_codes.shape[1],
- numpy_int_codes.shape[2],
- -1])
- numpy_codes = numpy_int_codes.astype(np.float32) * 2.0 - 1.0
-
- with tf.Graph().as_default() as graph:
- # TF tensor to hold the binary codes to losslessly compress.
- batch_size = 1
- codes = tf.placeholder(tf.float32, shape=numpy_codes.shape)
-
- # Create the entropy coder model.
- global_step = None
- optimizer = None
- model = model_factory.GetModelRegistry().CreateModel(FLAGS.model)
- model_config_string = config_helper.GetConfigString(FLAGS.model_config)
- model.Initialize(global_step, optimizer, model_config_string)
- model.BuildGraph(codes)
-
- saver = tf.train.Saver(sharded=True, keep_checkpoint_every_n_hours=12.0)
-
- with tf.Session(graph=graph) as sess:
- # Initialize local variables.
- sess.run(tf.local_variables_initializer())
-
- # Restore model variables.
- saver.restore(sess, FLAGS.checkpoint)
-
- tf_tensors = {
- 'code_length': model.average_code_length
- }
- feed_dict = {codes: numpy_codes}
- np_tensors = sess.run(tf_tensors, feed_dict=feed_dict)
-
- print('Additional compression ratio: {}'.format(
- np_tensors['code_length']))
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/compression/entropy_coder/core/entropy_coder_train.py b/research/compression/entropy_coder/core/entropy_coder_train.py
deleted file mode 100644
index 27c489037d27095b578aed6ad10a5a190ec49b18..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/core/entropy_coder_train.py
+++ /dev/null
@@ -1,184 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Train an entropy coder model."""
-
-import time
-
-import tensorflow as tf
-
-import code_loader
-import config_helper
-
-# pylint: disable=unused-import
-from entropy_coder.all_models import all_models
-# pylint: enable=unused-import
-from entropy_coder.model import model_factory
-
-
-FLAGS = tf.app.flags.FLAGS
-
-# Hardware resources configuration.
-tf.app.flags.DEFINE_string('master', '',
- """Name of the TensorFlow master to use.""")
-tf.app.flags.DEFINE_string('train_dir', None,
- """Directory where to write event logs.""")
-tf.app.flags.DEFINE_integer('task', None,
- """Task id of the replica running the training.""")
-tf.app.flags.DEFINE_integer('ps_tasks', 0, """Number of tasks in the ps job.
- If 0 no ps job is used.""")
-
-# Model selection and configuration.
-tf.app.flags.DEFINE_string('model', None, """Underlying encoder model.""")
-tf.app.flags.DEFINE_string('model_config', None,
- """Model config protobuf given as text file.""")
-
-# Training data and parameters configuration.
-tf.app.flags.DEFINE_string('input_config', None,
- """Path to the training input config file.""")
-tf.app.flags.DEFINE_string('train_config', None,
- """Path to the training experiment config file.""")
-
-
-def train():
- if FLAGS.train_dir is None:
- raise ValueError('Parameter train_dir must be provided')
- if FLAGS.task is None:
- raise ValueError('Parameter task must be provided')
- if FLAGS.model is None:
- raise ValueError('Parameter model must be provided')
-
- input_config_string = config_helper.GetConfigString(FLAGS.input_config)
- input_config = config_helper.InputConfig(input_config_string)
-
- # Training parameters.
- train_config_string = config_helper.GetConfigString(FLAGS.train_config)
- train_config = config_helper.TrainConfig(train_config_string)
-
- batch_size = train_config.batch_size
- initial_learning_rate = train_config.learning_rate
- decay_rate = train_config.decay_rate
- samples_per_decay = train_config.samples_per_decay
-
- # Parameters for learning-rate decay.
- # The formula is decay_rate ** floor(steps / decay_steps).
- decay_steps = samples_per_decay / batch_size
- decay_steps = max(decay_steps, 1)
-
- first_code = code_loader.ReadFirstCode(input_config.data)
- first_code_height = (
- first_code.features.feature['code_shape'].int64_list.value[0])
- first_code_width = (
- first_code.features.feature['code_shape'].int64_list.value[1])
- max_bit_depth = (
- first_code.features.feature['code_shape'].int64_list.value[2])
- print('Maximum code depth: {}'.format(max_bit_depth))
-
- with tf.Graph().as_default():
- ps_ops = ["Variable", "VariableV2", "AutoReloadVariable", "VarHandleOp"]
- with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks,
- ps_ops=ps_ops)):
- codes = code_loader.LoadBinaryCode(
- input_config=input_config,
- batch_size=batch_size)
- if input_config.unique_code_size:
- print('Input code size: {} x {}'.format(first_code_height,
- first_code_width))
- codes.set_shape(
- [batch_size, first_code_height, first_code_width, max_bit_depth])
- else:
- codes.set_shape([batch_size, None, None, max_bit_depth])
- codes_effective_shape = tf.shape(codes)
-
- global_step = tf.contrib.framework.create_global_step()
-
- # Apply learning-rate decay.
- learning_rate = tf.train.exponential_decay(
- learning_rate=initial_learning_rate,
- global_step=global_step,
- decay_steps=decay_steps,
- decay_rate=decay_rate,
- staircase=True)
- tf.summary.scalar('Learning Rate', learning_rate)
- optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
- epsilon=1.0)
-
- # Create the entropy coder model.
- model = model_factory.GetModelRegistry().CreateModel(FLAGS.model)
- model_config_string = config_helper.GetConfigString(FLAGS.model_config)
- model.Initialize(global_step, optimizer, model_config_string)
- model.BuildGraph(codes)
-
- summary_op = tf.summary.merge_all()
-
- # Verify that the model can actually be trained.
- if model.train_op is None:
- raise ValueError('Input model {} is not trainable'.format(FLAGS.model))
-
- # We disable the summary thread run by Supervisor class by passing
- # summary_op=None. We still pass save_summaries_secs because it is used by
- # the global step counter thread.
- is_chief = (FLAGS.task == 0)
- sv = tf.train.Supervisor(logdir=FLAGS.train_dir,
- is_chief=is_chief,
- global_step=global_step,
- # saver=model.saver,
- summary_op=None,
- save_summaries_secs=120,
- save_model_secs=600,
- recovery_wait_secs=30)
-
- sess = sv.PrepareSession(FLAGS.master)
- sv.StartQueueRunners(sess)
-
- step = sess.run(global_step)
- print('Trainer initial step: {}.'.format(step))
-
- # Once everything has been setup properly, save the configs.
- if is_chief:
- config_helper.SaveConfig(FLAGS.train_dir, 'input_config.json',
- input_config_string)
- config_helper.SaveConfig(FLAGS.train_dir, 'model_config.json',
- model_config_string)
- config_helper.SaveConfig(FLAGS.train_dir, 'train_config.json',
- train_config_string)
-
- # Train the model.
- next_summary_time = time.time()
- while not sv.ShouldStop():
- feed_dict = None
-
- # Once in a while, update the summaries on the chief worker.
- if is_chief and next_summary_time < time.time():
- summary_str = sess.run(summary_op, feed_dict=feed_dict)
- sv.SummaryComputed(sess, summary_str)
- next_summary_time = time.time() + sv.save_summaries_secs
- else:
- tf_tensors = {
- 'train': model.train_op,
- 'code_length': model.average_code_length
- }
- np_tensors = sess.run(tf_tensors, feed_dict=feed_dict)
- print(np_tensors['code_length'])
-
- sv.Stop()
-
-
-def main(argv=None): # pylint: disable=unused-argument
- train()
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/compression/entropy_coder/dataset/gen_synthetic_dataset.py b/research/compression/entropy_coder/dataset/gen_synthetic_dataset.py
deleted file mode 100644
index de60aee324d4a6209d00a873ee681aa59aae0d8e..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/dataset/gen_synthetic_dataset.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Generate a synthetic dataset."""
-
-import os
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import synthetic_model
-
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_string(
- 'dataset_dir', None,
- """Directory where to write the dataset and the configs.""")
-tf.app.flags.DEFINE_integer(
- 'count', 1000,
- """Number of samples to generate.""")
-
-
-def int64_feature(values):
- """Returns a TF-Feature of int64s.
-
- Args:
- values: A scalar or list of values.
-
- Returns:
- A TF-Feature.
- """
- if not isinstance(values, (tuple, list)):
- values = [values]
- return tf.train.Feature(int64_list=tf.train.Int64List(value=values))
-
-
-def float_feature(values):
- """Returns a TF-Feature of floats.
-
- Args:
- values: A scalar of list of values.
-
- Returns:
- A TF-Feature.
- """
- if not isinstance(values, (tuple, list)):
- values = [values]
- return tf.train.Feature(float_list=tf.train.FloatList(value=values))
-
-
-def AddToTFRecord(code, tfrecord_writer):
- example = tf.train.Example(features=tf.train.Features(feature={
- 'code_shape': int64_feature(code.shape),
- 'code': float_feature(code.flatten().tolist()),
- }))
- tfrecord_writer.write(example.SerializeToString())
-
-
-def GenerateDataset(filename, count, code_shape):
- with tf.python_io.TFRecordWriter(filename) as tfrecord_writer:
- for _ in xrange(count):
- code = synthetic_model.GenerateSingleCode(code_shape)
- # Convert {0,1} codes to {-1,+1} codes.
- code = 2.0 * code - 1.0
- AddToTFRecord(code, tfrecord_writer)
-
-
-def main(argv=None): # pylint: disable=unused-argument
- GenerateDataset(os.path.join(FLAGS.dataset_dir + '/synthetic_dataset'),
- FLAGS.count,
- [35, 48, 8])
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/compression/entropy_coder/dataset/gen_synthetic_single.py b/research/compression/entropy_coder/dataset/gen_synthetic_single.py
deleted file mode 100644
index b8c3821c38b6a0b95f01ad7ffb283cca4beb34b3..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/dataset/gen_synthetic_single.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Generate a single synthetic sample."""
-
-import io
-import os
-
-import numpy as np
-import tensorflow as tf
-
-import synthetic_model
-
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_string(
- 'sample_filename', None,
- """Output file to store the generated binary code.""")
-
-
-def GenerateSample(filename, code_shape, layer_depth):
- # {0, +1} binary codes.
- # No conversion since the output file is expected to store
- # codes using {0, +1} codes (and not {-1, +1}).
- code = synthetic_model.GenerateSingleCode(code_shape)
- code = np.round(code)
-
- # Reformat the code so as to be compatible with what is generated
- # by the image encoder.
- # The image encoder generates a tensor of size:
- # iteration_count x batch_size x height x width x iteration_depth.
- # Here: batch_size = 1
- if code_shape[-1] % layer_depth != 0:
- raise ValueError('Number of layers is not an integer')
- height = code_shape[0]
- width = code_shape[1]
- code = code.reshape([1, height, width, -1, layer_depth])
- code = np.transpose(code, [3, 0, 1, 2, 4])
-
- int_codes = code.astype(np.int8)
- exported_codes = np.packbits(int_codes.reshape(-1))
-
- output = io.BytesIO()
- np.savez_compressed(output, shape=int_codes.shape, codes=exported_codes)
- with tf.gfile.FastGFile(filename, 'wb') as code_file:
- code_file.write(output.getvalue())
-
-
-def main(argv=None): # pylint: disable=unused-argument
- # Note: the height and the width is different from the training dataset.
- # The main purpose is to show that the entropy coder model is fully
- # convolutional and can be used on any image size.
- layer_depth = 2
- GenerateSample(FLAGS.sample_filename, [31, 36, 8], layer_depth)
-
-
-if __name__ == '__main__':
- tf.app.run()
-
diff --git a/research/compression/entropy_coder/dataset/synthetic_model.py b/research/compression/entropy_coder/dataset/synthetic_model.py
deleted file mode 100644
index 9cccb64a136aba5a623c95e7c2dede2191d2cd62..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/dataset/synthetic_model.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Binary code sample generator."""
-
-import numpy as np
-from six.moves import xrange
-
-
-_CRC_LINE = [
- [0, 1, 0],
- [1, 1, 0],
- [1, 0, 0]
-]
-
-_CRC_DEPTH = [1, 1, 0, 1]
-
-
-def ComputeLineCrc(code, width, y, x, d):
- crc = 0
- for dy in xrange(len(_CRC_LINE)):
- i = y - 1 - dy
- if i < 0:
- continue
- for dx in xrange(len(_CRC_LINE[dy])):
- j = x - 2 + dx
- if j < 0 or j >= width:
- continue
- crc += 1 if (code[i, j, d] != _CRC_LINE[dy][dx]) else 0
- return crc
-
-
-def ComputeDepthCrc(code, y, x, d):
- crc = 0
- for delta in xrange(len(_CRC_DEPTH)):
- k = d - 1 - delta
- if k < 0:
- continue
- crc += 1 if (code[y, x, k] != _CRC_DEPTH[delta]) else 0
- return crc
-
-
-def GenerateSingleCode(code_shape):
- code = np.zeros(code_shape, dtype=np.int)
-
- keep_value_proba = 0.8
-
- height = code_shape[0]
- width = code_shape[1]
- depth = code_shape[2]
-
- for d in xrange(depth):
- for y in xrange(height):
- for x in xrange(width):
- v1 = ComputeLineCrc(code, width, y, x, d)
- v2 = ComputeDepthCrc(code, y, x, d)
- v = 1 if (v1 + v2 >= 6) else 0
- if np.random.rand() < keep_value_proba:
- code[y, x, d] = v
- else:
- code[y, x, d] = 1 - v
-
- return code
diff --git a/research/compression/entropy_coder/lib/__init__.py b/research/compression/entropy_coder/lib/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/compression/entropy_coder/lib/block_base.py b/research/compression/entropy_coder/lib/block_base.py
deleted file mode 100644
index 615dff82829dbbcab46c7217cd35f6259de01161..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/block_base.py
+++ /dev/null
@@ -1,258 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Base class for Tensorflow building blocks."""
-
-import collections
-import contextlib
-import itertools
-
-import tensorflow as tf
-
-_block_stacks = collections.defaultdict(lambda: [])
-
-
-class BlockBase(object):
- """Base class for transform wrappers of Tensorflow.
-
- To implement a Tensorflow transform block, inherit this class.
-
- 1. To create a variable, use NewVar() method. Do not overload this method!
- For example, use as follows.
- a_variable = self.NewVar(initial_value)
-
- 2. All Tensorflow-related code must be done inside 'with self._BlockScope().'
- Otherwise, name scoping and block hierarchy will not work. An exception
- is _Apply() method, which is already called inside the context manager
- by __call__() method.
-
- 3. Override and implement _Apply() method. This method is called by
- __call__() method.
-
- The users would use blocks like the following.
- nn1 = NN(128, bias=Bias(0), act=tf.nn.relu)
- y = nn1(x)
-
- Some things to consider.
-
- - Use lazy-initialization if possible. That is, initialize at first Apply()
- rather than at __init__().
-
- Note: if needed, the variables can be created on a specific parameter
- server by creating blocks in a scope like:
- with g.device(device):
- linear = Linear(...)
- """
-
- def __init__(self, name):
- self._variables = []
- self._subblocks = []
- self._called = False
-
- # Intentionally distinguishing empty string and None.
- # If name is an empty string, then do not use name scope.
- self.name = name if name is not None else self.__class__.__name__
- self._graph = tf.get_default_graph()
-
- if self.name:
- # Capture the scope string at the init time.
- with self._graph.name_scope(self.name) as scope:
- self._scope_str = scope
- else:
- self._scope_str = ''
-
- # Maintain hierarchy structure of blocks.
- self._stack = _block_stacks[self._graph]
- if self.__class__ is BlockBase:
- # This code is only executed to create the root, which starts in the
- # initialized state.
- assert not self._stack
- self._parent = None
- self._called = True # The root is initialized.
- return
-
- # Create a fake root if a root is not already present.
- if not self._stack:
- self._stack.append(BlockBase('NoOpRoot'))
-
- self._parent = self._stack[-1]
- self._parent._subblocks.append(self) # pylint: disable=protected-access
-
- def __repr__(self):
- return '"{}" ({})'.format(self._scope_str, self.__class__.__name__)
-
- @contextlib.contextmanager
- def _OptionalNameScope(self, scope_str):
- if scope_str:
- with self._graph.name_scope(scope_str):
- yield
- else:
- yield
-
- @contextlib.contextmanager
- def _BlockScope(self):
- """Context manager that handles graph, namescope, and nested blocks."""
- self._stack.append(self)
-
- try:
- with self._graph.as_default():
- with self._OptionalNameScope(self._scope_str):
- yield self
- finally: # Pop from the stack no matter exception is raised or not.
- # The following line is executed when leaving 'with self._BlockScope()'
- self._stack.pop()
-
- def __call__(self, *args, **kwargs):
- assert self._stack is _block_stacks[self._graph]
-
- with self._BlockScope():
- ret = self._Apply(*args, **kwargs)
-
- self._called = True
- return ret
-
- def _Apply(self, *args, **kwargs):
- """Implementation of __call__()."""
- raise NotImplementedError()
-
- # Redirect all variable creation to this single function, so that we can
- # switch to better variable creation scheme.
- def NewVar(self, value, **kwargs):
- """Creates a new variable.
-
- This function creates a variable, then returns a local copy created by
- Identity operation. To get the Variable class object, use LookupRef()
- method.
-
- Note that each time Variable class object is used as an input to an
- operation, Tensorflow will create a new Send/Recv pair. This hurts
- performance.
-
- If not for assign operations, use the local copy returned by this method.
-
- Args:
- value: Initialization value of the variable. The shape and the data type
- of the variable is determined by this initial value.
- **kwargs: Extra named arguments passed to Variable.__init__().
-
- Returns:
- A local copy of the new variable.
- """
- v = tf.Variable(value, **kwargs)
-
- self._variables.append(v)
- return v
-
- @property
- def initialized(self):
- """Returns bool if the block is initialized.
-
- By default, BlockBase assumes that a block is initialized when __call__()
- is executed for the first time. If this is an incorrect assumption for some
- subclasses, override this property in those subclasses.
-
- Returns:
- True if initialized, False otherwise.
- """
- return self._called
-
- def AssertInitialized(self):
- """Asserts initialized property."""
- if not self.initialized:
- raise RuntimeError('{} has not been initialized.'.format(self))
-
- def VariableList(self):
- """Returns the list of all tensorflow variables used inside this block."""
- variables = list(itertools.chain(
- itertools.chain.from_iterable(
- t.VariableList() for t in self._subblocks),
- self._VariableList()))
- return variables
-
- def _VariableList(self):
- """Returns the list of all tensorflow variables owned by this block."""
- self.AssertInitialized()
- return self._variables
-
- def CreateWeightLoss(self):
- """Returns L2 loss list of (almost) all variables used inside this block.
-
- When this method needs to be overridden, there are two choices.
-
- 1. Override CreateWeightLoss() to change the weight loss of all variables
- that belong to this block, both directly and indirectly.
- 2. Override _CreateWeightLoss() to change the weight loss of all
- variables that directly belong to this block but not to the sub-blocks.
-
- Returns:
- A Tensor object or None.
- """
- losses = list(itertools.chain(
- itertools.chain.from_iterable(
- t.CreateWeightLoss() for t in self._subblocks),
- self._CreateWeightLoss()))
- return losses
-
- def _CreateWeightLoss(self):
- """Returns weight loss list of variables that belong to this block."""
- self.AssertInitialized()
- with self._BlockScope():
- return [tf.nn.l2_loss(v) for v in self._variables]
-
- def CreateUpdateOps(self):
- """Creates update operations for this block and its sub-blocks."""
- ops = list(itertools.chain(
- itertools.chain.from_iterable(
- t.CreateUpdateOps() for t in self._subblocks),
- self._CreateUpdateOps()))
- return ops
-
- def _CreateUpdateOps(self):
- """Creates update operations for this block."""
- self.AssertInitialized()
- return []
-
- def MarkAsNonTrainable(self):
- """Mark all the variables of this block as non-trainable.
-
- All the variables owned directly or indirectly (through subblocks) are
- marked as non trainable.
-
- This function along with CheckpointInitOp can be used to load a pretrained
- model that consists in only one part of the whole graph.
- """
- assert self._called
-
- all_variables = self.VariableList()
- collection = tf.get_collection_ref(tf.GraphKeys.TRAINABLE_VARIABLES)
- for v in all_variables:
- if v in collection:
- collection.remove(v)
-
-
-def CreateWeightLoss():
- """Returns all weight losses from the blocks in the graph."""
- stack = _block_stacks[tf.get_default_graph()]
- if not stack:
- return []
- return stack[0].CreateWeightLoss()
-
-
-def CreateBlockUpdates():
- """Combines all updates from the blocks in the graph."""
- stack = _block_stacks[tf.get_default_graph()]
- if not stack:
- return []
- return stack[0].CreateUpdateOps()
diff --git a/research/compression/entropy_coder/lib/block_util.py b/research/compression/entropy_coder/lib/block_util.py
deleted file mode 100644
index 80479cc66df95338aa119ba1216cd213ecfbe08d..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/block_util.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utility functions for blocks."""
-
-from __future__ import division
-from __future__ import unicode_literals
-
-import math
-
-import numpy as np
-import six
-import tensorflow as tf
-
-
-class RsqrtInitializer(object):
- """Gaussian initializer with standard deviation 1/sqrt(n).
-
- Note that tf.truncated_normal is used internally. Therefore any random sample
- outside two-sigma will be discarded and re-sampled.
- """
-
- def __init__(self, dims=(0,), **kwargs):
- """Creates an initializer.
-
- Args:
- dims: Dimension(s) index to compute standard deviation:
- 1.0 / sqrt(product(shape[dims]))
- **kwargs: Extra keyword arguments to pass to tf.truncated_normal.
- """
- if isinstance(dims, six.integer_types):
- self._dims = [dims]
- else:
- self._dims = dims
- self._kwargs = kwargs
-
- def __call__(self, shape, dtype):
- stddev = 1.0 / np.sqrt(np.prod([shape[x] for x in self._dims]))
- return tf.truncated_normal(
- shape=shape, dtype=dtype, stddev=stddev, **self._kwargs)
-
-
-class RectifierInitializer(object):
- """Gaussian initializer with standard deviation sqrt(2/fan_in).
-
- Note that tf.random_normal is used internally to ensure the expected weight
- distribution. This is intended to be used with ReLU activations, specially
- in ResNets.
-
- For details please refer to:
- Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet
- Classification
- """
-
- def __init__(self, dims=(0,), scale=2.0, **kwargs):
- """Creates an initializer.
-
- Args:
- dims: Dimension(s) index to compute standard deviation:
- sqrt(scale / product(shape[dims]))
- scale: A constant scaling for the initialization used as
- sqrt(scale / product(shape[dims])).
- **kwargs: Extra keyword arguments to pass to tf.truncated_normal.
- """
- if isinstance(dims, six.integer_types):
- self._dims = [dims]
- else:
- self._dims = dims
- self._kwargs = kwargs
- self._scale = scale
-
- def __call__(self, shape, dtype):
- stddev = np.sqrt(self._scale / np.prod([shape[x] for x in self._dims]))
- return tf.random_normal(
- shape=shape, dtype=dtype, stddev=stddev, **self._kwargs)
-
-
-class GaussianInitializer(object):
- """Gaussian initializer with a given standard deviation.
-
- Note that tf.truncated_normal is used internally. Therefore any random sample
- outside two-sigma will be discarded and re-sampled.
- """
-
- def __init__(self, stddev=1.0):
- self._stddev = stddev
-
- def __call__(self, shape, dtype):
- return tf.truncated_normal(shape=shape, dtype=dtype, stddev=self._stddev)
diff --git a/research/compression/entropy_coder/lib/blocks.py b/research/compression/entropy_coder/lib/blocks.py
deleted file mode 100644
index 002384eb07045f1cad963d217a205ade51ba03b6..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from block_base import *
-from block_util import *
-from blocks_binarizer import *
-from blocks_entropy_coding import *
-from blocks_lstm import *
-from blocks_masked_conv2d import *
-from blocks_masked_conv2d_lstm import *
-from blocks_operator import *
-from blocks_std import *
diff --git a/research/compression/entropy_coder/lib/blocks_binarizer.py b/research/compression/entropy_coder/lib/blocks_binarizer.py
deleted file mode 100644
index 8206731610613af2cf3ec15210fd5b9977f4a916..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_binarizer.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Activation and weight binarizer implementations."""
-
-import math
-
-import numpy as np
-import tensorflow as tf
-
-
-def ConvertSignCodeToZeroOneCode(x):
- """Conversion from codes {-1, +1} to codes {0, 1}."""
- return 0.5 * (x + 1.0)
-
-
-def ConvertZeroOneCodeToSignCode(x):
- """Convert from codes {0, 1} to codes {-1, +1}."""
- return 2.0 * x - 1.0
-
-
-def CheckZeroOneCode(x):
- return tf.reduce_all(tf.equal(x * (x - 1.0), 0))
diff --git a/research/compression/entropy_coder/lib/blocks_entropy_coding.py b/research/compression/entropy_coder/lib/blocks_entropy_coding.py
deleted file mode 100644
index 6ee5d97926c1b50b12cb9853d16caa25ba31e8d7..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_entropy_coding.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Set of blocks related to entropy coding."""
-
-import math
-
-import tensorflow as tf
-
-import block_base
-
-# pylint does not recognize block_base.BlockBase.__call__().
-# pylint: disable=not-callable
-
-
-class CodeLength(block_base.BlockBase):
- """Theoretical bound for a code length given a probability distribution.
- """
-
- def __init__(self, name=None):
- super(CodeLength, self).__init__(name)
-
- def _Apply(self, c, p):
- """Theoretical bound of the coded length given a probability distribution.
-
- Args:
- c: The binary codes. Belong to {0, 1}.
- p: The probability of: P(code==+1)
-
- Returns:
- The average code length.
- Note: the average code length can be greater than 1 bit (e.g. when
- encoding the least likely symbol).
- """
- entropy = ((1.0 - c) * tf.log(1.0 - p) + c * tf.log(p)) / (-math.log(2))
- entropy = tf.reduce_mean(entropy)
- return entropy
diff --git a/research/compression/entropy_coder/lib/blocks_entropy_coding_test.py b/research/compression/entropy_coder/lib/blocks_entropy_coding_test.py
deleted file mode 100644
index 5209865f5991598ee873ed24a4be572e3f9fc515..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_entropy_coding_test.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for basic tensorflow blocks_entropy_coding."""
-
-from __future__ import division
-from __future__ import unicode_literals
-
-import math
-
-import numpy as np
-import tensorflow as tf
-
-import blocks_entropy_coding
-
-
-class BlocksEntropyCodingTest(tf.test.TestCase):
-
- def testCodeLength(self):
- shape = [2, 4]
- proba_feed = [[0.65, 0.25, 0.70, 0.10],
- [0.28, 0.20, 0.44, 0.54]]
- symbol_feed = [[1.0, 0.0, 1.0, 0.0],
- [0.0, 0.0, 0.0, 1.0]]
- mean_code_length = - (
- (math.log(0.65) + math.log(0.75) + math.log(0.70) + math.log(0.90) +
- math.log(0.72) + math.log(0.80) + math.log(0.56) + math.log(0.54)) /
- math.log(2.0)) / (shape[0] * shape[1])
-
- symbol = tf.placeholder(dtype=tf.float32, shape=shape)
- proba = tf.placeholder(dtype=tf.float32, shape=shape)
- code_length_calculator = blocks_entropy_coding.CodeLength()
- code_length = code_length_calculator(symbol, proba)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- code_length_eval = code_length.eval(
- feed_dict={symbol: symbol_feed, proba: proba_feed})
-
- self.assertAllClose(mean_code_length, code_length_eval)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/compression/entropy_coder/lib/blocks_lstm.py b/research/compression/entropy_coder/lib/blocks_lstm.py
deleted file mode 100644
index 6e474e3e3fcb6eeb3f18daf320e21a3acc88a2bf..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_lstm.py
+++ /dev/null
@@ -1,263 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Blocks of LSTM and its variants."""
-
-import numpy as np
-import tensorflow as tf
-
-import block_base
-import block_util
-import blocks_std
-
-# pylint does not recognize block_base.BlockBase.__call__().
-# pylint: disable=not-callable
-
-
-def LSTMBiasInit(shape, dtype):
- """Returns ones for forget-gate, and zeros for the others."""
- shape = np.array(shape)
-
- # Check internal consistencies.
- assert shape.shape == (1,), shape
- assert shape[0] % 4 == 0, shape
-
- n = shape[0] // 4
- ones = tf.fill([n], tf.constant(1, dtype=dtype))
- zeros = tf.fill([3 * n], tf.constant(0, dtype=dtype))
- return tf.concat([ones, zeros], 0)
-
-
-class LSTMBase(block_base.BlockBase):
- """Base class for LSTM implementations.
-
- These LSTM implementations use the pattern found in [1]. No peephole
- connection, i.e., cell content is not used in recurrence computation.
- Hidden units are also output units.
-
- [1] Zaremba, Sutskever, Vinyals. Recurrent Neural Network Regularization,
- 2015. arxiv:1409.2329.
- """
-
- def __init__(self, output_shape, name):
- """Initializes LSTMBase class object.
-
- Args:
- output_shape: List representing the LSTM output shape. This argument
- does not include batch dimension. For example, if the LSTM output has
- shape [batch, depth], then pass [depth].
- name: Name of this block.
- """
- super(LSTMBase, self).__init__(name)
-
- with self._BlockScope():
- self._output_shape = [None] + list(output_shape)
- self._hidden = None
- self._cell = None
-
- @property
- def hidden(self):
- """Returns the hidden units of this LSTM."""
- return self._hidden
-
- @hidden.setter
- def hidden(self, value):
- """Assigns to the hidden units of this LSTM.
-
- Args:
- value: The new value for the hidden units. If None, the hidden units are
- considered to be filled with zeros.
- """
- if value is not None:
- value.get_shape().assert_is_compatible_with(self._output_shape)
- self._hidden = value
-
- @property
- def cell(self):
- """Returns the cell units of this LSTM."""
- return self._cell
-
- @cell.setter
- def cell(self, value):
- """Assigns to the cell units of this LSTM.
-
- Args:
- value: The new value for the cell units. If None, the cell units are
- considered to be filled with zeros.
- """
- if value is not None:
- value.get_shape().assert_is_compatible_with(self._output_shape)
- self._cell = value
-
- # Consider moving bias terms to the base, and require this method to be
- # linear.
- def _TransformInputs(self, _):
- """Transforms the input units to (4 * depth) units.
-
- The forget-gate, input-gate, output-gate, and cell update is computed as
- f, i, j, o = T(h) + R(x)
- where h is hidden units, x is input units, and T, R are transforms of
- h, x, respectively.
-
- This method implements R. Note that T is strictly linear, so if LSTM is
- going to use bias, this method must include the bias to the transformation.
-
- Subclasses must implement this method. See _Apply() for more details.
- """
- raise NotImplementedError()
-
- def _TransformHidden(self, _):
- """Transforms the hidden units to (4 * depth) units.
-
- The forget-gate, input-gate, output-gate, and cell update is computed as
- f, i, j, o = T(h) + R(x)
- where h is hidden units, x is input units, and T, R are transforms of
- h, x, respectively.
-
- This method implements T in the equation. The method must implement a
- strictly linear transformation. For example, it may use MatMul or Conv2D,
- but must not add bias. This is because when hidden units are zeros, then
- the LSTM implementation will skip calling this method, instead of passing
- zeros to this function.
-
- Subclasses must implement this method. See _Apply() for more details.
- """
- raise NotImplementedError()
-
- def _Apply(self, *args):
- xtransform = self._TransformInputs(*args)
- depth_axis = len(self._output_shape) - 1
-
- if self.hidden is not None:
- htransform = self._TransformHidden(self.hidden)
- f, i, j, o = tf.split(
- value=htransform + xtransform, num_or_size_splits=4, axis=depth_axis)
- else:
- f, i, j, o = tf.split(
- value=xtransform, num_or_size_splits=4, axis=depth_axis)
-
- if self.cell is not None:
- self.cell = tf.sigmoid(f) * self.cell + tf.sigmoid(i) * tf.tanh(j)
- else:
- self.cell = tf.sigmoid(i) * tf.tanh(j)
-
- self.hidden = tf.sigmoid(o) * tf.tanh(self.cell)
- return self.hidden
-
-
-class LSTM(LSTMBase):
- """Efficient LSTM implementation used in [1].
-
- [1] Zaremba, Sutskever, Vinyals. Recurrent Neural Network Regularization,
- 2015. arxiv:1409.2329.
- """
-
- def __init__(self,
- depth,
- bias=LSTMBiasInit,
- initializer=block_util.RsqrtInitializer(),
- name=None):
- super(LSTM, self).__init__([depth], name)
-
- with self._BlockScope():
- self._depth = depth
- self._nn = blocks_std.NN(
- 4 * depth, bias=bias, act=None, initializer=initializer)
- self._hidden_linear = blocks_std.Linear(
- 4 * depth, initializer=initializer)
-
- def _TransformInputs(self, *args):
- return self._nn(*args)
-
- def _TransformHidden(self, h):
- return self._hidden_linear(h)
-
-
-class Conv2DLSTM(LSTMBase):
- """Convolutional LSTM implementation with optimizations inspired by [1].
-
- Note that when using the batch normalization feature, the bias initializer
- will not be used, since BN effectively cancels its effect out.
-
- [1] Zaremba, Sutskever, Vinyals. Recurrent Neural Network Regularization,
- 2015. arxiv:1409.2329.
- """
-
- def __init__(self,
- depth,
- filter_size,
- hidden_filter_size,
- strides,
- padding,
- bias=LSTMBiasInit,
- initializer=block_util.RsqrtInitializer(dims=(0, 1, 2)),
- use_moving_average=False,
- name=None):
- super(Conv2DLSTM, self).__init__([None, None, depth], name)
- self._iter = 0
-
- with self._BlockScope():
- self._input_conv = blocks_std.Conv2D(
- 4 * depth,
- filter_size,
- strides,
- padding,
- bias=None,
- act=None,
- initializer=initializer,
- name='input_conv2d')
-
- self._hidden_conv = blocks_std.Conv2D(
- 4 * depth,
- hidden_filter_size,
- [1, 1],
- 'SAME',
- bias=None,
- act=None,
- initializer=initializer,
- name='hidden_conv2d')
-
- if bias is not None:
- self._bias = blocks_std.BiasAdd(bias, name='biases')
- else:
- self._bias = blocks_std.PassThrough()
-
- def _TransformInputs(self, x):
- return self._bias(self._input_conv(x))
-
- def _TransformHidden(self, h):
- return self._hidden_conv(h)
-
- def _Apply(self, *args):
- xtransform = self._TransformInputs(*args)
- depth_axis = len(self._output_shape) - 1
-
- if self.hidden is not None:
- htransform = self._TransformHidden(self.hidden)
- f, i, j, o = tf.split(
- value=htransform + xtransform, num_or_size_splits=4, axis=depth_axis)
- else:
- f, i, j, o = tf.split(
- value=xtransform, num_or_size_splits=4, axis=depth_axis)
-
- if self.cell is not None:
- self.cell = tf.sigmoid(f) * self.cell + tf.sigmoid(i) * tf.tanh(j)
- else:
- self.cell = tf.sigmoid(i) * tf.tanh(j)
-
- self.hidden = tf.sigmoid(o) * tf.tanh(self.cell)
-
- self._iter += 1
- return self.hidden
diff --git a/research/compression/entropy_coder/lib/blocks_lstm_test.py b/research/compression/entropy_coder/lib/blocks_lstm_test.py
deleted file mode 100644
index 03c32dc136effda11163f2e35c5a48496f0187c0..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_lstm_test.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for LSTM tensorflow blocks."""
-from __future__ import division
-
-import numpy as np
-import tensorflow as tf
-
-import block_base
-import blocks_std
-import blocks_lstm
-
-
-class BlocksLSTMTest(tf.test.TestCase):
-
- def CheckUnary(self, y, op_type):
- self.assertEqual(op_type, y.op.type)
- self.assertEqual(1, len(y.op.inputs))
- return y.op.inputs[0]
-
- def CheckBinary(self, y, op_type):
- self.assertEqual(op_type, y.op.type)
- self.assertEqual(2, len(y.op.inputs))
- return y.op.inputs
-
- def testLSTM(self):
- lstm = blocks_lstm.LSTM(10)
- lstm.hidden = tf.zeros(shape=[10, 10], dtype=tf.float32)
- lstm.cell = tf.zeros(shape=[10, 10], dtype=tf.float32)
- x = tf.placeholder(dtype=tf.float32, shape=[10, 11])
- y = lstm(x)
-
- o, tanhc = self.CheckBinary(y, 'Mul')
- self.assertEqual(self.CheckUnary(o, 'Sigmoid').name, 'LSTM/split:3')
-
- self.assertIs(lstm.cell, self.CheckUnary(tanhc, 'Tanh'))
- fc, ij = self.CheckBinary(lstm.cell, 'Add')
-
- f, _ = self.CheckBinary(fc, 'Mul')
- self.assertEqual(self.CheckUnary(f, 'Sigmoid').name, 'LSTM/split:0')
-
- i, j = self.CheckBinary(ij, 'Mul')
- self.assertEqual(self.CheckUnary(i, 'Sigmoid').name, 'LSTM/split:1')
- j = self.CheckUnary(j, 'Tanh')
- self.assertEqual(j.name, 'LSTM/split:2')
-
- def testLSTMBiasInit(self):
- lstm = blocks_lstm.LSTM(9)
- x = tf.placeholder(dtype=tf.float32, shape=[15, 7])
- lstm(x)
- b = lstm._nn._bias
-
- with self.test_session():
- tf.global_variables_initializer().run()
- bias_var = b._bias.eval()
-
- comp = ([1.0] * 9) + ([0.0] * 27)
- self.assertAllEqual(bias_var, comp)
-
- def testConv2DLSTM(self):
- lstm = blocks_lstm.Conv2DLSTM(depth=10,
- filter_size=[1, 1],
- hidden_filter_size=[1, 1],
- strides=[1, 1],
- padding='SAME')
- lstm.hidden = tf.zeros(shape=[10, 11, 11, 10], dtype=tf.float32)
- lstm.cell = tf.zeros(shape=[10, 11, 11, 10], dtype=tf.float32)
- x = tf.placeholder(dtype=tf.float32, shape=[10, 11, 11, 1])
- y = lstm(x)
-
- o, tanhc = self.CheckBinary(y, 'Mul')
- self.assertEqual(self.CheckUnary(o, 'Sigmoid').name, 'Conv2DLSTM/split:3')
-
- self.assertIs(lstm.cell, self.CheckUnary(tanhc, 'Tanh'))
- fc, ij = self.CheckBinary(lstm.cell, 'Add')
-
- f, _ = self.CheckBinary(fc, 'Mul')
- self.assertEqual(self.CheckUnary(f, 'Sigmoid').name, 'Conv2DLSTM/split:0')
-
- i, j = self.CheckBinary(ij, 'Mul')
- self.assertEqual(self.CheckUnary(i, 'Sigmoid').name, 'Conv2DLSTM/split:1')
- j = self.CheckUnary(j, 'Tanh')
- self.assertEqual(j.name, 'Conv2DLSTM/split:2')
-
- def testConv2DLSTMBiasInit(self):
- lstm = blocks_lstm.Conv2DLSTM(9, 1, 1, [1, 1], 'SAME')
- x = tf.placeholder(dtype=tf.float32, shape=[1, 7, 7, 7])
- lstm(x)
- b = lstm._bias
-
- with self.test_session():
- tf.global_variables_initializer().run()
- bias_var = b._bias.eval()
-
- comp = ([1.0] * 9) + ([0.0] * 27)
- self.assertAllEqual(bias_var, comp)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/compression/entropy_coder/lib/blocks_masked_conv2d.py b/research/compression/entropy_coder/lib/blocks_masked_conv2d.py
deleted file mode 100644
index 3f562384a681964554ead02477da24c13715d4d1..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_masked_conv2d.py
+++ /dev/null
@@ -1,226 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Define some typical masked 2D convolutions."""
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import block_util
-import blocks_std
-
-# pylint does not recognize block_base.BlockBase.__call__().
-# pylint: disable=not-callable
-
-
-class RasterScanConv2D(blocks_std.Conv2DBase):
- """Conv2D with no dependency on future pixels (in raster scan order).
-
- For example, assuming a 5 x 5 kernel, the kernel is applied a spatial mask:
- T T T T T
- T T T T T
- T T x F F
- F F F F F
- F F F F F
- where 'T' are pixels which are available when computing the convolution
- for pixel 'x'. All the pixels marked with 'F' are not available.
- 'x' itself is not available if strict_order is True, otherwise, it is
- available.
- """
-
- def __init__(self, depth, filter_size, strides, padding,
- strict_order=True,
- bias=None, act=None, initializer=None, name=None):
- super(RasterScanConv2D, self).__init__(
- depth, filter_size, strides, padding, bias, act, name=name)
-
- if (filter_size[0] % 2) != 1 or (filter_size[1] % 2) != 1:
- raise ValueError('Kernel size should be odd.')
-
- with self._BlockScope():
- if initializer is None:
- initializer = block_util.RsqrtInitializer(dims=(0, 1, 2))
- self._initializer = initializer
- self._strict_order = strict_order
-
- def _CreateKernel(self, shape, dtype):
- init = self._initializer(shape, dtype)
- kernel = self.NewVar(init)
-
- mask = np.ones(shape[:2], dtype=dtype.as_numpy_dtype)
- center = shape[:2] // 2
- mask[center[0] + 1:, :] = 0
- if not self._strict_order:
- mask[center[0], center[1] + 1:] = 0
- else:
- mask[center[0], center[1]:] = 0
- mask = mask.reshape(mask.shape + (1, 1))
-
- return tf.convert_to_tensor(mask, dtype) * kernel
-
-
-class DepthOrderConv2D(blocks_std.Conv2DBase):
- """Conv2D with no dependency on higher depth dimensions.
-
- More precisely, the output depth #n has only dependencies on input depths #k
- for k < n (if strict_order is True) or for k <= n (if strict_order is False).
- """
-
- def __init__(self, depth, filter_size, strides, padding,
- strict_order=True,
- bias=None, act=None, initializer=None, name=None):
- super(DepthOrderConv2D, self).__init__(
- depth, filter_size, strides, padding, bias, act, name=name)
-
- with self._BlockScope():
- if initializer is None:
- initializer = block_util.RsqrtInitializer(dims=(0, 1, 2))
- self._initializer = initializer
- self._strict_order = strict_order
-
- def _CreateKernel(self, shape, dtype):
- init = self._initializer(shape, dtype)
- kernel = self.NewVar(init)
-
- mask = np.ones(shape[2:], dtype=dtype.as_numpy_dtype)
- depth_output = shape[3]
- for d in xrange(depth_output):
- if self._strict_order:
- mask[d:, d] = 0
- else:
- mask[d + 1:, d] = 0
- mask = mask.reshape((1, 1) + mask.shape)
-
- return tf.convert_to_tensor(mask, dtype) * kernel
-
-
-class GroupRasterScanConv2D(blocks_std.Conv2DBase):
- """Conv2D with no dependency on future pixels (in raster scan order).
-
- This version only introduces dependencies on previous pixels in raster scan
- order. It can also introduce some dependencies on previous depth positions
- of the current pixel (current pixel = center pixel of the kernel) in the
- following way:
- the depth dimension of the input is split into Ki groups of size
- |input_group_size|, the output dimension is split into Ko groups of size
- |output_group_size| (usually Ki == Ko). Each output group ko of the current
- pixel position can only depend on previous input groups ki
- (i.e. ki < ko if strict_order is True or ki <= ko if strict_order is False).
-
- Notes:
- - Block RasterScanConv2D is a special case of GroupRasterScanConv2D
- where Ki == Ko == 1 (i.e. input_group_size == input_depth and
- output_group_size == output_depth).
- - For 1x1 convolution, block DepthOrderConv2D is a special case of
- GroupRasterScanConv2D where input_group_size == 1 and
- output_group_size == 1.
- """
-
- def __init__(self, depth, filter_size, strides, padding,
- strict_order=True,
- input_group_size=1,
- output_group_size=1,
- bias=None, act=None, initializer=None, name=None):
- super(GroupRasterScanConv2D, self).__init__(
- depth, filter_size, strides, padding, bias, act, name=name)
-
- if (filter_size[0] % 2) != 1 or (filter_size[1] % 2) != 1:
- raise ValueError('Kernel size should be odd.')
-
- with self._BlockScope():
- if initializer is None:
- initializer = block_util.RsqrtInitializer(dims=(0, 1, 2))
- self._initializer = initializer
- self._input_group_size = input_group_size
- self._output_group_size = output_group_size
- self._strict_order = strict_order
-
- if depth % self._output_group_size != 0:
- raise ValueError(
- 'Invalid depth group size: {} for depth {}'.format(
- self._output_group_size, depth))
- self._output_group_count = depth // self._output_group_size
-
- def _CreateKernel(self, shape, dtype):
- init = self._initializer(shape, dtype)
- kernel = self.NewVar(init)
-
- depth_input = shape[2]
- if depth_input % self._input_group_size != 0:
- raise ValueError(
- 'Invalid depth group size: {} for depth {}'.format(
- self._input_group_size, depth_input))
- input_group_count = depth_input // self._input_group_size
- output_group_count = self._output_group_count
-
- # Set the mask to 0 for future pixels in raster scan order.
- center = shape[:2] // 2
- mask = np.ones([shape[0], shape[1],
- input_group_count, self._input_group_size,
- output_group_count, self._output_group_size],
- dtype=dtype.as_numpy_dtype)
- mask[center[0] + 1:, :, :, :, :, :] = 0
- mask[center[0], center[1] + 1:, :, :, :, :] = 0
-
- # Adjust the mask for the current position (the center position).
- depth_output = shape[3]
- for d in xrange(output_group_count):
- mask[center[0], center[1], d + 1:, :, d:d + 1, :] = 0
- if self._strict_order:
- mask[center[0], center[1], d, :, d:d + 1, :] = 0
-
- mask = mask.reshape([shape[0], shape[1], depth_input, depth_output])
- return tf.convert_to_tensor(mask, dtype) * kernel
-
-
-class InFillingConv2D(blocks_std.Conv2DBase):
- """Conv2D with kernel having no dependency on the current pixel.
-
- For example, assuming a 5 x 5 kernel, the kernel is applied a spatial mask:
- T T T T T
- T T T T T
- T T x T T
- T T T T T
- T T T T T
- where 'T' marks a pixel which is available when computing the convolution
- for pixel 'x'. 'x' itself is not available.
- """
-
- def __init__(self, depth, filter_size, strides, padding,
- bias=None, act=None, initializer=None, name=None):
- super(InFillingConv2D, self).__init__(
- depth, filter_size, strides, padding, bias, act, name=name)
-
- if (filter_size[0] % 2) != 1 or (filter_size[1] % 2) != 1:
- raise ValueError('Kernel size should be odd.')
- if filter_size[0] == 1 and filter_size[1] == 1:
- raise ValueError('Kernel size should be larger than 1x1.')
-
- with self._BlockScope():
- if initializer is None:
- initializer = block_util.RsqrtInitializer(dims=(0, 1, 2))
- self._initializer = initializer
-
- def _CreateKernel(self, shape, dtype):
- init = self._initializer(shape, dtype)
- kernel = self.NewVar(init)
-
- mask = np.ones(shape[:2], dtype=dtype.as_numpy_dtype)
- center = shape[:2] // 2
- mask[center[0], center[1]] = 0
- mask = mask.reshape(mask.shape + (1, 1))
-
- return tf.convert_to_tensor(mask, dtype) * kernel
diff --git a/research/compression/entropy_coder/lib/blocks_masked_conv2d_lstm.py b/research/compression/entropy_coder/lib/blocks_masked_conv2d_lstm.py
deleted file mode 100644
index 2d6dfeffcaff1289adf3bdec33cb0560db6b0416..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_masked_conv2d_lstm.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Masked conv2d LSTM."""
-
-import block_base
-import block_util
-import blocks_masked_conv2d
-import blocks_lstm
-import blocks_std
-
-# pylint: disable=not-callable
-
-
-class RasterScanConv2DLSTM(blocks_lstm.LSTMBase):
- """Convolutional LSTM implementation with optimizations inspired by [1].
-
- Note that when using the batch normalization feature, the bias initializer
- will not be used, since BN effectively cancels its effect out.
-
- [1] Zaremba, Sutskever, Vinyals. Recurrent Neural Network Regularization,
- 2015. arxiv:1409.2329.
- """
-
- def __init__(self,
- depth,
- filter_size,
- hidden_filter_size,
- strides,
- padding,
- bias=blocks_lstm.LSTMBiasInit,
- initializer=block_util.RsqrtInitializer(dims=(0, 1, 2)),
- name=None):
- super(RasterScanConv2DLSTM, self).__init__([None, None, depth], name)
-
- with self._BlockScope():
- self._input_conv = blocks_masked_conv2d.RasterScanConv2D(
- 4 * depth,
- filter_size,
- strides,
- padding,
- strict_order=False,
- bias=None,
- act=None,
- initializer=initializer,
- name='input_conv2d')
-
- self._hidden_conv = blocks_std.Conv2D(
- 4 * depth,
- hidden_filter_size,
- [1, 1],
- 'SAME',
- bias=None,
- act=None,
- initializer=initializer,
- name='hidden_conv2d')
-
- if bias is not None:
- self._bias = blocks_std.BiasAdd(bias, name='biases')
- else:
- self._bias = blocks_std.PassThrough()
-
- def _TransformInputs(self, x):
- return self._bias(self._input_conv(x))
-
- def _TransformHidden(self, h):
- return self._hidden_conv(h)
diff --git a/research/compression/entropy_coder/lib/blocks_masked_conv2d_test.py b/research/compression/entropy_coder/lib/blocks_masked_conv2d_test.py
deleted file mode 100644
index 1d284ebffe5a24b91c96936c17d6c23febdf76d5..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_masked_conv2d_test.py
+++ /dev/null
@@ -1,207 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests of the 2D masked convolution blocks."""
-
-from __future__ import division
-from __future__ import unicode_literals
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import blocks_masked_conv2d
-
-
-class MaskedConv2DTest(tf.test.TestCase):
-
- def testRasterScanKernel(self):
- kernel_size = 5
- input_depth = 1
- output_depth = 1
- kernel_shape = [kernel_size, kernel_size, input_depth, output_depth]
-
- # pylint: disable=bad-whitespace
- kernel_feed = [[ 1.0, 2.0, 3.0, 4.0, 5.0],
- [ 6.0, 7.0, 8.0, 9.0, 10.0],
- [11.0, 12.0, 13.0, 14.0, 15.0],
- [16.0, 17.0, 18.0, 19.0, 20.0],
- [21.0, 22.0, 23.0, 24.0, 25.0]]
- kernel_feed = np.reshape(kernel_feed, kernel_shape)
- kernel_expected = [[ 1.0, 2.0, 3.0, 4.0, 5.0],
- [ 6.0, 7.0, 8.0, 9.0, 10.0],
- [11.0, 12.0, 0.0, 0.0, 0.0],
- [ 0.0, 0.0, 0.0, 0.0, 0.0],
- [ 0.0, 0.0, 0.0, 0.0, 0.0]]
- kernel_expected = np.reshape(kernel_expected, kernel_shape)
- # pylint: enable=bad-whitespace
-
- init_kernel = lambda s, t: tf.constant(kernel_feed, dtype=t, shape=s)
- masked_conv2d = blocks_masked_conv2d.RasterScanConv2D(
- output_depth, [kernel_size] * 2, [1] * 2, 'SAME',
- initializer=init_kernel)
- x = tf.placeholder(dtype=tf.float32, shape=[10] * 3 + [input_depth])
- _ = masked_conv2d(x)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- kernel_value = masked_conv2d._kernel.eval()
-
- self.assertAllEqual(kernel_expected, kernel_value)
-
- def testDepthOrderKernel(self):
- kernel_size = 1
- input_depth = 7
- output_depth = input_depth
- kernel_shape = [kernel_size, kernel_size, input_depth, output_depth]
-
- kernel_feed = np.ones(kernel_shape)
- x_shape = [5] * 3 + [input_depth]
- x_feed = np.ones(x_shape)
- y_expected = np.zeros(x_shape[0:3] + [output_depth])
- y_expected[:, :, :] = np.arange(output_depth)
-
- init_kernel = lambda s, t: tf.constant(kernel_feed, dtype=t, shape=s)
- masked_conv2d = blocks_masked_conv2d.DepthOrderConv2D(
- output_depth, [kernel_size] * 2, [1] * 2, 'SAME',
- strict_order=True,
- initializer=init_kernel)
- x = tf.placeholder(dtype=tf.float32, shape=x_shape)
- y = masked_conv2d(x)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- y_value = y.eval(feed_dict={x: x_feed})
-
- self.assertAllEqual(y_expected, y_value)
-
- def testGroupRasterScanKernel(self):
- kernel_size = 3
- input_depth = 4
- input_group_size = 2
- output_depth = 2
- output_group_size = 1
- kernel_shape = [kernel_size, kernel_size, input_depth, output_depth]
- kernel_feed = np.ones(shape=kernel_shape)
-
- height = 5
- width = 5
- x_shape = [1, height, width, input_depth]
- x_feed = np.ones(shape=x_shape)
-
- # pylint: disable=bad-whitespace
- y_expected = [
- [[ 0, 2], [ 4, 6], [ 4, 6], [ 4, 6], [ 4, 6]],
- [[ 8, 10], [16, 18], [16, 18], [16, 18], [12, 14]],
- [[ 8, 10], [16, 18], [16, 18], [16, 18], [12, 14]],
- [[ 8, 10], [16, 18], [16, 18], [16, 18], [12, 14]],
- [[ 8, 10], [16, 18], [16, 18], [16, 18], [12, 14]],
- ]
- y_expected = np.reshape(y_expected, [1, height, width, output_depth])
- # pylint: enable=bad-whitespace
-
- init_kernel = lambda s, t: tf.constant(kernel_feed, dtype=t, shape=s)
- masked_conv2d = blocks_masked_conv2d.GroupRasterScanConv2D(
- output_depth, [kernel_size] * 2, [1] * 2, 'SAME',
- strict_order=True,
- input_group_size=input_group_size,
- output_group_size=output_group_size,
- initializer=init_kernel)
- x = tf.placeholder(dtype=tf.float32, shape=x_shape)
- y = masked_conv2d(x)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- y_value = y.eval(feed_dict={x: x_feed})
-
- self.assertAllEqual(y_expected, y_value)
-
- def testInFillingKernel(self):
- kernel_size = 5
- input_depth = 1
- output_depth = 1
- kernel_shape = [kernel_size, kernel_size, input_depth, output_depth]
-
- # pylint: disable=bad-whitespace
- kernel_feed = [[ 1.0, 2.0, 3.0, 4.0, 5.0],
- [ 6.0, 7.0, 8.0, 9.0, 10.0],
- [11.0, 12.0, 13.0, 14.0, 15.0],
- [16.0, 17.0, 18.0, 19.0, 20.0],
- [21.0, 22.0, 23.0, 24.0, 25.0]]
- kernel_feed = np.reshape(kernel_feed, kernel_shape)
- kernel_expected = [[ 1.0, 2.0, 3.0, 4.0, 5.0],
- [ 6.0, 7.0, 8.0, 9.0, 10.0],
- [11.0, 12.0, 0.0, 14.0, 15.0],
- [16.0, 17.0, 18.0, 19.0, 20.0],
- [21.0, 22.0, 23.0, 24.0, 25.0]]
- kernel_expected = np.reshape(kernel_expected, kernel_shape)
- # pylint: enable=bad-whitespace
-
- init_kernel = lambda s, t: tf.constant(kernel_feed, dtype=t, shape=s)
- masked_conv2d = blocks_masked_conv2d.InFillingConv2D(
- output_depth, [kernel_size] * 2, [1] * 2, 'SAME',
- initializer=init_kernel)
- x = tf.placeholder(dtype=tf.float32, shape=[10] * 3 + [input_depth])
- _ = masked_conv2d(x)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- kernel_value = masked_conv2d._kernel.eval()
-
- self.assertAllEqual(kernel_expected, kernel_value)
-
- def testConv2DMaskedNumerics(self):
- kernel_size = 5
- input_shape = [1, 10, 10, 1]
- filter_shape = [kernel_size, kernel_size, 1, 1]
- strides = [1, 1, 1, 1]
- output_shape = [1, 10, 10, 1]
-
- conv = blocks_masked_conv2d.RasterScanConv2D(
- depth=filter_shape[-1],
- filter_size=filter_shape[0:2],
- strides=strides[1:3],
- padding='SAME',
- initializer=tf.constant_initializer(value=1.0))
- x = tf.placeholder(dtype=tf.float32, shape=input_shape)
- y = conv(x)
-
- x_feed = - np.ones(input_shape, dtype=float)
- y_expected = np.ones(output_shape, dtype=float)
- for i in xrange(input_shape[1]):
- for j in xrange(input_shape[2]):
- x_feed[0, i, j, 0] = 10 * (j + 1) + i
- v = 0
- ki_start = max(i - kernel_size // 2, 0)
- kj_start = max(j - kernel_size // 2, 0)
- kj_end = min(j + kernel_size // 2, input_shape[2] - 1)
- for ki in range(ki_start, i + 1):
- for kj in range(kj_start, kj_end + 1):
- if ki > i:
- continue
- if ki == i and kj >= j:
- continue
- v += 10 * (kj + 1) + ki
- y_expected[0, i, j, 0] = v
-
- with self.test_session():
- tf.global_variables_initializer().run()
- y_value = y.eval(feed_dict={x: x_feed})
-
- self.assertAllEqual(y_expected, y_value)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/compression/entropy_coder/lib/blocks_operator.py b/research/compression/entropy_coder/lib/blocks_operator.py
deleted file mode 100644
index e35e37b27aa416ed48f91eda866d372601741cba..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_operator.py
+++ /dev/null
@@ -1,87 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Common blocks which work as operators on other blocks."""
-
-import tensorflow as tf
-
-import block_base
-
-# pylint: disable=not-callable
-
-
-class CompositionOperator(block_base.BlockBase):
- """Composition of several blocks."""
-
- def __init__(self, block_list, name=None):
- """Initialization of the composition operator.
-
- Args:
- block_list: List of blocks.BlockBase that are chained to create
- a new blocks.BlockBase.
- name: Name of this block.
- """
- super(CompositionOperator, self).__init__(name)
- self._blocks = block_list
-
- def _Apply(self, x):
- """Apply successively all the blocks on the given input tensor."""
- h = x
- for layer in self._blocks:
- h = layer(h)
- return h
-
-
-class LineOperator(block_base.BlockBase):
- """Repeat the same block over all the lines of an input tensor."""
-
- def __init__(self, block, name=None):
- super(LineOperator, self).__init__(name)
- self._block = block
-
- def _Apply(self, x):
- height = x.get_shape()[1].value
- if height is None:
- raise ValueError('Unknown tensor height')
- all_line_x = tf.split(value=x, num_or_size_splits=height, axis=1)
-
- y = []
- for line_x in all_line_x:
- y.append(self._block(line_x))
- y = tf.concat(values=y, axis=1)
-
- return y
-
-
-class TowerOperator(block_base.BlockBase):
- """Parallel execution with concatenation of several blocks."""
-
- def __init__(self, block_list, dim=3, name=None):
- """Initialization of the parallel exec + concat (Tower).
-
- Args:
- block_list: List of blocks.BlockBase that are chained to create
- a new blocks.BlockBase.
- dim: the dimension on which to concat.
- name: Name of this block.
- """
- super(TowerOperator, self).__init__(name)
- self._blocks = block_list
- self._concat_dim = dim
-
- def _Apply(self, x):
- """Apply successively all the blocks on the given input tensor."""
- outputs = [layer(x) for layer in self._blocks]
- return tf.concat(outputs, self._concat_dim)
diff --git a/research/compression/entropy_coder/lib/blocks_operator_test.py b/research/compression/entropy_coder/lib/blocks_operator_test.py
deleted file mode 100644
index 8b6d80da1d09102585e4725dd5c59f48d48eafcd..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_operator_test.py
+++ /dev/null
@@ -1,64 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests of the block operators."""
-
-import numpy as np
-import tensorflow as tf
-
-import block_base
-import blocks_operator
-
-
-class AddOneBlock(block_base.BlockBase):
-
- def __init__(self, name=None):
- super(AddOneBlock, self).__init__(name)
-
- def _Apply(self, x):
- return x + 1.0
-
-
-class SquareBlock(block_base.BlockBase):
-
- def __init__(self, name=None):
- super(SquareBlock, self).__init__(name)
-
- def _Apply(self, x):
- return x * x
-
-
-class BlocksOperatorTest(tf.test.TestCase):
-
- def testComposition(self):
- x_value = np.array([[1.0, 2.0, 3.0],
- [-1.0, -2.0, -3.0]])
- y_expected_value = np.array([[4.0, 9.0, 16.0],
- [0.0, 1.0, 4.0]])
-
- x = tf.placeholder(dtype=tf.float32, shape=[2, 3])
- complex_block = blocks_operator.CompositionOperator(
- [AddOneBlock(),
- SquareBlock()])
- y = complex_block(x)
-
- with self.test_session():
- y_value = y.eval(feed_dict={x: x_value})
-
- self.assertAllClose(y_expected_value, y_value)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/compression/entropy_coder/lib/blocks_std.py b/research/compression/entropy_coder/lib/blocks_std.py
deleted file mode 100644
index 2c617485342452f500d4b1b0b18e33b07d51e487..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_std.py
+++ /dev/null
@@ -1,363 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Basic blocks for building tensorflow models."""
-
-import numpy as np
-import tensorflow as tf
-
-import block_base
-import block_util
-
-# pylint does not recognize block_base.BlockBase.__call__().
-# pylint: disable=not-callable
-
-
-def HandleConvPaddingModes(x, padding, kernel_shape, strides):
- """Returns an updated tensor and padding type for REFLECT and SYMMETRIC.
-
- Args:
- x: A 4D tensor with shape [batch_size, height, width, depth].
- padding: Padding mode (SAME, VALID, REFLECT, or SYMMETRIC).
- kernel_shape: Shape of convolution kernel that will be applied.
- strides: Convolution stride that will be used.
-
- Returns:
- x and padding after adjustments for REFLECT and SYMMETRIC.
- """
- # For 1x1 convolution, all padding modes are the same.
- if np.all(kernel_shape[:2] == 1):
- return x, 'VALID'
-
- if padding == 'REFLECT' or padding == 'SYMMETRIC':
- # We manually compute the number of paddings as if 'SAME'.
- # From Tensorflow kernel, the formulas are as follows.
- # output_shape = ceil(input_shape / strides)
- # paddings = (output_shape - 1) * strides + filter_size - input_shape
- # Let x, y, s be a shorthand notations for input_shape, output_shape, and
- # strides, respectively. Let (x - 1) = sn + r where 0 <= r < s. Note that
- # y - 1 = ceil(x / s) - 1 = floor((x - 1) / s) = n
- # provided that x > 0. Therefore
- # paddings = n * s + filter_size - (sn + r + 1)
- # = filter_size - r - 1.
- input_shape = x.get_shape() # shape at graph construction time
- img_shape = tf.shape(x)[1:3] # image shape (no batch) at run time
- remainder = tf.mod(img_shape - 1, strides[1:3])
- pad_sizes = kernel_shape[:2] - remainder - 1
-
- pad_rows = pad_sizes[0]
- pad_cols = pad_sizes[1]
- pad = tf.stack([[0, 0], tf.stack([pad_rows // 2, (pad_rows + 1) // 2]),
- tf.stack([pad_cols // 2, (pad_cols + 1) // 2]), [0, 0]])
-
- # Manually pad the input and switch the padding mode to 'VALID'.
- x = tf.pad(x, pad, mode=padding)
- x.set_shape([input_shape[0], x.get_shape()[1],
- x.get_shape()[2], input_shape[3]])
- padding = 'VALID'
-
- return x, padding
-
-
-class PassThrough(block_base.BlockBase):
- """A dummy transform block that does nothing."""
-
- def __init__(self):
- # Pass an empty string to disable name scoping.
- super(PassThrough, self).__init__(name='')
-
- def _Apply(self, inp):
- return inp
-
- @property
- def initialized(self):
- """Always returns True."""
- return True
-
-
-class Bias(object):
- """An initialization helper class for BiasAdd block below."""
-
- def __init__(self, value=0):
- self.value = value
-
-
-class BiasAdd(block_base.BlockBase):
- """A tf.nn.bias_add wrapper.
-
- This wrapper may act as a PassThrough block depending on the initializer
- provided, to make easier optional bias applications in NN blocks, etc.
- See __init__() for the details.
- """
-
- def __init__(self, initializer=Bias(0), name=None):
- """Initializes Bias block.
-
- |initializer| parameter have two special cases.
-
- 1. If initializer is None, then this block works as a PassThrough.
- 2. If initializer is a Bias class object, then tf.constant_initializer is
- used with the stored value.
-
- Args:
- initializer: An initializer for the bias variable.
- name: Name of this block.
- """
- super(BiasAdd, self).__init__(name)
-
- with self._BlockScope():
- if isinstance(initializer, Bias):
- self._initializer = tf.constant_initializer(value=initializer.value)
- else:
- self._initializer = initializer
-
- self._bias = None
-
- def _Apply(self, x):
- if not self._bias:
- init = self._initializer([int(x.get_shape()[-1])], x.dtype)
- self._bias = self.NewVar(init)
-
- return tf.nn.bias_add(x, self._bias)
-
- def CreateWeightLoss(self):
- return []
-
-
-class LinearBase(block_base.BlockBase):
- """A matmul wrapper.
-
- Returns input * W, where matrix W can be customized through derivation.
- """
-
- def __init__(self, depth, name=None):
- super(LinearBase, self).__init__(name)
-
- with self._BlockScope():
- self._depth = depth
- self._matrix = None
-
- def _CreateKernel(self, shape, dtype):
- raise NotImplementedError('This method must be sub-classed.')
-
- def _Apply(self, x):
- if not self._matrix:
- shape = [int(x.get_shape()[-1]), self._depth]
- self._matrix = self._CreateKernel(shape, x.dtype)
-
- return tf.matmul(x, self._matrix)
-
-
-class Linear(LinearBase):
- """A matmul wrapper.
-
- Returns input * W, where matrix W is learned.
- """
-
- def __init__(self,
- depth,
- initializer=block_util.RsqrtInitializer(),
- name=None):
- super(Linear, self).__init__(depth, name)
-
- with self._BlockScope():
- self._initializer = initializer
-
- def _CreateKernel(self, shape, dtype):
- init = self._initializer(shape, dtype)
- return self.NewVar(init)
-
-
-class NN(block_base.BlockBase):
- """A neural network layer wrapper.
-
- Returns act(input * W + b), where matrix W, bias b are learned, and act is an
- optional activation function (i.e., nonlinearity).
-
- This transform block can handle multiple inputs. If x_1, x_2, ..., x_m are
- the inputs, then returns act(x_1 * W_1 + ... + x_m * W_m + b).
-
- Attributes:
- nunits: The dimension of the output.
- """
-
- def __init__(self,
- depth,
- bias=Bias(0),
- act=None, # e.g., tf.nn.relu
- initializer=block_util.RsqrtInitializer(),
- linear_block_factory=(lambda d, i: Linear(d, initializer=i)),
- name=None):
- """Initializes NN block.
-
- Args:
- depth: The depth of the output.
- bias: An initializer for the bias, or a Bias class object. If None, there
- will be no bias term for this NN block. See BiasAdd block.
- act: Optional activation function. If None, no activation is applied.
- initializer: The initialization method for the matrix weights.
- linear_block_factory: A function used to create a linear block.
- name: The name of this block.
- """
- super(NN, self).__init__(name)
-
- with self._BlockScope():
- self._linear_block_factory = linear_block_factory
- self._depth = depth
- self._initializer = initializer
- self._matrices = None
-
- self._bias = BiasAdd(bias) if bias else PassThrough()
- self._act = act if act else PassThrough()
-
- def _Apply(self, *args):
- if not self._matrices:
- self._matrices = [
- self._linear_block_factory(self._depth, self._initializer)
- for _ in args]
-
- if len(self._matrices) != len(args):
- raise ValueError('{} expected {} inputs, but observed {} inputs'.format(
- self.name, len(self._matrices), len(args)))
-
- if len(args) > 1:
- y = tf.add_n([m(x) for m, x in zip(self._matrices, args)])
- else:
- y = self._matrices[0](args[0])
-
- return self._act(self._bias(y))
-
-
-class Conv2DBase(block_base.BlockBase):
- """A tf.nn.conv2d operator."""
-
- def __init__(self, depth, filter_size, strides, padding,
- bias=None, act=None, atrous_rate=None, conv=tf.nn.conv2d,
- name=None):
- """Initializes a Conv2DBase block.
-
- Arguments:
- depth: The output depth of the block (i.e. #filters); if negative, the
- output depth will be set to be the same as the input depth.
- filter_size: The size of the 2D filter. If it's specified as an integer,
- it's going to create a square filter. Otherwise, this is a tuple
- specifying the height x width of the filter.
- strides: A tuple specifying the y and x stride.
- padding: One of the valid padding modes allowed by tf.nn.conv2d, or
- 'REFLECT'/'SYMMETRIC' for mirror padding.
- bias: An initializer for the bias, or a Bias class object. If None, there
- will be no bias in this block. See BiasAdd block.
- act: Optional activation function applied to the output.
- atrous_rate: optional input rate for ATrous convolution. If not None, this
- will be used and the strides will be ignored.
- conv: The convolution function to use (e.g. tf.nn.conv2d).
- name: The name for this conv2d op.
- """
- super(Conv2DBase, self).__init__(name)
-
- with self._BlockScope():
- self._act = act if act else PassThrough()
- self._bias = BiasAdd(bias) if bias else PassThrough()
-
- self._kernel_shape = np.zeros((4,), dtype=np.int32)
- self._kernel_shape[:2] = filter_size
- self._kernel_shape[3] = depth
-
- self._strides = np.ones((4,), dtype=np.int32)
- self._strides[1:3] = strides
- self._strides = list(self._strides)
-
- self._padding = padding
-
- self._kernel = None
- self._conv = conv
-
- self._atrous_rate = atrous_rate
-
- def _CreateKernel(self, shape, dtype):
- raise NotImplementedError('This method must be sub-classed')
-
- def _Apply(self, x):
- """Apply the self._conv op.
-
- Arguments:
- x: input tensor. It needs to be a 4D tensor of the form
- [batch, height, width, channels].
- Returns:
- The output of the convolution of x with the current convolutional
- kernel.
- Raises:
- ValueError: if number of channels is not defined at graph construction.
- """
- input_shape = x.get_shape().with_rank(4)
- input_shape[3:].assert_is_fully_defined() # channels must be defined
- if self._kernel is None:
- assert self._kernel_shape[2] == 0, self._kernel_shape
- self._kernel_shape[2] = input_shape[3].value
- if self._kernel_shape[3] < 0:
- # Make output depth be the same as input depth.
- self._kernel_shape[3] = self._kernel_shape[2]
- self._kernel = self._CreateKernel(self._kernel_shape, x.dtype)
-
- x, padding = HandleConvPaddingModes(
- x, self._padding, self._kernel_shape, self._strides)
- if self._atrous_rate is None:
- x = self._conv(x, self._kernel, strides=self._strides, padding=padding)
- else:
- x = self._conv(x, self._kernel, rate=self._atrous_rate, padding=padding)
-
- if self._padding != 'VALID':
- # Manually update shape. Known shape information can be lost by tf.pad().
- height = (1 + (input_shape[1].value - 1) // self._strides[1]
- if input_shape[1].value else None)
- width = (1 + (input_shape[2].value - 1) // self._strides[2]
- if input_shape[2].value else None)
- shape = x.get_shape()
- x.set_shape([shape[0], height, width, shape[3]])
-
- return self._act(self._bias(x))
-
-
-class Conv2D(Conv2DBase):
- """A tf.nn.conv2d operator."""
-
- def __init__(self, depth, filter_size, strides, padding,
- bias=None, act=None, initializer=None, name=None):
- """Initializes a Conv2D block.
-
- Arguments:
- depth: The output depth of the block (i.e., #filters)
- filter_size: The size of the 2D filter. If it's specified as an integer,
- it's going to create a square filter. Otherwise, this is a tuple
- specifying the height x width of the filter.
- strides: A tuple specifying the y and x stride.
- padding: One of the valid padding modes allowed by tf.nn.conv2d, or
- 'REFLECT'/'SYMMETRIC' for mirror padding.
- bias: An initializer for the bias, or a Bias class object. If None, there
- will be no bias in this block. See BiasAdd block.
- act: Optional activation function applied to the output.
- initializer: Optional initializer for weights.
- name: The name for this conv2d op.
- """
- super(Conv2D, self).__init__(depth, filter_size, strides, padding, bias,
- act, conv=tf.nn.conv2d, name=name)
-
- with self._BlockScope():
- if initializer is None:
- initializer = block_util.RsqrtInitializer(dims=(0, 1, 2))
- self._initializer = initializer
-
- def _CreateKernel(self, shape, dtype):
- return self.NewVar(self._initializer(shape, dtype))
diff --git a/research/compression/entropy_coder/lib/blocks_std_test.py b/research/compression/entropy_coder/lib/blocks_std_test.py
deleted file mode 100644
index 328ebc9d2173436b2108b343b98650128a4613e3..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/lib/blocks_std_test.py
+++ /dev/null
@@ -1,340 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for basic tensorflow blocks_std."""
-
-from __future__ import division
-from __future__ import unicode_literals
-
-import math
-import os
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import blocks_std
-
-
-def _NumpyConv2D(x, f, strides, padding, rate=1):
- assert strides[0] == 1 and strides[3] == 1, strides
-
- if rate > 1:
- f_shape = f.shape
- expand_f = np.zeros([f_shape[0], ((f_shape[1] - 1) * rate + 1),
- f_shape[2], f_shape[3]])
- expand_f[:, [y * rate for y in range(f_shape[1])], :, :] = f
- f = np.zeros([((f_shape[0] - 1) * rate + 1), expand_f.shape[1],
- f_shape[2], f_shape[3]])
- f[[y * rate for y in range(f_shape[0])], :, :, :] = expand_f
-
- if padding != 'VALID':
- assert x.shape[1] > 0 and x.shape[2] > 0, x.shape
- # Compute the number of padded rows and cols.
- # See Conv2D block comments for a math explanation.
- remainder = ((x.shape[1] - 1) % strides[1], (x.shape[2] - 1) % strides[2])
- pad_rows = f.shape[0] - remainder[0] - 1
- pad_cols = f.shape[1] - remainder[1] - 1
- pad = ((0, 0),
- (pad_rows // 2, (pad_rows + 1) // 2),
- (pad_cols // 2, (pad_cols + 1) // 2),
- (0, 0))
-
- # Pad the input using numpy.pad().
- mode = None
- if padding == 'SAME':
- mode = str('constant')
- if padding == 'REFLECT':
- mode = str('reflect')
- if padding == 'SYMMETRIC':
- mode = str('symmetric')
- x = np.pad(x, pad, mode=mode)
-
- # Since x is now properly padded, proceed as if padding mode is VALID.
- x_window = np.empty(
- (x.shape[0],
- int(math.ceil((x.shape[1] - f.shape[0] + 1) / strides[1])),
- int(math.ceil((x.shape[2] - f.shape[1] + 1) / strides[2])),
- np.prod(f.shape[:3])))
-
- # The output at pixel location (i, j) is the result of linear transformation
- # applied to the window whose top-left corner is at
- # (i * row_stride, j * col_stride).
- for i in xrange(x_window.shape[1]):
- k = i * strides[1]
- for j in xrange(x_window.shape[2]):
- l = j * strides[2]
- x_window[:, i, j, :] = x[:,
- k:(k + f.shape[0]),
- l:(l + f.shape[1]),
- :].reshape((x_window.shape[0], -1))
-
- y = np.tensordot(x_window, f.reshape((-1, f.shape[3])), axes=1)
- return y
-
-
-class BlocksStdTest(tf.test.TestCase):
-
- def CheckUnary(self, y, op_type):
- self.assertEqual(op_type, y.op.type)
- self.assertEqual(1, len(y.op.inputs))
- return y.op.inputs[0]
-
- def CheckBinary(self, y, op_type):
- self.assertEqual(op_type, y.op.type)
- self.assertEqual(2, len(y.op.inputs))
- return y.op.inputs
-
- def testPassThrough(self):
- p = blocks_std.PassThrough()
- x = tf.placeholder(dtype=tf.float32, shape=[1])
- self.assertIs(p(x), x)
-
- def CheckBiasAdd(self, y, b):
- x, u = self.CheckBinary(y, 'BiasAdd')
- self.assertIs(u, b._bias.value())
- self.assertEqual(x.dtype, u.dtype.base_dtype)
- return x
-
- def testBiasAdd(self):
- b = blocks_std.BiasAdd()
- x = tf.placeholder(dtype=tf.float32, shape=[4, 8])
- y = b(x)
- self.assertEqual(b._bias.get_shape(), x.get_shape()[-1:])
- self.assertIs(x, self.CheckBiasAdd(y, b))
-
- def testBiasRankTest(self):
- b = blocks_std.BiasAdd()
- x = tf.placeholder(dtype=tf.float32, shape=[10])
- with self.assertRaises(ValueError):
- b(x)
-
- def CheckLinear(self, y, m):
- x, w = self.CheckBinary(y, 'MatMul')
- self.assertIs(w, m._matrix.value())
- self.assertEqual(x.dtype, w.dtype.base_dtype)
- return x
-
- def testLinear(self):
- m = blocks_std.Linear(10)
- x = tf.placeholder(dtype=tf.float32, shape=[8, 9])
- y = m(x)
- self.assertEqual(m._matrix.get_shape(), [9, 10])
- self.assertIs(x, self.CheckLinear(y, m))
-
- def testLinearShared(self):
- # Create a linear map which is applied twice on different inputs
- # (i.e. the weights of the map are shared).
- linear_map = blocks_std.Linear(6)
- x1 = tf.random_normal(shape=[1, 5])
- x2 = tf.random_normal(shape=[1, 5])
- xs = x1 + x2
-
- # Apply the transform with the same weights.
- y1 = linear_map(x1)
- y2 = linear_map(x2)
- ys = linear_map(xs)
-
- with self.test_session() as sess:
- # Initialize all the variables of the graph.
- tf.global_variables_initializer().run()
-
- y1_res, y2_res, ys_res = sess.run([y1, y2, ys])
- self.assertAllClose(y1_res + y2_res, ys_res)
-
- def CheckNN(self, y, nn, act=None):
- if act:
- pre_act = self.CheckUnary(y, act)
- else:
- pre_act = y
-
- if not isinstance(nn._bias, blocks_std.PassThrough):
- pre_bias = self.CheckBiasAdd(pre_act, nn._bias)
- else:
- pre_bias = pre_act
-
- if len(nn._matrices) > 1:
- self.assertEqual('AddN', pre_bias.op.type)
- pre_bias = pre_bias.op.inputs
- else:
- pre_bias = [pre_bias]
-
- self.assertEqual(len(pre_bias), len(nn._matrices))
- return [self.CheckLinear(u, m) for u, m in zip(pre_bias, nn._matrices)]
-
- def testNNWithoutActWithoutBias(self):
- nn = blocks_std.NN(10, act=None, bias=None)
- x = tf.placeholder(dtype=tf.float32, shape=[5, 7])
- y = nn(x)
- self.assertIs(x, self.CheckNN(y, nn)[0])
-
- def testNNWithoutBiasWithAct(self):
- nn = blocks_std.NN(10, act=tf.nn.relu, bias=None)
- x = tf.placeholder(dtype=tf.float32, shape=[5, 7])
- y = nn(x)
- self.assertIs(x, self.CheckNN(y, nn, 'Relu')[0])
-
- def testNNWithBiasWithoutAct(self):
- nn = blocks_std.NN(10, bias=blocks_std.Bias(0), act=None)
- x = tf.placeholder(dtype=tf.float32, shape=[5, 7])
- y = nn(x)
- self.assertIs(x, self.CheckNN(y, nn)[0])
-
- def testNNWithBiasWithAct(self):
- nn = blocks_std.NN(10, bias=blocks_std.Bias(0), act=tf.square)
- x = tf.placeholder(dtype=tf.float32, shape=[5, 7])
- y = nn(x)
- self.assertIs(x, self.CheckNN(y, nn, 'Square')[0])
-
- def testNNMultipleInputs(self):
- nn = blocks_std.NN(10, bias=blocks_std.Bias(0), act=tf.tanh)
- x = [tf.placeholder(dtype=tf.float32, shape=[5, 7]),
- tf.placeholder(dtype=tf.float32, shape=[5, 3]),
- tf.placeholder(dtype=tf.float32, shape=[5, 5])]
- y = nn(*x)
- xs = self.CheckNN(y, nn, 'Tanh')
- self.assertEqual(len(x), len(xs))
- for u, v in zip(x, xs):
- self.assertIs(u, v)
-
- def testConv2DSAME(self):
- np.random.seed(142536)
-
- x_shape = [4, 16, 11, 5]
- f_shape = [4, 3, 5, 6]
- strides = [1, 2, 2, 1]
- padding = 'SAME'
-
- conv = blocks_std.Conv2D(depth=f_shape[-1],
- filter_size=f_shape[0:2],
- strides=strides[1:3],
- padding=padding,
- act=None,
- bias=None)
- x_value = np.random.normal(size=x_shape)
- x = tf.convert_to_tensor(x_value, dtype=tf.float32)
- y = conv(x)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- f_value = conv._kernel.eval()
- y_value = y.eval()
-
- y_expected = _NumpyConv2D(x_value, f_value,
- strides=strides, padding=padding)
- self.assertAllClose(y_expected, y_value)
-
- def testConv2DValid(self):
- np.random.seed(253647)
-
- x_shape = [4, 11, 12, 5]
- f_shape = [5, 2, 5, 5]
- strides = [1, 2, 2, 1]
- padding = 'VALID'
-
- conv = blocks_std.Conv2D(depth=f_shape[-1],
- filter_size=f_shape[0:2],
- strides=strides[1:3],
- padding=padding,
- act=None,
- bias=None)
- x_value = np.random.normal(size=x_shape)
- x = tf.convert_to_tensor(x_value, dtype=tf.float32)
- y = conv(x)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- f_value = conv._kernel.eval()
- y_value = y.eval()
-
- y_expected = _NumpyConv2D(x_value, f_value,
- strides=strides, padding=padding)
- self.assertAllClose(y_expected, y_value)
-
- def testConv2DSymmetric(self):
- np.random.seed(364758)
-
- x_shape = [4, 10, 12, 6]
- f_shape = [3, 4, 6, 5]
- strides = [1, 1, 1, 1]
- padding = 'SYMMETRIC'
-
- conv = blocks_std.Conv2D(depth=f_shape[-1],
- filter_size=f_shape[0:2],
- strides=strides[1:3],
- padding=padding,
- act=None,
- bias=None)
- x_value = np.random.normal(size=x_shape)
- x = tf.convert_to_tensor(x_value, dtype=tf.float32)
- y = conv(x)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- f_value = conv._kernel.eval()
- y_value = y.eval()
-
- y_expected = _NumpyConv2D(x_value, f_value,
- strides=strides, padding=padding)
- self.assertAllClose(y_expected, y_value)
-
- def testConv2DReflect(self):
- np.random.seed(768798)
-
- x_shape = [4, 10, 12, 6]
- f_shape = [3, 4, 6, 5]
- strides = [1, 2, 2, 1]
- padding = 'REFLECT'
-
- conv = blocks_std.Conv2D(depth=f_shape[-1],
- filter_size=f_shape[0:2],
- strides=strides[1:3],
- padding=padding,
- act=None,
- bias=None)
- x_value = np.random.normal(size=x_shape)
- x = tf.convert_to_tensor(x_value, dtype=tf.float32)
- y = conv(x)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- f_value = conv._kernel.eval()
- y_value = y.eval()
-
- y_expected = _NumpyConv2D(x_value, f_value,
- strides=strides, padding=padding)
- self.assertAllClose(y_expected, y_value)
-
- def testConv2DBias(self):
- input_shape = [19, 14, 14, 64]
- filter_shape = [3, 7, 64, 128]
- strides = [1, 2, 2, 1]
- output_shape = [19, 6, 4, 128]
-
- conv = blocks_std.Conv2D(depth=filter_shape[-1],
- filter_size=filter_shape[0:2],
- strides=strides[1:3],
- padding='VALID',
- act=None,
- bias=blocks_std.Bias(1))
- x = tf.placeholder(dtype=tf.float32, shape=input_shape)
-
- y = conv(x)
- self.CheckBiasAdd(y, conv._bias)
- self.assertEqual(output_shape, y.get_shape().as_list())
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/compression/entropy_coder/model/__init__.py b/research/compression/entropy_coder/model/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/compression/entropy_coder/model/entropy_coder_model.py b/research/compression/entropy_coder/model/entropy_coder_model.py
deleted file mode 100644
index 67f7eb5bc05f3df7363529c19fa77d176caaabc1..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/model/entropy_coder_model.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Entropy coder model."""
-
-
-class EntropyCoderModel(object):
- """Entropy coder model."""
-
- def __init__(self):
- # Loss used for training the model.
- self.loss = None
-
- # Tensorflow op to run to train the model.
- self.train_op = None
-
- # Tensor corresponding to the average code length of the input bit field
- # tensor. The average code length is a number of output bits per input bit.
- # To get an effective compression, this number should be between 0.0
- # and 1.0 (1.0 corresponds to no compression).
- self.average_code_length = None
-
- def Initialize(self, global_step, optimizer, config_string):
- raise NotImplementedError()
-
- def BuildGraph(self, input_codes):
- """Build the Tensorflow graph corresponding to the entropy coder model.
-
- Args:
- input_codes: Tensor of size: batch_size x height x width x bit_depth
- corresponding to the codes to compress.
- The input codes are {-1, +1} codes.
- """
- # TODO:
- # - consider switching to {0, 1} codes.
- # - consider passing an extra tensor which gives for each (b, y, x)
- # what is the actual depth (which would allow to use more or less bits
- # for each (y, x) location.
- raise NotImplementedError()
-
- def GetConfigStringForUnitTest(self):
- """Returns a default model configuration to be used for unit tests."""
- return None
diff --git a/research/compression/entropy_coder/model/model_factory.py b/research/compression/entropy_coder/model/model_factory.py
deleted file mode 100644
index e6f9902f3bb720e76f228f2774a9eaf7774ef191..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/model/model_factory.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Entropy coder model registrar."""
-
-
-class ModelFactory(object):
- """Factory of encoder/decoder models."""
-
- def __init__(self):
- self._model_dictionary = dict()
-
- def RegisterModel(self,
- entropy_coder_model_name,
- entropy_coder_model_factory):
- self._model_dictionary[entropy_coder_model_name] = (
- entropy_coder_model_factory)
-
- def CreateModel(self, model_name):
- current_model_factory = self._model_dictionary[model_name]
- return current_model_factory()
-
- def GetAvailableModels(self):
- return self._model_dictionary.keys()
-
-
-_model_registry = ModelFactory()
-
-
-def GetModelRegistry():
- return _model_registry
-
-
-class RegisterEntropyCoderModel(object):
-
- def __init__(self, model_name):
- self._model_name = model_name
-
- def __call__(self, f):
- _model_registry.RegisterModel(self._model_name, f)
- return f
diff --git a/research/compression/entropy_coder/progressive/__init__.py b/research/compression/entropy_coder/progressive/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/compression/entropy_coder/progressive/progressive.py b/research/compression/entropy_coder/progressive/progressive.py
deleted file mode 100644
index 7b03a07db055b62aa1c0f9cc89ddd2472899db3c..0000000000000000000000000000000000000000
--- a/research/compression/entropy_coder/progressive/progressive.py
+++ /dev/null
@@ -1,242 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Code probability model used for entropy coding."""
-
-import json
-
-from six.moves import xrange
-import tensorflow as tf
-
-from entropy_coder.lib import blocks
-from entropy_coder.model import entropy_coder_model
-from entropy_coder.model import model_factory
-
-# pylint: disable=not-callable
-
-
-class BrnnPredictor(blocks.BlockBase):
- """BRNN prediction applied on one layer."""
-
- def __init__(self, code_depth, name=None):
- super(BrnnPredictor, self).__init__(name)
-
- with self._BlockScope():
- hidden_depth = 2 * code_depth
-
- # What is coming from the previous layer/iteration
- # is going through a regular Conv2D layer as opposed to the binary codes
- # of the current layer/iteration which are going through a masked
- # convolution.
- self._adaptation0 = blocks.RasterScanConv2D(
- hidden_depth, [7, 7], [1, 1], 'SAME',
- strict_order=True,
- bias=blocks.Bias(0), act=tf.tanh)
- self._adaptation1 = blocks.Conv2D(
- hidden_depth, [3, 3], [1, 1], 'SAME',
- bias=blocks.Bias(0), act=tf.tanh)
- self._predictor = blocks.CompositionOperator([
- blocks.LineOperator(
- blocks.RasterScanConv2DLSTM(
- depth=hidden_depth,
- filter_size=[1, 3],
- hidden_filter_size=[1, 3],
- strides=[1, 1],
- padding='SAME')),
- blocks.Conv2D(hidden_depth, [1, 1], [1, 1], 'SAME',
- bias=blocks.Bias(0), act=tf.tanh),
- blocks.Conv2D(code_depth, [1, 1], [1, 1], 'SAME',
- bias=blocks.Bias(0), act=tf.tanh)
- ])
-
- def _Apply(self, x, s):
- # Code estimation using both:
- # - the state from the previous iteration/layer,
- # - the binary codes that are before in raster scan order.
- h = tf.concat(values=[self._adaptation0(x), self._adaptation1(s)], axis=3)
-
- estimated_codes = self._predictor(h)
-
- return estimated_codes
-
-
-class LayerPrediction(blocks.BlockBase):
- """Binary code prediction for one layer."""
-
- def __init__(self, layer_count, code_depth, name=None):
- super(LayerPrediction, self).__init__(name)
-
- self._layer_count = layer_count
-
- # No previous layer.
- self._layer_state = None
- self._current_layer = 0
-
- with self._BlockScope():
- # Layers used to do the conditional code prediction.
- self._brnn_predictors = []
- for _ in xrange(layer_count):
- self._brnn_predictors.append(BrnnPredictor(code_depth))
-
- # Layers used to generate the input of the LSTM operating on the
- # iteration/depth domain.
- hidden_depth = 2 * code_depth
- self._state_blocks = []
- for _ in xrange(layer_count):
- self._state_blocks.append(blocks.CompositionOperator([
- blocks.Conv2D(
- hidden_depth, [3, 3], [1, 1], 'SAME',
- bias=blocks.Bias(0), act=tf.tanh),
- blocks.Conv2D(
- code_depth, [3, 3], [1, 1], 'SAME',
- bias=blocks.Bias(0), act=tf.tanh)
- ]))
-
- # Memory of the RNN is equivalent to the size of 2 layers of binary
- # codes.
- hidden_depth = 2 * code_depth
- self._layer_rnn = blocks.CompositionOperator([
- blocks.Conv2DLSTM(
- depth=hidden_depth,
- filter_size=[1, 1],
- hidden_filter_size=[1, 1],
- strides=[1, 1],
- padding='SAME'),
- blocks.Conv2D(hidden_depth, [1, 1], [1, 1], 'SAME',
- bias=blocks.Bias(0), act=tf.tanh),
- blocks.Conv2D(code_depth, [1, 1], [1, 1], 'SAME',
- bias=blocks.Bias(0), act=tf.tanh)
- ])
-
- def _Apply(self, x):
- assert self._current_layer < self._layer_count
-
- # Layer state is set to 0 when there is no previous iteration.
- if self._layer_state is None:
- self._layer_state = tf.zeros_like(x, dtype=tf.float32)
-
- # Code estimation using both:
- # - the state from the previous iteration/layer,
- # - the binary codes that are before in raster scan order.
- estimated_codes = self._brnn_predictors[self._current_layer](
- x, self._layer_state)
-
- # Compute the updated layer state.
- h = self._state_blocks[self._current_layer](x)
- self._layer_state = self._layer_rnn(h)
- self._current_layer += 1
-
- return estimated_codes
-
-
-class ProgressiveModel(entropy_coder_model.EntropyCoderModel):
- """Progressive BRNN entropy coder model."""
-
- def __init__(self):
- super(ProgressiveModel, self).__init__()
-
- def Initialize(self, global_step, optimizer, config_string):
- if config_string is None:
- raise ValueError('The progressive model requires a configuration.')
- config = json.loads(config_string)
- if 'coded_layer_count' not in config:
- config['coded_layer_count'] = 0
-
- self._config = config
- self._optimizer = optimizer
- self._global_step = global_step
-
- def BuildGraph(self, input_codes):
- """Build the graph corresponding to the progressive BRNN model."""
- layer_depth = self._config['layer_depth']
- layer_count = self._config['layer_count']
-
- code_shape = input_codes.get_shape()
- code_depth = code_shape[-1].value
- if self._config['coded_layer_count'] > 0:
- prefix_depth = self._config['coded_layer_count'] * layer_depth
- if code_depth < prefix_depth:
- raise ValueError('Invalid prefix depth: {} VS {}'.format(
- prefix_depth, code_depth))
- input_codes = input_codes[:, :, :, :prefix_depth]
-
- code_shape = input_codes.get_shape()
- code_depth = code_shape[-1].value
- if code_depth % layer_depth != 0:
- raise ValueError(
- 'Code depth must be a multiple of the layer depth: {} vs {}'.format(
- code_depth, layer_depth))
- code_layer_count = code_depth // layer_depth
- if code_layer_count > layer_count:
- raise ValueError('Input codes have too many layers: {}, max={}'.format(
- code_layer_count, layer_count))
-
- # Block used to estimate binary codes.
- layer_prediction = LayerPrediction(layer_count, layer_depth)
-
- # Block used to compute code lengths.
- code_length_block = blocks.CodeLength()
-
- # Loop over all the layers.
- code_length = []
- code_layers = tf.split(
- value=input_codes, num_or_size_splits=code_layer_count, axis=3)
- for k in xrange(code_layer_count):
- x = code_layers[k]
- predicted_x = layer_prediction(x)
- # Saturate the prediction to avoid infinite code length.
- epsilon = 0.001
- predicted_x = tf.clip_by_value(
- predicted_x, -1 + epsilon, +1 - epsilon)
- code_length.append(code_length_block(
- blocks.ConvertSignCodeToZeroOneCode(x),
- blocks.ConvertSignCodeToZeroOneCode(predicted_x)))
- tf.summary.scalar('code_length_layer_{:02d}'.format(k), code_length[-1])
- code_length = tf.stack(code_length)
- self.loss = tf.reduce_mean(code_length)
- tf.summary.scalar('loss', self.loss)
-
- # Loop over all the remaining layers just to make sure they are
- # instantiated. Otherwise, loading model params could fail.
- dummy_x = tf.zeros_like(code_layers[0])
- for _ in xrange(layer_count - code_layer_count):
- dummy_predicted_x = layer_prediction(dummy_x)
-
- # Average bitrate over total_line_count.
- self.average_code_length = tf.reduce_mean(code_length)
-
- if self._optimizer:
- optim_op = self._optimizer.minimize(self.loss,
- global_step=self._global_step)
- block_updates = blocks.CreateBlockUpdates()
- if block_updates:
- with tf.get_default_graph().control_dependencies([optim_op]):
- self.train_op = tf.group(*block_updates)
- else:
- self.train_op = optim_op
- else:
- self.train_op = None
-
- def GetConfigStringForUnitTest(self):
- s = '{\n'
- s += '"layer_depth": 1,\n'
- s += '"layer_count": 8\n'
- s += '}\n'
- return s
-
-
-@model_factory.RegisterEntropyCoderModel('progressive')
-def CreateProgressiveModel():
- return ProgressiveModel()
diff --git a/research/compression/image_encoder/README.md b/research/compression/image_encoder/README.md
deleted file mode 100644
index a47da977aa4db4be26528c5ebfe030024f31291b..0000000000000000000000000000000000000000
--- a/research/compression/image_encoder/README.md
+++ /dev/null
@@ -1,105 +0,0 @@
-# Image Compression with Neural Networks
-
-This is a [TensorFlow](http://www.tensorflow.org/) model for compressing and
-decompressing images using an already trained Residual GRU model as descibed
-in [Full Resolution Image Compression with Recurrent Neural Networks](https://arxiv.org/abs/1608.05148). Please consult the paper for more details
-on the architecture and compression results.
-
-This code will allow you to perform the lossy compression on an model
-already trained on compression. This code doesn't not currently contain the
-Entropy Coding portions of our paper.
-
-
-## Prerequisites
-The only software requirements for running the encoder and decoder is having
-Tensorflow installed. You will also need to [download](http://download.tensorflow.org/models/compression_residual_gru-2016-08-23.tar.gz)
-and extract the model residual_gru.pb.
-
-If you want to generate the perceptual similarity under MS-SSIM, you will also
-need to [Install SciPy](https://www.scipy.org/install.html).
-
-## Encoding
-The Residual GRU network is fully convolutional, but requires the images
-height and width in pixels by a multiple of 32. There is an image in this folder
-called example.png that is 768x1024 if one is needed for testing. We also
-rely on TensorFlow's built in decoding ops, which support only PNG and JPEG at
-time of release.
-
-To encode an image, simply run the following command:
-
-`python encoder.py --input_image=/your/image/here.png
---output_codes=output_codes.npz --iteration=15
---model=/path/to/model/residual_gru.pb
-`
-
-The iteration parameter specifies the lossy-quality to target for compression.
-The quality can be [0-15], where 0 corresponds to a target of 1/8 (bits per
-pixel) bpp and every increment results in an additional 1/8 bpp.
-
-| Iteration | BPP | Compression Ratio |
-|---: |---: |---: |
-|0 | 0.125 | 192:1|
-|1 | 0.250 | 96:1|
-|2 | 0.375 | 64:1|
-|3 | 0.500 | 48:1|
-|4 | 0.625 | 38.4:1|
-|5 | 0.750 | 32:1|
-|6 | 0.875 | 27.4:1|
-|7 | 1.000 | 24:1|
-|8 | 1.125 | 21.3:1|
-|9 | 1.250 | 19.2:1|
-|10 | 1.375 | 17.4:1|
-|11 | 1.500 | 16:1|
-|12 | 1.625 | 14.7:1|
-|13 | 1.750 | 13.7:1|
-|14 | 1.875 | 12.8:1|
-|15 | 2.000 | 12:1|
-
-The output_codes file contains the numpy shape and a flattened, bit-packed
-array of the codes. These can be inspected in python by using numpy.load().
-
-
-## Decoding
-After generating codes for an image, the lossy reconstructions for that image
-can be done as follows:
-
-`python decoder.py --input_codes=codes.npz --output_directory=/tmp/decoded/
---model=residual_gru.pb`
-
-The output_directory will contain images decoded at each quality level.
-
-
-## Comparing Similarity
-One of our primary metrics for comparing how similar two images are
-is MS-SSIM.
-
-To generate these metrics on your images you can run:
-`python msssim.py --original_image=/path/to/your/image.png
---compared_image=/tmp/decoded/image_15.png`
-
-
-## Results
-CSV results containing the post-entropy bitrates and MS-SSIM over Kodak can
-are available for reference. Each row of the CSV represents each of the Kodak
-images in their dataset number (1-24). Each column of the CSV represents each
-iteration of the model (1-16).
-
-[Post Entropy Bitrates](https://storage.googleapis.com/compression-ml/residual_gru_results/bitrate.csv)
-
-[MS-SSIM](https://storage.googleapis.com/compression-ml/residual_gru_results/msssim.csv)
-
-
-## FAQ
-
-#### How do I train my own compression network?
-We currently don't provide the code to build and train a compression
-graph from scratch.
-
-#### I get an InvalidArgumentError: Incompatible shapes.
-This is usually due to the fact that our network only supports images that are
-both height and width divisible by 32 pixel. Try padding your images to 32
-pixel boundaries.
-
-
-## Contact Info
-Model repository maintained by Nick Johnston ([nmjohn](https://github.com/nmjohn)).
diff --git a/research/compression/image_encoder/decoder.py b/research/compression/image_encoder/decoder.py
deleted file mode 100644
index 75bc18cad0fdd4055df7b42d5440635365504774..0000000000000000000000000000000000000000
--- a/research/compression/image_encoder/decoder.py
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-r"""Neural Network Image Compression Decoder.
-
-Decompress an image from the numpy's npz format generated by the encoder.
-
-Example usage:
-python decoder.py --input_codes=output_codes.pkl --iteration=15 \
---output_directory=/tmp/compression_output/ --model=residual_gru.pb
-"""
-import io
-import os
-
-import numpy as np
-import tensorflow as tf
-
-tf.flags.DEFINE_string('input_codes', None, 'Location of binary code file.')
-tf.flags.DEFINE_integer('iteration', -1, 'The max quality level of '
- 'the images to output. Use -1 to infer from loaded '
- ' codes.')
-tf.flags.DEFINE_string('output_directory', None, 'Directory to save decoded '
- 'images.')
-tf.flags.DEFINE_string('model', None, 'Location of compression model.')
-
-FLAGS = tf.flags.FLAGS
-
-
-def get_input_tensor_names():
- name_list = ['GruBinarizer/SignBinarizer/Sign:0']
- for i in range(1, 16):
- name_list.append('GruBinarizer/SignBinarizer/Sign_{}:0'.format(i))
- return name_list
-
-
-def get_output_tensor_names():
- return ['loop_{0:02d}/add:0'.format(i) for i in range(0, 16)]
-
-
-def main(_):
- if (FLAGS.input_codes is None or FLAGS.output_directory is None or
- FLAGS.model is None):
- print('\nUsage: python decoder.py --input_codes=output_codes.pkl '
- '--iteration=15 --output_directory=/tmp/compression_output/ '
- '--model=residual_gru.pb\n\n')
- return
-
- if FLAGS.iteration < -1 or FLAGS.iteration > 15:
- print('\n--iteration must be between 0 and 15 inclusive, or -1 to infer '
- 'from file.\n')
- return
- iteration = FLAGS.iteration
-
- if not tf.gfile.Exists(FLAGS.output_directory):
- tf.gfile.MkDir(FLAGS.output_directory)
-
- if not tf.gfile.Exists(FLAGS.input_codes):
- print('\nInput codes not found.\n')
- return
-
- contents = ''
- with tf.gfile.FastGFile(FLAGS.input_codes, 'rb') as code_file:
- contents = code_file.read()
- loaded_codes = np.load(io.BytesIO(contents))
- assert ['codes', 'shape'] not in loaded_codes.files
- loaded_shape = loaded_codes['shape']
- loaded_array = loaded_codes['codes']
-
- # Unpack and recover code shapes.
- unpacked_codes = np.reshape(np.unpackbits(loaded_array)
- [:np.prod(loaded_shape)],
- loaded_shape)
-
- numpy_int_codes = np.split(unpacked_codes, len(unpacked_codes))
- if iteration == -1:
- iteration = len(unpacked_codes) - 1
- # Convert back to float and recover scale.
- numpy_codes = [np.squeeze(x.astype(np.float32), 0) * 2 - 1 for x in
- numpy_int_codes]
-
- with tf.Graph().as_default() as graph:
- # Load the inference model for decoding.
- with tf.gfile.FastGFile(FLAGS.model, 'rb') as model_file:
- graph_def = tf.GraphDef()
- graph_def.ParseFromString(model_file.read())
- _ = tf.import_graph_def(graph_def, name='')
-
- # For encoding the tensors into PNGs.
- input_image = tf.placeholder(tf.uint8)
- encoded_image = tf.image.encode_png(input_image)
-
- input_tensors = [graph.get_tensor_by_name(name) for name in
- get_input_tensor_names()][0:iteration+1]
- outputs = [graph.get_tensor_by_name(name) for name in
- get_output_tensor_names()][0:iteration+1]
-
- feed_dict = {key: value for (key, value) in zip(input_tensors,
- numpy_codes)}
-
- with tf.Session(graph=graph) as sess:
- results = sess.run(outputs, feed_dict=feed_dict)
-
- for index, result in enumerate(results):
- img = np.uint8(np.clip(result + 0.5, 0, 255))
- img = img.squeeze()
- png_img = sess.run(encoded_image, feed_dict={input_image: img})
-
- with tf.gfile.FastGFile(os.path.join(FLAGS.output_directory,
- 'image_{0:02d}.png'.format(index)),
- 'w') as output_image:
- output_image.write(png_img)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/compression/image_encoder/encoder.py b/research/compression/image_encoder/encoder.py
deleted file mode 100644
index 27754bdaea19779cea653408d17ed2e6a051f0c5..0000000000000000000000000000000000000000
--- a/research/compression/image_encoder/encoder.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-r"""Neural Network Image Compression Encoder.
-
-Compresses an image to a binarized numpy array. The image must be padded to a
-multiple of 32 pixels in height and width.
-
-Example usage:
-python encoder.py --input_image=/your/image/here.png \
---output_codes=output_codes.pkl --iteration=15 --model=residual_gru.pb
-"""
-import io
-import os
-
-import numpy as np
-import tensorflow as tf
-
-tf.flags.DEFINE_string('input_image', None, 'Location of input image. We rely '
- 'on tf.image to decode the image, so only PNG and JPEG '
- 'formats are currently supported.')
-tf.flags.DEFINE_integer('iteration', 15, 'Quality level for encoding image. '
- 'Must be between 0 and 15 inclusive.')
-tf.flags.DEFINE_string('output_codes', None, 'File to save output encoding.')
-tf.flags.DEFINE_string('model', None, 'Location of compression model.')
-
-FLAGS = tf.flags.FLAGS
-
-
-def get_output_tensor_names():
- name_list = ['GruBinarizer/SignBinarizer/Sign:0']
- for i in range(1, 16):
- name_list.append('GruBinarizer/SignBinarizer/Sign_{}:0'.format(i))
- return name_list
-
-
-def main(_):
- if (FLAGS.input_image is None or FLAGS.output_codes is None or
- FLAGS.model is None):
- print('\nUsage: python encoder.py --input_image=/your/image/here.png '
- '--output_codes=output_codes.pkl --iteration=15 '
- '--model=residual_gru.pb\n\n')
- return
-
- if FLAGS.iteration < 0 or FLAGS.iteration > 15:
- print('\n--iteration must be between 0 and 15 inclusive.\n')
- return
-
- with tf.gfile.FastGFile(FLAGS.input_image, 'rb') as input_image:
- input_image_str = input_image.read()
-
- with tf.Graph().as_default() as graph:
- # Load the inference model for encoding.
- with tf.gfile.FastGFile(FLAGS.model, 'rb') as model_file:
- graph_def = tf.GraphDef()
- graph_def.ParseFromString(model_file.read())
- _ = tf.import_graph_def(graph_def, name='')
-
- input_tensor = graph.get_tensor_by_name('Placeholder:0')
- outputs = [graph.get_tensor_by_name(name) for name in
- get_output_tensor_names()]
-
- input_image = tf.placeholder(tf.string)
- _, ext = os.path.splitext(FLAGS.input_image)
- if ext == '.png':
- decoded_image = tf.image.decode_png(input_image, channels=3)
- elif ext == '.jpeg' or ext == '.jpg':
- decoded_image = tf.image.decode_jpeg(input_image, channels=3)
- else:
- assert False, 'Unsupported file format {}'.format(ext)
- decoded_image = tf.expand_dims(decoded_image, 0)
-
- with tf.Session(graph=graph) as sess:
- img_array = sess.run(decoded_image, feed_dict={input_image:
- input_image_str})
- results = sess.run(outputs, feed_dict={input_tensor: img_array})
-
- results = results[0:FLAGS.iteration + 1]
- int_codes = np.asarray([x.astype(np.int8) for x in results])
-
- # Convert int codes to binary.
- int_codes = (int_codes + 1)//2
- export = np.packbits(int_codes.reshape(-1))
-
- output = io.BytesIO()
- np.savez_compressed(output, shape=int_codes.shape, codes=export)
- with tf.gfile.FastGFile(FLAGS.output_codes, 'w') as code_file:
- code_file.write(output.getvalue())
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/compression/image_encoder/example.png b/research/compression/image_encoder/example.png
deleted file mode 100644
index d3409b01a557fe8c3058fad21ed969f8af28cb97..0000000000000000000000000000000000000000
Binary files a/research/compression/image_encoder/example.png and /dev/null differ
diff --git a/research/compression/image_encoder/msssim.py b/research/compression/image_encoder/msssim.py
deleted file mode 100644
index f07a3712785c62feb261feb90016e0f621a3ee1d..0000000000000000000000000000000000000000
--- a/research/compression/image_encoder/msssim.py
+++ /dev/null
@@ -1,217 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Python implementation of MS-SSIM.
-
-Usage:
-
-python msssim.py --original_image=original.png --compared_image=distorted.png
-"""
-import numpy as np
-from scipy import signal
-from scipy.ndimage.filters import convolve
-import tensorflow as tf
-
-
-tf.flags.DEFINE_string('original_image', None, 'Path to PNG image.')
-tf.flags.DEFINE_string('compared_image', None, 'Path to PNG image.')
-FLAGS = tf.flags.FLAGS
-
-
-def _FSpecialGauss(size, sigma):
- """Function to mimic the 'fspecial' gaussian MATLAB function."""
- radius = size // 2
- offset = 0.0
- start, stop = -radius, radius + 1
- if size % 2 == 0:
- offset = 0.5
- stop -= 1
- x, y = np.mgrid[offset + start:stop, offset + start:stop]
- assert len(x) == size
- g = np.exp(-((x**2 + y**2)/(2.0 * sigma**2)))
- return g / g.sum()
-
-
-def _SSIMForMultiScale(img1, img2, max_val=255, filter_size=11,
- filter_sigma=1.5, k1=0.01, k2=0.03):
- """Return the Structural Similarity Map between `img1` and `img2`.
-
- This function attempts to match the functionality of ssim_index_new.m by
- Zhou Wang: http://www.cns.nyu.edu/~lcv/ssim/msssim.zip
-
- Arguments:
- img1: Numpy array holding the first RGB image batch.
- img2: Numpy array holding the second RGB image batch.
- max_val: the dynamic range of the images (i.e., the difference between the
- maximum the and minimum allowed values).
- filter_size: Size of blur kernel to use (will be reduced for small images).
- filter_sigma: Standard deviation for Gaussian blur kernel (will be reduced
- for small images).
- k1: Constant used to maintain stability in the SSIM calculation (0.01 in
- the original paper).
- k2: Constant used to maintain stability in the SSIM calculation (0.03 in
- the original paper).
-
- Returns:
- Pair containing the mean SSIM and contrast sensitivity between `img1` and
- `img2`.
-
- Raises:
- RuntimeError: If input images don't have the same shape or don't have four
- dimensions: [batch_size, height, width, depth].
- """
- if img1.shape != img2.shape:
- raise RuntimeError('Input images must have the same shape (%s vs. %s).',
- img1.shape, img2.shape)
- if img1.ndim != 4:
- raise RuntimeError('Input images must have four dimensions, not %d',
- img1.ndim)
-
- img1 = img1.astype(np.float64)
- img2 = img2.astype(np.float64)
- _, height, width, _ = img1.shape
-
- # Filter size can't be larger than height or width of images.
- size = min(filter_size, height, width)
-
- # Scale down sigma if a smaller filter size is used.
- sigma = size * filter_sigma / filter_size if filter_size else 0
-
- if filter_size:
- window = np.reshape(_FSpecialGauss(size, sigma), (1, size, size, 1))
- mu1 = signal.fftconvolve(img1, window, mode='valid')
- mu2 = signal.fftconvolve(img2, window, mode='valid')
- sigma11 = signal.fftconvolve(img1 * img1, window, mode='valid')
- sigma22 = signal.fftconvolve(img2 * img2, window, mode='valid')
- sigma12 = signal.fftconvolve(img1 * img2, window, mode='valid')
- else:
- # Empty blur kernel so no need to convolve.
- mu1, mu2 = img1, img2
- sigma11 = img1 * img1
- sigma22 = img2 * img2
- sigma12 = img1 * img2
-
- mu11 = mu1 * mu1
- mu22 = mu2 * mu2
- mu12 = mu1 * mu2
- sigma11 -= mu11
- sigma22 -= mu22
- sigma12 -= mu12
-
- # Calculate intermediate values used by both ssim and cs_map.
- c1 = (k1 * max_val) ** 2
- c2 = (k2 * max_val) ** 2
- v1 = 2.0 * sigma12 + c2
- v2 = sigma11 + sigma22 + c2
- ssim = np.mean((((2.0 * mu12 + c1) * v1) / ((mu11 + mu22 + c1) * v2)))
- cs = np.mean(v1 / v2)
- return ssim, cs
-
-
-def MultiScaleSSIM(img1, img2, max_val=255, filter_size=11, filter_sigma=1.5,
- k1=0.01, k2=0.03, weights=None):
- """Return the MS-SSIM score between `img1` and `img2`.
-
- This function implements Multi-Scale Structural Similarity (MS-SSIM) Image
- Quality Assessment according to Zhou Wang's paper, "Multi-scale structural
- similarity for image quality assessment" (2003).
- Link: https://ece.uwaterloo.ca/~z70wang/publications/msssim.pdf
-
- Author's MATLAB implementation:
- http://www.cns.nyu.edu/~lcv/ssim/msssim.zip
-
- Arguments:
- img1: Numpy array holding the first RGB image batch.
- img2: Numpy array holding the second RGB image batch.
- max_val: the dynamic range of the images (i.e., the difference between the
- maximum the and minimum allowed values).
- filter_size: Size of blur kernel to use (will be reduced for small images).
- filter_sigma: Standard deviation for Gaussian blur kernel (will be reduced
- for small images).
- k1: Constant used to maintain stability in the SSIM calculation (0.01 in
- the original paper).
- k2: Constant used to maintain stability in the SSIM calculation (0.03 in
- the original paper).
- weights: List of weights for each level; if none, use five levels and the
- weights from the original paper.
-
- Returns:
- MS-SSIM score between `img1` and `img2`.
-
- Raises:
- RuntimeError: If input images don't have the same shape or don't have four
- dimensions: [batch_size, height, width, depth].
- """
- if img1.shape != img2.shape:
- raise RuntimeError('Input images must have the same shape (%s vs. %s).',
- img1.shape, img2.shape)
- if img1.ndim != 4:
- raise RuntimeError('Input images must have four dimensions, not %d',
- img1.ndim)
-
- # Note: default weights don't sum to 1.0 but do match the paper / matlab code.
- weights = np.array(weights if weights else
- [0.0448, 0.2856, 0.3001, 0.2363, 0.1333])
- levels = weights.size
- downsample_filter = np.ones((1, 2, 2, 1)) / 4.0
- im1, im2 = [x.astype(np.float64) for x in [img1, img2]]
- mssim = np.array([])
- mcs = np.array([])
- for _ in range(levels):
- ssim, cs = _SSIMForMultiScale(
- im1, im2, max_val=max_val, filter_size=filter_size,
- filter_sigma=filter_sigma, k1=k1, k2=k2)
- mssim = np.append(mssim, ssim)
- mcs = np.append(mcs, cs)
- filtered = [convolve(im, downsample_filter, mode='reflect')
- for im in [im1, im2]]
- im1, im2 = [x[:, ::2, ::2, :] for x in filtered]
- return (np.prod(mcs[0:levels-1] ** weights[0:levels-1]) *
- (mssim[levels-1] ** weights[levels-1]))
-
-
-def main(_):
- if FLAGS.original_image is None or FLAGS.compared_image is None:
- print('\nUsage: python msssim.py --original_image=original.png '
- '--compared_image=distorted.png\n\n')
- return
-
- if not tf.gfile.Exists(FLAGS.original_image):
- print('\nCannot find --original_image.\n')
- return
-
- if not tf.gfile.Exists(FLAGS.compared_image):
- print('\nCannot find --compared_image.\n')
- return
-
- with tf.gfile.FastGFile(FLAGS.original_image) as image_file:
- img1_str = image_file.read('rb')
- with tf.gfile.FastGFile(FLAGS.compared_image) as image_file:
- img2_str = image_file.read('rb')
-
- input_img = tf.placeholder(tf.string)
- decoded_image = tf.expand_dims(tf.image.decode_png(input_img, channels=3), 0)
-
- with tf.Session() as sess:
- img1 = sess.run(decoded_image, feed_dict={input_img: img1_str})
- img2 = sess.run(decoded_image, feed_dict={input_img: img2_str})
-
- print((MultiScaleSSIM(img1, img2, max_val=255)))
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/deep_contextual_bandits/README.md b/research/deep_contextual_bandits/README.md
deleted file mode 100644
index b81309af5b08003eb727e079e70c3dd08eedb6f6..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/README.md
+++ /dev/null
@@ -1,444 +0,0 @@
-
-
-
-
-# Deep Bayesian Bandits Library
-
-This library corresponds to the *[Deep Bayesian Bandits Showdown: An Empirical
-Comparison of Bayesian Deep Networks for Thompson
-Sampling](https://arxiv.org/abs/1802.09127)* paper, published in
-[ICLR](https://iclr.cc/) 2018. We provide a benchmark to test decision-making
-algorithms for contextual-bandits. In particular, the current library implements
-a variety of algorithms (many of them based on approximate Bayesian Neural
-Networks and Thompson sampling), and a number of real and syntethic data
-problems exhibiting a diverse set of properties.
-
-It is a Python library that uses [TensorFlow](https://www.tensorflow.org/).
-
-We encourage contributors to add new approximate Bayesian Neural Networks or,
-more generally, contextual bandits algorithms to the library. Also, we would
-like to extend the data sources over time, so we warmly encourage contributions
-in this front too!
-
-Please, use the following when citing the code or the paper:
-
-```
-@article{riquelme2018deep, title={Deep Bayesian Bandits Showdown: An Empirical
-Comparison of Bayesian Deep Networks for Thompson Sampling},
-author={Riquelme, Carlos and Tucker, George and Snoek, Jasper},
-journal={International Conference on Learning Representations, ICLR.}, year={2018}}
-```
-
-**Contact**. This repository is maintained by [Carlos Riquelme](http://rikel.me) ([rikel](https://github.com/rikel)). Feel free to reach out directly at [rikel@google.com](mailto:rikel@google.com) with any questions or comments.
-
-
-We first briefly introduce contextual bandits, Thompson sampling, enumerate the
-implemented algorithms, and the available data sources. Then, we provide a
-simple complete example illustrating how to use the library.
-
-## Contextual Bandits
-
-Contextual bandits are a rich decision-making framework where an algorithm has
-to choose among a set of *k* actions at every time step *t*, after observing
-a context (or side-information) denoted by *Xt*. The general pseudocode for
-the process if we use algorithm **A** is as follows:
-
-```
-At time t = 1, ..., T:
- 1. Observe new context: X_t
- 2. Choose action: a_t = A.action(X_t)
- 3. Observe reward: r_t
- 4. Update internal state of the algorithm: A.update((X_t, a_t, r_t))
-```
-
-The goal is to maximize the total sum of rewards: ∑t rt
-
-For example, each *Xt* could encode the properties of a specific user (and
-the time or day), and we may have to choose an ad, discount coupon, treatment,
-hyper-parameters, or version of a website to show or provide to the user.
-Hopefully, over time, we will learn how to match each type of user to the most
-beneficial personalized action under some metric (the reward).
-
-## Thompson Sampling
-
-Thompson Sampling is a meta-algorithm that chooses an action for the contextual
-bandit in a statistically efficient manner, simultaneously finding the best arm
-while attempting to incur low cost. Informally speaking, we assume the expected
-reward is given by some function
-**E**[rt | Xt, at] = f(Xt, at).
-Unfortunately, function **f** is unknown, as otherwise we could just choose the
-action with highest expected value:
-at* = arg maxi f(Xt, at).
-
-The idea behind Thompson Sampling is based on keeping a posterior distribution
-πt over functions in some family f ∈ F after observing the first
-*t-1* datapoints. Then, at time *t*, we sample one potential explanation of
-the underlying process: ft ∼ πt, and act optimally (i.e., greedily)
-*according to ft*. In other words, we choose
-at = arg maxi ft(Xt, ai).
-Finally, we update our posterior distribution with the new collected
-datapoint (Xt, at, rt).
-
-The main issue is that keeping an updated posterior πt (or, even,
-sampling from it) is often intractable for highly parameterized models like deep
-neural networks. The algorithms we list in the next section provide tractable
-*approximations* that can be used in combination with Thompson Sampling to solve
-the contextual bandit problem.
-
-## Algorithms
-
-The Deep Bayesian Bandits library includes the following algorithms (see the
-[paper](https://arxiv.org/abs/1802.09127) for further details):
-
-1. **Linear Algorithms**. As a powerful baseline, we provide linear algorithms.
- In particular, we focus on the exact Bayesian linear regression
- implementation, while it is easy to derive the greedy OLS version (possibly,
- with epsilon-greedy exploration). The algorithm is implemented in
- *linear_full_posterior_sampling.py*, and it is instantiated as follows:
-
- ```
- linear_full = LinearFullPosteriorSampling('MyLinearTS', my_hparams)
- ```
-
-2. **Neural Linear**. We introduce an algorithm we call Neural Linear, which
- operates by learning a neural network to map contexts to rewards for each
- action, and ---simultaneously--- it updates a Bayesian linear regression in
- the last layer (i.e., the one that maps the final representation **z** to
- the rewards **r**). Thompson Sampling samples the linear parameters
- βi for each action *i*, but keeps the network that computes the
- representation. Then, both parts (network and Bayesian linear regression)
- are updated, possibly at different frequencies. The algorithm is implemented
- in *neural_linear_sampling.py*, and we create an algorithm instance like
- this:
-
- ```
- neural_linear = NeuralLinearPosteriorSampling('MyNLinear', my_hparams)
- ```
-
-3. **Neural Greedy**. Another standard benchmark is to train a neural network
- that maps contexts to rewards, and at each time *t* just acts greedily
- according to the current model. In particular, this approach does *not*
- explicitly use Thompson Sampling. However, due to stochastic gradient
- descent, there is still some randomness in its output. It is
- straight-forward to add epsilon-greedy exploration to choose random
- actions with probability ε ∈ (0, 1). The algorithm is
- implemented in *neural_bandit_model.py*, and it is used together with
- *PosteriorBNNSampling* (defined in *posterior_bnn_sampling.py*) by calling:
-
- ```
- neural_greedy = PosteriorBNNSampling('MyNGreedy', my_hparams, 'RMSProp')
- ```
-
-4. **Stochastic Variational Inference**, Bayes by Backpropagation. We implement
- a Bayesian neural network by modeling each individual weight posterior as a
- univariate Gaussian distribution: wij ∼ N(μij, σij2).
- Thompson sampling then samples a network at each time step
- by sampling each weight independently. The variational approach consists in
- maximizing a proxy for maximum likelihood of the observed data, the ELBO or
- variational lower bound, to fit the values of μij, σij2
- for every *i, j*.
-
- See [Weight Uncertainty in Neural
- Networks](https://arxiv.org/abs/1505.05424).
-
- The BNN algorithm is implemented in *variational_neural_bandit_model.py*,
- and it is used together with *PosteriorBNNSampling* (defined in
- *posterior_bnn_sampling.py*) by calling:
-
- ```
- bbb = PosteriorBNNSampling('myBBB', my_hparams, 'Variational')
- ```
-
-5. **Expectation-Propagation**, Black-box alpha-divergence minimization.
- The family of expectation-propagation algorithms is based on the message
- passing framework . They iteratively approximate the posterior by updating a
- single approximation factor (or site) at a time, which usually corresponds
- to the likelihood of one data point. We focus on methods that directly
- optimize the global EP objective via stochastic gradient descent, as, for
- instance, Power EP. For further details see original paper below.
-
- See [Black-box alpha-divergence
- Minimization](https://arxiv.org/abs/1511.03243).
-
- We create an instance of the algorithm like this:
-
- ```
- bb_adiv = PosteriorBNNSampling('MyEP', my_hparams, 'AlphaDiv')
- ```
-
-6. **Dropout**. Dropout is a training technique where the output of each neuron
- is independently zeroed out with probability *p* at each forward pass.
- Once the network has been trained, dropout can still be used to obtain a
- distribution of predictions for a specific input. Following the best action
- with respect to the random dropout prediction can be interpreted as an
- implicit form of Thompson sampling. The code for dropout is the same as for
- Neural Greedy (see above), but we need to set two hyper-parameters:
- *use_dropout=True* and *keep_prob=p* where *p* takes the desired value in
- (0, 1). Then:
-
- ```
- dropout = PosteriorBNNSampling('MyDropout', my_hparams, 'RMSProp')
- ```
-
-7. **Monte Carlo Methods**. To be added soon.
-
-8. **Bootstrapped Networks**. This algorithm trains simultaneously and in
- parallel **q** neural networks based on different datasets D1, ..., Dq. The way those datasets are collected is by adding each new collected
- datapoint (Xt, at, rt) to each dataset *Di* independently and with
- probability p ∈ (0, 1]. Therefore, the main hyperparameters of the
- algorithm are **(q, p)**. In order to choose an action for a new context,
- one of the **q** networks is first selected with uniform probability (i.e.,
- *1/q*). Then, the best action according to the *selected* network is
- played.
-
- See [Deep Exploration via Bootstrapped
- DQN](https://arxiv.org/abs/1602.04621).
-
- The algorithm is implemented in *bootstrapped_bnn_sampling.py*, and we
- instantiate it as (where *my_hparams* contains both **q** and **p**):
-
- ```
- bootstrap = BootstrappedBNNSampling('MyBoot', my_hparams)
- ```
-
-9. **Parameter-Noise**. Another approach to approximate a distribution over
- neural networks (or more generally, models) that map contexts to rewards,
- consists in randomly perturbing a point estimate trained by Stochastic
- Gradient Descent on the data. The Parameter-Noise algorithm uses a heuristic
- to control the amount of noise σt2 it adds independently to the
- parameters representing a neural network: θt' = θt + ε where
- ε ∼ N(0, σt2 Id).
- After using θt' for decision making, the following SGD
- training steps start again from θt. The key hyperparameters to set
- are those controlling the noise heuristic.
-
- See [Parameter Space Noise for
- Exploration](https://arxiv.org/abs/1706.01905).
-
- The algorithm is implemented in *parameter_noise_sampling.py*, and we create
- an instance by calling:
-
- ```
- parameter_noise = ParameterNoiseSampling('MyParamNoise', my_hparams)
- ```
-
-10. **Gaussian Processes**. Another standard benchmark are Gaussian Processes,
- see *Gaussian Processes for Machine Learning* by Rasmussen and Williams for
- an introduction. To model the expected reward of different actions, we fit a
- multitask GP.
-
- See [Multi-task Gaussian Process
- Prediction](http://papers.nips.cc/paper/3189-multi-task-gaussian-process-prediction.pdf).
-
- Our implementation is provided in *multitask_gp.py*, and it is instantiated
- as follows:
-
- ```
- gp = PosteriorBNNSampling('MyMultitaskGP', my_hparams, 'GP')
- ```
-
-In the code snippet at the bottom, we show how to instantiate some of these
-algorithms, and how to run the contextual bandit simulator, and display the
-high-level results.
-
-## Data
-
-In the paper we use two types of contextual datasets: synthetic and based on
-real-world data.
-
-We provide functions that sample problems from those datasets. In the case of
-real-world data, you first need to download the raw datasets, and pass the route
-to the functions. Links for the datasets are provided below.
-
-### Synthetic Datasets
-
-Synthetic datasets are contained in the *synthetic_data_sampler.py* file. In
-particular, it includes:
-
-1. **Linear data**. Provides a number of linear arms, and Gaussian contexts.
-
-2. **Sparse linear data**. Provides a number of sparse linear arms, and
- Gaussian contexts.
-
-3. **Wheel bandit data**. Provides sampled data from the wheel bandit data, see
- [Section 5.4](https://arxiv.org/abs/1802.09127) in the paper.
-
-### Real-World Datasets
-
-Real-world data generating functions are contained in the *data_sampler.py*
-file.
-
-In particular, it includes:
-
-1. **Mushroom data**. Each incoming context represents a different type of
- mushroom, and the actions are eat or no-eat. Eating an edible mushroom
- provides positive reward, while eating a poisonous one provides positive
- reward with probability *p*, and a large negative reward with probability
- *1-p*. All the rewards, and the value of *p* are customizable. The
- [dataset](https://archive.ics.uci.edu/ml/datasets/mushroom) is part of the
- UCI repository, and the bandit problem was proposed in Blundell et al.
- (2015). Data is available [here](https://storage.googleapis.com/bandits_datasets/mushroom.data)
- or alternatively [here](https://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/),
- use the *agaricus-lepiota.data* file.
-
-2. **Stock data**. We created the Financial Dataset by pulling the stock prices
- of *d = 21* publicly traded companies in NYSE and Nasdaq, for the last 14
- years (*n = 3713*). For each day, the context was the price difference
- between the beginning and end of the session for each stock. We
- synthetically created the arms to be a linear combination of the contexts,
- representing *k = 8* different potential portfolios. Data is available
- [here](https://storage.googleapis.com/bandits_datasets/raw_stock_contexts).
-
-3. **Jester data**. We create a recommendation system bandit problem as
- follows. The Jester Dataset (Goldberg et al., 2001) provides continuous
- ratings in *[-10, 10]* for 100 jokes from a total of 73421 users. We find
- a *complete* subset of *n = 19181* users rating all 40 jokes. Following
- Riquelme et al. (2017), we take *d = 32* of the ratings as the context of
- the user, and *k = 8* as the arms. The agent recommends one joke, and
- obtains the reward corresponding to the rating of the user for the selected
- joke. Data is available [here](https://storage.googleapis.com/bandits_datasets/jester_data_40jokes_19181users.npy).
-
-4. **Statlog data**. The Shuttle Statlog Dataset (Asuncion & Newman, 2007)
- provides the value of *d = 9* indicators during a space shuttle flight,
- and the goal is to predict the state of the radiator subsystem of the
- shuttle. There are *k = 7* possible states, and if the agent selects the
- right state, then reward 1 is generated. Otherwise, the agent obtains no
- reward (*r = 0*). The most interesting aspect of the dataset is that one
- action is the optimal one in 80% of the cases, and some algorithms may
- commit to this action instead of further exploring. In this case, the number
- of contexts is *n = 43500*. Data is available [here](https://storage.googleapis.com/bandits_datasets/shuttle.trn) or alternatively
- [here](https://archive.ics.uci.edu/ml/datasets/Statlog+\(Shuttle\)), use
- *shuttle.trn* file.
-
-5. **Adult data**. The Adult Dataset (Kohavi, 1996; Asuncion & Newman, 2007)
- comprises personal information from the US Census Bureau database, and the
- standard prediction task is to determine if a person makes over 50K a year
- or not. However, we consider the *k = 14* different occupations as
- feasible actions, based on *d = 94* covariates (many of them binarized).
- As in previous datasets, the agent obtains a reward of 1 for making the
- right prediction, and 0 otherwise. The total number of contexts is *n =
- 45222*. Data is available [here](https://storage.googleapis.com/bandits_datasets/adult.full) or alternatively
- [here](https://archive.ics.uci.edu/ml/datasets/adult), use *adult.data*
- file.
-
-6. **Census data**. The US Census (1990) Dataset (Asuncion & Newman, 2007)
- contains a number of personal features (age, native language, education...)
- which we summarize in *d = 389* covariates, including binary dummy
- variables for categorical features. Our goal again is to predict the
- occupation of the individual among *k = 9* classes. The agent obtains
- reward 1 for making the right prediction, and 0 otherwise. Data is available
- [here](https://storage.googleapis.com/bandits_datasets/USCensus1990.data.txt) or alternatively [here](https://archive.ics.uci.edu/ml/datasets/US+Census+Data+\(1990\)), use
- *USCensus1990.data.txt* file.
-
-7. **Covertype data**. The Covertype Dataset (Asuncion & Newman, 2007)
- classifies the cover type of northern Colorado forest areas in *k = 7*
- classes, based on *d = 54* features, including elevation, slope, aspect,
- and soil type. Again, the agent obtains reward 1 if the correct class is
- selected, and 0 otherwise. Data is available [here](https://storage.googleapis.com/bandits_datasets/covtype.data) or alternatively
- [here](https://archive.ics.uci.edu/ml/datasets/covertype), use
- *covtype.data* file.
-
-In datasets 4-7, each feature of the dataset is normalized first.
-
-## Usage: Basic Example
-
-This library requires Tensorflow, Numpy, and Pandas.
-
-The file *example_main.py* provides a complete example on how to use the
-library. We run the code:
-
-```
- python example_main.py
-```
-
-**Do not forget to** configure the routes to the data files at the top of *example_main.py*.
-
-For example, we can run the Mushroom bandit for 2000 contexts on a few
-algorithms as follows:
-
-```
- # Problem parameters
- num_contexts = 2000
-
- # Choose data source among:
- # {linear, sparse_linear, mushroom, financial, jester,
- # statlog, adult, covertype, census, wheel}
- data_type = 'mushroom'
-
- # Create dataset
- sampled_vals = sample_data(data_type, num_contexts)
- dataset, opt_rewards, opt_actions, num_actions, context_dim = sampled_vals
-
- # Define hyperparameters and algorithms
- hparams_linear = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- a0=6,
- b0=6,
- lambda_prior=0.25,
- initial_pulls=2)
-
- hparams_dropout = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- init_scale=0.3,
- activation=tf.nn.relu,
- layer_sizes=[50],
- batch_size=512,
- activate_decay=True,
- initial_lr=0.1,
- max_grad_norm=5.0,
- show_training=False,
- freq_summary=1000,
- buffer_s=-1,
- initial_pulls=2,
- optimizer='RMS',
- reset_lr=True,
- lr_decay_rate=0.5,
- training_freq=50,
- training_epochs=100,
- keep_prob=0.80,
- use_dropout=True)
-
- ### Create hyper-parameter configurations for other algorithms
- [...]
-
- algos = [
- UniformSampling('Uniform Sampling', hparams),
- PosteriorBNNSampling('Dropout', hparams_dropout, 'RMSProp'),
- PosteriorBNNSampling('BBB', hparams_bbb, 'Variational'),
- NeuralLinearPosteriorSampling('NeuralLinear', hparams_nlinear),
- LinearFullPosteriorSampling('LinFullPost', hparams_linear),
- BootstrappedBNNSampling('BootRMS', hparams_boot),
- ParameterNoiseSampling('ParamNoise', hparams_pnoise),
- ]
-
- # Run contextual bandit problem
- t_init = time.time()
- results = run_contextual_bandit(context_dim, num_actions, dataset, algos)
- _, h_rewards = results
-
- # Display results
- display_results(algos, opt_rewards, opt_actions, h_rewards, t_init, data_type)
-
-```
-
-The previous code leads to final results that look like:
-
-```
----------------------------------------------------
----------------------------------------------------
-mushroom bandit completed after 69.8401839733 seconds.
----------------------------------------------------
- 0) LinFullPost | total reward = 4365.0.
- 1) NeuralLinear | total reward = 4110.0.
- 2) Dropout | total reward = 3430.0.
- 3) ParamNoise | total reward = 3270.0.
- 4) BootRMS | total reward = 3050.0.
- 5) BBB | total reward = 2505.0.
- 6) Uniform Sampling | total reward = -4930.0.
----------------------------------------------------
-Optimal total reward = 5235.
-Frequency of optimal actions (action, frequency):
-[[0, 953], [1, 1047]]
----------------------------------------------------
----------------------------------------------------
-```
diff --git a/research/deep_contextual_bandits/bandits/algorithms/bb_alpha_divergence_model.py b/research/deep_contextual_bandits/bandits/algorithms/bb_alpha_divergence_model.py
deleted file mode 100644
index 5b9c0ebd0988873eaf97d8d68d25dae5e5b9cd71..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/bb_alpha_divergence_model.py
+++ /dev/null
@@ -1,373 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Bayesian NN using expectation propagation (Black-Box Alpha-Divergence).
-
-See https://arxiv.org/abs/1511.03243 for details.
-All formulas used in this implementation are derived in:
-https://www.overleaf.com/12837696kwzjxkyhdytk#/49028744/.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import sys
-import numpy as np
-import tensorflow as tf
-from absl import flags
-
-from bandits.core.bayesian_nn import BayesianNN
-
-
-FLAGS = flags.FLAGS
-tfd = tf.contrib.distributions # update to: tensorflow_probability.distributions
-
-
-def log_gaussian(x, mu, sigma, reduce_sum=True):
- res = tfd.Normal(mu, sigma).log_prob(x)
- if reduce_sum:
- return tf.reduce_sum(res)
- else:
- return res
-
-
-class BBAlphaDivergence(BayesianNN):
- """Implements an approximate Bayesian NN via Black-Box Alpha-Divergence."""
-
- def __init__(self, hparams, name):
-
- self.name = name
- self.hparams = hparams
-
- self.alpha = getattr(self.hparams, 'alpha', 1.0)
- self.num_mc_nn_samples = getattr(self.hparams, 'num_mc_nn_samples', 10)
-
- self.n_in = self.hparams.context_dim
- self.n_out = self.hparams.num_actions
- self.layers = self.hparams.layer_sizes
- self.batch_size = self.hparams.batch_size
-
- self.show_training = self.hparams.show_training
- self.freq_summary = self.hparams.freq_summary
- self.verbose = getattr(self.hparams, 'verbose', True)
-
- self.cleared_times_trained = self.hparams.cleared_times_trained
- self.initial_training_steps = self.hparams.initial_training_steps
- self.training_schedule = np.linspace(self.initial_training_steps,
- self.hparams.training_epochs,
- self.cleared_times_trained)
-
- self.times_trained = 0
- self.initialize_model()
-
- def initialize_model(self):
- """Builds and initialize the model."""
-
- self.num_w = 0
- self.num_b = 0
-
- self.weights_m = {}
- self.weights_std = {}
- self.biases_m = {}
- self.biases_std = {}
-
- self.h_max_var = []
-
- if self.hparams.use_sigma_exp_transform:
- self.sigma_transform = tfd.bijectors.Exp()
- else:
- self.sigma_transform = tfd.bijectors.Softplus()
-
- # Build the graph corresponding to the Bayesian NN instance.
- self.graph = tf.Graph()
-
- with self.graph.as_default():
-
- self.sess = tf.Session()
- self.x = tf.placeholder(shape=[None, self.n_in],
- dtype=tf.float32, name='x')
- self.y = tf.placeholder(shape=[None, self.n_out],
- dtype=tf.float32, name='y')
- self.weights = tf.placeholder(shape=[None, self.n_out],
- dtype=tf.float32, name='w')
- self.data_size = tf.placeholder(tf.float32, shape=(), name='data_size')
-
- self.prior_variance = self.hparams.prior_variance
- if self.prior_variance < 0:
- # if not fixed, we learn the prior.
- self.prior_variance = self.sigma_transform.forward(
- self.build_mu_variable([1, 1]))
-
- self.build_model()
- self.sess.run(tf.global_variables_initializer())
-
- def build_mu_variable(self, shape):
- """Returns a mean variable initialized as N(0, 0.05)."""
- return tf.Variable(tf.random_normal(shape, 0.0, 0.05))
-
- def build_sigma_variable(self, shape, init=-5.):
- """Returns a sigma variable initialized as N(init, 0.05)."""
- # Initialize sigma to be very small initially to encourage MAP opt first
- return tf.Variable(tf.random_normal(shape, init, 0.05))
-
- def build_layer(self, input_x, shape, layer_id, activation_fn=tf.nn.relu):
- """Builds a layer with N(mean, std) for each weight, and samples from it."""
-
- w_mu = self.build_mu_variable(shape)
- w_sigma = self.sigma_transform.forward(self.build_sigma_variable(shape))
-
- w_noise = tf.random_normal(shape)
- w = w_mu + w_sigma * w_noise
-
- b_mu = self.build_mu_variable([1, shape[1]])
- b_sigma = self.sigma_transform.forward(
- self.build_sigma_variable([1, shape[1]]))
-
- b_noise = tf.random_normal([1, shape[1]])
- b = b_mu + b_sigma * b_noise
-
- # Create outputs
- output_h = activation_fn(tf.matmul(input_x, w) + b)
-
- # Store means and stds
- self.weights_m[layer_id] = w_mu
- self.weights_std[layer_id] = w_sigma
- self.biases_m[layer_id] = b_mu
- self.biases_std[layer_id] = b_sigma
-
- return output_h
-
- def sample_neural_network(self, activation_fn=tf.nn.relu):
- """Samples a nn from posterior, computes data log lk and log f factor."""
-
- with self.graph.as_default():
-
- log_f = 0
- n = self.data_size
- input_x = self.x
-
- for layer_id in range(self.total_layers):
-
- # load mean and std of each weight
- w_mu = self.weights_m[layer_id]
- w_sigma = self.weights_std[layer_id]
- b_mu = self.biases_m[layer_id]
- b_sigma = self.biases_std[layer_id]
-
- # sample weights from Gaussian distribution
- shape = w_mu.shape
- w_noise = tf.random_normal(shape)
- b_noise = tf.random_normal([1, int(shape[1])])
- w = w_mu + w_sigma * w_noise
- b = b_mu + b_sigma * b_noise
-
- # compute contribution to log_f
- t1 = w * w_mu / (n * w_sigma ** 2)
- t2 = (0.5 * w ** 2 / n) * (1 / self.prior_variance - 1 / w_sigma ** 2)
- log_f += tf.reduce_sum(t1 + t2)
-
- t1 = b * b_mu / (n * b_sigma ** 2)
- t2 = (0.5 * b ** 2 / n) * (1 / self.prior_variance - 1 / b_sigma ** 2)
- log_f += tf.reduce_sum(t1 + t2)
-
- if layer_id < self.total_layers - 1:
- output_h = activation_fn(tf.matmul(input_x, w) + b)
- else:
- output_h = tf.matmul(input_x, w) + b
-
- input_x = output_h
-
- # compute log likelihood of the observed reward under the sampled nn
- log_likelihood = log_gaussian(
- self.y, output_h, self.noise_sigma, reduce_sum=False)
- weighted_log_likelihood = tf.reduce_sum(log_likelihood * self.weights, -1)
-
- return log_f, weighted_log_likelihood
-
- def log_z_q(self):
- """Computes log-partition function of current posterior parameters."""
-
- with self.graph.as_default():
-
- log_z_q = 0
-
- for layer_id in range(self.total_layers):
-
- w_mu = self.weights_m[layer_id]
- w_sigma = self.weights_std[layer_id]
- b_mu = self.biases_m[layer_id]
- b_sigma = self.biases_std[layer_id]
-
- w_term = 0.5 * tf.reduce_sum(w_mu ** 2 / w_sigma ** 2)
- w_term += 0.5 * tf.reduce_sum(tf.log(2 * np.pi) + 2 * tf.log(w_sigma))
-
- b_term = 0.5 * tf.reduce_sum(b_mu ** 2 / b_sigma ** 2)
- b_term += 0.5 * tf.reduce_sum(tf.log(2 * np.pi) + 2 * tf.log(b_sigma))
-
- log_z_q += w_term + b_term
-
- return log_z_q
-
- def log_z_prior(self):
- """Computes log-partition function of the prior parameters."""
- num_params = self.num_w + self.num_b
- return num_params * 0.5 * tf.log(2 * np.pi * self.prior_variance)
-
- def log_alpha_likelihood_ratio(self, activation_fn=tf.nn.relu):
-
- # each nn sample returns (log f, log likelihoods)
- nn_samples = [
- self.sample_neural_network(activation_fn)
- for _ in range(self.num_mc_nn_samples)
- ]
- nn_log_f_samples = [elt[0] for elt in nn_samples]
- nn_log_lk_samples = [elt[1] for elt in nn_samples]
-
- # we stack the (log f, log likelihoods) from the k nn samples
- nn_log_f_stack = tf.stack(nn_log_f_samples) # k x 1
- nn_log_lk_stack = tf.stack(nn_log_lk_samples) # k x N
- nn_f_tile = tf.tile(nn_log_f_stack, [self.batch_size])
- nn_f_tile = tf.reshape(nn_f_tile,
- [self.num_mc_nn_samples, self.batch_size])
-
- # now both the log f and log likelihood terms have shape: k x N
- # apply formula in https://www.overleaf.com/12837696kwzjxkyhdytk#/49028744/
- nn_log_ratio = nn_log_lk_stack - nn_f_tile
- nn_log_ratio = self.alpha * tf.transpose(nn_log_ratio)
- logsumexp_value = tf.reduce_logsumexp(nn_log_ratio, -1)
- log_k_scalar = tf.log(tf.cast(self.num_mc_nn_samples, tf.float32))
- log_k = log_k_scalar * tf.ones([self.batch_size])
-
- return tf.reduce_sum(logsumexp_value - log_k, -1)
-
- def build_model(self, activation_fn=tf.nn.relu):
- """Defines the actual NN model with fully connected layers.
-
- Args:
- activation_fn: Activation function for the neural network.
-
- The loss is computed for partial feedback settings (bandits), so only
- the observed outcome is backpropagated (see weighted loss).
- Selects the optimizer and, finally, it also initializes the graph.
- """
-
- print('Initializing model {}.'.format(self.name))
-
- # Build terms for the noise sigma estimation for each action.
- noise_sigma_mu = (self.build_mu_variable([1, self.n_out])
- + self.sigma_transform.inverse(self.hparams.noise_sigma))
- noise_sigma_sigma = self.sigma_transform.forward(
- self.build_sigma_variable([1, self.n_out]))
-
- pre_noise_sigma = noise_sigma_mu + tf.random_normal(
- [1, self.n_out]) * noise_sigma_sigma
- self.noise_sigma = self.sigma_transform.forward(pre_noise_sigma)
-
- # Build network
- input_x = self.x
- n_in = self.n_in
- self.total_layers = len(self.layers) + 1
- if self.layers[0] == 0:
- self.total_layers = 1
-
- for l_number, n_nodes in enumerate(self.layers):
- if n_nodes > 0:
- h = self.build_layer(input_x, [n_in, n_nodes], l_number)
- input_x = h
- n_in = n_nodes
- self.num_w += n_in * n_nodes
- self.num_b += n_nodes
-
- self.y_pred = self.build_layer(input_x, [n_in, self.n_out],
- self.total_layers - 1,
- activation_fn=lambda x: x)
-
- # Compute energy function based on sampled nn's
- log_coeff = self.data_size / (self.batch_size * self.alpha)
- log_ratio = log_coeff * self.log_alpha_likelihood_ratio(activation_fn)
- logzprior = self.log_z_prior()
- logzq = self.log_z_q()
- energy = logzprior - logzq - log_ratio
-
- self.loss = energy
- self.global_step = tf.train.get_or_create_global_step()
- self.train_op = tf.train.AdamOptimizer(self.hparams.initial_lr).minimize(
- self.loss, global_step=self.global_step)
-
- # Useful for debugging
- sq_loss = tf.squared_difference(self.y_pred, self.y)
- weighted_sq_loss = self.weights * sq_loss
- self.cost = tf.reduce_sum(weighted_sq_loss) / self.batch_size
-
- # Create tensorboard metrics
- self.create_summaries()
- self.summary_writer = tf.summary.FileWriter('{}/graph_{}'.format(
- FLAGS.logdir, self.name), self.sess.graph)
-
- def create_summaries(self):
- tf.summary.scalar('loss', self.loss)
- tf.summary.scalar('cost', self.cost)
- self.summary_op = tf.summary.merge_all()
-
- def assign_lr(self):
- """Resets the learning rate in dynamic schedules for subsequent trainings.
-
- In bandits settings, we do expand our dataset over time. Then, we need to
- re-train the network with the new data. Those algorithms that do not keep
- the step constant, can reset it at the start of each training process.
- """
-
- decay_steps = 1
- if self.hparams.activate_decay:
- current_gs = self.sess.run(self.global_step)
- with self.graph.as_default():
- self.lr = tf.train.inverse_time_decay(self.hparams.initial_lr,
- self.global_step - current_gs,
- decay_steps,
- self.hparams.lr_decay_rate)
-
- def train(self, data, num_steps):
- """Trains the BNN for num_steps, using the data in 'data'.
-
- Args:
- data: ContextualDataset object that provides the data.
- num_steps: Number of minibatches to train the network for.
- """
-
- if self.times_trained < self.cleared_times_trained:
- num_steps = int(self.training_schedule[self.times_trained])
- self.times_trained += 1
-
- if self.verbose:
- print('Training {} for {} steps...'.format(self.name, num_steps))
-
- with self.graph.as_default():
-
- for step in range(num_steps):
- x, y, w = data.get_batch_with_weights(self.hparams.batch_size)
- _, summary, global_step, loss = self.sess.run(
- [self.train_op, self.summary_op, self.global_step, self.loss],
- feed_dict={self.x: x, self.y: y, self.weights: w,
- self.data_size: data.num_points()})
-
- weights_l = self.sess.run(self.weights_std[0])
- self.h_max_var.append(np.max(weights_l))
-
- if step % self.freq_summary == 0:
- if self.show_training:
- print('step: {}, loss: {}'.format(step, loss))
- sys.stdout.flush()
- self.summary_writer.add_summary(summary, global_step)
diff --git a/research/deep_contextual_bandits/bandits/algorithms/bf_variational_neural_bandit_model.py b/research/deep_contextual_bandits/bandits/algorithms/bf_variational_neural_bandit_model.py
deleted file mode 100644
index cb87c23358f27bd93e30528b20f7a3bb3ba876dd..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/bf_variational_neural_bandit_model.py
+++ /dev/null
@@ -1,352 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Bayesian NN using factorized VI (Bayes By Backprop. Blundell et al. 2014).
-
-See https://arxiv.org/abs/1505.05424 for details.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-# import tensorflow_probability as tfp
-
-from absl import flags
-from bandits.core.bayesian_nn import BayesianNN
-
-
-FLAGS = flags.FLAGS
-# tfd = tfp.distributions
-tfd = tf.contrib.distributions
-tfl = tf.contrib.layers
-
-
-def log_gaussian(x, mu, sigma, reduce_sum=True):
- """Returns log Gaussian pdf."""
- res = tfd.Normal(mu, sigma).log_prob(x)
- if reduce_sum:
- return tf.reduce_sum(res)
- else:
- return res
-
-
-def analytic_kl(mu_1, sigma_1, mu_2, sigma_2):
- """KL for two Gaussian distributions with diagonal covariance matrix."""
- kl = tfd.kl_divergence(tfd.MVNDiag(mu_1, sigma_1), tfd.MVNDiag(mu_2, sigma_2))
- return kl
-
-
-class BfVariationalNeuralBanditModel(BayesianNN):
- """Implements an approximate Bayesian NN using Variational Inference."""
-
- def __init__(self, hparams, name="BBBNN"):
-
- self.name = name
- self.hparams = hparams
-
- self.n_in = self.hparams.context_dim
- self.n_out = self.hparams.num_actions
- self.layers = self.hparams.layer_sizes
- self.init_scale = self.hparams.init_scale
- self.f_num_points = None
- if "f_num_points" in hparams:
- self.f_num_points = self.hparams.f_num_points
-
- self.cleared_times_trained = self.hparams.cleared_times_trained
- self.initial_training_steps = self.hparams.initial_training_steps
- self.training_schedule = np.linspace(self.initial_training_steps,
- self.hparams.training_epochs,
- self.cleared_times_trained)
- self.verbose = getattr(self.hparams, "verbose", True)
-
- self.weights_m = {}
- self.weights_std = {}
- self.biases_m = {}
- self.biases_std = {}
-
- self.times_trained = 0
-
- if self.hparams.use_sigma_exp_transform:
- self.sigma_transform = tf.exp
- self.inverse_sigma_transform = np.log
- else:
- self.sigma_transform = tf.nn.softplus
- self.inverse_sigma_transform = lambda y: y + np.log(1. - np.exp(-y))
-
- # Whether to use the local reparameterization trick to compute the loss.
- # See details in https://arxiv.org/abs/1506.02557
- self.use_local_reparameterization = True
-
- self.build_graph()
-
- def build_mu_variable(self, shape):
- """Returns a mean variable initialized as N(0, 0.05)."""
- return tf.Variable(tf.random_normal(shape, 0.0, 0.05))
-
- def build_sigma_variable(self, shape, init=-5.):
- """Returns a sigma variable initialized as N(init, 0.05)."""
- # Initialize sigma to be very small initially to encourage MAP opt first
- return tf.Variable(tf.random_normal(shape, init, 0.05))
-
- def build_layer(self, input_x, input_x_local, shape,
- layer_id, activation_fn=tf.nn.relu):
- """Builds a variational layer, and computes KL term.
-
- Args:
- input_x: Input to the variational layer.
- input_x_local: Input when the local reparameterization trick was applied.
- shape: [number_inputs, number_outputs] for the layer.
- layer_id: Number of layer in the architecture.
- activation_fn: Activation function to apply.
-
- Returns:
- output_h: Output of the variational layer.
- output_h_local: Output when local reparameterization trick was applied.
- neg_kl: Negative KL term for the layer.
- """
-
- w_mu = self.build_mu_variable(shape)
- w_sigma = self.sigma_transform(self.build_sigma_variable(shape))
- w_noise = tf.random_normal(shape)
- w = w_mu + w_sigma * w_noise
-
- b_mu = self.build_mu_variable([1, shape[1]])
- b_sigma = self.sigma_transform(self.build_sigma_variable([1, shape[1]]))
- b = b_mu
-
- # Store means and stds
- self.weights_m[layer_id] = w_mu
- self.weights_std[layer_id] = w_sigma
- self.biases_m[layer_id] = b_mu
- self.biases_std[layer_id] = b_sigma
-
- # Create outputs
- output_h = activation_fn(tf.matmul(input_x, w) + b)
-
- if self.use_local_reparameterization:
- # Use analytic KL divergence wrt the prior
- neg_kl = -analytic_kl(w_mu, w_sigma,
- 0., tf.to_float(np.sqrt(2./shape[0])))
- else:
- # Create empirical KL loss terms
- log_p = log_gaussian(w, 0., tf.to_float(np.sqrt(2./shape[0])))
- log_q = log_gaussian(w, tf.stop_gradient(w_mu), tf.stop_gradient(w_sigma))
- neg_kl = log_p - log_q
-
- # Apply local reparameterization trick: sample activations pre nonlinearity
- m_h = tf.matmul(input_x_local, w_mu) + b
- v_h = tf.matmul(tf.square(input_x_local), tf.square(w_sigma))
- output_h_local = m_h + tf.sqrt(v_h + 1e-6) * tf.random_normal(tf.shape(v_h))
- output_h_local = activation_fn(output_h_local)
-
- return output_h, output_h_local, neg_kl
-
- def build_action_noise(self):
- """Defines a model for additive noise per action, and its KL term."""
-
- # Define mean and std variables (log-normal dist) for each action.
- noise_sigma_mu = (self.build_mu_variable([1, self.n_out])
- + self.inverse_sigma_transform(self.hparams.noise_sigma))
- noise_sigma_sigma = self.sigma_transform(
- self.build_sigma_variable([1, self.n_out]))
-
- pre_noise_sigma = (noise_sigma_mu
- + tf.random_normal([1, self.n_out]) * noise_sigma_sigma)
- self.noise_sigma = self.sigma_transform(pre_noise_sigma)
-
- # Compute KL for additive noise sigma terms.
- if getattr(self.hparams, "infer_noise_sigma", False):
- neg_kl_term = log_gaussian(
- pre_noise_sigma,
- self.inverse_sigma_transform(self.hparams.noise_sigma),
- self.hparams.prior_sigma
- )
- neg_kl_term -= log_gaussian(pre_noise_sigma,
- noise_sigma_mu,
- noise_sigma_sigma)
- else:
- neg_kl_term = 0.
-
- return neg_kl_term
-
- def build_model(self, activation_fn=tf.nn.relu):
- """Defines the actual NN model with fully connected layers.
-
- The loss is computed for partial feedback settings (bandits), so only
- the observed outcome is backpropagated (see weighted loss).
- Selects the optimizer and, finally, it also initializes the graph.
-
- Args:
- activation_fn: the activation function used in the nn layers.
- """
-
- def weight_prior(dtype, shape, c, d, e):
- del c, d, e
- return tfd.Independent(
- tfd.Normal(loc=tf.zeros(shape, dtype),
- scale=tf.to_float(np.sqrt(2) / shape[0])),
- reinterpreted_batch_ndims=tf.size(shape))
-
- if self.verbose:
- print("Initializing model {}.".format(self.name))
-
- # Compute model additive noise for each action with log-normal distribution
- neg_kl_term = self.build_action_noise()
-
- # Build variational network using self.x as input.
- input_x = self.x
-
- # Create Keras model using DenseLocalReparameterization (prior N(0, 1)).
- model_layers = [
- tfl.DenseLocalReparameterization(
- n_nodes,
- activation=tf.nn.relu,
- kernel_prior_fn=weight_prior
- )
- for n_nodes in self.layers if n_nodes > 0
- ]
-
- output_layer = tfl.DenseLocalReparameterization(
- self.n_out,
- activation=lambda x: x,
- kernel_prior_fn=weight_prior
- )
- model_layers.append(output_layer)
-
- model = tf.keras.Sequential(model_layers)
- self.y_pred = model(input_x)
-
- # Compute KL term
- neg_kl_term -= tf.add_n(model.losses)
-
- # Compute log likelihood (with learned or fixed noise level)
- if getattr(self.hparams, "infer_noise_sigma", False):
- log_likelihood = log_gaussian(
- self.y, self.y_pred, self.noise_sigma, reduce_sum=False)
- else:
- log_likelihood = log_gaussian(
- self.y, self.y_pred, self.hparams.noise_sigma, reduce_sum=False)
-
- # Only take into account observed outcomes (bandits setting)
- batch_size = tf.to_float(tf.shape(self.x)[0])
- weighted_log_likelihood = tf.reduce_sum(
- log_likelihood * self.weights) / batch_size
-
- # The objective is 1/n * (\sum_i log_like_i - KL); neg_kl_term estimates -KL
- elbo = weighted_log_likelihood + (neg_kl_term / self.n)
-
- self.loss = -elbo
- self.global_step = tf.train.get_or_create_global_step()
- self.train_op = tf.train.AdamOptimizer(self.hparams.initial_lr).minimize(
- self.loss, global_step=self.global_step)
-
- # Create tensorboard metrics
- self.create_summaries()
- self.summary_writer = tf.summary.FileWriter(
- "{}/graph_{}".format(FLAGS.logdir, self.name), self.sess.graph)
-
- def build_graph(self):
- """Defines graph, session, placeholders, and model.
-
- Placeholders are: n (size of the dataset), x and y (context and observed
- reward for each action), and weights (one-hot encoding of selected action
- for each context, i.e., only possibly non-zero element in each y).
- """
-
- self.graph = tf.Graph()
- with self.graph.as_default():
-
- self.sess = tf.Session()
-
- self.n = tf.placeholder(shape=[], dtype=tf.float32)
-
- self.x = tf.placeholder(shape=[None, self.n_in], dtype=tf.float32)
- self.y = tf.placeholder(shape=[None, self.n_out], dtype=tf.float32)
- self.weights = tf.placeholder(shape=[None, self.n_out], dtype=tf.float32)
-
- self.build_model()
- self.sess.run(tf.global_variables_initializer())
-
- def create_summaries(self):
- """Defines summaries including mean loss, and global step."""
-
- with self.graph.as_default():
- with tf.name_scope(self.name + "_summaries"):
- tf.summary.scalar("loss", self.loss)
- tf.summary.scalar("global_step", self.global_step)
- self.summary_op = tf.summary.merge_all()
-
- def assign_lr(self):
- """Resets the learning rate in dynamic schedules for subsequent trainings.
-
- In bandits settings, we do expand our dataset over time. Then, we need to
- re-train the network with the new data. The algorithms that do not keep
- the step constant, can reset it at the start of each *training* process.
- """
-
- decay_steps = 1
- if self.hparams.activate_decay:
- current_gs = self.sess.run(self.global_step)
- with self.graph.as_default():
- self.lr = tf.train.inverse_time_decay(self.hparams.initial_lr,
- self.global_step - current_gs,
- decay_steps,
- self.hparams.lr_decay_rate)
-
- def train(self, data, num_steps):
- """Trains the BNN for num_steps, using the data in 'data'.
-
- Args:
- data: ContextualDataset object that provides the data.
- num_steps: Number of minibatches to train the network for.
-
- Returns:
- losses: Loss history during training.
- """
-
- if self.times_trained < self.cleared_times_trained:
- num_steps = int(self.training_schedule[self.times_trained])
- self.times_trained += 1
-
- losses = []
-
- with self.graph.as_default():
-
- if self.verbose:
- print("Training {} for {} steps...".format(self.name, num_steps))
-
- for step in range(num_steps):
- x, y, weights = data.get_batch_with_weights(self.hparams.batch_size)
- _, summary, global_step, loss = self.sess.run(
- [self.train_op, self.summary_op, self.global_step, self.loss],
- feed_dict={
- self.x: x,
- self.y: y,
- self.weights: weights,
- self.n: data.num_points(self.f_num_points),
- })
-
- losses.append(loss)
-
- if step % self.hparams.freq_summary == 0:
- if self.hparams.show_training:
- print("{} | step: {}, loss: {}".format(
- self.name, global_step, loss))
- self.summary_writer.add_summary(summary, global_step)
-
- return losses
diff --git a/research/deep_contextual_bandits/bandits/algorithms/bootstrapped_bnn_sampling.py b/research/deep_contextual_bandits/bandits/algorithms/bootstrapped_bnn_sampling.py
deleted file mode 100644
index 7c44b681c7bd1da113ec29c1bb6d370c88d7053f..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/bootstrapped_bnn_sampling.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contextual algorithm based on boostrapping neural networks."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from bandits.core.bandit_algorithm import BanditAlgorithm
-from bandits.core.contextual_dataset import ContextualDataset
-from bandits.algorithms.neural_bandit_model import NeuralBanditModel
-
-
-class BootstrappedBNNSampling(BanditAlgorithm):
- """Thompson Sampling algorithm based on training several neural networks."""
-
- def __init__(self, name, hparams, optimizer='RMS'):
- """Creates a BootstrappedSGDSampling object based on a specific optimizer.
-
- hparams.q: Number of models that are independently trained.
- hparams.p: Prob of independently including each datapoint in each model.
-
- Args:
- name: Name given to the instance.
- hparams: Hyperparameters for each individual model.
- optimizer: Neural network optimization algorithm.
- """
-
- self.name = name
- self.hparams = hparams
- self.optimizer_n = optimizer
-
- self.training_freq = hparams.training_freq
- self.training_epochs = hparams.training_epochs
- self.t = 0
-
- self.q = hparams.q
- self.p = hparams.p
-
- self.datasets = [
- ContextualDataset(hparams.context_dim,
- hparams.num_actions,
- hparams.buffer_s)
- for _ in range(self.q)
- ]
-
- self.bnn_boot = [
- NeuralBanditModel(optimizer, hparams, '{}-{}-bnn'.format(name, i))
- for i in range(self.q)
- ]
-
- def action(self, context):
- """Selects action for context based on Thompson Sampling using one BNN."""
-
- if self.t < self.hparams.num_actions * self.hparams.initial_pulls:
- # round robin until each action has been taken "initial_pulls" times
- return self.t % self.hparams.num_actions
-
- # choose model uniformly at random
- model_index = np.random.randint(self.q)
-
- with self.bnn_boot[model_index].graph.as_default():
- c = context.reshape((1, self.hparams.context_dim))
- output = self.bnn_boot[model_index].sess.run(
- self.bnn_boot[model_index].y_pred,
- feed_dict={self.bnn_boot[model_index].x: c})
- return np.argmax(output)
-
- def update(self, context, action, reward):
- """Updates the data buffer, and re-trains the BNN every self.freq_update."""
-
- self.t += 1
- for i in range(self.q):
- # include the data point with probability p independently in each dataset
- if np.random.random() < self.p or self.t < 2:
- self.datasets[i].add(context, action, reward)
-
- if self.t % self.training_freq == 0:
- # update all the models:
- for i in range(self.q):
- if self.hparams.reset_lr:
- self.bnn_boot[i].assign_lr()
- self.bnn_boot[i].train(self.datasets[i], self.training_epochs)
diff --git a/research/deep_contextual_bandits/bandits/algorithms/fixed_policy_sampling.py b/research/deep_contextual_bandits/bandits/algorithms/fixed_policy_sampling.py
deleted file mode 100644
index d5ad6e3ed9ed9d1478e6ac132b41cfb5ae1bb47a..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/fixed_policy_sampling.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contextual bandit algorithm that selects an action at random."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from bandits.core.bandit_algorithm import BanditAlgorithm
-
-
-class FixedPolicySampling(BanditAlgorithm):
- """Defines a baseline; returns an action at random with probs given by p."""
-
- def __init__(self, name, p, hparams):
- """Creates a FixedPolicySampling object.
-
- Args:
- name: Name of the algorithm.
- p: Vector of normalized probabilities corresponding to sampling each arm.
- hparams: Hyper-parameters, including the number of arms (num_actions).
-
- Raises:
- ValueError: when p dimension does not match the number of actions.
- """
-
- self.name = name
- self.p = p
- self.hparams = hparams
-
- if len(p) != self.hparams.num_actions:
- raise ValueError('Policy needs k probabilities.')
-
- def action(self, context):
- """Selects an action at random according to distribution p."""
- return np.random.choice(range(self.hparams.num_actions), p=self.p)
diff --git a/research/deep_contextual_bandits/bandits/algorithms/linear_full_posterior_sampling.py b/research/deep_contextual_bandits/bandits/algorithms/linear_full_posterior_sampling.py
deleted file mode 100644
index 15ef8fa9b562101111042dc2ce7b17174018ab6e..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/linear_full_posterior_sampling.py
+++ /dev/null
@@ -1,164 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contextual algorithm that keeps a full linear posterior for each arm."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-from scipy.stats import invgamma
-
-from bandits.core.bandit_algorithm import BanditAlgorithm
-from bandits.core.contextual_dataset import ContextualDataset
-
-
-class LinearFullPosteriorSampling(BanditAlgorithm):
- """Thompson Sampling with independent linear models and unknown noise var."""
-
- def __init__(self, name, hparams):
- """Initialize posterior distributions and hyperparameters.
-
- Assume a linear model for each action i: reward = context^T beta_i + noise
- Each beta_i has a Gaussian prior (lambda parameter), each sigma2_i (noise
- level) has an inverse Gamma prior (a0, b0 parameters). Mean, covariance,
- and precision matrices are initialized, and the ContextualDataset created.
-
- Args:
- name: Name of the algorithm.
- hparams: Hyper-parameters of the algorithm.
- """
-
- self.name = name
- self.hparams = hparams
-
- # Gaussian prior for each beta_i
- self._lambda_prior = self.hparams.lambda_prior
-
- self.mu = [
- np.zeros(self.hparams.context_dim + 1)
- for _ in range(self.hparams.num_actions)
- ]
-
- self.cov = [(1.0 / self.lambda_prior) * np.eye(self.hparams.context_dim + 1)
- for _ in range(self.hparams.num_actions)]
-
- self.precision = [
- self.lambda_prior * np.eye(self.hparams.context_dim + 1)
- for _ in range(self.hparams.num_actions)
- ]
-
- # Inverse Gamma prior for each sigma2_i
- self._a0 = self.hparams.a0
- self._b0 = self.hparams.b0
-
- self.a = [self._a0 for _ in range(self.hparams.num_actions)]
- self.b = [self._b0 for _ in range(self.hparams.num_actions)]
-
- self.t = 0
- self.data_h = ContextualDataset(hparams.context_dim,
- hparams.num_actions,
- intercept=True)
-
- def action(self, context):
- """Samples beta's from posterior, and chooses best action accordingly.
-
- Args:
- context: Context for which the action need to be chosen.
-
- Returns:
- action: Selected action for the context.
- """
-
- # Round robin until each action has been selected "initial_pulls" times
- if self.t < self.hparams.num_actions * self.hparams.initial_pulls:
- return self.t % self.hparams.num_actions
-
- # Sample sigma2, and beta conditional on sigma2
- sigma2_s = [
- self.b[i] * invgamma.rvs(self.a[i])
- for i in range(self.hparams.num_actions)
- ]
-
- try:
- beta_s = [
- np.random.multivariate_normal(self.mu[i], sigma2_s[i] * self.cov[i])
- for i in range(self.hparams.num_actions)
- ]
- except np.linalg.LinAlgError as e:
- # Sampling could fail if covariance is not positive definite
- print('Exception when sampling from {}.'.format(self.name))
- print('Details: {} | {}.'.format(e.message, e.args))
- d = self.hparams.context_dim + 1
- beta_s = [
- np.random.multivariate_normal(np.zeros((d)), np.eye(d))
- for i in range(self.hparams.num_actions)
- ]
-
- # Compute sampled expected values, intercept is last component of beta
- vals = [
- np.dot(beta_s[i][:-1], context.T) + beta_s[i][-1]
- for i in range(self.hparams.num_actions)
- ]
-
- return np.argmax(vals)
-
- def update(self, context, action, reward):
- """Updates action posterior using the linear Bayesian regression formula.
-
- Args:
- context: Last observed context.
- action: Last observed action.
- reward: Last observed reward.
- """
-
- self.t += 1
- self.data_h.add(context, action, reward)
-
- # Update posterior of action with formulas: \beta | x,y ~ N(mu_q, cov_q)
- x, y = self.data_h.get_data(action)
-
- # The algorithm could be improved with sequential update formulas (cheaper)
- s = np.dot(x.T, x)
-
- # Some terms are removed as we assume prior mu_0 = 0.
- precision_a = s + self.lambda_prior * np.eye(self.hparams.context_dim + 1)
- cov_a = np.linalg.inv(precision_a)
- mu_a = np.dot(cov_a, np.dot(x.T, y))
-
- # Inverse Gamma posterior update
- a_post = self.a0 + x.shape[0] / 2.0
- b_upd = 0.5 * (np.dot(y.T, y) - np.dot(mu_a.T, np.dot(precision_a, mu_a)))
- b_post = self.b0 + b_upd
-
- # Store new posterior distributions
- self.mu[action] = mu_a
- self.cov[action] = cov_a
- self.precision[action] = precision_a
- self.a[action] = a_post
- self.b[action] = b_post
-
- @property
- def a0(self):
- return self._a0
-
- @property
- def b0(self):
- return self._b0
-
- @property
- def lambda_prior(self):
- return self._lambda_prior
diff --git a/research/deep_contextual_bandits/bandits/algorithms/multitask_gp.py b/research/deep_contextual_bandits/bandits/algorithms/multitask_gp.py
deleted file mode 100644
index 0c35dfaeaf9e30993d49d807f16dd64e15d3fc66..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/multitask_gp.py
+++ /dev/null
@@ -1,374 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A Multitask Gaussian process."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from absl import flags
-from absl import logging
-
-import numpy as np
-import tensorflow as tf
-from bandits.core.bayesian_nn import BayesianNN
-
-FLAGS = flags.FLAGS
-tfd = tf.contrib.distributions
-
-class MultitaskGP(BayesianNN):
- """Implements a Gaussian process with multi-task outputs.
-
- Optimizes the hyperparameters over the log marginal likelihood.
- Uses a Matern 3/2 + linear covariance and returns
- sampled predictions for test inputs. The outputs are optionally
- correlated where the correlation structure is learned through latent
- embeddings of the tasks.
- """
-
- def __init__(self, hparams):
- self.name = "MultiTaskGP"
- self.hparams = hparams
-
- self.n_in = self.hparams.context_dim
- self.n_out = self.hparams.num_outputs
- self.keep_fixed_after_max_obs = self.hparams.keep_fixed_after_max_obs
-
- self._show_training = self.hparams.show_training
- self._freq_summary = self.hparams.freq_summary
-
- # Dimensionality of the latent task vectors
- self.task_latent_dim = self.hparams.task_latent_dim
-
- # Maximum number of observations to include
- self.max_num_points = self.hparams.max_num_points
-
- if self.hparams.learn_embeddings:
- self.learn_embeddings = self.hparams.learn_embeddings
- else:
- self.learn_embeddings = False
-
- # create the graph corresponding to the BNN instance
- self.graph = tf.Graph()
- with self.graph.as_default():
- # store a new session for the graph
- self.sess = tf.Session()
-
- with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE):
- self.n = tf.placeholder(shape=[], dtype=tf.float64)
- self.x = tf.placeholder(shape=[None, self.n_in], dtype=tf.float64)
- self.x_in = tf.placeholder(shape=[None, self.n_in], dtype=tf.float64)
- self.y = tf.placeholder(shape=[None, self.n_out], dtype=tf.float64)
- self.weights = tf.placeholder(shape=[None, self.n_out],
- dtype=tf.float64)
-
- self.build_model()
- self.sess.run(tf.global_variables_initializer())
-
- def atleast_2d(self, x, dims):
- return tf.reshape(tf.expand_dims(x, axis=0), (-1, dims))
-
- def sq_dist(self, x, x2):
- a2 = tf.reduce_sum(tf.square(x), 1)
- b2 = tf.reduce_sum(tf.square(x2), 1)
- sqdists = tf.expand_dims(a2, 1) + b2 - 2.0 * tf.matmul(x, tf.transpose(x2))
- return sqdists
-
- # Covariance between outputs
- def task_cov(self, x, x2):
- """Squared Exponential Covariance Kernel over latent task embeddings."""
- # Index into latent task vectors
- x_vecs = tf.gather(self.task_vectors, tf.argmax(x, axis=1), axis=0)
- x2_vecs = tf.gather(self.task_vectors, tf.argmax(x2, axis=1), axis=0)
- r = self.sq_dist(self.atleast_2d(x_vecs, self.task_latent_dim),
- self.atleast_2d(x2_vecs, self.task_latent_dim))
- return tf.exp(-r)
-
- def cov(self, x, x2):
- """Matern 3/2 + Linear Gaussian Process Covariance Function."""
- ls = tf.clip_by_value(self.length_scales, -5.0, 5.0)
- ls_lin = tf.clip_by_value(self.length_scales_lin, -5.0, 5.0)
- r = self.sq_dist(self.atleast_2d(x, self.n_in)/tf.nn.softplus(ls),
- self.atleast_2d(x2, self.n_in)/tf.nn.softplus(ls))
- r = tf.clip_by_value(r, 0, 1e8)
-
- # Matern 3/2 Covariance
- matern = (1.0 + tf.sqrt(3.0*r + 1e-16)) * tf.exp(-tf.sqrt(3.0*r + 1e-16))
- # Linear Covariance
- lin = tf.matmul(x / tf.nn.softplus(ls_lin),
- x2 / tf.nn.softplus(ls_lin), transpose_b=True)
- return (tf.nn.softplus(self.amplitude) * matern +
- tf.nn.softplus(self.amplitude_linear) * lin)
-
- def build_model(self):
- """Defines the GP model.
-
- The loss is computed for partial feedback settings (bandits), so only
- the observed outcome is backpropagated (see weighted loss).
- Selects the optimizer and, finally, it also initializes the graph.
- """
-
- logging.info("Initializing model %s.", self.name)
- self.global_step = tf.train.get_or_create_global_step()
-
- # Define state for the model (inputs, etc.)
- self.x_train = tf.get_variable(
- "training_data",
- initializer=tf.ones(
- [self.hparams.batch_size, self.n_in], dtype=tf.float64),
- validate_shape=False,
- trainable=False)
- self.y_train = tf.get_variable(
- "training_labels",
- initializer=tf.zeros([self.hparams.batch_size, 1], dtype=tf.float64),
- validate_shape=False,
- trainable=False)
- self.weights_train = tf.get_variable(
- "weights_train",
- initializer=tf.ones(
- [self.hparams.batch_size, self.n_out], dtype=tf.float64),
- validate_shape=False,
- trainable=False)
- self.input_op = tf.assign(self.x_train, self.x_in, validate_shape=False)
- self.input_w_op = tf.assign(
- self.weights_train, self.weights, validate_shape=False)
-
- self.input_std = tf.get_variable(
- "data_standard_deviation",
- initializer=tf.ones([1, self.n_out], dtype=tf.float64),
- dtype=tf.float64,
- trainable=False)
- self.input_mean = tf.get_variable(
- "data_mean",
- initializer=tf.zeros([1, self.n_out], dtype=tf.float64),
- dtype=tf.float64,
- trainable=True)
-
- # GP Hyperparameters
- self.noise = tf.get_variable(
- "noise", initializer=tf.cast(0.0, dtype=tf.float64))
- self.amplitude = tf.get_variable(
- "amplitude", initializer=tf.cast(1.0, dtype=tf.float64))
- self.amplitude_linear = tf.get_variable(
- "linear_amplitude", initializer=tf.cast(1.0, dtype=tf.float64))
- self.length_scales = tf.get_variable(
- "length_scales", initializer=tf.zeros([1, self.n_in], dtype=tf.float64))
- self.length_scales_lin = tf.get_variable(
- "length_scales_linear",
- initializer=tf.zeros([1, self.n_in], dtype=tf.float64))
-
- # Latent embeddings of the different outputs for task covariance
- self.task_vectors = tf.get_variable(
- "latent_task_vectors",
- initializer=tf.random_normal(
- [self.n_out, self.task_latent_dim], dtype=tf.float64))
-
- # Normalize outputs across each dimension
- # Since we have different numbers of observations across each task, we
- # normalize by their respective counts.
- index_counts = self.atleast_2d(tf.reduce_sum(self.weights, axis=0),
- self.n_out)
- index_counts = tf.where(index_counts > 0, index_counts,
- tf.ones(tf.shape(index_counts), dtype=tf.float64))
- self.mean_op = tf.assign(self.input_mean,
- tf.reduce_sum(self.y, axis=0) / index_counts)
- self.var_op = tf.assign(
- self.input_std, tf.sqrt(1e-4 + tf.reduce_sum(tf.square(
- self.y - tf.reduce_sum(self.y, axis=0) / index_counts), axis=0)
- / index_counts))
-
- with tf.control_dependencies([self.var_op]):
- y_normed = self.atleast_2d(
- (self.y - self.input_mean) / self.input_std, self.n_out)
- y_normed = self.atleast_2d(tf.boolean_mask(y_normed, self.weights > 0), 1)
- self.out_op = tf.assign(self.y_train, y_normed, validate_shape=False)
-
- # Observation noise
- alpha = tf.nn.softplus(self.noise) + 1e-6
-
- # Covariance
- with tf.control_dependencies([self.input_op, self.input_w_op, self.out_op]):
- self.self_cov = (self.cov(self.x_in, self.x_in) *
- self.task_cov(self.weights, self.weights) +
- tf.eye(tf.shape(self.x_in)[0], dtype=tf.float64) * alpha)
-
- self.chol = tf.cholesky(self.self_cov)
- self.kinv = tf.cholesky_solve(self.chol, tf.eye(tf.shape(self.x_in)[0],
- dtype=tf.float64))
-
- self.input_inv = tf.Variable(
- tf.eye(self.hparams.batch_size, dtype=tf.float64),
- validate_shape=False,
- trainable=False)
- self.input_cov_op = tf.assign(self.input_inv, self.kinv,
- validate_shape=False)
-
- # Log determinant by taking the singular values along the diagonal
- # of self.chol
- with tf.control_dependencies([self.input_cov_op]):
- logdet = 2.0 * tf.reduce_sum(tf.log(tf.diag_part(self.chol) + 1e-16))
-
- # Log Marginal likelihood
- self.marginal_ll = -tf.reduce_sum(-0.5 * tf.matmul(
- tf.transpose(y_normed), tf.matmul(self.kinv, y_normed)) - 0.5 * logdet -
- 0.5 * self.n * np.log(2 * np.pi))
-
- zero = tf.cast(0., dtype=tf.float64)
- one = tf.cast(1., dtype=tf.float64)
- standard_normal = tfd.Normal(loc=zero, scale=one)
-
- # Loss is marginal likelihood and priors
- self.loss = tf.reduce_sum(
- self.marginal_ll -
- (standard_normal.log_prob(self.amplitude) +
- standard_normal.log_prob(tf.exp(self.noise)) +
- standard_normal.log_prob(self.amplitude_linear) +
- tfd.Normal(loc=zero, scale=one * 10.).log_prob(
- self.task_vectors))
- )
-
- # Optimizer for hyperparameters
- optimizer = tf.train.AdamOptimizer(learning_rate=self.hparams.lr)
- vars_to_optimize = [
- self.amplitude, self.length_scales, self.length_scales_lin,
- self.amplitude_linear, self.noise, self.input_mean
- ]
-
- if self.learn_embeddings:
- vars_to_optimize.append(self.task_vectors)
- grads = optimizer.compute_gradients(self.loss, vars_to_optimize)
- self.train_op = optimizer.apply_gradients(grads,
- global_step=self.global_step)
-
- # Predictions for test data
- self.y_mean, self.y_pred = self.posterior_mean_and_sample(self.x)
-
- # create tensorboard metrics
- self.create_summaries()
- self.summary_writer = tf.summary.FileWriter("{}/graph_{}".format(
- FLAGS.logdir, self.name), self.sess.graph)
- self.check = tf.add_check_numerics_ops()
-
- def posterior_mean_and_sample(self, candidates):
- """Draw samples for test predictions.
-
- Given a Tensor of 'candidates' inputs, returns samples from the posterior
- and the posterior mean prediction for those inputs.
-
- Args:
- candidates: A (num-examples x num-dims) Tensor containing the inputs for
- which to return predictions.
- Returns:
- y_mean: The posterior mean prediction given these inputs
- y_sample: A sample from the posterior of the outputs given these inputs
- """
- # Cross-covariance for test predictions
- w = tf.identity(self.weights_train)
- inds = tf.squeeze(
- tf.reshape(
- tf.tile(
- tf.reshape(tf.range(self.n_out), (self.n_out, 1)),
- (1, tf.shape(candidates)[0])), (-1, 1)))
-
- cross_cov = self.cov(tf.tile(candidates, [self.n_out, 1]), self.x_train)
- cross_task_cov = self.task_cov(tf.one_hot(inds, self.n_out), w)
- cross_cov *= cross_task_cov
-
- # Test mean prediction
- y_mean = tf.matmul(cross_cov, tf.matmul(self.input_inv, self.y_train))
-
- # Test sample predictions
- # Note this can be done much more efficiently using Kronecker products
- # if all tasks are fully observed (which we won't assume)
- test_cov = (
- self.cov(tf.tile(candidates, [self.n_out, 1]),
- tf.tile(candidates, [self.n_out, 1])) *
- self.task_cov(tf.one_hot(inds, self.n_out),
- tf.one_hot(inds, self.n_out)) -
- tf.matmul(cross_cov,
- tf.matmul(self.input_inv,
- tf.transpose(cross_cov))))
-
- # Get the matrix square root through an SVD for drawing samples
- # This seems more numerically stable than the Cholesky
- s, _, v = tf.svd(test_cov, full_matrices=True)
- test_sqrt = tf.matmul(v, tf.matmul(tf.diag(s), tf.transpose(v)))
-
- y_sample = (
- tf.matmul(
- test_sqrt,
- tf.random_normal([tf.shape(test_sqrt)[0], 1], dtype=tf.float64)) +
- y_mean)
-
- y_sample = (
- tf.transpose(tf.reshape(y_sample,
- (self.n_out, -1))) * self.input_std +
- self.input_mean)
-
- return y_mean, y_sample
-
- def create_summaries(self):
- with self.graph.as_default():
- tf.summary.scalar("loss", self.loss)
- tf.summary.scalar("log_noise", self.noise)
- tf.summary.scalar("log_amp", self.amplitude)
- tf.summary.scalar("log_amp_lin", self.amplitude_linear)
- tf.summary.histogram("length_scales", self.length_scales)
- tf.summary.histogram("length_scales_lin", self.length_scales_lin)
- self.summary_op = tf.summary.merge_all()
-
- def train(self, data, num_steps):
- """Trains the GP for num_steps, using the data in 'data'.
-
- Args:
- data: ContextualDataset object that provides the data.
- num_steps: Number of minibatches to train the network for.
- """
-
- logging.info("Training %s for %d steps...", self.name, num_steps)
- for step in range(num_steps):
- numpts = min(data.num_points(None), self.max_num_points)
- if numpts >= self.max_num_points and self.keep_fixed_after_max_obs:
- x = data.contexts[:numpts, :]
- y = data.rewards[:numpts, :]
- weights = np.zeros((x.shape[0], self.n_out))
- for i, val in enumerate(data.actions[:numpts]):
- weights[i, val] = 1.0
- else:
- x, y, weights = data.get_batch_with_weights(numpts)
-
- ops = [
- self.global_step, self.summary_op, self.loss, self.noise,
- self.amplitude, self.amplitude_linear, self.length_scales,
- self.length_scales_lin, self.input_cov_op, self.input_op, self.var_op,
- self.input_w_op, self.out_op, self.train_op
- ]
-
- res = self.sess.run(ops,
- feed_dict={self.x: x,
- self.x_in: x,
- self.y: y,
- self.weights: weights,
- self.n: numpts,
- })
-
- if step % self._freq_summary == 0:
- if self._show_training:
- logging.info("step: %d, loss: %g noise: %f amp: %f amp_lin: %f",
- step, res[2], res[3], res[4], res[5])
- summary = res[1]
- global_step = res[0]
- self.summary_writer.add_summary(summary, global_step=global_step)
diff --git a/research/deep_contextual_bandits/bandits/algorithms/neural_bandit_model.py b/research/deep_contextual_bandits/bandits/algorithms/neural_bandit_model.py
deleted file mode 100644
index 99d7cd4dc8e2c35571f82bbb79ea1564a148ff5d..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/neural_bandit_model.py
+++ /dev/null
@@ -1,220 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Define a family of neural network architectures for bandits.
-
-The network accepts different type of optimizers that could lead to different
-approximations of the posterior distribution or simply to point estimates.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from absl import flags
-from bandits.core.bayesian_nn import BayesianNN
-
-FLAGS = flags.FLAGS
-
-
-class NeuralBanditModel(BayesianNN):
- """Implements a neural network for bandit problems."""
-
- def __init__(self, optimizer, hparams, name):
- """Saves hyper-params and builds the Tensorflow graph."""
-
- self.opt_name = optimizer
- self.name = name
- self.hparams = hparams
- self.verbose = getattr(self.hparams, "verbose", True)
- self.times_trained = 0
- self.build_model()
-
- def build_layer(self, x, num_units):
- """Builds a layer with input x; dropout and layer norm if specified."""
-
- init_s = self.hparams.init_scale
-
- layer_n = getattr(self.hparams, "layer_norm", False)
- dropout = getattr(self.hparams, "use_dropout", False)
-
- nn = tf.contrib.layers.fully_connected(
- x,
- num_units,
- activation_fn=self.hparams.activation,
- normalizer_fn=None if not layer_n else tf.contrib.layers.layer_norm,
- normalizer_params={},
- weights_initializer=tf.random_uniform_initializer(-init_s, init_s)
- )
-
- if dropout:
- nn = tf.nn.dropout(nn, self.hparams.keep_prob)
-
- return nn
-
- def forward_pass(self):
-
- init_s = self.hparams.init_scale
-
- scope_name = "prediction_{}".format(self.name)
- with tf.variable_scope(scope_name, reuse=tf.AUTO_REUSE):
- nn = self.x
- for num_units in self.hparams.layer_sizes:
- if num_units > 0:
- nn = self.build_layer(nn, num_units)
-
- y_pred = tf.layers.dense(
- nn,
- self.hparams.num_actions,
- kernel_initializer=tf.random_uniform_initializer(-init_s, init_s))
-
- return nn, y_pred
-
- def build_model(self):
- """Defines the actual NN model with fully connected layers.
-
- The loss is computed for partial feedback settings (bandits), so only
- the observed outcome is backpropagated (see weighted loss).
- Selects the optimizer and, finally, it also initializes the graph.
- """
-
- # create and store the graph corresponding to the BNN instance
- self.graph = tf.Graph()
-
- with self.graph.as_default():
-
- # create and store a new session for the graph
- self.sess = tf.Session()
-
- with tf.name_scope(self.name):
-
- self.global_step = tf.train.get_or_create_global_step()
-
- # context
- self.x = tf.placeholder(
- shape=[None, self.hparams.context_dim],
- dtype=tf.float32,
- name="{}_x".format(self.name))
-
- # reward vector
- self.y = tf.placeholder(
- shape=[None, self.hparams.num_actions],
- dtype=tf.float32,
- name="{}_y".format(self.name))
-
- # weights (1 for selected action, 0 otherwise)
- self.weights = tf.placeholder(
- shape=[None, self.hparams.num_actions],
- dtype=tf.float32,
- name="{}_w".format(self.name))
-
- # with tf.variable_scope("prediction_{}".format(self.name)):
- self.nn, self.y_pred = self.forward_pass()
- self.loss = tf.squared_difference(self.y_pred, self.y)
- self.weighted_loss = tf.multiply(self.weights, self.loss)
- self.cost = tf.reduce_sum(self.weighted_loss) / self.hparams.batch_size
-
- if self.hparams.activate_decay:
- self.lr = tf.train.inverse_time_decay(
- self.hparams.initial_lr, self.global_step,
- 1, self.hparams.lr_decay_rate)
- else:
- self.lr = tf.Variable(self.hparams.initial_lr, trainable=False)
-
- # create tensorboard metrics
- self.create_summaries()
- self.summary_writer = tf.summary.FileWriter(
- "{}/graph_{}".format(FLAGS.logdir, self.name), self.sess.graph)
-
- tvars = tf.trainable_variables()
- grads, _ = tf.clip_by_global_norm(
- tf.gradients(self.cost, tvars), self.hparams.max_grad_norm)
-
- self.optimizer = self.select_optimizer()
-
- self.train_op = self.optimizer.apply_gradients(
- zip(grads, tvars), global_step=self.global_step)
-
- self.init = tf.global_variables_initializer()
-
- self.initialize_graph()
-
- def initialize_graph(self):
- """Initializes all variables."""
-
- with self.graph.as_default():
- if self.verbose:
- print("Initializing model {}.".format(self.name))
- self.sess.run(self.init)
-
- def assign_lr(self):
- """Resets the learning rate in dynamic schedules for subsequent trainings.
-
- In bandits settings, we do expand our dataset over time. Then, we need to
- re-train the network with the new data. The algorithms that do not keep
- the step constant, can reset it at the start of each *training* process.
- """
-
- decay_steps = 1
- if self.hparams.activate_decay:
- current_gs = self.sess.run(self.global_step)
- with self.graph.as_default():
- self.lr = tf.train.inverse_time_decay(self.hparams.initial_lr,
- self.global_step - current_gs,
- decay_steps,
- self.hparams.lr_decay_rate)
-
- def select_optimizer(self):
- """Selects optimizer. To be extended (SGLD, KFAC, etc)."""
- return tf.train.RMSPropOptimizer(self.lr)
-
- def create_summaries(self):
- """Defines summaries including mean loss, learning rate, and global step."""
-
- with self.graph.as_default():
- with tf.name_scope(self.name + "_summaries"):
- tf.summary.scalar("cost", self.cost)
- tf.summary.scalar("lr", self.lr)
- tf.summary.scalar("global_step", self.global_step)
- self.summary_op = tf.summary.merge_all()
-
- def train(self, data, num_steps):
- """Trains the network for num_steps, using the provided data.
-
- Args:
- data: ContextualDataset object that provides the data.
- num_steps: Number of minibatches to train the network for.
- """
-
- if self.verbose:
- print("Training {} for {} steps...".format(self.name, num_steps))
-
- with self.graph.as_default():
-
- for step in range(num_steps):
- x, y, w = data.get_batch_with_weights(self.hparams.batch_size)
- _, cost, summary, lr = self.sess.run(
- [self.train_op, self.cost, self.summary_op, self.lr],
- feed_dict={self.x: x, self.y: y, self.weights: w})
-
- if step % self.hparams.freq_summary == 0:
- if self.hparams.show_training:
- print("{} | step: {}, lr: {}, loss: {}".format(
- self.name, step, lr, cost))
- self.summary_writer.add_summary(summary, step)
-
- self.times_trained += 1
diff --git a/research/deep_contextual_bandits/bandits/algorithms/neural_linear_sampling.py b/research/deep_contextual_bandits/bandits/algorithms/neural_linear_sampling.py
deleted file mode 100644
index 43fc551614b49ad34538aa64090bcda5f823a60f..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/neural_linear_sampling.py
+++ /dev/null
@@ -1,180 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Thompson Sampling with linear posterior over a learnt deep representation."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-from scipy.stats import invgamma
-
-from bandits.core.bandit_algorithm import BanditAlgorithm
-from bandits.core.contextual_dataset import ContextualDataset
-from bandits.algorithms.neural_bandit_model import NeuralBanditModel
-
-
-class NeuralLinearPosteriorSampling(BanditAlgorithm):
- """Full Bayesian linear regression on the last layer of a deep neural net."""
-
- def __init__(self, name, hparams, optimizer='RMS'):
-
- self.name = name
- self.hparams = hparams
- self.latent_dim = self.hparams.layer_sizes[-1]
-
- # Gaussian prior for each beta_i
- self._lambda_prior = self.hparams.lambda_prior
-
- self.mu = [
- np.zeros(self.latent_dim)
- for _ in range(self.hparams.num_actions)
- ]
-
- self.cov = [(1.0 / self.lambda_prior) * np.eye(self.latent_dim)
- for _ in range(self.hparams.num_actions)]
-
- self.precision = [
- self.lambda_prior * np.eye(self.latent_dim)
- for _ in range(self.hparams.num_actions)
- ]
-
- # Inverse Gamma prior for each sigma2_i
- self._a0 = self.hparams.a0
- self._b0 = self.hparams.b0
-
- self.a = [self._a0 for _ in range(self.hparams.num_actions)]
- self.b = [self._b0 for _ in range(self.hparams.num_actions)]
-
- # Regression and NN Update Frequency
- self.update_freq_lr = hparams.training_freq
- self.update_freq_nn = hparams.training_freq_network
-
- self.t = 0
- self.optimizer_n = optimizer
-
- self.num_epochs = hparams.training_epochs
- self.data_h = ContextualDataset(hparams.context_dim,
- hparams.num_actions,
- intercept=False)
- self.latent_h = ContextualDataset(self.latent_dim,
- hparams.num_actions,
- intercept=False)
- self.bnn = NeuralBanditModel(optimizer, hparams, '{}-bnn'.format(name))
-
- def action(self, context):
- """Samples beta's from posterior, and chooses best action accordingly."""
-
- # Round robin until each action has been selected "initial_pulls" times
- if self.t < self.hparams.num_actions * self.hparams.initial_pulls:
- return self.t % self.hparams.num_actions
-
- # Sample sigma2, and beta conditional on sigma2
- sigma2_s = [
- self.b[i] * invgamma.rvs(self.a[i])
- for i in range(self.hparams.num_actions)
- ]
-
- try:
- beta_s = [
- np.random.multivariate_normal(self.mu[i], sigma2_s[i] * self.cov[i])
- for i in range(self.hparams.num_actions)
- ]
- except np.linalg.LinAlgError as e:
- # Sampling could fail if covariance is not positive definite
- print('Exception when sampling for {}.'.format(self.name))
- print('Details: {} | {}.'.format(e.message, e.args))
- d = self.latent_dim
- beta_s = [
- np.random.multivariate_normal(np.zeros((d)), np.eye(d))
- for i in range(self.hparams.num_actions)
- ]
-
- # Compute last-layer representation for the current context
- with self.bnn.graph.as_default():
- c = context.reshape((1, self.hparams.context_dim))
- z_context = self.bnn.sess.run(self.bnn.nn, feed_dict={self.bnn.x: c})
-
- # Apply Thompson Sampling to last-layer representation
- vals = [
- np.dot(beta_s[i], z_context.T) for i in range(self.hparams.num_actions)
- ]
- return np.argmax(vals)
-
- def update(self, context, action, reward):
- """Updates the posterior using linear bayesian regression formula."""
-
- self.t += 1
- self.data_h.add(context, action, reward)
- c = context.reshape((1, self.hparams.context_dim))
- z_context = self.bnn.sess.run(self.bnn.nn, feed_dict={self.bnn.x: c})
- self.latent_h.add(z_context, action, reward)
-
- # Retrain the network on the original data (data_h)
- if self.t % self.update_freq_nn == 0:
-
- if self.hparams.reset_lr:
- self.bnn.assign_lr()
- self.bnn.train(self.data_h, self.num_epochs)
-
- # Update the latent representation of every datapoint collected so far
- new_z = self.bnn.sess.run(self.bnn.nn,
- feed_dict={self.bnn.x: self.data_h.contexts})
- self.latent_h.replace_data(contexts=new_z)
-
- # Update the Bayesian Linear Regression
- if self.t % self.update_freq_lr == 0:
-
- # Find all the actions to update
- actions_to_update = self.latent_h.actions[:-self.update_freq_lr]
-
- for action_v in np.unique(actions_to_update):
-
- # Update action posterior with formulas: \beta | z,y ~ N(mu_q, cov_q)
- z, y = self.latent_h.get_data(action_v)
-
- # The algorithm could be improved with sequential formulas (cheaper)
- s = np.dot(z.T, z)
-
- # Some terms are removed as we assume prior mu_0 = 0.
- precision_a = s + self.lambda_prior * np.eye(self.latent_dim)
- cov_a = np.linalg.inv(precision_a)
- mu_a = np.dot(cov_a, np.dot(z.T, y))
-
- # Inverse Gamma posterior update
- a_post = self.a0 + z.shape[0] / 2.0
- b_upd = 0.5 * np.dot(y.T, y)
- b_upd -= 0.5 * np.dot(mu_a.T, np.dot(precision_a, mu_a))
- b_post = self.b0 + b_upd
-
- # Store new posterior distributions
- self.mu[action_v] = mu_a
- self.cov[action_v] = cov_a
- self.precision[action_v] = precision_a
- self.a[action_v] = a_post
- self.b[action_v] = b_post
-
- @property
- def a0(self):
- return self._a0
-
- @property
- def b0(self):
- return self._b0
-
- @property
- def lambda_prior(self):
- return self._lambda_prior
diff --git a/research/deep_contextual_bandits/bandits/algorithms/parameter_noise_sampling.py b/research/deep_contextual_bandits/bandits/algorithms/parameter_noise_sampling.py
deleted file mode 100644
index 19944ad577372b6971f03f1117fc33d5a2a276b1..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/parameter_noise_sampling.py
+++ /dev/null
@@ -1,187 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contextual algorithm based on Thompson Sampling + direct noise injection."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-from scipy.special import logsumexp
-import tensorflow as tf
-
-from absl import flags
-
-from bandits.core.bandit_algorithm import BanditAlgorithm
-from bandits.core.contextual_dataset import ContextualDataset
-from bandits.algorithms.neural_bandit_model import NeuralBanditModel
-
-FLAGS = flags.FLAGS
-
-
-class ParameterNoiseSampling(BanditAlgorithm):
- """Parameter Noise Sampling algorithm based on adding noise to net params.
-
- Described in https://arxiv.org/abs/1706.01905
- """
-
- def __init__(self, name, hparams):
- """Creates the algorithm, and sets up the adaptive Gaussian noise."""
-
- self.name = name
- self.hparams = hparams
- self.verbose = getattr(self.hparams, 'verbose', True)
- self.noise_std = getattr(self.hparams, 'noise_std', 0.005)
- self.eps = getattr(self.hparams, 'eps', 0.05)
- self.d_samples = getattr(self.hparams, 'd_samples', 300)
- self.optimizer = getattr(self.hparams, 'optimizer', 'RMS')
-
- # keep track of noise heuristic statistics
- self.std_h = [self.noise_std]
- self.eps_h = [self.eps]
- self.kl_h = []
- self.t = 0
-
- self.freq_update = hparams.training_freq
- self.num_epochs = hparams.training_epochs
-
- self.data_h = ContextualDataset(hparams.context_dim, hparams.num_actions,
- hparams.buffer_s)
- self.bnn = NeuralBanditModel(self.optimizer, hparams, '{}-bnn'.format(name))
-
- with self.bnn.graph.as_default():
-
- # noise-injection std placeholder
- self.bnn.noise_std_ph = tf.placeholder(tf.float32, shape=())
-
- # create noise corruption op; adds noise to all weights
- tvars = tf.trainable_variables()
- self.bnn.noisy_grads = [
- tf.random_normal(v.get_shape(), 0, self.bnn.noise_std_ph)
- for v in tvars
- ]
-
- # add noise to all params, then compute prediction, then subtract.
- with tf.control_dependencies(self.bnn.noisy_grads):
- self.bnn.noise_add_ops = [
- tvars[i].assign_add(n) for i, n in enumerate(self.bnn.noisy_grads)
- ]
- with tf.control_dependencies(self.bnn.noise_add_ops):
- # we force the prediction for 'y' to be recomputed after adding noise
- self.bnn.noisy_nn, self.bnn.noisy_pred_val = self.bnn.forward_pass()
-
- self.bnn.noisy_pred = tf.identity(self.bnn.noisy_pred_val)
- with tf.control_dependencies([tf.identity(self.bnn.noisy_pred)]):
- self.bnn.noise_sub_ops = [
- tvars[i].assign_add(-n)
- for i, n in enumerate(self.bnn.noisy_grads)
- ]
-
- def action(self, context):
- """Selects action based on Thompson Sampling *after* adding noise."""
-
- if self.t < self.hparams.num_actions * self.hparams.initial_pulls:
- # round robin until each action has been taken "initial_pulls" times
- return self.t % self.hparams.num_actions
-
- with self.bnn.graph.as_default():
- # run noise prediction op to choose action, and subtract noise op after.
- c = context.reshape((1, self.hparams.context_dim))
- output, _ = self.bnn.sess.run(
- [self.bnn.noisy_pred, self.bnn.noise_sub_ops],
- feed_dict={self.bnn.x: c,
- self.bnn.noise_std_ph: self.noise_std})
- return np.argmax(output)
-
- def update(self, context, action, reward):
- """Updates the data buffer, and re-trains the BNN and noise level."""
-
- self.t += 1
- self.data_h.add(context, action, reward)
-
- if self.t % self.freq_update == 0:
- self.bnn.train(self.data_h, self.num_epochs)
- self.update_noise()
-
- def update_noise(self):
- """Increase noise if distance btw original and corrupted distrib small."""
-
- kl = self.compute_distance()
- delta = -np.log1p(- self.eps + self.eps / self.hparams.num_actions)
-
- if kl < delta:
- self.noise_std *= 1.01
- else:
- self.noise_std /= 1.01
-
- self.eps *= 0.99
-
- if self.verbose:
- print('Update eps={} | kl={} | std={} | delta={} | increase={}.'.format(
- self.eps, kl, self.noise_std, delta, kl < delta))
-
- # store noise-injection statistics for inspection: std, KL, eps.
- self.std_h.append(self.noise_std)
- self.kl_h.append(kl)
- self.eps_h.append(self.eps)
-
- def compute_distance(self):
- """Computes empirical KL for original and corrupted output distributions."""
-
- random_inputs, _ = self.data_h.get_batch(self.d_samples)
- y_model = self.bnn.sess.run(
- self.bnn.y_pred,
- feed_dict={
- self.bnn.x: random_inputs,
- self.bnn.noise_std_ph: self.noise_std
- })
- y_noisy, _ = self.bnn.sess.run(
- [self.bnn.noisy_pred, self.bnn.noise_sub_ops],
- feed_dict={
- self.bnn.x: random_inputs,
- self.bnn.noise_std_ph: self.noise_std
- })
-
- if self.verbose:
- # display how often original & perturbed models propose different actions
- s = np.sum([np.argmax(y_model[i, :]) == np.argmax(y_noisy[i, :])
- for i in range(y_model.shape[0])])
- print('{} | % of agreement btw original / corrupted actions: {}.'.format(
- self.name, s / self.d_samples))
-
- kl = self.compute_kl_with_logits(y_model, y_noisy)
- return kl
-
- def compute_kl_with_logits(self, logits1, logits2):
- """Computes KL from logits samples from two distributions."""
-
- def exp_times_diff(a, b):
- return np.multiply(np.exp(a), a - b)
-
- logsumexp1 = logsumexp(logits1, axis=1)
- logsumexp2 = logsumexp(logits2, axis=1)
- logsumexp_diff = logsumexp2 - logsumexp1
-
- exp_diff = exp_times_diff(logits1, logits2)
- exp_diff = np.sum(exp_diff, axis=1)
-
- inv_exp_sum = np.sum(np.exp(logits1), axis=1)
- term1 = np.divide(exp_diff, inv_exp_sum)
-
- kl = term1 + logsumexp_diff
- kl = np.maximum(kl, 0.0)
- kl = np.nan_to_num(kl)
- return np.mean(kl)
diff --git a/research/deep_contextual_bandits/bandits/algorithms/posterior_bnn_sampling.py b/research/deep_contextual_bandits/bandits/algorithms/posterior_bnn_sampling.py
deleted file mode 100644
index 0f0c5d365a3a3e48006fe6b4e7e47ab73ea756cf..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/posterior_bnn_sampling.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contextual bandit algorithm based on Thompson Sampling and a Bayesian NN."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from bandits.core.bandit_algorithm import BanditAlgorithm
-from bandits.algorithms.bb_alpha_divergence_model import BBAlphaDivergence
-from bandits.algorithms.bf_variational_neural_bandit_model import BfVariationalNeuralBanditModel
-from bandits.core.contextual_dataset import ContextualDataset
-from bandits.algorithms.multitask_gp import MultitaskGP
-from bandits.algorithms.neural_bandit_model import NeuralBanditModel
-from bandits.algorithms.variational_neural_bandit_model import VariationalNeuralBanditModel
-
-
-class PosteriorBNNSampling(BanditAlgorithm):
- """Posterior Sampling algorithm based on a Bayesian neural network."""
-
- def __init__(self, name, hparams, bnn_model='RMSProp'):
- """Creates a PosteriorBNNSampling object based on a specific optimizer.
-
- The algorithm has two basic tools: an Approx BNN and a Contextual Dataset.
- The Bayesian Network keeps the posterior based on the optimizer iterations.
-
- Args:
- name: Name of the algorithm.
- hparams: Hyper-parameters of the algorithm.
- bnn_model: Type of BNN. By default RMSProp (point estimate).
- """
-
- self.name = name
- self.hparams = hparams
- self.optimizer_n = hparams.optimizer
-
- self.training_freq = hparams.training_freq
- self.training_epochs = hparams.training_epochs
- self.t = 0
- self.data_h = ContextualDataset(hparams.context_dim, hparams.num_actions,
- hparams.buffer_s)
-
- # to be extended with more BNNs (BB alpha-div, GPs, SGFS, constSGD...)
- bnn_name = '{}-bnn'.format(name)
- if bnn_model == 'Variational':
- self.bnn = VariationalNeuralBanditModel(hparams, bnn_name)
- elif bnn_model == 'AlphaDiv':
- self.bnn = BBAlphaDivergence(hparams, bnn_name)
- elif bnn_model == 'Variational_BF':
- self.bnn = BfVariationalNeuralBanditModel(hparams, bnn_name)
- elif bnn_model == 'GP':
- self.bnn = MultitaskGP(hparams)
- else:
- self.bnn = NeuralBanditModel(self.optimizer_n, hparams, bnn_name)
-
- def action(self, context):
- """Selects action for context based on Thompson Sampling using the BNN."""
-
- if self.t < self.hparams.num_actions * self.hparams.initial_pulls:
- # round robin until each action has been taken "initial_pulls" times
- return self.t % self.hparams.num_actions
-
- with self.bnn.graph.as_default():
- c = context.reshape((1, self.hparams.context_dim))
- output = self.bnn.sess.run(self.bnn.y_pred, feed_dict={self.bnn.x: c})
- return np.argmax(output)
-
- def update(self, context, action, reward):
- """Updates data buffer, and re-trains the BNN every training_freq steps."""
-
- self.t += 1
- self.data_h.add(context, action, reward)
-
- if self.t % self.training_freq == 0:
- if self.hparams.reset_lr:
- self.bnn.assign_lr()
- self.bnn.train(self.data_h, self.training_epochs)
diff --git a/research/deep_contextual_bandits/bandits/algorithms/uniform_sampling.py b/research/deep_contextual_bandits/bandits/algorithms/uniform_sampling.py
deleted file mode 100644
index 15c073fbe89da4e9aef595c8772ceaa3667e1952..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/uniform_sampling.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contextual bandit algorithm that selects an action uniformly at random."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from bandits.core.bandit_algorithm import BanditAlgorithm
-
-
-class UniformSampling(BanditAlgorithm):
- """Defines a baseline; returns one action uniformly at random."""
-
- def __init__(self, name, hparams):
- """Creates a UniformSampling object.
-
- Args:
- name: Name of the algorithm.
- hparams: Hyper-parameters, including the number of arms (num_actions).
- """
-
- self.name = name
- self.hparams = hparams
-
- def action(self, context):
- """Selects an action uniformly at random."""
- return np.random.choice(range(self.hparams.num_actions))
diff --git a/research/deep_contextual_bandits/bandits/algorithms/variational_neural_bandit_model.py b/research/deep_contextual_bandits/bandits/algorithms/variational_neural_bandit_model.py
deleted file mode 100644
index 7700c08ba9f7861aac522ba6da9f7371b5e203af..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/algorithms/variational_neural_bandit_model.py
+++ /dev/null
@@ -1,346 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Bayesian NN using factorized VI (Bayes By Backprop. Blundell et al. 2014).
-
-See https://arxiv.org/abs/1505.05424 for details.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-from absl import flags
-from bandits.core.bayesian_nn import BayesianNN
-
-FLAGS = flags.FLAGS
-
-
-def log_gaussian(x, mu, sigma, reduce_sum=True):
- """Returns log Gaussian pdf."""
- res = (-0.5 * np.log(2 * np.pi) - tf.log(sigma) - tf.square(x - mu) /
- (2 * tf.square(sigma)))
- if reduce_sum:
- return tf.reduce_sum(res)
- else:
- return res
-
-
-def analytic_kl(mu_1, sigma_1, mu_2, sigma_2):
- """KL for two Gaussian distributions with diagonal covariance matrix."""
- sigma_1_sq = tf.square(sigma_1)
- sigma_2_sq = tf.square(sigma_2)
-
- t1 = tf.square(mu_1 - mu_2) / (2. * sigma_2_sq)
- t2 = (sigma_1_sq/sigma_2_sq - 1. - tf.log(sigma_1_sq) + tf.log(sigma_2_sq))/2.
- return tf.reduce_sum(t1 + t2)
-
-
-class VariationalNeuralBanditModel(BayesianNN):
- """Implements an approximate Bayesian NN using Variational Inference."""
-
- def __init__(self, hparams, name="BBBNN"):
-
- self.name = name
- self.hparams = hparams
-
- self.n_in = self.hparams.context_dim
- self.n_out = self.hparams.num_actions
- self.layers = self.hparams.layer_sizes
- self.init_scale = self.hparams.init_scale
- self.f_num_points = None
- if "f_num_points" in hparams:
- self.f_num_points = self.hparams.f_num_points
-
- self.cleared_times_trained = self.hparams.cleared_times_trained
- self.initial_training_steps = self.hparams.initial_training_steps
- self.training_schedule = np.linspace(self.initial_training_steps,
- self.hparams.training_epochs,
- self.cleared_times_trained)
- self.verbose = getattr(self.hparams, "verbose", True)
-
- self.weights_m = {}
- self.weights_std = {}
- self.biases_m = {}
- self.biases_std = {}
-
- self.times_trained = 0
-
- if self.hparams.use_sigma_exp_transform:
- self.sigma_transform = tf.exp
- self.inverse_sigma_transform = np.log
- else:
- self.sigma_transform = tf.nn.softplus
- self.inverse_sigma_transform = lambda y: y + np.log(1. - np.exp(-y))
-
- # Whether to use the local reparameterization trick to compute the loss.
- # See details in https://arxiv.org/abs/1506.02557
- self.use_local_reparameterization = True
-
- self.build_graph()
-
- def build_mu_variable(self, shape):
- """Returns a mean variable initialized as N(0, 0.05)."""
- return tf.Variable(tf.random_normal(shape, 0.0, 0.05))
-
- def build_sigma_variable(self, shape, init=-5.):
- """Returns a sigma variable initialized as N(init, 0.05)."""
- # Initialize sigma to be very small initially to encourage MAP opt first
- return tf.Variable(tf.random_normal(shape, init, 0.05))
-
- def build_layer(self, input_x, input_x_local, shape,
- layer_id, activation_fn=tf.nn.relu):
- """Builds a variational layer, and computes KL term.
-
- Args:
- input_x: Input to the variational layer.
- input_x_local: Input when the local reparameterization trick was applied.
- shape: [number_inputs, number_outputs] for the layer.
- layer_id: Number of layer in the architecture.
- activation_fn: Activation function to apply.
-
- Returns:
- output_h: Output of the variational layer.
- output_h_local: Output when local reparameterization trick was applied.
- neg_kl: Negative KL term for the layer.
- """
-
- w_mu = self.build_mu_variable(shape)
- w_sigma = self.sigma_transform(self.build_sigma_variable(shape))
- w_noise = tf.random_normal(shape)
- w = w_mu + w_sigma * w_noise
-
- b_mu = self.build_mu_variable([1, shape[1]])
- b_sigma = self.sigma_transform(self.build_sigma_variable([1, shape[1]]))
- b = b_mu
-
- # Store means and stds
- self.weights_m[layer_id] = w_mu
- self.weights_std[layer_id] = w_sigma
- self.biases_m[layer_id] = b_mu
- self.biases_std[layer_id] = b_sigma
-
- # Create outputs
- output_h = activation_fn(tf.matmul(input_x, w) + b)
-
- if self.use_local_reparameterization:
- # Use analytic KL divergence wrt the prior
- neg_kl = -analytic_kl(w_mu, w_sigma,
- 0., tf.to_float(np.sqrt(2./shape[0])))
- else:
- # Create empirical KL loss terms
- log_p = log_gaussian(w, 0., tf.to_float(np.sqrt(2./shape[0])))
- log_q = log_gaussian(w, tf.stop_gradient(w_mu), tf.stop_gradient(w_sigma))
- neg_kl = log_p - log_q
-
- # Apply local reparameterization trick: sample activations pre nonlinearity
- m_h = tf.matmul(input_x_local, w_mu) + b
- v_h = tf.matmul(tf.square(input_x_local), tf.square(w_sigma))
- output_h_local = m_h + tf.sqrt(v_h + 1e-6) * tf.random_normal(tf.shape(v_h))
- output_h_local = activation_fn(output_h_local)
-
- return output_h, output_h_local, neg_kl
-
- def build_action_noise(self):
- """Defines a model for additive noise per action, and its KL term."""
-
- # Define mean and std variables (log-normal dist) for each action.
- noise_sigma_mu = (self.build_mu_variable([1, self.n_out])
- + self.inverse_sigma_transform(self.hparams.noise_sigma))
- noise_sigma_sigma = self.sigma_transform(
- self.build_sigma_variable([1, self.n_out]))
-
- pre_noise_sigma = (noise_sigma_mu
- + tf.random_normal([1, self.n_out]) * noise_sigma_sigma)
- self.noise_sigma = self.sigma_transform(pre_noise_sigma)
-
- # Compute KL for additive noise sigma terms.
- if getattr(self.hparams, "infer_noise_sigma", False):
- neg_kl_term = log_gaussian(
- pre_noise_sigma,
- self.inverse_sigma_transform(self.hparams.noise_sigma),
- self.hparams.prior_sigma
- )
- neg_kl_term -= log_gaussian(pre_noise_sigma,
- noise_sigma_mu,
- noise_sigma_sigma)
- else:
- neg_kl_term = 0.
-
- return neg_kl_term
-
- def build_model(self, activation_fn=tf.nn.relu):
- """Defines the actual NN model with fully connected layers.
-
- The loss is computed for partial feedback settings (bandits), so only
- the observed outcome is backpropagated (see weighted loss).
- Selects the optimizer and, finally, it also initializes the graph.
-
- Args:
- activation_fn: the activation function used in the nn layers.
- """
-
- if self.verbose:
- print("Initializing model {}.".format(self.name))
- neg_kl_term, l_number = 0, 0
- use_local_reparameterization = self.use_local_reparameterization
-
- # Compute model additive noise for each action with log-normal distribution
- neg_kl_term += self.build_action_noise()
-
- # Build network.
- input_x = self.x
- input_local = self.x
- n_in = self.n_in
-
- for l_number, n_nodes in enumerate(self.layers):
- if n_nodes > 0:
- h, h_local, neg_kl = self.build_layer(input_x, input_local,
- [n_in, n_nodes], l_number)
-
- neg_kl_term += neg_kl
- input_x, input_local = h, h_local
- n_in = n_nodes
-
- # Create last linear layer
- h, h_local, neg_kl = self.build_layer(input_x, input_local,
- [n_in, self.n_out],
- l_number + 1,
- activation_fn=lambda x: x)
- neg_kl_term += neg_kl
-
- self.y_pred = h
- self.y_pred_local = h_local
-
- # Compute log likelihood (with learned or fixed noise level)
- if getattr(self.hparams, "infer_noise_sigma", False):
- log_likelihood = log_gaussian(
- self.y, self.y_pred_local, self.noise_sigma, reduce_sum=False)
- else:
- y_hat = self.y_pred_local if use_local_reparameterization else self.y_pred
- log_likelihood = log_gaussian(
- self.y, y_hat, self.hparams.noise_sigma, reduce_sum=False)
-
- # Only take into account observed outcomes (bandits setting)
- batch_size = tf.to_float(tf.shape(self.x)[0])
- weighted_log_likelihood = tf.reduce_sum(
- log_likelihood * self.weights) / batch_size
-
- # The objective is 1/n * (\sum_i log_like_i - KL); neg_kl_term estimates -KL
- elbo = weighted_log_likelihood + (neg_kl_term / self.n)
-
- self.loss = -elbo
- self.global_step = tf.train.get_or_create_global_step()
- self.train_op = tf.train.AdamOptimizer(self.hparams.initial_lr).minimize(
- self.loss, global_step=self.global_step)
-
- # Create tensorboard metrics
- self.create_summaries()
- self.summary_writer = tf.summary.FileWriter(
- "{}/graph_{}".format(FLAGS.logdir, self.name), self.sess.graph)
-
- def build_graph(self):
- """Defines graph, session, placeholders, and model.
-
- Placeholders are: n (size of the dataset), x and y (context and observed
- reward for each action), and weights (one-hot encoding of selected action
- for each context, i.e., only possibly non-zero element in each y).
- """
-
- self.graph = tf.Graph()
- with self.graph.as_default():
-
- self.sess = tf.Session()
-
- self.n = tf.placeholder(shape=[], dtype=tf.float32)
-
- self.x = tf.placeholder(shape=[None, self.n_in], dtype=tf.float32)
- self.y = tf.placeholder(shape=[None, self.n_out], dtype=tf.float32)
- self.weights = tf.placeholder(shape=[None, self.n_out], dtype=tf.float32)
-
- self.build_model()
- self.sess.run(tf.global_variables_initializer())
-
- def create_summaries(self):
- """Defines summaries including mean loss, and global step."""
-
- with self.graph.as_default():
- with tf.name_scope(self.name + "_summaries"):
- tf.summary.scalar("loss", self.loss)
- tf.summary.scalar("global_step", self.global_step)
- self.summary_op = tf.summary.merge_all()
-
- def assign_lr(self):
- """Resets the learning rate in dynamic schedules for subsequent trainings.
-
- In bandits settings, we do expand our dataset over time. Then, we need to
- re-train the network with the new data. The algorithms that do not keep
- the step constant, can reset it at the start of each *training* process.
- """
-
- decay_steps = 1
- if self.hparams.activate_decay:
- current_gs = self.sess.run(self.global_step)
- with self.graph.as_default():
- self.lr = tf.train.inverse_time_decay(self.hparams.initial_lr,
- self.global_step - current_gs,
- decay_steps,
- self.hparams.lr_decay_rate)
-
- def train(self, data, num_steps):
- """Trains the BNN for num_steps, using the data in 'data'.
-
- Args:
- data: ContextualDataset object that provides the data.
- num_steps: Number of minibatches to train the network for.
-
- Returns:
- losses: Loss history during training.
- """
-
- if self.times_trained < self.cleared_times_trained:
- num_steps = int(self.training_schedule[self.times_trained])
- self.times_trained += 1
-
- losses = []
-
- with self.graph.as_default():
-
- if self.verbose:
- print("Training {} for {} steps...".format(self.name, num_steps))
-
- for step in range(num_steps):
- x, y, weights = data.get_batch_with_weights(self.hparams.batch_size)
- _, summary, global_step, loss = self.sess.run(
- [self.train_op, self.summary_op, self.global_step, self.loss],
- feed_dict={
- self.x: x,
- self.y: y,
- self.weights: weights,
- self.n: data.num_points(self.f_num_points),
- })
-
- losses.append(loss)
-
- if step % self.hparams.freq_summary == 0:
- if self.hparams.show_training:
- print("{} | step: {}, loss: {}".format(
- self.name, global_step, loss))
- self.summary_writer.add_summary(summary, global_step)
-
- return losses
diff --git a/research/deep_contextual_bandits/bandits/core/bandit_algorithm.py b/research/deep_contextual_bandits/bandits/core/bandit_algorithm.py
deleted file mode 100644
index cae4e1676a865d538fa41936feb9118283b92a2c..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/core/bandit_algorithm.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Define the abstract class for contextual bandit algorithms."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-class BanditAlgorithm(object):
- """A bandit algorithm must be able to do two basic operations.
-
- 1. Choose an action given a context.
- 2. Update its internal model given a triple (context, played action, reward).
- """
-
- def action(self, context):
- pass
-
- def update(self, context, action, reward):
- pass
diff --git a/research/deep_contextual_bandits/bandits/core/bayesian_nn.py b/research/deep_contextual_bandits/bandits/core/bayesian_nn.py
deleted file mode 100644
index 310961591317f8c9ff958a5178e81e0422385baf..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/core/bayesian_nn.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Define the abstract class for Bayesian Neural Networks."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-class BayesianNN(object):
- """A Bayesian neural network keeps a distribution over neural nets."""
-
- def __init__(self, optimizer):
- pass
-
- def build_model(self):
- pass
-
- def train(self, data):
- pass
-
- def sample(self, steps):
- pass
diff --git a/research/deep_contextual_bandits/bandits/core/contextual_bandit.py b/research/deep_contextual_bandits/bandits/core/contextual_bandit.py
deleted file mode 100644
index 98467378953b9f3e38057be8a0068fdbc7b59a84..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/core/contextual_bandit.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Define a contextual bandit from which we can sample and compute rewards.
-
-We can feed the data, sample a context, its reward for a specific action, and
-also the optimal action for a given context.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-
-def run_contextual_bandit(context_dim, num_actions, dataset, algos):
- """Run a contextual bandit problem on a set of algorithms.
-
- Args:
- context_dim: Dimension of the context.
- num_actions: Number of available actions.
- dataset: Matrix where every row is a context + num_actions rewards.
- algos: List of algorithms to use in the contextual bandit instance.
-
- Returns:
- h_actions: Matrix with actions: size (num_context, num_algorithms).
- h_rewards: Matrix with rewards: size (num_context, num_algorithms).
- """
-
- num_contexts = dataset.shape[0]
-
- # Create contextual bandit
- cmab = ContextualBandit(context_dim, num_actions)
- cmab.feed_data(dataset)
-
- h_actions = np.empty((0, len(algos)), float)
- h_rewards = np.empty((0, len(algos)), float)
-
- # Run the contextual bandit process
- for i in range(num_contexts):
- context = cmab.context(i)
- actions = [a.action(context) for a in algos]
- rewards = [cmab.reward(i, action) for action in actions]
-
- for j, a in enumerate(algos):
- a.update(context, actions[j], rewards[j])
-
- h_actions = np.vstack((h_actions, np.array(actions)))
- h_rewards = np.vstack((h_rewards, np.array(rewards)))
-
- return h_actions, h_rewards
-
-
-class ContextualBandit(object):
- """Implements a Contextual Bandit with d-dimensional contexts and k arms."""
-
- def __init__(self, context_dim, num_actions):
- """Creates a contextual bandit object.
-
- Args:
- context_dim: Dimension of the contexts.
- num_actions: Number of arms for the multi-armed bandit.
- """
-
- self._context_dim = context_dim
- self._num_actions = num_actions
-
- def feed_data(self, data):
- """Feeds the data (contexts + rewards) to the bandit object.
-
- Args:
- data: Numpy array with shape [n, d+k], where n is the number of contexts,
- d is the dimension of each context, and k the number of arms (rewards).
-
- Raises:
- ValueError: when data dimensions do not correspond to the object values.
- """
-
- if data.shape[1] != self.context_dim + self.num_actions:
- raise ValueError('Data dimensions do not match.')
-
- self._number_contexts = data.shape[0]
- self.data = data
- self.order = range(self.number_contexts)
-
- def reset(self):
- """Randomly shuffle the order of the contexts to deliver."""
- self.order = np.random.permutation(self.number_contexts)
-
- def context(self, number):
- """Returns the number-th context."""
- return self.data[self.order[number]][:self.context_dim]
-
- def reward(self, number, action):
- """Returns the reward for the number-th context and action."""
- return self.data[self.order[number]][self.context_dim + action]
-
- def optimal(self, number):
- """Returns the optimal action (in hindsight) for the number-th context."""
- return np.argmax(self.data[self.order[number]][self.context_dim:])
-
- @property
- def context_dim(self):
- return self._context_dim
-
- @property
- def num_actions(self):
- return self._num_actions
-
- @property
- def number_contexts(self):
- return self._number_contexts
diff --git a/research/deep_contextual_bandits/bandits/core/contextual_dataset.py b/research/deep_contextual_bandits/bandits/core/contextual_dataset.py
deleted file mode 100644
index 9fae7629c7c2ee39ab6b98ddac73876b5fca421a..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/core/contextual_dataset.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Define a data buffer for contextual bandit algorithms."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-
-class ContextualDataset(object):
- """The buffer is able to append new data, and sample random minibatches."""
-
- def __init__(self, context_dim, num_actions, buffer_s=-1, intercept=False):
- """Creates a ContextualDataset object.
-
- The data is stored in attributes: contexts and rewards.
- The sequence of taken actions are stored in attribute actions.
-
- Args:
- context_dim: Dimension of the contexts.
- num_actions: Number of arms for the multi-armed bandit.
- buffer_s: Size of buffer for training. Only last buffer_s will be
- returned as minibatch. If buffer_s = -1, all data will be used.
- intercept: If True, it adds a constant (1.0) dimension to each context X,
- at the end.
- """
-
- self._context_dim = context_dim
- self._num_actions = num_actions
- self._contexts = None
- self._rewards = None
- self.actions = []
- self.buffer_s = buffer_s
- self.intercept = intercept
-
- def add(self, context, action, reward):
- """Adds a new triplet (context, action, reward) to the dataset.
-
- The reward for the actions that weren't played is assumed to be zero.
-
- Args:
- context: A d-dimensional vector with the context.
- action: Integer between 0 and k-1 representing the chosen arm.
- reward: Real number representing the reward for the (context, action).
- """
-
- if self.intercept:
- c = np.array(context[:])
- c = np.append(c, 1.0).reshape((1, self.context_dim + 1))
- else:
- c = np.array(context[:]).reshape((1, self.context_dim))
-
- if self.contexts is None:
- self.contexts = c
- else:
- self.contexts = np.vstack((self.contexts, c))
-
- r = np.zeros((1, self.num_actions))
- r[0, action] = reward
- if self.rewards is None:
- self.rewards = r
- else:
- self.rewards = np.vstack((self.rewards, r))
-
- self.actions.append(action)
-
- def replace_data(self, contexts=None, actions=None, rewards=None):
- if contexts is not None:
- self.contexts = contexts
- if actions is not None:
- self.actions = actions
- if rewards is not None:
- self.rewards = rewards
-
- def get_batch(self, batch_size):
- """Returns a random minibatch of (contexts, rewards) with batch_size."""
- n, _ = self.contexts.shape
- if self.buffer_s == -1:
- # use all the data
- ind = np.random.choice(range(n), batch_size)
- else:
- # use only buffer (last buffer_s observations)
- ind = np.random.choice(range(max(0, n - self.buffer_s), n), batch_size)
- return self.contexts[ind, :], self.rewards[ind, :]
-
- def get_data(self, action):
- """Returns all (context, reward) where the action was played."""
- n, _ = self.contexts.shape
- ind = np.array([i for i in range(n) if self.actions[i] == action])
- return self.contexts[ind, :], self.rewards[ind, action]
-
- def get_data_with_weights(self):
- """Returns all observations with one-hot weights for actions."""
- weights = np.zeros((self.contexts.shape[0], self.num_actions))
- a_ind = np.array([(i, val) for i, val in enumerate(self.actions)])
- weights[a_ind[:, 0], a_ind[:, 1]] = 1.0
- return self.contexts, self.rewards, weights
-
- def get_batch_with_weights(self, batch_size):
- """Returns a random mini-batch with one-hot weights for actions."""
- n, _ = self.contexts.shape
- if self.buffer_s == -1:
- # use all the data
- ind = np.random.choice(range(n), batch_size)
- else:
- # use only buffer (last buffer_s obs)
- ind = np.random.choice(range(max(0, n - self.buffer_s), n), batch_size)
-
- weights = np.zeros((batch_size, self.num_actions))
- sampled_actions = np.array(self.actions)[ind]
- a_ind = np.array([(i, val) for i, val in enumerate(sampled_actions)])
- weights[a_ind[:, 0], a_ind[:, 1]] = 1.0
- return self.contexts[ind, :], self.rewards[ind, :], weights
-
- def num_points(self, f=None):
- """Returns number of points in the buffer (after applying function f)."""
- if f is not None:
- return f(self.contexts.shape[0])
- return self.contexts.shape[0]
-
- @property
- def context_dim(self):
- return self._context_dim
-
- @property
- def num_actions(self):
- return self._num_actions
-
- @property
- def contexts(self):
- return self._contexts
-
- @contexts.setter
- def contexts(self, value):
- self._contexts = value
-
- @property
- def actions(self):
- return self._actions
-
- @actions.setter
- def actions(self, value):
- self._actions = value
-
- @property
- def rewards(self):
- return self._rewards
-
- @rewards.setter
- def rewards(self, value):
- self._rewards = value
diff --git a/research/deep_contextual_bandits/bandits/data/data_sampler.py b/research/deep_contextual_bandits/bandits/data/data_sampler.py
deleted file mode 100644
index 55d1bae383637485182a9524ba8a3cb37b76bd0d..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/data/data_sampler.py
+++ /dev/null
@@ -1,374 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Functions to create bandit problems from datasets."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import pandas as pd
-import tensorflow as tf
-
-
-def one_hot(df, cols):
- """Returns one-hot encoding of DataFrame df including columns in cols."""
- for col in cols:
- dummies = pd.get_dummies(df[col], prefix=col, drop_first=False)
- df = pd.concat([df, dummies], axis=1)
- df = df.drop(col, axis=1)
- return df
-
-
-def sample_mushroom_data(file_name,
- num_contexts,
- r_noeat=0,
- r_eat_safe=5,
- r_eat_poison_bad=-35,
- r_eat_poison_good=5,
- prob_poison_bad=0.5):
- """Samples bandit game from Mushroom UCI Dataset.
-
- Args:
- file_name: Route of file containing the original Mushroom UCI dataset.
- num_contexts: Number of points to sample, i.e. (context, action rewards).
- r_noeat: Reward for not eating a mushroom.
- r_eat_safe: Reward for eating a non-poisonous mushroom.
- r_eat_poison_bad: Reward for eating a poisonous mushroom if harmed.
- r_eat_poison_good: Reward for eating a poisonous mushroom if not harmed.
- prob_poison_bad: Probability of being harmed by eating a poisonous mushroom.
-
- Returns:
- dataset: Sampled matrix with n rows: (context, eat_reward, no_eat_reward).
- opt_vals: Vector of expected optimal (reward, action) for each context.
-
- We assume r_eat_safe > r_noeat, and r_eat_poison_good > r_eat_poison_bad.
- """
-
- # first two cols of df encode whether mushroom is edible or poisonous
- df = pd.read_csv(file_name, header=None)
- df = one_hot(df, df.columns)
- ind = np.random.choice(range(df.shape[0]), num_contexts, replace=True)
-
- contexts = df.iloc[ind, 2:]
- no_eat_reward = r_noeat * np.ones((num_contexts, 1))
- random_poison = np.random.choice(
- [r_eat_poison_bad, r_eat_poison_good],
- p=[prob_poison_bad, 1 - prob_poison_bad],
- size=num_contexts)
- eat_reward = r_eat_safe * df.iloc[ind, 0]
- eat_reward += np.multiply(random_poison, df.iloc[ind, 1])
- eat_reward = eat_reward.values.reshape((num_contexts, 1))
-
- # compute optimal expected reward and optimal actions
- exp_eat_poison_reward = r_eat_poison_bad * prob_poison_bad
- exp_eat_poison_reward += r_eat_poison_good * (1 - prob_poison_bad)
- opt_exp_reward = r_eat_safe * df.iloc[ind, 0] + max(
- r_noeat, exp_eat_poison_reward) * df.iloc[ind, 1]
-
- if r_noeat > exp_eat_poison_reward:
- # actions: no eat = 0 ; eat = 1
- opt_actions = df.iloc[ind, 0] # indicator of edible
- else:
- # should always eat (higher expected reward)
- opt_actions = np.ones((num_contexts, 1))
-
- opt_vals = (opt_exp_reward.values, opt_actions.values)
-
- return np.hstack((contexts, no_eat_reward, eat_reward)), opt_vals
-
-
-def sample_stock_data(file_name, context_dim, num_actions, num_contexts,
- sigma, shuffle_rows=True):
- """Samples linear bandit game from stock prices dataset.
-
- Args:
- file_name: Route of file containing the stock prices dataset.
- context_dim: Context dimension (i.e. vector with the price of each stock).
- num_actions: Number of actions (different linear portfolio strategies).
- num_contexts: Number of contexts to sample.
- sigma: Vector with additive noise levels for each action.
- shuffle_rows: If True, rows from original dataset are shuffled.
-
- Returns:
- dataset: Sampled matrix with rows: (context, reward_1, ..., reward_k).
- opt_vals: Vector of expected optimal (reward, action) for each context.
- """
-
- with tf.gfile.Open(file_name, 'r') as f:
- contexts = np.loadtxt(f, skiprows=1)
-
- if shuffle_rows:
- np.random.shuffle(contexts)
- contexts = contexts[:num_contexts, :]
-
- betas = np.random.uniform(-1, 1, (context_dim, num_actions))
- betas /= np.linalg.norm(betas, axis=0)
-
- mean_rewards = np.dot(contexts, betas)
- noise = np.random.normal(scale=sigma, size=mean_rewards.shape)
- rewards = mean_rewards + noise
-
- opt_actions = np.argmax(mean_rewards, axis=1)
- opt_rewards = [mean_rewards[i, a] for i, a in enumerate(opt_actions)]
- return np.hstack((contexts, rewards)), (np.array(opt_rewards), opt_actions)
-
-
-def sample_jester_data(file_name, context_dim, num_actions, num_contexts,
- shuffle_rows=True, shuffle_cols=False):
- """Samples bandit game from (user, joke) dense subset of Jester dataset.
-
- Args:
- file_name: Route of file containing the modified Jester dataset.
- context_dim: Context dimension (i.e. vector with some ratings from a user).
- num_actions: Number of actions (number of joke ratings to predict).
- num_contexts: Number of contexts to sample.
- shuffle_rows: If True, rows from original dataset are shuffled.
- shuffle_cols: Whether or not context/action jokes are randomly shuffled.
-
- Returns:
- dataset: Sampled matrix with rows: (context, rating_1, ..., rating_k).
- opt_vals: Vector of deterministic optimal (reward, action) for each context.
- """
-
- with tf.gfile.Open(file_name, 'rb') as f:
- dataset = np.load(f)
-
- if shuffle_cols:
- dataset = dataset[:, np.random.permutation(dataset.shape[1])]
- if shuffle_rows:
- np.random.shuffle(dataset)
- dataset = dataset[:num_contexts, :]
-
- assert context_dim + num_actions == dataset.shape[1], 'Wrong data dimensions.'
-
- opt_actions = np.argmax(dataset[:, context_dim:], axis=1)
- opt_rewards = np.array([dataset[i, context_dim + a]
- for i, a in enumerate(opt_actions)])
-
- return dataset, (opt_rewards, opt_actions)
-
-
-def sample_statlog_data(file_name, num_contexts, shuffle_rows=True,
- remove_underrepresented=False):
- """Returns bandit problem dataset based on the UCI statlog data.
-
- Args:
- file_name: Route of file containing the Statlog dataset.
- num_contexts: Number of contexts to sample.
- shuffle_rows: If True, rows from original dataset are shuffled.
- remove_underrepresented: If True, removes arms with very few rewards.
-
- Returns:
- dataset: Sampled matrix with rows: (context, action rewards).
- opt_vals: Vector of deterministic optimal (reward, action) for each context.
-
- https://archive.ics.uci.edu/ml/datasets/Statlog+(Shuttle)
- """
-
- with tf.gfile.Open(file_name, 'r') as f:
- data = np.loadtxt(f)
-
- num_actions = 7 # some of the actions are very rarely optimal.
-
- # Shuffle data
- if shuffle_rows:
- np.random.shuffle(data)
- data = data[:num_contexts, :]
-
- # Last column is label, rest are features
- contexts = data[:, :-1]
- labels = data[:, -1].astype(int) - 1 # convert to 0 based index
-
- if remove_underrepresented:
- contexts, labels = remove_underrepresented_classes(contexts, labels)
-
- return classification_to_bandit_problem(contexts, labels, num_actions)
-
-
-def sample_adult_data(file_name, num_contexts, shuffle_rows=True,
- remove_underrepresented=False):
- """Returns bandit problem dataset based on the UCI adult data.
-
- Args:
- file_name: Route of file containing the Adult dataset.
- num_contexts: Number of contexts to sample.
- shuffle_rows: If True, rows from original dataset are shuffled.
- remove_underrepresented: If True, removes arms with very few rewards.
-
- Returns:
- dataset: Sampled matrix with rows: (context, action rewards).
- opt_vals: Vector of deterministic optimal (reward, action) for each context.
-
- Preprocessing:
- * drop rows with missing values
- * convert categorical variables to 1 hot encoding
-
- https://archive.ics.uci.edu/ml/datasets/census+income
- """
- with tf.gfile.Open(file_name, 'r') as f:
- df = pd.read_csv(f, header=None,
- na_values=[' ?']).dropna()
-
- num_actions = 14
-
- if shuffle_rows:
- df = df.sample(frac=1)
- df = df.iloc[:num_contexts, :]
-
- labels = df[6].astype('category').cat.codes.as_matrix()
- df = df.drop([6], axis=1)
-
- # Convert categorical variables to 1 hot encoding
- cols_to_transform = [1, 3, 5, 7, 8, 9, 13, 14]
- df = pd.get_dummies(df, columns=cols_to_transform)
-
- if remove_underrepresented:
- df, labels = remove_underrepresented_classes(df, labels)
- contexts = df.as_matrix()
-
- return classification_to_bandit_problem(contexts, labels, num_actions)
-
-
-def sample_census_data(file_name, num_contexts, shuffle_rows=True,
- remove_underrepresented=False):
- """Returns bandit problem dataset based on the UCI census data.
-
- Args:
- file_name: Route of file containing the Census dataset.
- num_contexts: Number of contexts to sample.
- shuffle_rows: If True, rows from original dataset are shuffled.
- remove_underrepresented: If True, removes arms with very few rewards.
-
- Returns:
- dataset: Sampled matrix with rows: (context, action rewards).
- opt_vals: Vector of deterministic optimal (reward, action) for each context.
-
- Preprocessing:
- * drop rows with missing labels
- * convert categorical variables to 1 hot encoding
-
- Note: this is the processed (not the 'raw') dataset. It contains a subset
- of the raw features and they've all been discretized.
-
- https://archive.ics.uci.edu/ml/datasets/US+Census+Data+%281990%29
- """
- # Note: this dataset is quite large. It will be slow to load and preprocess.
- with tf.gfile.Open(file_name, 'r') as f:
- df = (pd.read_csv(f, header=0, na_values=['?'])
- .dropna())
-
- num_actions = 9
-
- if shuffle_rows:
- df = df.sample(frac=1)
- df = df.iloc[:num_contexts, :]
-
- # Assuming what the paper calls response variable is the label?
- labels = df['dOccup'].astype('category').cat.codes.as_matrix()
- # In addition to label, also drop the (unique?) key.
- df = df.drop(['dOccup', 'caseid'], axis=1)
-
- # All columns are categorical. Convert to 1 hot encoding.
- df = pd.get_dummies(df, columns=df.columns)
-
- if remove_underrepresented:
- df, labels = remove_underrepresented_classes(df, labels)
- contexts = df.as_matrix()
-
- return classification_to_bandit_problem(contexts, labels, num_actions)
-
-
-def sample_covertype_data(file_name, num_contexts, shuffle_rows=True,
- remove_underrepresented=False):
- """Returns bandit problem dataset based on the UCI Cover_Type data.
-
- Args:
- file_name: Route of file containing the Covertype dataset.
- num_contexts: Number of contexts to sample.
- shuffle_rows: If True, rows from original dataset are shuffled.
- remove_underrepresented: If True, removes arms with very few rewards.
-
- Returns:
- dataset: Sampled matrix with rows: (context, action rewards).
- opt_vals: Vector of deterministic optimal (reward, action) for each context.
-
- Preprocessing:
- * drop rows with missing labels
- * convert categorical variables to 1 hot encoding
-
- https://archive.ics.uci.edu/ml/datasets/Covertype
- """
- with tf.gfile.Open(file_name, 'r') as f:
- df = (pd.read_csv(f, header=0, na_values=['?'])
- .dropna())
-
- num_actions = 7
-
- if shuffle_rows:
- df = df.sample(frac=1)
- df = df.iloc[:num_contexts, :]
-
- # Assuming what the paper calls response variable is the label?
- # Last column is label.
- labels = df[df.columns[-1]].astype('category').cat.codes.as_matrix()
- df = df.drop([df.columns[-1]], axis=1)
-
- # All columns are either quantitative or already converted to 1 hot.
- if remove_underrepresented:
- df, labels = remove_underrepresented_classes(df, labels)
- contexts = df.as_matrix()
-
- return classification_to_bandit_problem(contexts, labels, num_actions)
-
-
-def classification_to_bandit_problem(contexts, labels, num_actions=None):
- """Normalize contexts and encode deterministic rewards."""
-
- if num_actions is None:
- num_actions = np.max(labels) + 1
- num_contexts = contexts.shape[0]
-
- # Due to random subsampling in small problems, some features may be constant
- sstd = safe_std(np.std(contexts, axis=0, keepdims=True)[0, :])
-
- # Normalize features
- contexts = ((contexts - np.mean(contexts, axis=0, keepdims=True)) / sstd)
-
- # One hot encode labels as rewards
- rewards = np.zeros((num_contexts, num_actions))
- rewards[np.arange(num_contexts), labels] = 1.0
-
- return contexts, rewards, (np.ones(num_contexts), labels)
-
-
-def safe_std(values):
- """Remove zero std values for ones."""
- return np.array([val if val != 0.0 else 1.0 for val in values])
-
-
-def remove_underrepresented_classes(features, labels, thresh=0.0005):
- """Removes classes when number of datapoints fraction is below a threshold."""
-
- # Threshold doesn't seem to agree with https://arxiv.org/pdf/1706.04687.pdf
- # Example: for Covertype, they report 4 classes after filtering, we get 7?
- total_count = labels.shape[0]
- unique, counts = np.unique(labels, return_counts=True)
- ratios = counts.astype('float') / total_count
- vals_and_ratios = dict(zip(unique, ratios))
- print('Unique classes and their ratio of total: %s' % vals_and_ratios)
- keep = [vals_and_ratios[v] >= thresh for v in labels]
- return features[keep], labels[np.array(keep)]
diff --git a/research/deep_contextual_bandits/bandits/data/synthetic_data_sampler.py b/research/deep_contextual_bandits/bandits/data/synthetic_data_sampler.py
deleted file mode 100644
index c7de48aba4de109392aa8efad06071886cf67964..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/bandits/data/synthetic_data_sampler.py
+++ /dev/null
@@ -1,179 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Several functions to sample contextual data."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-
-def sample_contextual_data(num_contexts, dim_context, num_actions, sigma):
- """Samples independent Gaussian data.
-
- There is nothing to learn here as the rewards do not depend on the context.
-
- Args:
- num_contexts: Number of contexts to sample.
- dim_context: Dimension of the contexts.
- num_actions: Number of arms for the multi-armed bandit.
- sigma: Standard deviation of the independent Gaussian samples.
-
- Returns:
- data: A [num_contexts, dim_context + num_actions] numpy array with the data.
- """
- size_data = [num_contexts, dim_context + num_actions]
- return np.random.normal(scale=sigma, size=size_data)
-
-
-def sample_linear_data(num_contexts, dim_context, num_actions, sigma=0.0):
- """Samples data from linearly parameterized arms.
-
- The reward for context X and arm j is given by X^T beta_j, for some latent
- set of parameters {beta_j : j = 1, ..., k}. The beta's are sampled uniformly
- at random, the contexts are Gaussian, and sigma-noise is added to the rewards.
-
- Args:
- num_contexts: Number of contexts to sample.
- dim_context: Dimension of the contexts.
- num_actions: Number of arms for the multi-armed bandit.
- sigma: Standard deviation of the additive noise. Set to zero for no noise.
-
- Returns:
- data: A [n, d+k] numpy array with the data.
- betas: Latent parameters that determine expected reward for each arm.
- opt: (optimal_rewards, optimal_actions) for all contexts.
- """
-
- betas = np.random.uniform(-1, 1, (dim_context, num_actions))
- betas /= np.linalg.norm(betas, axis=0)
- contexts = np.random.normal(size=[num_contexts, dim_context])
- rewards = np.dot(contexts, betas)
- opt_actions = np.argmax(rewards, axis=1)
- rewards += np.random.normal(scale=sigma, size=rewards.shape)
- opt_rewards = np.array([rewards[i, act] for i, act in enumerate(opt_actions)])
- return np.hstack((contexts, rewards)), betas, (opt_rewards, opt_actions)
-
-
-def sample_sparse_linear_data(num_contexts, dim_context, num_actions,
- sparse_dim, sigma=0.0):
- """Samples data from sparse linearly parameterized arms.
-
- The reward for context X and arm j is given by X^T beta_j, for some latent
- set of parameters {beta_j : j = 1, ..., k}. The beta's are sampled uniformly
- at random, the contexts are Gaussian, and sigma-noise is added to the rewards.
- Only s components out of d are non-zero for each arm's beta.
-
- Args:
- num_contexts: Number of contexts to sample.
- dim_context: Dimension of the contexts.
- num_actions: Number of arms for the multi-armed bandit.
- sparse_dim: Dimension of the latent subspace (sparsity pattern dimension).
- sigma: Standard deviation of the additive noise. Set to zero for no noise.
-
- Returns:
- data: A [num_contexts, dim_context+num_actions] numpy array with the data.
- betas: Latent parameters that determine expected reward for each arm.
- opt: (optimal_rewards, optimal_actions) for all contexts.
- """
-
- flatten = lambda l: [item for sublist in l for item in sublist]
- sparse_pattern = flatten(
- [[(j, i) for j in np.random.choice(range(dim_context),
- sparse_dim,
- replace=False)]
- for i in range(num_actions)])
- betas = np.random.uniform(-1, 1, (dim_context, num_actions))
- mask = np.zeros((dim_context, num_actions))
- for elt in sparse_pattern:
- mask[elt] = 1
- betas = np.multiply(betas, mask)
- betas /= np.linalg.norm(betas, axis=0)
- contexts = np.random.normal(size=[num_contexts, dim_context])
- rewards = np.dot(contexts, betas)
- opt_actions = np.argmax(rewards, axis=1)
- rewards += np.random.normal(scale=sigma, size=rewards.shape)
- opt_rewards = np.array([rewards[i, act] for i, act in enumerate(opt_actions)])
- return np.hstack((contexts, rewards)), betas, (opt_rewards, opt_actions)
-
-
-def sample_wheel_bandit_data(num_contexts, delta, mean_v, std_v,
- mu_large, std_large):
- """Samples from Wheel bandit game (see https://arxiv.org/abs/1802.09127).
-
- Args:
- num_contexts: Number of points to sample, i.e. (context, action rewards).
- delta: Exploration parameter: high reward in one region if norm above delta.
- mean_v: Mean reward for each action if context norm is below delta.
- std_v: Gaussian reward std for each action if context norm is below delta.
- mu_large: Mean reward for optimal action if context norm is above delta.
- std_large: Reward std for optimal action if context norm is above delta.
-
- Returns:
- dataset: Sampled matrix with n rows: (context, action rewards).
- opt_vals: Vector of expected optimal (reward, action) for each context.
- """
-
- context_dim = 2
- num_actions = 5
-
- data = []
- rewards = []
- opt_actions = []
- opt_rewards = []
-
- # sample uniform contexts in unit ball
- while len(data) < num_contexts:
- raw_data = np.random.uniform(-1, 1, (int(num_contexts / 3), context_dim))
-
- for i in range(raw_data.shape[0]):
- if np.linalg.norm(raw_data[i, :]) <= 1:
- data.append(raw_data[i, :])
-
- contexts = np.stack(data)[:num_contexts, :]
-
- # sample rewards
- for i in range(num_contexts):
- r = [np.random.normal(mean_v[j], std_v[j]) for j in range(num_actions)]
- if np.linalg.norm(contexts[i, :]) >= delta:
- # large reward in the right region for the context
- r_big = np.random.normal(mu_large, std_large)
- if contexts[i, 0] > 0:
- if contexts[i, 1] > 0:
- r[0] = r_big
- opt_actions.append(0)
- else:
- r[1] = r_big
- opt_actions.append(1)
- else:
- if contexts[i, 1] > 0:
- r[2] = r_big
- opt_actions.append(2)
- else:
- r[3] = r_big
- opt_actions.append(3)
- else:
- opt_actions.append(np.argmax(mean_v))
-
- opt_rewards.append(r[opt_actions[-1]])
- rewards.append(r)
-
- rewards = np.stack(rewards)
- opt_rewards = np.array(opt_rewards)
- opt_actions = np.array(opt_actions)
-
- return np.hstack((contexts, rewards)), (opt_rewards, opt_actions)
diff --git a/research/deep_contextual_bandits/example_main.py b/research/deep_contextual_bandits/example_main.py
deleted file mode 100644
index c71a5aa26f94adbf5989d002fd5c768582c14e14..0000000000000000000000000000000000000000
--- a/research/deep_contextual_bandits/example_main.py
+++ /dev/null
@@ -1,454 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple example of contextual bandits simulation.
-
-Code corresponding to:
-Deep Bayesian Bandits Showdown: An Empirical Comparison of Bayesian Deep Networks
-for Thompson Sampling, by Carlos Riquelme, George Tucker, and Jasper Snoek.
-https://arxiv.org/abs/1802.09127
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import time
-from absl import app
-from absl import flags
-import numpy as np
-import os
-import tensorflow as tf
-
-from bandits.algorithms.bootstrapped_bnn_sampling import BootstrappedBNNSampling
-from bandits.core.contextual_bandit import run_contextual_bandit
-from bandits.data.data_sampler import sample_adult_data
-from bandits.data.data_sampler import sample_census_data
-from bandits.data.data_sampler import sample_covertype_data
-from bandits.data.data_sampler import sample_jester_data
-from bandits.data.data_sampler import sample_mushroom_data
-from bandits.data.data_sampler import sample_statlog_data
-from bandits.data.data_sampler import sample_stock_data
-from bandits.algorithms.fixed_policy_sampling import FixedPolicySampling
-from bandits.algorithms.linear_full_posterior_sampling import LinearFullPosteriorSampling
-from bandits.algorithms.neural_linear_sampling import NeuralLinearPosteriorSampling
-from bandits.algorithms.parameter_noise_sampling import ParameterNoiseSampling
-from bandits.algorithms.posterior_bnn_sampling import PosteriorBNNSampling
-from bandits.data.synthetic_data_sampler import sample_linear_data
-from bandits.data.synthetic_data_sampler import sample_sparse_linear_data
-from bandits.data.synthetic_data_sampler import sample_wheel_bandit_data
-from bandits.algorithms.uniform_sampling import UniformSampling
-
-# Set up your file routes to the data files.
-base_route = os.getcwd()
-data_route = 'contextual_bandits/datasets'
-
-FLAGS = flags.FLAGS
-FLAGS.set_default('alsologtostderr', True)
-flags.DEFINE_string('logdir', '/tmp/bandits/', 'Base directory to save output')
-flags.DEFINE_string(
- 'mushroom_data',
- os.path.join(base_route, data_route, 'mushroom.data'),
- 'Directory where Mushroom data is stored.')
-flags.DEFINE_string(
- 'financial_data',
- os.path.join(base_route, data_route, 'raw_stock_contexts'),
- 'Directory where Financial data is stored.')
-flags.DEFINE_string(
- 'jester_data',
- os.path.join(base_route, data_route, 'jester_data_40jokes_19181users.npy'),
- 'Directory where Jester data is stored.')
-flags.DEFINE_string(
- 'statlog_data',
- os.path.join(base_route, data_route, 'shuttle.trn'),
- 'Directory where Statlog data is stored.')
-flags.DEFINE_string(
- 'adult_data',
- os.path.join(base_route, data_route, 'adult.full'),
- 'Directory where Adult data is stored.')
-flags.DEFINE_string(
- 'covertype_data',
- os.path.join(base_route, data_route, 'covtype.data'),
- 'Directory where Covertype data is stored.')
-flags.DEFINE_string(
- 'census_data',
- os.path.join(base_route, data_route, 'USCensus1990.data.txt'),
- 'Directory where Census data is stored.')
-
-
-def sample_data(data_type, num_contexts=None):
- """Sample data from given 'data_type'.
-
- Args:
- data_type: Dataset from which to sample.
- num_contexts: Number of contexts to sample.
-
- Returns:
- dataset: Sampled matrix with rows: (context, reward_1, ..., reward_num_act).
- opt_rewards: Vector of expected optimal reward for each context.
- opt_actions: Vector of optimal action for each context.
- num_actions: Number of available actions.
- context_dim: Dimension of each context.
- """
-
- if data_type == 'linear':
- # Create linear dataset
- num_actions = 8
- context_dim = 10
- noise_stds = [0.01 * (i + 1) for i in range(num_actions)]
- dataset, _, opt_linear = sample_linear_data(num_contexts, context_dim,
- num_actions, sigma=noise_stds)
- opt_rewards, opt_actions = opt_linear
- elif data_type == 'sparse_linear':
- # Create sparse linear dataset
- num_actions = 7
- context_dim = 10
- noise_stds = [0.01 * (i + 1) for i in range(num_actions)]
- num_nnz_dims = int(context_dim / 3.0)
- dataset, _, opt_sparse_linear = sample_sparse_linear_data(
- num_contexts, context_dim, num_actions, num_nnz_dims, sigma=noise_stds)
- opt_rewards, opt_actions = opt_sparse_linear
- elif data_type == 'mushroom':
- # Create mushroom dataset
- num_actions = 2
- context_dim = 117
- file_name = FLAGS.mushroom_data
- dataset, opt_mushroom = sample_mushroom_data(file_name, num_contexts)
- opt_rewards, opt_actions = opt_mushroom
- elif data_type == 'financial':
- num_actions = 8
- context_dim = 21
- num_contexts = min(3713, num_contexts)
- noise_stds = [0.01 * (i + 1) for i in range(num_actions)]
- file_name = FLAGS.financial_data
- dataset, opt_financial = sample_stock_data(file_name, context_dim,
- num_actions, num_contexts,
- noise_stds, shuffle_rows=True)
- opt_rewards, opt_actions = opt_financial
- elif data_type == 'jester':
- num_actions = 8
- context_dim = 32
- num_contexts = min(19181, num_contexts)
- file_name = FLAGS.jester_data
- dataset, opt_jester = sample_jester_data(file_name, context_dim,
- num_actions, num_contexts,
- shuffle_rows=True,
- shuffle_cols=True)
- opt_rewards, opt_actions = opt_jester
- elif data_type == 'statlog':
- file_name = FLAGS.statlog_data
- num_actions = 7
- num_contexts = min(43500, num_contexts)
- sampled_vals = sample_statlog_data(file_name, num_contexts,
- shuffle_rows=True)
- contexts, rewards, (opt_rewards, opt_actions) = sampled_vals
- dataset = np.hstack((contexts, rewards))
- context_dim = contexts.shape[1]
- elif data_type == 'adult':
- file_name = FLAGS.adult_data
- num_actions = 14
- num_contexts = min(45222, num_contexts)
- sampled_vals = sample_adult_data(file_name, num_contexts,
- shuffle_rows=True)
- contexts, rewards, (opt_rewards, opt_actions) = sampled_vals
- dataset = np.hstack((contexts, rewards))
- context_dim = contexts.shape[1]
- elif data_type == 'covertype':
- file_name = FLAGS.covertype_data
- num_actions = 7
- num_contexts = min(150000, num_contexts)
- sampled_vals = sample_covertype_data(file_name, num_contexts,
- shuffle_rows=True)
- contexts, rewards, (opt_rewards, opt_actions) = sampled_vals
- dataset = np.hstack((contexts, rewards))
- context_dim = contexts.shape[1]
- elif data_type == 'census':
- file_name = FLAGS.census_data
- num_actions = 9
- num_contexts = min(150000, num_contexts)
- sampled_vals = sample_census_data(file_name, num_contexts,
- shuffle_rows=True)
- contexts, rewards, (opt_rewards, opt_actions) = sampled_vals
- dataset = np.hstack((contexts, rewards))
- context_dim = contexts.shape[1]
- elif data_type == 'wheel':
- delta = 0.95
- num_actions = 5
- context_dim = 2
- mean_v = [1.0, 1.0, 1.0, 1.0, 1.2]
- std_v = [0.05, 0.05, 0.05, 0.05, 0.05]
- mu_large = 50
- std_large = 0.01
- dataset, opt_wheel = sample_wheel_bandit_data(num_contexts, delta,
- mean_v, std_v,
- mu_large, std_large)
- opt_rewards, opt_actions = opt_wheel
-
- return dataset, opt_rewards, opt_actions, num_actions, context_dim
-
-
-def display_results(algos, opt_rewards, opt_actions, h_rewards, t_init, name):
- """Displays summary statistics of the performance of each algorithm."""
-
- print('---------------------------------------------------')
- print('---------------------------------------------------')
- print('{} bandit completed after {} seconds.'.format(
- name, time.time() - t_init))
- print('---------------------------------------------------')
-
- performance_pairs = []
- for j, a in enumerate(algos):
- performance_pairs.append((a.name, np.sum(h_rewards[:, j])))
- performance_pairs = sorted(performance_pairs,
- key=lambda elt: elt[1],
- reverse=True)
- for i, (name, reward) in enumerate(performance_pairs):
- print('{:3}) {:20}| \t \t total reward = {:10}.'.format(i, name, reward))
-
- print('---------------------------------------------------')
- print('Optimal total reward = {}.'.format(np.sum(opt_rewards)))
- print('Frequency of optimal actions (action, frequency):')
- print([[elt, list(opt_actions).count(elt)] for elt in set(opt_actions)])
- print('---------------------------------------------------')
- print('---------------------------------------------------')
-
-
-def main(_):
-
- # Problem parameters
- num_contexts = 2000
-
- # Data type in {linear, sparse_linear, mushroom, financial, jester,
- # statlog, adult, covertype, census, wheel}
- data_type = 'mushroom'
-
- # Create dataset
- sampled_vals = sample_data(data_type, num_contexts)
- dataset, opt_rewards, opt_actions, num_actions, context_dim = sampled_vals
-
- # Define hyperparameters and algorithms
- hparams = tf.contrib.training.HParams(num_actions=num_actions)
-
- hparams_linear = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- a0=6,
- b0=6,
- lambda_prior=0.25,
- initial_pulls=2)
-
- hparams_rms = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- init_scale=0.3,
- activation=tf.nn.relu,
- layer_sizes=[50],
- batch_size=512,
- activate_decay=True,
- initial_lr=0.1,
- max_grad_norm=5.0,
- show_training=False,
- freq_summary=1000,
- buffer_s=-1,
- initial_pulls=2,
- optimizer='RMS',
- reset_lr=True,
- lr_decay_rate=0.5,
- training_freq=50,
- training_epochs=100,
- p=0.95,
- q=3)
-
- hparams_dropout = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- init_scale=0.3,
- activation=tf.nn.relu,
- layer_sizes=[50],
- batch_size=512,
- activate_decay=True,
- initial_lr=0.1,
- max_grad_norm=5.0,
- show_training=False,
- freq_summary=1000,
- buffer_s=-1,
- initial_pulls=2,
- optimizer='RMS',
- reset_lr=True,
- lr_decay_rate=0.5,
- training_freq=50,
- training_epochs=100,
- use_dropout=True,
- keep_prob=0.80)
-
- hparams_bbb = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- init_scale=0.3,
- activation=tf.nn.relu,
- layer_sizes=[50],
- batch_size=512,
- activate_decay=True,
- initial_lr=0.1,
- max_grad_norm=5.0,
- show_training=False,
- freq_summary=1000,
- buffer_s=-1,
- initial_pulls=2,
- optimizer='RMS',
- use_sigma_exp_transform=True,
- cleared_times_trained=10,
- initial_training_steps=100,
- noise_sigma=0.1,
- reset_lr=False,
- training_freq=50,
- training_epochs=100)
-
- hparams_nlinear = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- init_scale=0.3,
- activation=tf.nn.relu,
- layer_sizes=[50],
- batch_size=512,
- activate_decay=True,
- initial_lr=0.1,
- max_grad_norm=5.0,
- show_training=False,
- freq_summary=1000,
- buffer_s=-1,
- initial_pulls=2,
- reset_lr=True,
- lr_decay_rate=0.5,
- training_freq=1,
- training_freq_network=50,
- training_epochs=100,
- a0=6,
- b0=6,
- lambda_prior=0.25)
-
- hparams_nlinear2 = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- init_scale=0.3,
- activation=tf.nn.relu,
- layer_sizes=[50],
- batch_size=512,
- activate_decay=True,
- initial_lr=0.1,
- max_grad_norm=5.0,
- show_training=False,
- freq_summary=1000,
- buffer_s=-1,
- initial_pulls=2,
- reset_lr=True,
- lr_decay_rate=0.5,
- training_freq=10,
- training_freq_network=50,
- training_epochs=100,
- a0=6,
- b0=6,
- lambda_prior=0.25)
-
- hparams_pnoise = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- init_scale=0.3,
- activation=tf.nn.relu,
- layer_sizes=[50],
- batch_size=512,
- activate_decay=True,
- initial_lr=0.1,
- max_grad_norm=5.0,
- show_training=False,
- freq_summary=1000,
- buffer_s=-1,
- initial_pulls=2,
- optimizer='RMS',
- reset_lr=True,
- lr_decay_rate=0.5,
- training_freq=50,
- training_epochs=100,
- noise_std=0.05,
- eps=0.1,
- d_samples=300,
- )
-
- hparams_alpha_div = tf.contrib.training.HParams(num_actions=num_actions,
- context_dim=context_dim,
- init_scale=0.3,
- activation=tf.nn.relu,
- layer_sizes=[50],
- batch_size=512,
- activate_decay=True,
- initial_lr=0.1,
- max_grad_norm=5.0,
- show_training=False,
- freq_summary=1000,
- buffer_s=-1,
- initial_pulls=2,
- optimizer='RMS',
- use_sigma_exp_transform=True,
- cleared_times_trained=10,
- initial_training_steps=100,
- noise_sigma=0.1,
- reset_lr=False,
- training_freq=50,
- training_epochs=100,
- alpha=1.0,
- k=20,
- prior_variance=0.1)
-
- hparams_gp = tf.contrib.training.HParams(num_actions=num_actions,
- num_outputs=num_actions,
- context_dim=context_dim,
- reset_lr=False,
- learn_embeddings=True,
- max_num_points=1000,
- show_training=False,
- freq_summary=1000,
- batch_size=512,
- keep_fixed_after_max_obs=True,
- training_freq=50,
- initial_pulls=2,
- training_epochs=100,
- lr=0.01,
- buffer_s=-1,
- initial_lr=0.001,
- lr_decay_rate=0.0,
- optimizer='RMS',
- task_latent_dim=5,
- activate_decay=False)
-
- algos = [
- UniformSampling('Uniform Sampling', hparams),
- UniformSampling('Uniform Sampling 2', hparams),
- FixedPolicySampling('fixed1', [0.75, 0.25], hparams),
- FixedPolicySampling('fixed2', [0.25, 0.75], hparams),
- PosteriorBNNSampling('RMS', hparams_rms, 'RMSProp'),
- PosteriorBNNSampling('Dropout', hparams_dropout, 'RMSProp'),
- PosteriorBNNSampling('BBB', hparams_bbb, 'Variational'),
- NeuralLinearPosteriorSampling('NeuralLinear', hparams_nlinear),
- NeuralLinearPosteriorSampling('NeuralLinear2', hparams_nlinear2),
- LinearFullPosteriorSampling('LinFullPost', hparams_linear),
- BootstrappedBNNSampling('BootRMS', hparams_rms),
- ParameterNoiseSampling('ParamNoise', hparams_pnoise),
- PosteriorBNNSampling('BBAlphaDiv', hparams_alpha_div, 'AlphaDiv'),
- PosteriorBNNSampling('MultitaskGP', hparams_gp, 'GP'),
- ]
-
- # Run contextual bandit problem
- t_init = time.time()
- results = run_contextual_bandit(context_dim, num_actions, dataset, algos)
- _, h_rewards = results
-
- # Display results
- display_results(algos, opt_rewards, opt_actions, h_rewards, t_init, data_type)
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/deep_speech/README.md b/research/deep_speech/README.md
index 59a9dea7f81e3963372b23d8c4436e3e04d763ac..06ded0c912773f477e9b8176f6655b609adeb56a 100644
--- a/research/deep_speech/README.md
+++ b/research/deep_speech/README.md
@@ -1,6 +1,6 @@
-
-
+[](https://github.com/tensorflow/tensorflow/releases/tag/v1.15.3)
+[](https://github.com/tensorflow/tensorflow/releases/tag/v2.3.0)
# DeepSpeech2 Model
diff --git a/research/deep_speech/data/dataset.py b/research/deep_speech/data/dataset.py
index 4a8cb59955c4608e20dcec6d9c5e38d705f24311..32391773dfe978b4bda2fd7e9552300c11126007 100644
--- a/research/deep_speech/data/dataset.py
+++ b/research/deep_speech/data/dataset.py
@@ -24,6 +24,7 @@ import numpy as np
from six.moves import xrange # pylint: disable=redefined-builtin
import soundfile
import tensorflow as tf
+from absl import logging
# pylint: enable=g-bad-import-order
import data.featurizer as featurizer # pylint: disable=g-bad-import-order
@@ -71,8 +72,8 @@ class DatasetConfig(object):
"""
self.audio_config = audio_config
- assert tf.gfile.Exists(data_path)
- assert tf.gfile.Exists(vocab_file_path)
+ assert tf.io.gfile.exists(data_path)
+ assert tf.io.gfile.exists(vocab_file_path)
self.data_path = data_path
self.vocab_file_path = vocab_file_path
self.sortagrad = sortagrad
@@ -125,8 +126,8 @@ def _preprocess_data(file_path):
A list of tuples (wav_filename, wav_filesize, transcript) sorted by
file_size.
"""
- tf.logging.info("Loading data set {}".format(file_path))
- with tf.gfile.Open(file_path, "r") as f:
+ logging.info("Loading data set {}".format(file_path))
+ with tf.io.gfile.GFile(file_path, "r") as f:
lines = f.read().splitlines()
# Skip the csv header in lines[0].
lines = lines[1:]
diff --git a/research/deep_speech/data/download.py b/research/deep_speech/data/download.py
index 5ded03762138a0006c36585e2e8da450baaccac7..3ea6e2f3e54a98c95198638d28a4dfc0d96fff97 100644
--- a/research/deep_speech/data/download.py
+++ b/research/deep_speech/data/download.py
@@ -32,6 +32,7 @@ import pandas
from six.moves import urllib
from sox import Transformer
import tensorflow as tf
+from absl import logging
LIBRI_SPEECH_URLS = {
"train-clean-100":
@@ -59,13 +60,13 @@ def download_and_extract(directory, url):
url: the url to download the data file.
"""
- if not tf.gfile.Exists(directory):
- tf.gfile.MakeDirs(directory)
+ if not tf.io.gfile.exists(directory):
+ tf.io.gfile.makedirs(directory)
_, tar_filepath = tempfile.mkstemp(suffix=".tar.gz")
try:
- tf.logging.info("Downloading %s to %s" % (url, tar_filepath))
+ logging.info("Downloading %s to %s" % (url, tar_filepath))
def _progress(count, block_size, total_size):
sys.stdout.write("\r>> Downloading {} {:.1f}%".format(
@@ -75,12 +76,12 @@ def download_and_extract(directory, url):
urllib.request.urlretrieve(url, tar_filepath, _progress)
print()
statinfo = os.stat(tar_filepath)
- tf.logging.info(
+ logging.info(
"Successfully downloaded %s, size(bytes): %d" % (url, statinfo.st_size))
with tarfile.open(tar_filepath, "r") as tar:
tar.extractall(directory)
finally:
- tf.gfile.Remove(tar_filepath)
+ tf.io.gfile.remove(tar_filepath)
def convert_audio_and_split_transcript(input_dir, source_name, target_name,
@@ -112,18 +113,18 @@ def convert_audio_and_split_transcript(input_dir, source_name, target_name,
output_file: the name of the newly generated csv file. e.g. test-clean.csv
"""
- tf.logging.info("Preprocessing audio and transcript for %s" % source_name)
+ logging.info("Preprocessing audio and transcript for %s" % source_name)
source_dir = os.path.join(input_dir, source_name)
target_dir = os.path.join(input_dir, target_name)
- if not tf.gfile.Exists(target_dir):
- tf.gfile.MakeDirs(target_dir)
+ if not tf.io.gfile.exists(target_dir):
+ tf.io.gfile.makedirs(target_dir)
files = []
tfm = Transformer()
# Convert all FLAC file into WAV format. At the same time, generate the csv
# file.
- for root, _, filenames in tf.gfile.Walk(source_dir):
+ for root, _, filenames in tf.io.gfile.walk(source_dir):
for filename in fnmatch.filter(filenames, "*.trans.txt"):
trans_file = os.path.join(root, filename)
with codecs.open(trans_file, "r", "utf-8") as fin:
@@ -137,7 +138,7 @@ def convert_audio_and_split_transcript(input_dir, source_name, target_name,
# Convert FLAC to WAV.
flac_file = os.path.join(root, seqid + ".flac")
wav_file = os.path.join(target_dir, seqid + ".wav")
- if not tf.gfile.Exists(wav_file):
+ if not tf.io.gfile.exists(wav_file):
tfm.build(flac_file, wav_file)
wav_filesize = os.path.getsize(wav_file)
@@ -149,7 +150,7 @@ def convert_audio_and_split_transcript(input_dir, source_name, target_name,
df = pandas.DataFrame(
data=files, columns=["wav_filename", "wav_filesize", "transcript"])
df.to_csv(csv_file_path, index=False, sep="\t")
- tf.logging.info("Successfully generated csv file {}".format(csv_file_path))
+ logging.info("Successfully generated csv file {}".format(csv_file_path))
def download_and_process_datasets(directory, datasets):
@@ -160,10 +161,10 @@ def download_and_process_datasets(directory, datasets):
datasets: list of dataset names that will be downloaded and processed.
"""
- tf.logging.info("Preparing LibriSpeech dataset: {}".format(
+ logging.info("Preparing LibriSpeech dataset: {}".format(
",".join(datasets)))
for dataset in datasets:
- tf.logging.info("Preparing dataset %s", dataset)
+ logging.info("Preparing dataset %s", dataset)
dataset_dir = os.path.join(directory, dataset)
download_and_extract(dataset_dir, LIBRI_SPEECH_URLS[dataset])
convert_audio_and_split_transcript(
@@ -185,8 +186,8 @@ def define_data_download_flags():
def main(_):
- if not tf.gfile.Exists(FLAGS.data_dir):
- tf.gfile.MakeDirs(FLAGS.data_dir)
+ if not tf.io.gfile.exists(FLAGS.data_dir):
+ tf.io.gfile.makedirs(FLAGS.data_dir)
if FLAGS.train_only:
download_and_process_datasets(
@@ -202,7 +203,7 @@ def main(_):
if __name__ == "__main__":
- tf.logging.set_verbosity(tf.logging.INFO)
+ logging.set_verbosity(logging.INFO)
define_data_download_flags()
FLAGS = absl_flags.FLAGS
absl_app.run(main)
diff --git a/research/deep_speech/decoder.py b/research/deep_speech/decoder.py
index f46983170f5885385942be1b24e262f30b4be8e0..bf618bcb63c82f3d041d1b93e74b0ef4b3a1b6c8 100644
--- a/research/deep_speech/decoder.py
+++ b/research/deep_speech/decoder.py
@@ -30,7 +30,7 @@ class DeepSpeechDecoder(object):
def __init__(self, labels, blank_index=28):
"""Decoder initialization.
- Arguments:
+ Args:
labels: a string specifying the speech labels for the decoder to use.
blank_index: an integer specifying index for the blank character.
Defaults to 28.
diff --git a/research/deep_speech/deep_speech.py b/research/deep_speech/deep_speech.py
index 3d809c3cbc245e20b752bf2d2e45823f33521d64..468c7133115528d7114745c25d46274b34cd7207 100644
--- a/research/deep_speech/deep_speech.py
+++ b/research/deep_speech/deep_speech.py
@@ -21,6 +21,7 @@ import os
# pylint: disable=g-bad-import-order
from absl import app as absl_app
from absl import flags
+from absl import logging
import tensorflow as tf
# pylint: enable=g-bad-import-order
@@ -61,25 +62,10 @@ def compute_length_after_conv(max_time_steps, ctc_time_steps, input_length):
Returns:
the ctc_input_length after convolution layer.
"""
- ctc_input_length = tf.to_float(tf.multiply(
- input_length, ctc_time_steps))
- return tf.to_int32(tf.floordiv(
- ctc_input_length, tf.to_float(max_time_steps)))
-
-
-def ctc_loss(label_length, ctc_input_length, labels, logits):
- """Computes the ctc loss for the current batch of predictions."""
- label_length = tf.to_int32(tf.squeeze(label_length))
- ctc_input_length = tf.to_int32(tf.squeeze(ctc_input_length))
- sparse_labels = tf.to_int32(
- tf.keras.backend.ctc_label_dense_to_sparse(labels, label_length))
- y_pred = tf.log(tf.transpose(
- logits, perm=[1, 0, 2]) + tf.keras.backend.epsilon())
-
- return tf.expand_dims(
- tf.nn.ctc_loss(labels=sparse_labels, inputs=y_pred,
- sequence_length=ctc_input_length),
- axis=1)
+ ctc_input_length = tf.cast(tf.multiply(
+ input_length, ctc_time_steps), dtype=tf.float32)
+ return tf.cast(tf.math.floordiv(
+ ctc_input_length, tf.cast(max_time_steps, dtype=tf.float32)), dtype=tf.int32)
def evaluate_model(estimator, speech_labels, entries, input_fn_eval):
@@ -123,11 +109,11 @@ def evaluate_model(estimator, speech_labels, entries, input_fn_eval):
total_cer /= num_of_examples
total_wer /= num_of_examples
- global_step = estimator.get_variable_value(tf.GraphKeys.GLOBAL_STEP)
+ global_step = estimator.get_variable_value(tf.compat.v1.GraphKeys.GLOBAL_STEP)
eval_results = {
_WER_KEY: total_wer,
_CER_KEY: total_cer,
- tf.GraphKeys.GLOBAL_STEP: global_step,
+ tf.compat.v1.GraphKeys.GLOBAL_STEP: global_step,
}
return eval_results
@@ -163,7 +149,7 @@ def model_fn(features, labels, mode, params):
logits = model(features, training=False)
predictions = {
"classes": tf.argmax(logits, axis=2),
- "probabilities": tf.nn.softmax(logits),
+ "probabilities": logits,
"logits": logits
}
return tf.estimator.EstimatorSpec(
@@ -172,17 +158,16 @@ def model_fn(features, labels, mode, params):
# In training mode.
logits = model(features, training=True)
- probs = tf.nn.softmax(logits)
ctc_input_length = compute_length_after_conv(
- tf.shape(features)[1], tf.shape(probs)[1], input_length)
+ tf.shape(features)[1], tf.shape(logits)[1], input_length)
# Compute CTC loss
- loss = tf.reduce_mean(ctc_loss(
- label_length, ctc_input_length, labels, probs))
+ loss = tf.reduce_mean(tf.keras.backend.ctc_batch_cost(
+ labels, logits, ctc_input_length, label_length))
- optimizer = tf.train.AdamOptimizer(learning_rate=flags_obj.learning_rate)
- global_step = tf.train.get_or_create_global_step()
+ optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=flags_obj.learning_rate)
+ global_step = tf.compat.v1.train.get_or_create_global_step()
minimize_op = optimizer.minimize(loss, global_step=global_step)
- update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
+ update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS)
# Create the train_op that groups both minimize_ops and update_ops
train_op = tf.group(minimize_op, update_ops)
@@ -239,9 +224,9 @@ def per_device_batch_size(batch_size, num_gpus):
def run_deep_speech(_):
"""Run deep speech training and eval loop."""
- tf.set_random_seed(flags_obj.seed)
+ tf.compat.v1.set_random_seed(flags_obj.seed)
# Data preprocessing
- tf.logging.info("Data preprocessing...")
+ logging.info("Data preprocessing...")
train_speech_dataset = generate_dataset(flags_obj.train_data_dir)
eval_speech_dataset = generate_dataset(flags_obj.eval_data_dir)
@@ -287,7 +272,7 @@ def run_deep_speech(_):
total_training_cycle = (flags_obj.train_epochs //
flags_obj.epochs_between_evals)
for cycle_index in range(total_training_cycle):
- tf.logging.info("Starting a training cycle: %d/%d",
+ logging.info("Starting a training cycle: %d/%d",
cycle_index + 1, total_training_cycle)
# Perform batch_wise dataset shuffling
@@ -298,7 +283,7 @@ def run_deep_speech(_):
estimator.train(input_fn=input_fn_train)
# Evaluation
- tf.logging.info("Starting to evaluate...")
+ logging.info("Starting to evaluate...")
eval_results = evaluate_model(
estimator, eval_speech_dataset.speech_labels,
@@ -306,7 +291,7 @@ def run_deep_speech(_):
# Log the WER and CER results.
benchmark_logger.log_evaluation_result(eval_results)
- tf.logging.info(
+ logging.info(
"Iteration {}: WER = {:.2f}, CER = {:.2f}".format(
cycle_index + 1, eval_results[_WER_KEY], eval_results[_CER_KEY]))
@@ -425,7 +410,7 @@ def main(_):
if __name__ == "__main__":
- tf.logging.set_verbosity(tf.logging.INFO)
+ logging.set_verbosity(logging.INFO)
define_deep_speech_flags()
flags_obj = flags.FLAGS
absl_app.run(main)
diff --git a/research/deep_speech/deep_speech_model.py b/research/deep_speech/deep_speech_model.py
index dd768f825c792eb9f8f8ca7dbef6a25f5ce81091..7860f379f0d7431b29fcdb7bc65aa1e831cc8073 100644
--- a/research/deep_speech/deep_speech_model.py
+++ b/research/deep_speech/deep_speech_model.py
@@ -22,9 +22,9 @@ import tensorflow as tf
# Supported rnn cells.
SUPPORTED_RNNS = {
- "lstm": tf.contrib.rnn.BasicLSTMCell,
- "rnn": tf.contrib.rnn.RNNCell,
- "gru": tf.contrib.rnn.GRUCell,
+ "lstm": tf.keras.layers.LSTMCell,
+ "rnn": tf.keras.layers.SimpleRNNCell,
+ "gru": tf.keras.layers.GRUCell,
}
# Parameters for batch normalization.
@@ -53,9 +53,8 @@ def batch_norm(inputs, training):
Returns:
tensor output from batch norm layer.
"""
- return tf.layers.batch_normalization(
- inputs=inputs, momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON,
- fused=True, training=training)
+ return tf.keras.layers.BatchNormalization(
+ momentum=_BATCH_NORM_DECAY, epsilon=_BATCH_NORM_EPSILON)(inputs, training=training)
def _conv_bn_layer(inputs, padding, filters, kernel_size, strides, layer_id,
@@ -81,10 +80,10 @@ def _conv_bn_layer(inputs, padding, filters, kernel_size, strides, layer_id,
inputs = tf.pad(
inputs,
[[0, 0], [padding[0], padding[0]], [padding[1], padding[1]], [0, 0]])
- inputs = tf.layers.conv2d(
- inputs=inputs, filters=filters, kernel_size=kernel_size, strides=strides,
+ inputs = tf.keras.layers.Conv2D(
+ filters=filters, kernel_size=kernel_size, strides=strides,
padding="valid", use_bias=False, activation=tf.nn.relu6,
- name="cnn_{}".format(layer_id))
+ name="cnn_{}".format(layer_id))(inputs)
return batch_norm(inputs, training)
@@ -109,24 +108,16 @@ def _rnn_layer(inputs, rnn_cell, rnn_hidden_size, layer_id, is_batch_norm,
if is_batch_norm:
inputs = batch_norm(inputs, training)
- # Construct forward/backward RNN cells.
- fw_cell = rnn_cell(num_units=rnn_hidden_size,
- name="rnn_fw_{}".format(layer_id))
- bw_cell = rnn_cell(num_units=rnn_hidden_size,
- name="rnn_bw_{}".format(layer_id))
-
if is_bidirectional:
- outputs, _ = tf.nn.bidirectional_dynamic_rnn(
- cell_fw=fw_cell, cell_bw=bw_cell, inputs=inputs, dtype=tf.float32,
- swap_memory=True)
- rnn_outputs = tf.concat(outputs, -1)
+ rnn_outputs = tf.keras.layers.Bidirectional(
+ tf.keras.layers.RNN(rnn_cell(rnn_hidden_size),
+ return_sequences=True))(inputs)
else:
- rnn_outputs = tf.nn.dynamic_rnn(
- fw_cell, inputs, dtype=tf.float32, swap_memory=True)
+ rnn_outputs = tf.keras.layers.RNN(
+ rnn_cell(rnn_hidden_size), return_sequences=True)(inputs)
return rnn_outputs
-
class DeepSpeech2(object):
"""Define DeepSpeech2 model."""
@@ -179,7 +170,8 @@ class DeepSpeech2(object):
# FC layer with batch norm.
inputs = batch_norm(inputs, training)
- logits = tf.layers.dense(inputs, self.num_classes, use_bias=self.use_bias)
+ logits = tf.keras.layers.Dense(
+ self.num_classes, use_bias=self.use_bias, activation="softmax")(inputs)
return logits
diff --git a/research/deeplab/README.md b/research/deeplab/README.md
index 8609432b5bd78b56bf18a557dd5b813ed6a3fc77..f29002ea59a29ca6906e8728cc81f605dcdc7651 100644
--- a/research/deeplab/README.md
+++ b/research/deeplab/README.md
@@ -246,7 +246,7 @@ PASCAL VOC 2012 and Cityscapes.
### March 5, 2018
* First release of DeepLab in TensorFlow including deeper Xception network
-backbone. Included chekcpoints that have been pretrained on PASCAL VOC 2012
+backbone. Included checkpoints that have been pretrained on PASCAL VOC 2012
and Cityscapes.
## References
diff --git a/research/deeplab/convert_to_tflite.py b/research/deeplab/convert_to_tflite.py
new file mode 100644
index 0000000000000000000000000000000000000000..d23ce9e2337829d7a71a8cde487fb73ab068b664
--- /dev/null
+++ b/research/deeplab/convert_to_tflite.py
@@ -0,0 +1,112 @@
+# Copyright 2018 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tools to convert a quantized deeplab model to tflite."""
+
+from absl import app
+from absl import flags
+import numpy as np
+from PIL import Image
+import tensorflow as tf
+
+
+flags.DEFINE_string('quantized_graph_def_path', None,
+ 'Path to quantized graphdef.')
+flags.DEFINE_string('output_tflite_path', None, 'Output TFlite model path.')
+flags.DEFINE_string(
+ 'input_tensor_name', None,
+ 'Input tensor to TFlite model. This usually should be the input tensor to '
+ 'model backbone.'
+)
+flags.DEFINE_string(
+ 'output_tensor_name', 'ArgMax:0',
+ 'Output tensor name of TFlite model. By default we output the raw semantic '
+ 'label predictions.'
+)
+flags.DEFINE_string(
+ 'test_image_path', None,
+ 'Path to an image to test the consistency between input graphdef / '
+ 'converted tflite model.'
+)
+
+FLAGS = flags.FLAGS
+
+
+def convert_to_tflite(quantized_graphdef,
+ backbone_input_tensor,
+ output_tensor):
+ """Helper method to convert quantized deeplab model to TFlite."""
+ with tf.Graph().as_default() as graph:
+ tf.graph_util.import_graph_def(quantized_graphdef, name='')
+ sess = tf.compat.v1.Session()
+
+ tflite_input = graph.get_tensor_by_name(backbone_input_tensor)
+ tflite_output = graph.get_tensor_by_name(output_tensor)
+ converter = tf.compat.v1.lite.TFLiteConverter.from_session(
+ sess, [tflite_input], [tflite_output])
+ converter.inference_type = tf.compat.v1.lite.constants.QUANTIZED_UINT8
+ input_arrays = converter.get_input_arrays()
+ converter.quantized_input_stats = {input_arrays[0]: (127.5, 127.5)}
+ return converter.convert()
+
+
+def check_tflite_consistency(graph_def, tflite_model, image_path):
+ """Runs tflite and frozen graph on same input, check their outputs match."""
+ # Load tflite model and check input size.
+ interpreter = tf.lite.Interpreter(model_content=tflite_model)
+ interpreter.allocate_tensors()
+ input_details = interpreter.get_input_details()
+ output_details = interpreter.get_output_details()
+ height, width = input_details[0]['shape'][1:3]
+
+ # Prepare input image data.
+ with tf.io.gfile.GFile(image_path, 'rb') as f:
+ image = Image.open(f)
+ image = np.asarray(image.convert('RGB').resize((width, height)))
+ image = np.expand_dims(image, 0)
+
+ # Output from tflite model.
+ interpreter.set_tensor(input_details[0]['index'], image)
+ interpreter.invoke()
+ output_tflite = interpreter.get_tensor(output_details[0]['index'])
+
+ with tf.Graph().as_default():
+ tf.graph_util.import_graph_def(graph_def, name='')
+ with tf.compat.v1.Session() as sess:
+ # Note here the graph will include preprocessing part of the graph
+ # (e.g. resize, pad, normalize). Given the input image size is at the
+ # crop size (backbone input size), resize / pad should be an identity op.
+ output_graph = sess.run(
+ FLAGS.output_tensor_name, feed_dict={'ImageTensor:0': image})
+
+ print('%.2f%% pixels have matched semantic labels.' % (
+ 100 * np.mean(output_graph == output_tflite)))
+
+
+def main(unused_argv):
+ with tf.io.gfile.GFile(FLAGS.quantized_graph_def_path, 'rb') as f:
+ graph_def = tf.compat.v1.GraphDef.FromString(f.read())
+ tflite_model = convert_to_tflite(
+ graph_def, FLAGS.input_tensor_name, FLAGS.output_tensor_name)
+
+ if FLAGS.output_tflite_path:
+ with tf.io.gfile.GFile(FLAGS.output_tflite_path, 'wb') as f:
+ f.write(tflite_model)
+
+ if FLAGS.test_image_path:
+ check_tflite_consistency(graph_def, tflite_model, FLAGS.test_image_path)
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/research/deeplab/datasets/build_cityscapes_data.py b/research/deeplab/datasets/build_cityscapes_data.py
index ce81baef20a460abaa634d3f1dcb6760a0858dec..53c11e30310f38a8abadeeeae78a0d71f5f7f8cb 100644
--- a/research/deeplab/datasets/build_cityscapes_data.py
+++ b/research/deeplab/datasets/build_cityscapes_data.py
@@ -113,17 +113,23 @@ def _get_files(data, dataset_split):
Args:
data: String, desired data ('image' or 'label').
- dataset_split: String, dataset split ('train', 'val', 'test')
+ dataset_split: String, dataset split ('train_fine', 'val_fine', 'test_fine')
Returns:
A list of sorted file names or None when getting label for
test set.
"""
- if data == 'label' and dataset_split == 'test':
- return None
+ if dataset_split == 'train_fine':
+ split_dir = 'train'
+ elif dataset_split == 'val_fine':
+ split_dir = 'val'
+ elif dataset_split == 'test_fine':
+ split_dir = 'test'
+ else:
+ raise RuntimeError("Split {} is not supported".format(dataset_split))
pattern = '*%s.%s' % (_POSTFIX_MAP[data], _DATA_FORMAT_MAP[data])
search_files = os.path.join(
- FLAGS.cityscapes_root, _FOLDERS_MAP[data], dataset_split, '*', pattern)
+ FLAGS.cityscapes_root, _FOLDERS_MAP[data], split_dir, '*', pattern)
filenames = glob.glob(search_files)
return sorted(filenames)
@@ -132,7 +138,7 @@ def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
- dataset_split: The dataset split (e.g., train, val).
+ dataset_split: The dataset split (e.g., train_fine, val_fine).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
@@ -142,8 +148,12 @@ def _convert_dataset(dataset_split):
label_files = _get_files('label', dataset_split)
num_images = len(image_files)
+ num_labels = len(label_files)
num_per_shard = int(math.ceil(num_images / _NUM_SHARDS))
+ if num_images != num_labels:
+ raise RuntimeError("The number of images and labels doesn't match: {} {}".format(num_images, num_labels))
+
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
@@ -179,8 +189,8 @@ def _convert_dataset(dataset_split):
def main(unused_argv):
- # Only support converting 'train' and 'val' sets for now.
- for dataset_split in ['train', 'val']:
+ # Only support converting 'train_fine', 'val_fine' and 'test_fine' sets for now.
+ for dataset_split in ['train_fine', 'val_fine', 'test_fine']:
_convert_dataset(dataset_split)
diff --git a/research/deeplab/datasets/convert_cityscapes.sh b/research/deeplab/datasets/convert_cityscapes.sh
index a95b5d66aad79ae7cbd6ad2d3ee60550ab7f6239..ddc39fb11ddfed38e0b9daf5974b96286a4aa43b 100644
--- a/research/deeplab/datasets/convert_cityscapes.sh
+++ b/research/deeplab/datasets/convert_cityscapes.sh
@@ -42,6 +42,8 @@ WORK_DIR="."
# Root path for Cityscapes dataset.
CITYSCAPES_ROOT="${WORK_DIR}/cityscapes"
+export PYTHONPATH="${CITYSCAPES_ROOT}:${PYTHONPATH}"
+
# Create training labels.
python "${CITYSCAPES_ROOT}/cityscapesscripts/preparation/createTrainIdLabelImgs.py"
diff --git a/research/deeplab/datasets/download_and_convert_voc2012.sh b/research/deeplab/datasets/download_and_convert_voc2012.sh
index c02235182d427dfb1d63154a8266ad37b0a1d53f..3126f729decac4b684b237d6402d6c29aabc7541 100644
--- a/research/deeplab/datasets/download_and_convert_voc2012.sh
+++ b/research/deeplab/datasets/download_and_convert_voc2012.sh
@@ -51,19 +51,20 @@ download_and_uncompress() {
wget -nd -c "${BASE_URL}/${FILENAME}"
fi
echo "Uncompressing ${FILENAME}"
- tar -xf "${FILENAME}"
+ sudo apt install unzip
+ unzip "${FILENAME}"
}
# Download the images.
-BASE_URL="http://host.robots.ox.ac.uk/pascal/VOC/voc2012/"
-FILENAME="VOCtrainval_11-May-2012.tar"
+BASE_URL="https://data.deepai.org/"
+FILENAME="PascalVOC2012.zip"
download_and_uncompress "${BASE_URL}" "${FILENAME}"
cd "${CURRENT_DIR}"
# Root path for PASCAL VOC 2012 dataset.
-PASCAL_ROOT="${WORK_DIR}/VOCdevkit/VOC2012"
+PASCAL_ROOT="${WORK_DIR}/VOC2012"
# Remove the colormap in the ground truth annotations.
SEG_FOLDER="${PASCAL_ROOT}/SegmentationClass"
diff --git a/research/deeplab/deprecated/segmentation_dataset.py b/research/deeplab/deprecated/segmentation_dataset.py
index 2a5980b1d940878cd1aead4a5d301cca7b4a642b..8a6a8c766e40f06e6bee4f7550018b7933a51b92 100644
--- a/research/deeplab/deprecated/segmentation_dataset.py
+++ b/research/deeplab/deprecated/segmentation_dataset.py
@@ -81,8 +81,8 @@ DatasetDescriptor = collections.namedtuple(
_CITYSCAPES_INFORMATION = DatasetDescriptor(
splits_to_sizes={
- 'train': 2975,
- 'val': 500,
+ 'train_fine': 2975,
+ 'val_fine': 500,
},
num_classes=19,
ignore_label=255,
diff --git a/research/deeplab/g3doc/cityscapes.md b/research/deeplab/g3doc/cityscapes.md
index af703088e61b49aa81bf62b536469b410f0fb352..5a660aaca342d16461b2355cae058900a212db1a 100644
--- a/research/deeplab/g3doc/cityscapes.md
+++ b/research/deeplab/g3doc/cityscapes.md
@@ -43,7 +43,7 @@ A local training job using `xception_65` can be run with the following command:
python deeplab/train.py \
--logtostderr \
--training_number_of_steps=90000 \
- --train_split="train" \
+ --train_split="train_fine" \
--model_variant="xception_65" \
--atrous_rates=6 \
--atrous_rates=12 \
@@ -95,7 +95,7 @@ command:
# From tensorflow/models/research/
python deeplab/eval.py \
--logtostderr \
- --eval_split="val" \
+ --eval_split="val_fine" \
--model_variant="xception_65" \
--atrous_rates=6 \
--atrous_rates=12 \
@@ -121,7 +121,7 @@ command:
# From tensorflow/models/research/
python deeplab/vis.py \
--logtostderr \
- --vis_split="val" \
+ --vis_split="val_fine" \
--model_variant="xception_65" \
--atrous_rates=6 \
--atrous_rates=12 \
diff --git a/research/deeplab/g3doc/installation.md b/research/deeplab/g3doc/installation.md
index 8629aba42207fc6e35c907024485c0e7f29f5e10..591a1f8da50d139f48cb3cd3c299535fcdab16a9 100644
--- a/research/deeplab/g3doc/installation.md
+++ b/research/deeplab/g3doc/installation.md
@@ -68,6 +68,6 @@ Quick running the whole code on the PASCAL VOC 2012 dataset:
```bash
# From tensorflow/models/research/deeplab
-sh local_test.sh
+bash local_test.sh
```
diff --git a/research/deeplab/g3doc/quantize.md b/research/deeplab/g3doc/quantize.md
index d88a2e9a8acbac4a0de6e3ea2bed65cb44535665..65dbdd70b4dd67838326b434dd5fe9753a5afc1c 100644
--- a/research/deeplab/g3doc/quantize.md
+++ b/research/deeplab/g3doc/quantize.md
@@ -42,7 +42,6 @@ python deeplab/train.py \
--train_batch_size=8 \
--base_learning_rate=3e-5 \
--dataset="pascal_voc_seg" \
- --initialize_last_layer \
--quantize_delay_step=0 \
--tf_initial_checkpoint=${PATH_TO_TRAINED_FLOAT_MODEL} \
--train_logdir=${PATH_TO_TRAIN_DIR} \
@@ -65,18 +64,12 @@ python deeplab/export_model.py \
Commandline below shows how to convert exported graphdef to TFlite model.
```
-tflite_convert \
- --graph_def_file=${OUTPUT_DIR}/frozen_inference_graph.pb \
- --output_file=${OUTPUT_DIR}/frozen_inference_graph.tflite \
- --output_format=TFLITE \
- --input_shape=1,513,513,3 \
- --input_arrays="MobilenetV2/MobilenetV2/input" \
- --inference_type=QUANTIZED_UINT8 \
- --inference_input_type=QUANTIZED_UINT8 \
- --std_dev_values=128 \
- --mean_values=128 \
- --change_concat_input_ranges=true \
- --output_arrays="ArgMax"
+# From tensorflow/models/research/
+python deeplab/convert_to_tflite.py \
+ --quantized_graph_def_path=${OUTPUT_DIR}/frozen_inference_graph.pb \
+ --input_tensor_name=MobilenetV2/MobilenetV2/input:0 \
+ --output_tflite_path=${OUTPUT_DIR}/frozen_inference_graph.tflite \
+ --test_image_path=${PATH_TO_TEST_IMAGE}
```
**[Important]** Note that converted model expects 513x513 RGB input and doesn't
diff --git a/research/deeplab/local_test.sh b/research/deeplab/local_test.sh
index d5e4a5f42bb4241d4b6dd1b9d8a2619c4ca9dc8b..c9ad75f69280a9179a891b52f310b05a5b744def 100644
--- a/research/deeplab/local_test.sh
+++ b/research/deeplab/local_test.sh
@@ -19,7 +19,7 @@
#
# Usage:
# # From the tensorflow/models/research/deeplab directory.
-# sh ./local_test.sh
+# bash ./local_test.sh
#
#
@@ -42,7 +42,7 @@ python "${WORK_DIR}"/model_test.py
# Go to datasets folder and download PASCAL VOC 2012 segmentation dataset.
DATASET_DIR="datasets"
cd "${WORK_DIR}/${DATASET_DIR}"
-sh download_and_convert_voc2012.sh
+bash download_and_convert_voc2012.sh
# Go back to original directory.
cd "${CURRENT_DIR}"
diff --git a/research/delf/README.md b/research/delf/README.md
index a8bea62000a0f8a03c529284172107607d83192b..f89d71d555f214c4492e2dcedf9370cb9daf9bef 100644
--- a/research/delf/README.md
+++ b/research/delf/README.md
@@ -112,11 +112,17 @@ in the [Detect-to-Retrieve paper](https://arxiv.org/abs/1812.01584). Boosts
performance by ~4% mAP compared to ICCV'17 DELF model.
**DELG pre-trained on the Google-Landmarks dataset v1**
-([link](http://storage.googleapis.com/delf/delg_gld_20200520.tar.gz)). Presented
-in the [DELG paper](https://arxiv.org/abs/2001.05027).
+([R101-DELG](http://storage.googleapis.com/delf/r101delg_gld_20200814.tar.gz),
+[R50-DELG](http://storage.googleapis.com/delf/r50delg_gld_20200814.tar.gz)).
+Presented in the [DELG paper](https://arxiv.org/abs/2001.05027).
+
+**DELG pre-trained on the Google-Landmarks dataset v2 (clean)**
+([R101-DELG](https://storage.googleapis.com/delf/r101delg_gldv2clean_20200914.tar.gz),
+[R50-DELG](https://storage.googleapis.com/delf/r50delg_gldv2clean_20200914.tar.gz)).
+Presented in the [DELG paper](https://arxiv.org/abs/2001.05027).
**RN101-ArcFace pre-trained on the Google-Landmarks dataset v2 (train-clean)**
-([link](https://storage.googleapis.com/delf/rn101_af_gldv2clean_20200521.tar.gz)).
+([link](https://storage.googleapis.com/delf/rn101_af_gldv2clean_20200814.tar.gz)).
Presented in the [GLDv2 paper](https://arxiv.org/abs/2004.01804).
**DELF pre-trained on Landmarks-Clean/Landmarks-Full dataset**
diff --git a/research/delf/delf/__init__.py b/research/delf/delf/__init__.py
index a52df3c4546414e61f479357d06b65d4c132c753..a3c5d37bc44e10b74ce383f15c5979115d875fb1 100644
--- a/research/delf/delf/__init__.py
+++ b/research/delf/delf/__init__.py
@@ -30,10 +30,13 @@ from delf.python import feature_aggregation_similarity
from delf.python import feature_extractor
from delf.python import feature_io
from delf.python import utils
+from delf.python import whiten
from delf.python.examples import detector
from delf.python.examples import extractor
from delf.python import detect_to_retrieve
from delf.python import training
from delf.python.training import model
-from delf.python.training import datasets
+from delf.python import datasets
+from delf.python.datasets import google_landmarks_dataset
+from delf.python.datasets import revisited_op
# pylint: enable=unused-import
diff --git a/research/delf/delf/protos/delf_config.proto b/research/delf/delf/protos/delf_config.proto
index d13d911b48b7a9fe46e7c8fcb3c1c20d5a927e04..c7cd5b1ce27b227a0ff4c3d4536087d1151a6396 100644
--- a/research/delf/delf/protos/delf_config.proto
+++ b/research/delf/delf/protos/delf_config.proto
@@ -49,6 +49,12 @@ message DelfLocalFeatureConfig {
// PCA parameters for DELF local feature. This is used only if use_pca is
// true.
optional DelfPcaParameters pca_parameters = 6;
+
+ // If true, the returned keypoint locations are grounded to coordinates of the
+ // resized image used for extraction. If false (default), the returned
+ // keypoint locations are grounded to coordinates of the original image that
+ // is fed into feature extraction.
+ optional bool use_resized_coordinates = 7 [default = false];
}
message DelfGlobalFeatureConfig {
diff --git a/research/delf/delf/python/datasets/google_landmarks_dataset/README.md b/research/delf/delf/python/datasets/google_landmarks_dataset/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..4f34b59aaa4bf36040e49e67fd27adc80f62cb2a
--- /dev/null
+++ b/research/delf/delf/python/datasets/google_landmarks_dataset/README.md
@@ -0,0 +1,123 @@
+## GLDv2 code/models
+
+[](https://arxiv.org/abs/2004.01804)
+
+These instructions can be used to reproduce results from the
+[GLDv2 paper](https://arxiv.org/abs/2004.01804). We present here results on the
+Revisited Oxford/Paris datasets since they are smaller and quicker to
+reproduce -- but note that a very similar procedure can be used to obtain
+results on the GLDv2 retrieval or recognition datasets.
+
+Note that this directory also contains code to compute GLDv2 metrics: see
+`compute_retrieval_metrics.py`, `compute_recognition_metrics.py` and associated
+file reading / metric computation modules.
+
+For more details on the dataset, please refer to its
+[website](https://github.com/cvdfoundation/google-landmark).
+
+### Install DELF library
+
+To be able to use this code, please follow
+[these instructions](../../../../INSTALL_INSTRUCTIONS.md) to properly install the
+DELF library.
+
+### Download Revisited Oxford/Paris datasets
+
+```bash
+mkdir -p ~/revisitop/data && cd ~/revisitop/data
+
+# Oxford dataset.
+wget http://www.robots.ox.ac.uk/~vgg/data/oxbuildings/oxbuild_images.tgz
+mkdir oxford5k_images
+tar -xvzf oxbuild_images.tgz -C oxford5k_images/
+
+# Paris dataset. Download and move all images to same directory.
+wget http://www.robots.ox.ac.uk/~vgg/data/parisbuildings/paris_1.tgz
+wget http://www.robots.ox.ac.uk/~vgg/data/parisbuildings/paris_2.tgz
+mkdir paris6k_images_tmp
+tar -xvzf paris_1.tgz -C paris6k_images_tmp/
+tar -xvzf paris_2.tgz -C paris6k_images_tmp/
+mkdir paris6k_images
+mv paris6k_images_tmp/paris/*/*.jpg paris6k_images/
+
+# Revisited annotations.
+wget http://cmp.felk.cvut.cz/revisitop/data/datasets/roxford5k/gnd_roxford5k.mat
+wget http://cmp.felk.cvut.cz/revisitop/data/datasets/rparis6k/gnd_rparis6k.mat
+```
+
+### Download model
+
+```bash
+# From models/research/delf/delf/python/datasets/google_landmarks_dataset
+mkdir parameters && cd parameters
+
+# RN101-ArcFace model trained on GLDv2-clean.
+wget https://storage.googleapis.com/delf/rn101_af_gldv2clean_20200814.tar.gz
+tar -xvzf rn101_af_gldv2clean_20200814.tar.gz
+```
+
+### Feature extraction
+
+We present here commands for extraction on `roxford5k`. To extract on `rparis6k`
+instead, please edit the arguments accordingly (especially the
+`dataset_file_path` argument).
+
+#### Query feature extraction
+
+In the Revisited Oxford/Paris experimental protocol, query images must be the
+cropped before feature extraction (this is done in the `extract_features`
+script, when setting `image_set=query`). Note that this is specific to these
+datasets, and not required for the GLDv2 retrieval/recognition datasets.
+
+Run query feature extraction as follows:
+
+```bash
+# From models/research/delf/delf/python/datasets/google_landmarks_dataset
+python3 ../../delg/extract_features.py \
+ --delf_config_path rn101_af_gldv2clean_config.pbtxt \
+ --dataset_file_path ~/revisitop/data/gnd_roxford5k.mat \
+ --images_dir ~/revisitop/data/oxford5k_images \
+ --image_set query \
+ --output_features_dir ~/revisitop/data/oxford5k_features/query
+```
+
+#### Index feature extraction
+
+Run index feature extraction as follows:
+
+```bash
+# From models/research/delf/delf/python/datasets/google_landmarks_dataset
+python3 ../../delg/extract_features.py \
+ --delf_config_path rn101_af_gldv2clean_config.pbtxt \
+ --dataset_file_path ~/revisitop/data/gnd_roxford5k.mat \
+ --images_dir ~/revisitop/data/oxford5k_images \
+ --image_set index \
+ --output_features_dir ~/revisitop/data/oxford5k_features/index
+```
+
+### Perform retrieval
+
+To run retrieval on `roxford5k`, the following command can be used:
+
+```bash
+# From models/research/delf/delf/python/datasets/google_landmarks_dataset
+python3 ../../delg/perform_retrieval.py \
+ --dataset_file_path ~/revisitop/data/gnd_roxford5k.mat \
+ --query_features_dir ~/revisitop/data/oxford5k_features/query \
+ --index_features_dir ~/revisitop/data/oxford5k_features/index \
+ --output_dir ~/revisitop/results/oxford5k
+```
+
+A file with named `metrics.txt` will be written to the path given in
+`output_dir`. The contents should look approximately like:
+
+```
+hard
+ mAP=55.54
+ mP@k[ 1 5 10] [88.57 80.86 70.14]
+ mR@k[ 1 5 10] [19.46 33.65 42.44]
+medium
+ mAP=76.23
+ mP@k[ 1 5 10] [95.71 92.86 90.43]
+ mR@k[ 1 5 10] [10.17 25.96 35.29]
+```
diff --git a/research/delf/delf/python/datasets/google_landmarks_dataset/__init__.py b/research/delf/delf/python/datasets/google_landmarks_dataset/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e24e0fb7c56dc1ece37ff53f8a1a7dd5ba4ddfd
--- /dev/null
+++ b/research/delf/delf/python/datasets/google_landmarks_dataset/__init__.py
@@ -0,0 +1,22 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Module exposing Google Landmarks dataset for training."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint: disable=unused-import
+from delf.python.datasets.google_landmarks_dataset import googlelandmarks
+# pylint: enable=unused-import
diff --git a/research/delf/delf/python/google_landmarks_dataset/compute_recognition_metrics.py b/research/delf/delf/python/datasets/google_landmarks_dataset/compute_recognition_metrics.py
similarity index 95%
rename from research/delf/delf/python/google_landmarks_dataset/compute_recognition_metrics.py
rename to research/delf/delf/python/datasets/google_landmarks_dataset/compute_recognition_metrics.py
index f80cf47de7487d4cd584c969d994f7d3f1135cae..4c241ed5380cacd635016f8a197156d389a62a63 100644
--- a/research/delf/delf/python/google_landmarks_dataset/compute_recognition_metrics.py
+++ b/research/delf/delf/python/datasets/google_landmarks_dataset/compute_recognition_metrics.py
@@ -24,9 +24,9 @@ from __future__ import print_function
import argparse
import sys
-from tensorflow.python.platform import app
-from delf.python.google_landmarks_dataset import dataset_file_io
-from delf.python.google_landmarks_dataset import metrics
+from absl import app
+from delf.python.datasets.google_landmarks_dataset import dataset_file_io
+from delf.python.datasets.google_landmarks_dataset import metrics
cmd_args = None
diff --git a/research/delf/delf/python/google_landmarks_dataset/compute_retrieval_metrics.py b/research/delf/delf/python/datasets/google_landmarks_dataset/compute_retrieval_metrics.py
similarity index 96%
rename from research/delf/delf/python/google_landmarks_dataset/compute_retrieval_metrics.py
rename to research/delf/delf/python/datasets/google_landmarks_dataset/compute_retrieval_metrics.py
index adcee356e5d64d094236cda9656c86164c24faf8..231c320168cb61d175b6a486ec7a91de08d5fe6f 100644
--- a/research/delf/delf/python/google_landmarks_dataset/compute_retrieval_metrics.py
+++ b/research/delf/delf/python/datasets/google_landmarks_dataset/compute_retrieval_metrics.py
@@ -24,9 +24,9 @@ from __future__ import print_function
import argparse
import sys
-from tensorflow.python.platform import app
-from delf.python.google_landmarks_dataset import dataset_file_io
-from delf.python.google_landmarks_dataset import metrics
+from absl import app
+from delf.python.datasets.google_landmarks_dataset import dataset_file_io
+from delf.python.datasets.google_landmarks_dataset import metrics
cmd_args = None
diff --git a/research/delf/delf/python/google_landmarks_dataset/dataset_file_io.py b/research/delf/delf/python/datasets/google_landmarks_dataset/dataset_file_io.py
similarity index 100%
rename from research/delf/delf/python/google_landmarks_dataset/dataset_file_io.py
rename to research/delf/delf/python/datasets/google_landmarks_dataset/dataset_file_io.py
diff --git a/research/delf/delf/python/google_landmarks_dataset/dataset_file_io_test.py b/research/delf/delf/python/datasets/google_landmarks_dataset/dataset_file_io_test.py
similarity index 92%
rename from research/delf/delf/python/google_landmarks_dataset/dataset_file_io_test.py
rename to research/delf/delf/python/datasets/google_landmarks_dataset/dataset_file_io_test.py
index 0101d989fba85e487842e65b3b1aec4e728c101c..8bd2ac5e0f363cd15a386b50c0ecae6c9825b61e 100644
--- a/research/delf/delf/python/google_landmarks_dataset/dataset_file_io_test.py
+++ b/research/delf/delf/python/datasets/google_landmarks_dataset/dataset_file_io_test.py
@@ -23,7 +23,7 @@ import os
from absl import flags
import tensorflow as tf
-from delf.python.google_landmarks_dataset import dataset_file_io
+from delf.python.datasets.google_landmarks_dataset import dataset_file_io
FLAGS = flags.FLAGS
@@ -32,8 +32,7 @@ class DatasetFileIoTest(tf.test.TestCase):
def testReadRecognitionSolutionWorks(self):
# Define inputs.
- file_path = os.path.join(FLAGS.test_tmpdir,
- 'recognition_solution.csv')
+ file_path = os.path.join(FLAGS.test_tmpdir, 'recognition_solution.csv')
with tf.io.gfile.GFile(file_path, 'w') as f:
f.write('id,landmarks,Usage\n')
f.write('0123456789abcdef,0 12,Public\n')
@@ -64,8 +63,7 @@ class DatasetFileIoTest(tf.test.TestCase):
def testReadRetrievalSolutionWorks(self):
# Define inputs.
- file_path = os.path.join(FLAGS.test_tmpdir,
- 'retrieval_solution.csv')
+ file_path = os.path.join(FLAGS.test_tmpdir, 'retrieval_solution.csv')
with tf.io.gfile.GFile(file_path, 'w') as f:
f.write('id,images,Usage\n')
f.write('0123456789abcdef,None,Ignored\n')
@@ -96,8 +94,7 @@ class DatasetFileIoTest(tf.test.TestCase):
def testReadRecognitionPredictionsWorks(self):
# Define inputs.
- file_path = os.path.join(FLAGS.test_tmpdir,
- 'recognition_predictions.csv')
+ file_path = os.path.join(FLAGS.test_tmpdir, 'recognition_predictions.csv')
with tf.io.gfile.GFile(file_path, 'w') as f:
f.write('id,landmarks\n')
f.write('0123456789abcdef,12 0.1 \n')
@@ -134,8 +131,7 @@ class DatasetFileIoTest(tf.test.TestCase):
def testReadRetrievalPredictionsWorks(self):
# Define inputs.
- file_path = os.path.join(FLAGS.test_tmpdir,
- 'retrieval_predictions.csv')
+ file_path = os.path.join(FLAGS.test_tmpdir, 'retrieval_predictions.csv')
with tf.io.gfile.GFile(file_path, 'w') as f:
f.write('id,images\n')
f.write('0123456789abcdef,fedcba9876543250 \n')
diff --git a/research/delf/delf/python/training/datasets/googlelandmarks.py b/research/delf/delf/python/datasets/google_landmarks_dataset/googlelandmarks.py
similarity index 96%
rename from research/delf/delf/python/training/datasets/googlelandmarks.py
rename to research/delf/delf/python/datasets/google_landmarks_dataset/googlelandmarks.py
index 9d184aceca875bdba4109384141701049d6b389e..b6122f5c79c28a1eb159f2f8b838af10d7ea6276 100644
--- a/research/delf/delf/python/training/datasets/googlelandmarks.py
+++ b/research/delf/delf/python/datasets/google_landmarks_dataset/googlelandmarks.py
@@ -46,8 +46,6 @@ class _DataAugmentationParams(object):
central_fraction = 0.875
random_reflection = False
- input_rows = 321
- input_cols = 321
def NormalizeImages(images, pixel_value_scale=0.5, pixel_value_offset=0.5):
@@ -70,11 +68,12 @@ def NormalizeImages(images, pixel_value_scale=0.5, pixel_value_offset=0.5):
return normalized_images
-def _ImageNetCrop(image):
+def _ImageNetCrop(image, image_size):
"""Imagenet-style crop with random bbox and aspect ratio.
Args:
image: a `Tensor`, image to crop.
+ image_size: an `int`. The image size for the decoded image, on each side.
Returns:
cropped_image: `Tensor`, cropped image.
@@ -95,7 +94,7 @@ def _ImageNetCrop(image):
cropped_image.set_shape([None, None, 3])
cropped_image = tf.image.resize(
- cropped_image, [params.input_rows, params.input_cols], method='area')
+ cropped_image, [image_size, image_size], method='area')
if params.random_reflection:
cropped_image = tf.image.random_flip_left_right(cropped_image)
@@ -122,7 +121,7 @@ def _ParseFunction(example, name_to_features, image_size, augmentation):
image = NormalizeImages(
image, pixel_value_scale=128.0, pixel_value_offset=128.0)
if augmentation:
- image = _ImageNetCrop(image)
+ image = _ImageNetCrop(image, image_size)
else:
image = tf.image.resize(image, [image_size, image_size])
image.set_shape([image_size, image_size, 3])
diff --git a/research/delf/delf/python/google_landmarks_dataset/metrics.py b/research/delf/delf/python/datasets/google_landmarks_dataset/metrics.py
similarity index 100%
rename from research/delf/delf/python/google_landmarks_dataset/metrics.py
rename to research/delf/delf/python/datasets/google_landmarks_dataset/metrics.py
diff --git a/research/delf/delf/python/google_landmarks_dataset/metrics_test.py b/research/delf/delf/python/datasets/google_landmarks_dataset/metrics_test.py
similarity index 98%
rename from research/delf/delf/python/google_landmarks_dataset/metrics_test.py
rename to research/delf/delf/python/datasets/google_landmarks_dataset/metrics_test.py
index 50838cae2b5bfaa8f6f0c5cbfab2a07aa20b7c52..ee8a443de1644f83031d2a32300bbe7f7b447732 100644
--- a/research/delf/delf/python/google_landmarks_dataset/metrics_test.py
+++ b/research/delf/delf/python/datasets/google_landmarks_dataset/metrics_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
import tensorflow as tf
-from delf.python.google_landmarks_dataset import metrics
+from delf.python.datasets.google_landmarks_dataset import metrics
def _CreateRecognitionSolution():
diff --git a/research/delf/delf/python/google_landmarks_dataset/rn101_af_gldv2clean_config.pbtxt b/research/delf/delf/python/datasets/google_landmarks_dataset/rn101_af_gldv2clean_config.pbtxt
similarity index 76%
rename from research/delf/delf/python/google_landmarks_dataset/rn101_af_gldv2clean_config.pbtxt
rename to research/delf/delf/python/datasets/google_landmarks_dataset/rn101_af_gldv2clean_config.pbtxt
index 992cb0fd142ba8c6d89c763d5e323a0d33e7a3a5..6a065d51280d0705f62969f9b28f711d4982819e 100644
--- a/research/delf/delf/python/google_landmarks_dataset/rn101_af_gldv2clean_config.pbtxt
+++ b/research/delf/delf/python/datasets/google_landmarks_dataset/rn101_af_gldv2clean_config.pbtxt
@@ -1,6 +1,6 @@
use_local_features: false
use_global_features: true
-model_path: "parameters/rn101_af_gldv2clean_20200521"
+model_path: "parameters/rn101_af_gldv2clean_20200814"
image_scales: 0.70710677
image_scales: 1.0
image_scales: 1.4142135
diff --git a/research/delf/delf/python/datasets/revisited_op/__init__.py b/research/delf/delf/python/datasets/revisited_op/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a8b35fb1b46399bb735624a354fc7947d6ec242
--- /dev/null
+++ b/research/delf/delf/python/datasets/revisited_op/__init__.py
@@ -0,0 +1,22 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Module for revisited Oxford and Paris datasets."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint: disable=unused-import
+from delf.python.datasets.revisited_op import dataset
+# pylint: enable=unused-import
diff --git a/research/delf/delf/python/datasets/revisited_op/dataset.py b/research/delf/delf/python/datasets/revisited_op/dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae3020cd3451955147dbcd41754fba15e0d9f3c7
--- /dev/null
+++ b/research/delf/delf/python/datasets/revisited_op/dataset.py
@@ -0,0 +1,535 @@
+# Copyright 2019 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Python library to parse ground-truth/evaluate on Revisited datasets."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import pickle
+
+import numpy as np
+from scipy.io import matlab
+import tensorflow as tf
+
+_GROUND_TRUTH_KEYS = ['easy', 'hard', 'junk']
+
+DATASET_NAMES = ['roxford5k', 'rparis6k']
+
+
+def ReadDatasetFile(dataset_file_path):
+ """Reads dataset file in Revisited Oxford/Paris ".mat" format.
+
+ Args:
+ dataset_file_path: Path to dataset file, in .mat format.
+
+ Returns:
+ query_list: List of query image names.
+ index_list: List of index image names.
+ ground_truth: List containing ground-truth information for dataset. Each
+ entry is a dict corresponding to the ground-truth information for a query.
+ The dict may have keys 'easy', 'hard', or 'junk', mapping to a NumPy
+ array of integers; additionally, it has a key 'bbx' mapping to a NumPy
+ array of floats with bounding box coordinates.
+ """
+ with tf.io.gfile.GFile(dataset_file_path, 'rb') as f:
+ cfg = matlab.loadmat(f)
+
+ # Parse outputs according to the specificities of the dataset file.
+ query_list = [str(im_array[0]) for im_array in np.squeeze(cfg['qimlist'])]
+ index_list = [str(im_array[0]) for im_array in np.squeeze(cfg['imlist'])]
+ ground_truth_raw = np.squeeze(cfg['gnd'])
+ ground_truth = []
+ for query_ground_truth_raw in ground_truth_raw:
+ query_ground_truth = {}
+ for ground_truth_key in _GROUND_TRUTH_KEYS:
+ if ground_truth_key in query_ground_truth_raw.dtype.names:
+ adjusted_labels = query_ground_truth_raw[ground_truth_key] - 1
+ query_ground_truth[ground_truth_key] = adjusted_labels.flatten()
+
+ query_ground_truth['bbx'] = np.squeeze(query_ground_truth_raw['bbx'])
+ ground_truth.append(query_ground_truth)
+
+ return query_list, index_list, ground_truth
+
+
+def _ParseGroundTruth(ok_list, junk_list):
+ """Constructs dictionary of ok/junk indices for a data subset and query.
+
+ Args:
+ ok_list: List of NumPy arrays containing true positive indices for query.
+ junk_list: List of NumPy arrays containing ignored indices for query.
+
+ Returns:
+ ok_junk_dict: Dict mapping 'ok' and 'junk' strings to NumPy array of
+ indices.
+ """
+ ok_junk_dict = {}
+ ok_junk_dict['ok'] = np.concatenate(ok_list)
+ ok_junk_dict['junk'] = np.concatenate(junk_list)
+ return ok_junk_dict
+
+
+def ParseEasyMediumHardGroundTruth(ground_truth):
+ """Parses easy/medium/hard ground-truth from Revisited datasets.
+
+ Args:
+ ground_truth: Usually the output from ReadDatasetFile(). List containing
+ ground-truth information for dataset. Each entry is a dict corresponding
+ to the ground-truth information for a query. The dict must have keys
+ 'easy', 'hard', and 'junk', mapping to a NumPy array of integers.
+
+ Returns:
+ easy_ground_truth: List containing ground-truth information for easy subset
+ of dataset. Each entry is a dict corresponding to the ground-truth
+ information for a query. The dict has keys 'ok' and 'junk', mapping to a
+ NumPy array of integers.
+ medium_ground_truth: Same as `easy_ground_truth`, but for the medium subset.
+ hard_ground_truth: Same as `easy_ground_truth`, but for the hard subset.
+ """
+ num_queries = len(ground_truth)
+
+ easy_ground_truth = []
+ medium_ground_truth = []
+ hard_ground_truth = []
+ for i in range(num_queries):
+ easy_ground_truth.append(
+ _ParseGroundTruth([ground_truth[i]['easy']],
+ [ground_truth[i]['junk'], ground_truth[i]['hard']]))
+ medium_ground_truth.append(
+ _ParseGroundTruth([ground_truth[i]['easy'], ground_truth[i]['hard']],
+ [ground_truth[i]['junk']]))
+ hard_ground_truth.append(
+ _ParseGroundTruth([ground_truth[i]['hard']],
+ [ground_truth[i]['junk'], ground_truth[i]['easy']]))
+
+ return easy_ground_truth, medium_ground_truth, hard_ground_truth
+
+
+def AdjustPositiveRanks(positive_ranks, junk_ranks):
+ """Adjusts positive ranks based on junk ranks.
+
+ Args:
+ positive_ranks: Sorted 1D NumPy integer array.
+ junk_ranks: Sorted 1D NumPy integer array.
+
+ Returns:
+ adjusted_positive_ranks: Sorted 1D NumPy array.
+ """
+ if not junk_ranks.size:
+ return positive_ranks
+
+ adjusted_positive_ranks = positive_ranks
+ j = 0
+ for i, positive_index in enumerate(positive_ranks):
+ while (j < len(junk_ranks) and positive_index > junk_ranks[j]):
+ j += 1
+
+ adjusted_positive_ranks[i] -= j
+
+ return adjusted_positive_ranks
+
+
+def ComputeAveragePrecision(positive_ranks):
+ """Computes average precision according to dataset convention.
+
+ It assumes that `positive_ranks` contains the ranks for all expected positive
+ index images to be retrieved. If `positive_ranks` is empty, returns
+ `average_precision` = 0.
+
+ Note that average precision computation here does NOT use the finite sum
+ method (see
+ https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision)
+ which is common in information retrieval literature. Instead, the method
+ implemented here integrates over the precision-recall curve by averaging two
+ adjacent precision points, then multiplying by the recall step. This is the
+ convention for the Revisited Oxford/Paris datasets.
+
+ Args:
+ positive_ranks: Sorted 1D NumPy integer array, zero-indexed.
+
+ Returns:
+ average_precision: Float.
+ """
+ average_precision = 0.0
+
+ num_expected_positives = len(positive_ranks)
+ if not num_expected_positives:
+ return average_precision
+
+ recall_step = 1.0 / num_expected_positives
+ for i, rank in enumerate(positive_ranks):
+ if not rank:
+ left_precision = 1.0
+ else:
+ left_precision = i / rank
+
+ right_precision = (i + 1) / (rank + 1)
+ average_precision += (left_precision + right_precision) * recall_step / 2
+
+ return average_precision
+
+
+def ComputePRAtRanks(positive_ranks, desired_pr_ranks):
+ """Computes precision/recall at desired ranks.
+
+ It assumes that `positive_ranks` contains the ranks for all expected positive
+ index images to be retrieved. If `positive_ranks` is empty, return all-zeros
+ `precisions`/`recalls`.
+
+ If a desired rank is larger than the last positive rank, its precision is
+ computed based on the last positive rank. For example, if `desired_pr_ranks`
+ is [10] and `positive_ranks` = [0, 7] --> `precisions` = [0.25], `recalls` =
+ [1.0].
+
+ Args:
+ positive_ranks: 1D NumPy integer array, zero-indexed.
+ desired_pr_ranks: List of integers containing the desired precision/recall
+ ranks to be reported. Eg, if precision@1/recall@1 and
+ precision@10/recall@10 are desired, this should be set to [1, 10].
+
+ Returns:
+ precisions: Precision @ `desired_pr_ranks` (NumPy array of
+ floats, with shape [len(desired_pr_ranks)]).
+ recalls: Recall @ `desired_pr_ranks` (NumPy array of floats, with
+ shape [len(desired_pr_ranks)]).
+ """
+ num_desired_pr_ranks = len(desired_pr_ranks)
+ precisions = np.zeros([num_desired_pr_ranks])
+ recalls = np.zeros([num_desired_pr_ranks])
+
+ num_expected_positives = len(positive_ranks)
+ if not num_expected_positives:
+ return precisions, recalls
+
+ positive_ranks_one_indexed = positive_ranks + 1
+ for i, desired_pr_rank in enumerate(desired_pr_ranks):
+ recalls[i] = np.sum(
+ positive_ranks_one_indexed <= desired_pr_rank) / num_expected_positives
+
+ # If `desired_pr_rank` is larger than last positive's rank, only compute
+ # precision with respect to last positive's position.
+ precision_rank = min(max(positive_ranks_one_indexed), desired_pr_rank)
+ precisions[i] = np.sum(
+ positive_ranks_one_indexed <= precision_rank) / precision_rank
+
+ return precisions, recalls
+
+
+def ComputeMetrics(sorted_index_ids, ground_truth, desired_pr_ranks):
+ """Computes metrics for retrieval results on the Revisited datasets.
+
+ If there are no valid ground-truth index images for a given query, the metric
+ results for the given query (`average_precisions`, `precisions` and `recalls`)
+ are set to NaN, and they are not taken into account when computing the
+ aggregated metrics (`mean_average_precision`, `mean_precisions` and
+ `mean_recalls`) over all queries.
+
+ Args:
+ sorted_index_ids: Integer NumPy array of shape [#queries, #index_images].
+ For each query, contains an array denoting the most relevant index images,
+ sorted from most to least relevant.
+ ground_truth: List containing ground-truth information for dataset. Each
+ entry is a dict corresponding to the ground-truth information for a query.
+ The dict has keys 'ok' and 'junk', mapping to a NumPy array of integers.
+ desired_pr_ranks: List of integers containing the desired precision/recall
+ ranks to be reported. Eg, if precision@1/recall@1 and
+ precision@10/recall@10 are desired, this should be set to [1, 10]. The
+ largest item should be <= #index_images.
+
+ Returns:
+ mean_average_precision: Mean average precision (float).
+ mean_precisions: Mean precision @ `desired_pr_ranks` (NumPy array of
+ floats, with shape [len(desired_pr_ranks)]).
+ mean_recalls: Mean recall @ `desired_pr_ranks` (NumPy array of floats, with
+ shape [len(desired_pr_ranks)]).
+ average_precisions: Average precision for each query (NumPy array of floats,
+ with shape [#queries]).
+ precisions: Precision @ `desired_pr_ranks`, for each query (NumPy array of
+ floats, with shape [#queries, len(desired_pr_ranks)]).
+ recalls: Recall @ `desired_pr_ranks`, for each query (NumPy array of
+ floats, with shape [#queries, len(desired_pr_ranks)]).
+
+ Raises:
+ ValueError: If largest desired PR rank in `desired_pr_ranks` >
+ #index_images.
+ """
+ num_queries, num_index_images = sorted_index_ids.shape
+ num_desired_pr_ranks = len(desired_pr_ranks)
+
+ sorted_desired_pr_ranks = sorted(desired_pr_ranks)
+
+ if sorted_desired_pr_ranks[-1] > num_index_images:
+ raise ValueError(
+ 'Requested PR ranks up to %d, however there are only %d images' %
+ (sorted_desired_pr_ranks[-1], num_index_images))
+
+ # Instantiate all outputs, then loop over each query and gather metrics.
+ mean_average_precision = 0.0
+ mean_precisions = np.zeros([num_desired_pr_ranks])
+ mean_recalls = np.zeros([num_desired_pr_ranks])
+ average_precisions = np.zeros([num_queries])
+ precisions = np.zeros([num_queries, num_desired_pr_ranks])
+ recalls = np.zeros([num_queries, num_desired_pr_ranks])
+ num_empty_gt_queries = 0
+ for i in range(num_queries):
+ ok_index_images = ground_truth[i]['ok']
+ junk_index_images = ground_truth[i]['junk']
+
+ if not ok_index_images.size:
+ average_precisions[i] = float('nan')
+ precisions[i, :] = float('nan')
+ recalls[i, :] = float('nan')
+ num_empty_gt_queries += 1
+ continue
+
+ positive_ranks = np.arange(num_index_images)[np.in1d(
+ sorted_index_ids[i], ok_index_images)]
+ junk_ranks = np.arange(num_index_images)[np.in1d(sorted_index_ids[i],
+ junk_index_images)]
+
+ adjusted_positive_ranks = AdjustPositiveRanks(positive_ranks, junk_ranks)
+
+ average_precisions[i] = ComputeAveragePrecision(adjusted_positive_ranks)
+ precisions[i, :], recalls[i, :] = ComputePRAtRanks(adjusted_positive_ranks,
+ desired_pr_ranks)
+
+ mean_average_precision += average_precisions[i]
+ mean_precisions += precisions[i, :]
+ mean_recalls += recalls[i, :]
+
+ # Normalize aggregated metrics by number of queries.
+ num_valid_queries = num_queries - num_empty_gt_queries
+ mean_average_precision /= num_valid_queries
+ mean_precisions /= num_valid_queries
+ mean_recalls /= num_valid_queries
+
+ return (mean_average_precision, mean_precisions, mean_recalls,
+ average_precisions, precisions, recalls)
+
+
+def SaveMetricsFile(mean_average_precision, mean_precisions, mean_recalls,
+ pr_ranks, output_path):
+ """Saves aggregated retrieval metrics to text file.
+
+ Args:
+ mean_average_precision: Dict mapping each dataset protocol to a float.
+ mean_precisions: Dict mapping each dataset protocol to a NumPy array of
+ floats with shape [len(pr_ranks)].
+ mean_recalls: Dict mapping each dataset protocol to a NumPy array of floats
+ with shape [len(pr_ranks)].
+ pr_ranks: List of integers.
+ output_path: Full file path.
+ """
+ with tf.io.gfile.GFile(output_path, 'w') as f:
+ for k in sorted(mean_average_precision.keys()):
+ f.write('{}\n mAP={}\n mP@k{} {}\n mR@k{} {}\n'.format(
+ k, np.around(mean_average_precision[k] * 100, decimals=2),
+ np.array(pr_ranks), np.around(mean_precisions[k] * 100, decimals=2),
+ np.array(pr_ranks), np.around(mean_recalls[k] * 100, decimals=2)))
+
+
+def _ParseSpaceSeparatedStringsInBrackets(line, prefixes, ind):
+ """Parses line containing space-separated strings in brackets.
+
+ Args:
+ line: String, containing line in metrics file with mP@k or mR@k figures.
+ prefixes: Tuple/list of strings, containing valid prefixes.
+ ind: Integer indicating which field within brackets is parsed.
+
+ Yields:
+ entry: String format entry.
+
+ Raises:
+ ValueError: If input line does not contain a valid prefix.
+ """
+ for prefix in prefixes:
+ if line.startswith(prefix):
+ line = line[len(prefix):]
+ break
+ else:
+ raise ValueError('Line %s is malformed, cannot find valid prefixes' % line)
+
+ for entry in line.split('[')[ind].split(']')[0].split():
+ yield entry
+
+
+def _ParsePrRanks(line):
+ """Parses PR ranks from mP@k line in metrics file.
+
+ Args:
+ line: String, containing line in metrics file with mP@k figures.
+
+ Returns:
+ pr_ranks: List of integers, containing used ranks.
+
+ Raises:
+ ValueError: If input line is malformed.
+ """
+ return [
+ int(pr_rank) for pr_rank in _ParseSpaceSeparatedStringsInBrackets(
+ line, [' mP@k['], 0) if pr_rank
+ ]
+
+
+def _ParsePrScores(line, num_pr_ranks):
+ """Parses PR scores from line in metrics file.
+
+ Args:
+ line: String, containing line in metrics file with mP@k or mR@k figures.
+ num_pr_ranks: Integer, number of scores that should be in output list.
+
+ Returns:
+ pr_scores: List of floats, containing scores.
+
+ Raises:
+ ValueError: If input line is malformed.
+ """
+ pr_scores = [
+ float(pr_score) for pr_score in _ParseSpaceSeparatedStringsInBrackets(
+ line, (' mP@k[', ' mR@k['), 1) if pr_score
+ ]
+
+ if len(pr_scores) != num_pr_ranks:
+ raise ValueError('Line %s is malformed, expected %d scores but found %d' %
+ (line, num_pr_ranks, len(pr_scores)))
+
+ return pr_scores
+
+
+def ReadMetricsFile(metrics_path):
+ """Reads aggregated retrieval metrics from text file.
+
+ Args:
+ metrics_path: Full file path, containing aggregated retrieval metrics.
+
+ Returns:
+ mean_average_precision: Dict mapping each dataset protocol to a float.
+ pr_ranks: List of integer ranks used in aggregated recall/precision metrics.
+ mean_precisions: Dict mapping each dataset protocol to a NumPy array of
+ floats with shape [len(`pr_ranks`)].
+ mean_recalls: Dict mapping each dataset protocol to a NumPy array of floats
+ with shape [len(`pr_ranks`)].
+
+ Raises:
+ ValueError: If input file is malformed.
+ """
+ with tf.io.gfile.GFile(metrics_path, 'r') as f:
+ file_contents_stripped = [l.rstrip() for l in f]
+
+ if len(file_contents_stripped) % 4:
+ raise ValueError(
+ 'Malformed input %s: number of lines must be a multiple of 4, '
+ 'but it is %d' % (metrics_path, len(file_contents_stripped)))
+
+ mean_average_precision = {}
+ pr_ranks = []
+ mean_precisions = {}
+ mean_recalls = {}
+ protocols = set()
+ for i in range(0, len(file_contents_stripped), 4):
+ protocol = file_contents_stripped[i]
+ if protocol in protocols:
+ raise ValueError(
+ 'Malformed input %s: protocol %s is found a second time' %
+ (metrics_path, protocol))
+ protocols.add(protocol)
+
+ # Parse mAP.
+ mean_average_precision[protocol] = float(
+ file_contents_stripped[i + 1].split('=')[1]) / 100.0
+
+ # Parse (or check consistency of) pr_ranks.
+ parsed_pr_ranks = _ParsePrRanks(file_contents_stripped[i + 2])
+ if not pr_ranks:
+ pr_ranks = parsed_pr_ranks
+ else:
+ if parsed_pr_ranks != pr_ranks:
+ raise ValueError('Malformed input %s: inconsistent PR ranks' %
+ metrics_path)
+
+ # Parse mean precisions.
+ mean_precisions[protocol] = np.array(
+ _ParsePrScores(file_contents_stripped[i + 2], len(pr_ranks)),
+ dtype=float) / 100.0
+
+ # Parse mean recalls.
+ mean_recalls[protocol] = np.array(
+ _ParsePrScores(file_contents_stripped[i + 3], len(pr_ranks)),
+ dtype=float) / 100.0
+
+ return mean_average_precision, pr_ranks, mean_precisions, mean_recalls
+
+
+def CreateConfigForTestDataset(dataset, dir_main):
+ """Creates the configuration dictionary for the test dataset.
+
+ Args:
+ dataset: String, dataset name: either 'roxford5k' or 'rparis6k'.
+ dir_main: String, path to the folder containing ground truth files.
+
+ Returns:
+ cfg: Dataset configuration in a form of dictionary. The configuration
+ includes:
+ `gnd_fname` - path to the ground truth file for the dataset,
+ `ext` and `qext` - image extensions for the images in the test dataset
+ and the query images,
+ `dir_data` - path to the folder containing ground truth files,
+ `dir_images` - path to the folder containing images,
+ `n` and `nq` - number of images and query images in the dataset
+ respectively,
+ `im_fname` and `qim_fname` - functions providing paths for the dataset
+ and query images respectively,
+ `dataset` - test dataset name.
+
+ Raises:
+ ValueError: If an unknown dataset name is provided as an argument.
+ """
+ dataset = dataset.lower()
+
+ def _ConfigImname(cfg, i):
+ return os.path.join(cfg['dir_images'], cfg['imlist'][i] + cfg['ext'])
+
+ def _ConfigQimname(cfg, i):
+ return os.path.join(cfg['dir_images'], cfg['qimlist'][i] + cfg['qext'])
+
+ if dataset not in DATASET_NAMES:
+ raise ValueError('Unknown dataset: {}!'.format(dataset))
+
+ # Loading imlist, qimlist, and gnd in configuration as a dictionary.
+ gnd_fname = os.path.join(dir_main, 'gnd_{}.pkl'.format(dataset))
+ with tf.io.gfile.GFile(gnd_fname, 'rb') as f:
+ cfg = pickle.load(f)
+ cfg['gnd_fname'] = gnd_fname
+ if dataset == 'rparis6k':
+ dir_images = 'paris6k_images'
+ elif dataset == 'roxford5k':
+ dir_images = 'oxford5k_images'
+
+ cfg['ext'] = '.jpg'
+ cfg['qext'] = '.jpg'
+ cfg['dir_data'] = os.path.join(dir_main)
+ cfg['dir_images'] = os.path.join(cfg['dir_data'], dir_images)
+
+ cfg['n'] = len(cfg['imlist'])
+ cfg['nq'] = len(cfg['qimlist'])
+
+ cfg['im_fname'] = _ConfigImname
+ cfg['qim_fname'] = _ConfigQimname
+
+ cfg['dataset'] = dataset
+
+ return cfg
diff --git a/research/delf/delf/python/detect_to_retrieve/dataset_test.py b/research/delf/delf/python/datasets/revisited_op/dataset_test.py
similarity index 99%
rename from research/delf/delf/python/detect_to_retrieve/dataset_test.py
rename to research/delf/delf/python/datasets/revisited_op/dataset_test.py
index 8e742703b04210787ede0bfc945a9f305d59efc7..04caa64f098040a2d01f18b24d218a1b3478e257 100644
--- a/research/delf/delf/python/detect_to_retrieve/dataset_test.py
+++ b/research/delf/delf/python/datasets/revisited_op/dataset_test.py
@@ -24,7 +24,7 @@ from absl import flags
import numpy as np
import tensorflow as tf
-from delf.python.detect_to_retrieve import dataset
+from delf.python.datasets.revisited_op import dataset
FLAGS = flags.FLAGS
diff --git a/research/delf/delf/python/datasets/utils.py b/research/delf/delf/python/datasets/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..596fca99a5d13492dd107b51db628fd975253b2d
--- /dev/null
+++ b/research/delf/delf/python/datasets/utils.py
@@ -0,0 +1,74 @@
+# Lint as: python3
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Supporting functions for data loading."""
+
+import numpy as np
+from PIL import Image
+
+import tensorflow as tf
+from delf import utils as image_loading_utils
+
+
+def pil_imagenet_loader(path, imsize, bounding_box=None, preprocess=True):
+ """Pillow loader for the images.
+
+ Args:
+ path: Path to image to be loaded.
+ imsize: Integer, defines the maximum size of longer image side.
+ bounding_box: (x1,y1,x2,y2) tuple to crop the query image.
+ preprocess: Bool, whether to preprocess the images in respect to the
+ ImageNet dataset.
+
+ Returns:
+ image: `Tensor`, image in ImageNet suitable format.
+ """
+ img = image_loading_utils.RgbLoader(path)
+
+ if bounding_box is not None:
+ imfullsize = max(img.size)
+ img = img.crop(bounding_box)
+ imsize = imsize * max(img.size) / imfullsize
+
+ # Unlike `resize`, `thumbnail` resizes to the largest size that preserves
+ # the aspect ratio, making sure that the output image does not exceed the
+ # original image size and the size specified in the arguments of thumbnail.
+ img.thumbnail((imsize, imsize), Image.ANTIALIAS)
+ img = np.array(img)
+
+ if preprocess:
+ # Preprocessing for ImageNet data. Converts the images from RGB to BGR,
+ # then zero-centers each color channel with respect to the ImageNet
+ # dataset, without scaling.
+ tf.keras.applications.imagenet_utils.preprocess_input(img, mode='caffe')
+
+ return img
+
+
+def default_loader(path, imsize, bounding_box=None, preprocess=True):
+ """Default loader for the images is using Pillow.
+
+ Args:
+ path: Path to image to be loaded.
+ imsize: Integer, defines the maximum size of longer image side.
+ bounding_box: (x1,y1,x2,y2) tuple to crop the query image.
+ preprocess: Bool, whether to preprocess the images in respect to the
+ ImageNet dataset.
+
+ Returns:
+ image: `Tensor`, image in ImageNet suitable format.
+ """
+ img = pil_imagenet_loader(path, imsize, bounding_box, preprocess)
+ return img
diff --git a/research/delf/delf/python/datasets/utils_test.py b/research/delf/delf/python/datasets/utils_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..e38671bc1b73e6fc98be1ff05d87b49bd4235813
--- /dev/null
+++ b/research/delf/delf/python/datasets/utils_test.py
@@ -0,0 +1,76 @@
+# Lint as: python3
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for dataset utilities."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from absl import flags
+import numpy as np
+from PIL import Image
+import tensorflow as tf
+
+from delf.python.datasets import utils as image_loading_utils
+
+FLAGS = flags.FLAGS
+
+
+class UtilsTest(tf.test.TestCase):
+
+ def testDefaultLoader(self):
+ # Create a dummy image.
+ dummy_image = np.random.rand(1024, 750, 3) * 255
+ img_out = Image.fromarray(dummy_image.astype('uint8')).convert('RGB')
+ filename = os.path.join(FLAGS.test_tmpdir, 'test_image.png')
+ # Save the dummy image.
+ img_out.save(filename)
+
+ max_img_size = 1024
+ # Load the saved dummy image.
+ img = image_loading_utils.default_loader(
+ filename, imsize=max_img_size, preprocess=False)
+
+ # Make sure the values are the same before and after loading.
+ self.assertAllEqual(np.array(img_out), img)
+
+ self.assertAllLessEqual(tf.shape(img), max_img_size)
+
+ def testDefaultLoaderWithBoundingBox(self):
+ # Create a dummy image.
+ dummy_image = np.random.rand(1024, 750, 3) * 255
+ img_out = Image.fromarray(dummy_image.astype('uint8')).convert('RGB')
+ filename = os.path.join(FLAGS.test_tmpdir, 'test_image.png')
+ # Save the dummy image.
+ img_out.save(filename)
+
+ max_img_size = 1024
+ # Load the saved dummy image.
+ expected_size = 400
+ img = image_loading_utils.default_loader(
+ filename,
+ imsize=max_img_size,
+ bounding_box=[120, 120, 120 + expected_size, 120 + expected_size],
+ preprocess=False)
+
+ # Check that the final shape is as expected.
+ self.assertAllEqual(tf.shape(img), [expected_size, expected_size, 3])
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/delf/delf/python/delg/DELG_INSTRUCTIONS.md b/research/delf/delf/python/delg/DELG_INSTRUCTIONS.md
index 2b62ac29003e3d4b13a3ccc8fad5b43e236f96da..dc72422e87cb143a4e7029609f9354181c7540ef 100644
--- a/research/delf/delf/python/delg/DELG_INSTRUCTIONS.md
+++ b/research/delf/delf/python/delg/DELG_INSTRUCTIONS.md
@@ -34,26 +34,42 @@ mv paris6k_images_tmp/paris/*/*.jpg paris6k_images/
# Revisited annotations.
wget http://cmp.felk.cvut.cz/revisitop/data/datasets/roxford5k/gnd_roxford5k.mat
wget http://cmp.felk.cvut.cz/revisitop/data/datasets/rparis6k/gnd_rparis6k.mat
+wget http://cmp.felk.cvut.cz/cnnimageretrieval/data/test/roxford5k/gnd_roxford5k.pkl
+wget http://cmp.felk.cvut.cz/cnnimageretrieval/data/test/rparis6k/gnd_rparis6k.pkl
```
### Download model
-This is necessary to reproduce the main paper results:
+This is necessary to reproduce the main paper results. This example shows the
+R50-DELG model, pretrained on GLD; see the available pre-trained models
+[here](../../../README.md#pre-trained-models), for other variants (eg, R101,
+trained on GLDv2-clean).
```bash
# From models/research/delf/delf/python/delg
mkdir parameters && cd parameters
-# DELG-GLD model.
-wget http://storage.googleapis.com/delf/delg_gld_20200520.tar.gz
-tar -xvzf delg_gld_20200520.tar.gz
+# R50-DELG-GLD model.
+wget http://storage.googleapis.com/delf/r50delg_gld_20200814.tar.gz
+tar -xvzf r50delg_gld_20200814.tar.gz
```
### Feature extraction
-We present here commands for extraction on `roxford5k`. To extract on `rparis6k`
-instead, please edit the arguments accordingly (especially the
-`dataset_file_path` argument).
+We present here commands for R50-DELG (pretrained on GLD) extraction on
+`roxford5k`.
+
+- To use the R101-DELG model pretrained on GLD, first download it as mentioned
+ above; then, replace the below argument `delf_config_path` by
+ `r101delg_gld_config.pbtxt`
+- To use the R50-DELG model pretrained on GLDv2-clean, first download it as
+ mentioned above; then, replace the below argument `delf_config_path` by
+ `r50delg_gldv2clean_config.pbtxt`
+- To use the R101-DELG model pretrained on GLDv2-clean, first download it as
+ mentioned above; then, replace the below argument `delf_config_path` by
+ `r101delg_gldv2clean_config.pbtxt`
+- To extract on `rparis6k` instead, please edit the arguments accordingly
+ (especially the `dataset_file_path` argument).
#### Query feature extraction
@@ -67,7 +83,7 @@ Query feature extraction can be run as follows:
```bash
# From models/research/delf/delf/python/delg
python3 extract_features.py \
- --delf_config_path delg_gld_config.pbtxt \
+ --delf_config_path r50delg_gld_config.pbtxt \
--dataset_file_path ~/delg/data/gnd_roxford5k.mat \
--images_dir ~/delg/data/oxford5k_images \
--image_set query \
@@ -81,7 +97,7 @@ Run index feature extraction as follows:
```bash
# From models/research/delf/delf/python/delg
python3 extract_features.py \
- --delf_config_path delg_gld_config.pbtxt \
+ --delf_config_path r50delg_gld_config.pbtxt \
--dataset_file_path ~/delg/data/gnd_roxford5k.mat \
--images_dir ~/delg/data/oxford5k_images \
--image_set index \
diff --git a/research/delf/delf/python/delg/delg_gld_config.pbtxt b/research/delf/delf/python/delg/delg_gld_config.pbtxt
deleted file mode 100644
index a659a0a3ee502c31f7d4b71fd634803f94d425b7..0000000000000000000000000000000000000000
--- a/research/delf/delf/python/delg/delg_gld_config.pbtxt
+++ /dev/null
@@ -1,22 +0,0 @@
-use_local_features: true
-use_global_features: true
-model_path: "parameters/delg_gld_20200520"
-image_scales: 0.25
-image_scales: 0.35355338
-image_scales: 0.5
-image_scales: 0.70710677
-image_scales: 1.0
-image_scales: 1.4142135
-image_scales: 2.0
-delf_local_config {
- use_pca: false
- max_feature_num: 1000
- score_threshold: 175.0
-}
-delf_global_config {
- use_pca: false
- image_scales_ind: 3
- image_scales_ind: 4
- image_scales_ind: 5
-}
-max_image_size: 1024
diff --git a/research/delf/delf/python/delg/extract_features.py b/research/delf/delf/python/delg/extract_features.py
index ad65d66e69ddaa032d1201b34a2f10a04fe61eb5..4ef10dc9415b32015f359b3139fc847edb931d0b 100644
--- a/research/delf/delf/python/delg/extract_features.py
+++ b/research/delf/delf/python/delg/extract_features.py
@@ -1,3 +1,4 @@
+# Lint as: python3
# Copyright 2020 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,7 +41,7 @@ from delf import delf_config_pb2
from delf import datum_io
from delf import feature_io
from delf import utils
-from delf.python.detect_to_retrieve import dataset
+from delf.python.datasets.revisited_op import dataset
from delf import extractor
FLAGS = flags.FLAGS
diff --git a/research/delf/delf/python/delg/measure_latency.py b/research/delf/delf/python/delg/measure_latency.py
index 21ffbda4179a191139ae35244c8ae34693594fd9..966964d1072006ab78fef034a87d0ff0b9cc4aa6 100644
--- a/research/delf/delf/python/delg/measure_latency.py
+++ b/research/delf/delf/python/delg/measure_latency.py
@@ -42,6 +42,11 @@ flags.DEFINE_string('list_images_path', '/tmp/list_images.txt',
'Path to list of images whose features will be extracted.')
flags.DEFINE_integer('repeat_per_image', 10,
'Number of times to repeat extraction per image.')
+flags.DEFINE_boolean(
+ 'binary_local_features', False,
+ 'Whether to binarize local features after extraction, and take this extra '
+ 'latency into account. This should only be used if use_local_features is '
+ 'set in the input DelfConfig from `delf_config_path`.')
# Pace to report extraction log.
_STATUS_CHECK_ITERATIONS = 100
@@ -103,6 +108,12 @@ def main(argv):
# Extract and save features.
extracted_features = extractor_fn(im)
+ # Binarize local features, if desired (and if there are local features).
+ if (config.use_local_features and FLAGS.binary_local_features and
+ extracted_features['local_features']['attention'].size):
+ packed_descriptors = np.packbits(
+ extracted_features['local_features']['descriptors'] > 0, axis=1)
+
if __name__ == '__main__':
app.run(main)
diff --git a/research/delf/delf/python/delg/perform_retrieval.py b/research/delf/delf/python/delg/perform_retrieval.py
index fb53abb1a9e15a5d5a040be42213f325ab345163..dc380077c56eac410679bec1df06d365a58767e6 100644
--- a/research/delf/delf/python/delg/perform_retrieval.py
+++ b/research/delf/delf/python/delg/perform_retrieval.py
@@ -1,3 +1,4 @@
+# Lint as: python3
# Copyright 2020 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,7 +28,7 @@ import numpy as np
import tensorflow as tf
from delf import datum_io
-from delf.python.detect_to_retrieve import dataset
+from delf.python.datasets.revisited_op import dataset
from delf.python.detect_to_retrieve import image_reranking
FLAGS = flags.FLAGS
@@ -44,15 +45,19 @@ flags.DEFINE_boolean(
'If True, performs re-ranking using local feature-based geometric '
'verification.')
flags.DEFINE_float(
- 'local_feature_distance_threshold', 1.0,
+ 'local_descriptor_matching_threshold', 1.0,
'Optional, only used if `use_geometric_verification` is True. '
- 'Distance threshold below which a pair of local descriptors is considered '
+ 'Threshold below which a pair of local descriptors is considered '
'a potential match, and will be fed into RANSAC.')
flags.DEFINE_float(
'ransac_residual_threshold', 20.0,
'Optional, only used if `use_geometric_verification` is True. '
'Residual error threshold for considering matches as inliers, used in '
'RANSAC algorithm.')
+flags.DEFINE_boolean(
+ 'use_ratio_test', False,
+ 'Optional, only used if `use_geometric_verification` is True. '
+ 'Whether to use ratio test for local feature matching.')
flags.DEFINE_string(
'output_dir', '/tmp/retrieval',
'Directory where retrieval output will be written to. A file containing '
@@ -152,8 +157,10 @@ def main(argv):
junk_ids=set(medium_ground_truth[i]['junk']),
local_feature_extension=_DELG_LOCAL_EXTENSION,
ransac_seed=0,
- feature_distance_threshold=FLAGS.local_feature_distance_threshold,
- ransac_residual_threshold=FLAGS.ransac_residual_threshold)
+ descriptor_matching_threshold=FLAGS
+ .local_descriptor_matching_threshold,
+ ransac_residual_threshold=FLAGS.ransac_residual_threshold,
+ use_ratio_test=FLAGS.use_ratio_test)
hard_ranks_after_gv[i] = image_reranking.RerankByGeometricVerification(
input_ranks=ranks_before_gv[i],
initial_scores=similarities,
@@ -164,8 +171,10 @@ def main(argv):
junk_ids=set(hard_ground_truth[i]['junk']),
local_feature_extension=_DELG_LOCAL_EXTENSION,
ransac_seed=0,
- feature_distance_threshold=FLAGS.local_feature_distance_threshold,
- ransac_residual_threshold=FLAGS.ransac_residual_threshold)
+ descriptor_matching_threshold=FLAGS
+ .local_descriptor_matching_threshold,
+ ransac_residual_threshold=FLAGS.ransac_residual_threshold,
+ use_ratio_test=FLAGS.use_ratio_test)
elapsed = (time.time() - start)
print('done! Retrieval for query %d took %f seconds' % (i, elapsed))
diff --git a/research/delf/delf/python/delg/r101delg_gld_config.pbtxt b/research/delf/delf/python/delg/r101delg_gld_config.pbtxt
new file mode 100644
index 0000000000000000000000000000000000000000..ea8a70b53df6f94758428d108097e5a1d5a132fa
--- /dev/null
+++ b/research/delf/delf/python/delg/r101delg_gld_config.pbtxt
@@ -0,0 +1,22 @@
+use_local_features: true
+use_global_features: true
+model_path: "parameters/r101delg_gld_20200814"
+image_scales: 0.25
+image_scales: 0.35355338
+image_scales: 0.5
+image_scales: 0.70710677
+image_scales: 1.0
+image_scales: 1.4142135
+image_scales: 2.0
+delf_local_config {
+ use_pca: false
+ max_feature_num: 1000
+ score_threshold: 166.1
+}
+delf_global_config {
+ use_pca: false
+ image_scales_ind: 3
+ image_scales_ind: 4
+ image_scales_ind: 5
+}
+max_image_size: 1024
diff --git a/research/delf/delf/python/delg/r101delg_gldv2clean_config.pbtxt b/research/delf/delf/python/delg/r101delg_gldv2clean_config.pbtxt
new file mode 100644
index 0000000000000000000000000000000000000000..d34a039a4eada86b8820c6bb7106f1c0f8fcd25e
--- /dev/null
+++ b/research/delf/delf/python/delg/r101delg_gldv2clean_config.pbtxt
@@ -0,0 +1,22 @@
+use_local_features: true
+use_global_features: true
+model_path: "parameters/r101delg_gldv2clean_20200914"
+image_scales: 0.25
+image_scales: 0.35355338
+image_scales: 0.5
+image_scales: 0.70710677
+image_scales: 1.0
+image_scales: 1.4142135
+image_scales: 2.0
+delf_local_config {
+ use_pca: false
+ max_feature_num: 1000
+ score_threshold: 357.48
+}
+delf_global_config {
+ use_pca: false
+ image_scales_ind: 3
+ image_scales_ind: 4
+ image_scales_ind: 5
+}
+max_image_size: 1024
diff --git a/research/delf/delf/python/delg/r50delg_gld_config.pbtxt b/research/delf/delf/python/delg/r50delg_gld_config.pbtxt
new file mode 100644
index 0000000000000000000000000000000000000000..4457810b57514b6a3ba7d7289d1af327ce23282f
--- /dev/null
+++ b/research/delf/delf/python/delg/r50delg_gld_config.pbtxt
@@ -0,0 +1,22 @@
+use_local_features: true
+use_global_features: true
+model_path: "parameters/r50delg_gld_20200814"
+image_scales: 0.25
+image_scales: 0.35355338
+image_scales: 0.5
+image_scales: 0.70710677
+image_scales: 1.0
+image_scales: 1.4142135
+image_scales: 2.0
+delf_local_config {
+ use_pca: false
+ max_feature_num: 1000
+ score_threshold: 175.0
+}
+delf_global_config {
+ use_pca: false
+ image_scales_ind: 3
+ image_scales_ind: 4
+ image_scales_ind: 5
+}
+max_image_size: 1024
diff --git a/research/delf/delf/python/delg/r50delg_gldv2clean_config.pbtxt b/research/delf/delf/python/delg/r50delg_gldv2clean_config.pbtxt
new file mode 100644
index 0000000000000000000000000000000000000000..358d7cbe56c70b3433bd48f8f81b75694306482f
--- /dev/null
+++ b/research/delf/delf/python/delg/r50delg_gldv2clean_config.pbtxt
@@ -0,0 +1,22 @@
+use_local_features: true
+use_global_features: true
+model_path: "parameters/r50delg_gldv2clean_20200914"
+image_scales: 0.25
+image_scales: 0.35355338
+image_scales: 0.5
+image_scales: 0.70710677
+image_scales: 1.0
+image_scales: 1.4142135
+image_scales: 2.0
+delf_local_config {
+ use_pca: false
+ max_feature_num: 1000
+ score_threshold: 454.6
+}
+delf_global_config {
+ use_pca: false
+ image_scales_ind: 3
+ image_scales_ind: 4
+ image_scales_ind: 5
+}
+max_image_size: 1024
diff --git a/research/delf/delf/python/detect_to_retrieve/__init__.py b/research/delf/delf/python/detect_to_retrieve/__init__.py
index 06972a7d06738da1dc50e832c4e8443b0e6fb5b6..82a78321eb8d1c78eb1a2b8cd3969623d5071c5c 100644
--- a/research/delf/delf/python/detect_to_retrieve/__init__.py
+++ b/research/delf/delf/python/detect_to_retrieve/__init__.py
@@ -20,5 +20,4 @@ from __future__ import print_function
# pylint: disable=unused-import
from delf.python.detect_to_retrieve import aggregation_extraction
from delf.python.detect_to_retrieve import boxes_and_features_extraction
-from delf.python.detect_to_retrieve import dataset
# pylint: enable=unused-import
diff --git a/research/delf/delf/python/detect_to_retrieve/cluster_delf_features.py b/research/delf/delf/python/detect_to_retrieve/cluster_delf_features.py
index ec18f306b20d2a702c8b488726d164de7817c262..f77d47b1db9eadc0124e69775c5420c318e9adae 100644
--- a/research/delf/delf/python/detect_to_retrieve/cluster_delf_features.py
+++ b/research/delf/delf/python/detect_to_retrieve/cluster_delf_features.py
@@ -1,3 +1,4 @@
+# Lint as: python3
# Copyright 2019 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,12 +34,12 @@ import os
import sys
import time
+from absl import app
import numpy as np
import tensorflow as tf
-from tensorflow.python.platform import app
from delf import feature_io
-from delf.python.detect_to_retrieve import dataset
+from delf.python.datasets.revisited_op import dataset
cmd_args = None
diff --git a/research/delf/delf/python/detect_to_retrieve/dataset.py b/research/delf/delf/python/detect_to_retrieve/dataset.py
deleted file mode 100644
index 9a1e6b247895aa7bd8022d3a2fb87b878bbb3b38..0000000000000000000000000000000000000000
--- a/research/delf/delf/python/detect_to_retrieve/dataset.py
+++ /dev/null
@@ -1,469 +0,0 @@
-# Copyright 2019 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Python library to parse ground-truth/evaluate on Revisited datasets."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-from scipy.io import matlab
-import tensorflow as tf
-
-_GROUND_TRUTH_KEYS = ['easy', 'hard', 'junk']
-
-
-def ReadDatasetFile(dataset_file_path):
- """Reads dataset file in Revisited Oxford/Paris ".mat" format.
-
- Args:
- dataset_file_path: Path to dataset file, in .mat format.
-
- Returns:
- query_list: List of query image names.
- index_list: List of index image names.
- ground_truth: List containing ground-truth information for dataset. Each
- entry is a dict corresponding to the ground-truth information for a query.
- The dict may have keys 'easy', 'hard', or 'junk', mapping to a NumPy
- array of integers; additionally, it has a key 'bbx' mapping to a NumPy
- array of floats with bounding box coordinates.
- """
- with tf.io.gfile.GFile(dataset_file_path, 'rb') as f:
- cfg = matlab.loadmat(f)
-
- # Parse outputs according to the specificities of the dataset file.
- query_list = [str(im_array[0]) for im_array in np.squeeze(cfg['qimlist'])]
- index_list = [str(im_array[0]) for im_array in np.squeeze(cfg['imlist'])]
- ground_truth_raw = np.squeeze(cfg['gnd'])
- ground_truth = []
- for query_ground_truth_raw in ground_truth_raw:
- query_ground_truth = {}
- for ground_truth_key in _GROUND_TRUTH_KEYS:
- if ground_truth_key in query_ground_truth_raw.dtype.names:
- adjusted_labels = query_ground_truth_raw[ground_truth_key] - 1
- query_ground_truth[ground_truth_key] = adjusted_labels.flatten()
-
- query_ground_truth['bbx'] = np.squeeze(query_ground_truth_raw['bbx'])
- ground_truth.append(query_ground_truth)
-
- return query_list, index_list, ground_truth
-
-
-def _ParseGroundTruth(ok_list, junk_list):
- """Constructs dictionary of ok/junk indices for a data subset and query.
-
- Args:
- ok_list: List of NumPy arrays containing true positive indices for query.
- junk_list: List of NumPy arrays containing ignored indices for query.
-
- Returns:
- ok_junk_dict: Dict mapping 'ok' and 'junk' strings to NumPy array of
- indices.
- """
- ok_junk_dict = {}
- ok_junk_dict['ok'] = np.concatenate(ok_list)
- ok_junk_dict['junk'] = np.concatenate(junk_list)
- return ok_junk_dict
-
-
-def ParseEasyMediumHardGroundTruth(ground_truth):
- """Parses easy/medium/hard ground-truth from Revisited datasets.
-
- Args:
- ground_truth: Usually the output from ReadDatasetFile(). List containing
- ground-truth information for dataset. Each entry is a dict corresponding
- to the ground-truth information for a query. The dict must have keys
- 'easy', 'hard', and 'junk', mapping to a NumPy array of integers.
-
- Returns:
- easy_ground_truth: List containing ground-truth information for easy subset
- of dataset. Each entry is a dict corresponding to the ground-truth
- information for a query. The dict has keys 'ok' and 'junk', mapping to a
- NumPy array of integers.
- medium_ground_truth: Same as `easy_ground_truth`, but for the medium subset.
- hard_ground_truth: Same as `easy_ground_truth`, but for the hard subset.
- """
- num_queries = len(ground_truth)
-
- easy_ground_truth = []
- medium_ground_truth = []
- hard_ground_truth = []
- for i in range(num_queries):
- easy_ground_truth.append(
- _ParseGroundTruth([ground_truth[i]['easy']],
- [ground_truth[i]['junk'], ground_truth[i]['hard']]))
- medium_ground_truth.append(
- _ParseGroundTruth([ground_truth[i]['easy'], ground_truth[i]['hard']],
- [ground_truth[i]['junk']]))
- hard_ground_truth.append(
- _ParseGroundTruth([ground_truth[i]['hard']],
- [ground_truth[i]['junk'], ground_truth[i]['easy']]))
-
- return easy_ground_truth, medium_ground_truth, hard_ground_truth
-
-
-def AdjustPositiveRanks(positive_ranks, junk_ranks):
- """Adjusts positive ranks based on junk ranks.
-
- Args:
- positive_ranks: Sorted 1D NumPy integer array.
- junk_ranks: Sorted 1D NumPy integer array.
-
- Returns:
- adjusted_positive_ranks: Sorted 1D NumPy array.
- """
- if not junk_ranks.size:
- return positive_ranks
-
- adjusted_positive_ranks = positive_ranks
- j = 0
- for i, positive_index in enumerate(positive_ranks):
- while (j < len(junk_ranks) and positive_index > junk_ranks[j]):
- j += 1
-
- adjusted_positive_ranks[i] -= j
-
- return adjusted_positive_ranks
-
-
-def ComputeAveragePrecision(positive_ranks):
- """Computes average precision according to dataset convention.
-
- It assumes that `positive_ranks` contains the ranks for all expected positive
- index images to be retrieved. If `positive_ranks` is empty, returns
- `average_precision` = 0.
-
- Note that average precision computation here does NOT use the finite sum
- method (see
- https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision)
- which is common in information retrieval literature. Instead, the method
- implemented here integrates over the precision-recall curve by averaging two
- adjacent precision points, then multiplying by the recall step. This is the
- convention for the Revisited Oxford/Paris datasets.
-
- Args:
- positive_ranks: Sorted 1D NumPy integer array, zero-indexed.
-
- Returns:
- average_precision: Float.
- """
- average_precision = 0.0
-
- num_expected_positives = len(positive_ranks)
- if not num_expected_positives:
- return average_precision
-
- recall_step = 1.0 / num_expected_positives
- for i, rank in enumerate(positive_ranks):
- if not rank:
- left_precision = 1.0
- else:
- left_precision = i / rank
-
- right_precision = (i + 1) / (rank + 1)
- average_precision += (left_precision + right_precision) * recall_step / 2
-
- return average_precision
-
-
-def ComputePRAtRanks(positive_ranks, desired_pr_ranks):
- """Computes precision/recall at desired ranks.
-
- It assumes that `positive_ranks` contains the ranks for all expected positive
- index images to be retrieved. If `positive_ranks` is empty, return all-zeros
- `precisions`/`recalls`.
-
- If a desired rank is larger than the last positive rank, its precision is
- computed based on the last positive rank. For example, if `desired_pr_ranks`
- is [10] and `positive_ranks` = [0, 7] --> `precisions` = [0.25], `recalls` =
- [1.0].
-
- Args:
- positive_ranks: 1D NumPy integer array, zero-indexed.
- desired_pr_ranks: List of integers containing the desired precision/recall
- ranks to be reported. Eg, if precision@1/recall@1 and
- precision@10/recall@10 are desired, this should be set to [1, 10].
-
- Returns:
- precisions: Precision @ `desired_pr_ranks` (NumPy array of
- floats, with shape [len(desired_pr_ranks)]).
- recalls: Recall @ `desired_pr_ranks` (NumPy array of floats, with
- shape [len(desired_pr_ranks)]).
- """
- num_desired_pr_ranks = len(desired_pr_ranks)
- precisions = np.zeros([num_desired_pr_ranks])
- recalls = np.zeros([num_desired_pr_ranks])
-
- num_expected_positives = len(positive_ranks)
- if not num_expected_positives:
- return precisions, recalls
-
- positive_ranks_one_indexed = positive_ranks + 1
- for i, desired_pr_rank in enumerate(desired_pr_ranks):
- recalls[i] = np.sum(
- positive_ranks_one_indexed <= desired_pr_rank) / num_expected_positives
-
- # If `desired_pr_rank` is larger than last positive's rank, only compute
- # precision with respect to last positive's position.
- precision_rank = min(max(positive_ranks_one_indexed), desired_pr_rank)
- precisions[i] = np.sum(
- positive_ranks_one_indexed <= precision_rank) / precision_rank
-
- return precisions, recalls
-
-
-def ComputeMetrics(sorted_index_ids, ground_truth, desired_pr_ranks):
- """Computes metrics for retrieval results on the Revisited datasets.
-
- If there are no valid ground-truth index images for a given query, the metric
- results for the given query (`average_precisions`, `precisions` and `recalls`)
- are set to NaN, and they are not taken into account when computing the
- aggregated metrics (`mean_average_precision`, `mean_precisions` and
- `mean_recalls`) over all queries.
-
- Args:
- sorted_index_ids: Integer NumPy array of shape [#queries, #index_images].
- For each query, contains an array denoting the most relevant index images,
- sorted from most to least relevant.
- ground_truth: List containing ground-truth information for dataset. Each
- entry is a dict corresponding to the ground-truth information for a query.
- The dict has keys 'ok' and 'junk', mapping to a NumPy array of integers.
- desired_pr_ranks: List of integers containing the desired precision/recall
- ranks to be reported. Eg, if precision@1/recall@1 and
- precision@10/recall@10 are desired, this should be set to [1, 10]. The
- largest item should be <= #index_images.
-
- Returns:
- mean_average_precision: Mean average precision (float).
- mean_precisions: Mean precision @ `desired_pr_ranks` (NumPy array of
- floats, with shape [len(desired_pr_ranks)]).
- mean_recalls: Mean recall @ `desired_pr_ranks` (NumPy array of floats, with
- shape [len(desired_pr_ranks)]).
- average_precisions: Average precision for each query (NumPy array of floats,
- with shape [#queries]).
- precisions: Precision @ `desired_pr_ranks`, for each query (NumPy array of
- floats, with shape [#queries, len(desired_pr_ranks)]).
- recalls: Recall @ `desired_pr_ranks`, for each query (NumPy array of
- floats, with shape [#queries, len(desired_pr_ranks)]).
-
- Raises:
- ValueError: If largest desired PR rank in `desired_pr_ranks` >
- #index_images.
- """
- num_queries, num_index_images = sorted_index_ids.shape
- num_desired_pr_ranks = len(desired_pr_ranks)
-
- sorted_desired_pr_ranks = sorted(desired_pr_ranks)
-
- if sorted_desired_pr_ranks[-1] > num_index_images:
- raise ValueError(
- 'Requested PR ranks up to %d, however there are only %d images' %
- (sorted_desired_pr_ranks[-1], num_index_images))
-
- # Instantiate all outputs, then loop over each query and gather metrics.
- mean_average_precision = 0.0
- mean_precisions = np.zeros([num_desired_pr_ranks])
- mean_recalls = np.zeros([num_desired_pr_ranks])
- average_precisions = np.zeros([num_queries])
- precisions = np.zeros([num_queries, num_desired_pr_ranks])
- recalls = np.zeros([num_queries, num_desired_pr_ranks])
- num_empty_gt_queries = 0
- for i in range(num_queries):
- ok_index_images = ground_truth[i]['ok']
- junk_index_images = ground_truth[i]['junk']
-
- if not ok_index_images.size:
- average_precisions[i] = float('nan')
- precisions[i, :] = float('nan')
- recalls[i, :] = float('nan')
- num_empty_gt_queries += 1
- continue
-
- positive_ranks = np.arange(num_index_images)[np.in1d(
- sorted_index_ids[i], ok_index_images)]
- junk_ranks = np.arange(num_index_images)[np.in1d(sorted_index_ids[i],
- junk_index_images)]
-
- adjusted_positive_ranks = AdjustPositiveRanks(positive_ranks, junk_ranks)
-
- average_precisions[i] = ComputeAveragePrecision(adjusted_positive_ranks)
- precisions[i, :], recalls[i, :] = ComputePRAtRanks(adjusted_positive_ranks,
- desired_pr_ranks)
-
- mean_average_precision += average_precisions[i]
- mean_precisions += precisions[i, :]
- mean_recalls += recalls[i, :]
-
- # Normalize aggregated metrics by number of queries.
- num_valid_queries = num_queries - num_empty_gt_queries
- mean_average_precision /= num_valid_queries
- mean_precisions /= num_valid_queries
- mean_recalls /= num_valid_queries
-
- return (mean_average_precision, mean_precisions, mean_recalls,
- average_precisions, precisions, recalls)
-
-
-def SaveMetricsFile(mean_average_precision, mean_precisions, mean_recalls,
- pr_ranks, output_path):
- """Saves aggregated retrieval metrics to text file.
-
- Args:
- mean_average_precision: Dict mapping each dataset protocol to a float.
- mean_precisions: Dict mapping each dataset protocol to a NumPy array of
- floats with shape [len(pr_ranks)].
- mean_recalls: Dict mapping each dataset protocol to a NumPy array of floats
- with shape [len(pr_ranks)].
- pr_ranks: List of integers.
- output_path: Full file path.
- """
- with tf.io.gfile.GFile(output_path, 'w') as f:
- for k in sorted(mean_average_precision.keys()):
- f.write('{}\n mAP={}\n mP@k{} {}\n mR@k{} {}\n'.format(
- k, np.around(mean_average_precision[k] * 100, decimals=2),
- np.array(pr_ranks), np.around(mean_precisions[k] * 100, decimals=2),
- np.array(pr_ranks), np.around(mean_recalls[k] * 100, decimals=2)))
-
-
-def _ParseSpaceSeparatedStringsInBrackets(line, prefixes, ind):
- """Parses line containing space-separated strings in brackets.
-
- Args:
- line: String, containing line in metrics file with mP@k or mR@k figures.
- prefixes: Tuple/list of strings, containing valid prefixes.
- ind: Integer indicating which field within brackets is parsed.
-
- Yields:
- entry: String format entry.
-
- Raises:
- ValueError: If input line does not contain a valid prefix.
- """
- for prefix in prefixes:
- if line.startswith(prefix):
- line = line[len(prefix):]
- break
- else:
- raise ValueError('Line %s is malformed, cannot find valid prefixes' % line)
-
- for entry in line.split('[')[ind].split(']')[0].split():
- yield entry
-
-
-def _ParsePrRanks(line):
- """Parses PR ranks from mP@k line in metrics file.
-
- Args:
- line: String, containing line in metrics file with mP@k figures.
-
- Returns:
- pr_ranks: List of integers, containing used ranks.
-
- Raises:
- ValueError: If input line is malformed.
- """
- return [
- int(pr_rank) for pr_rank in _ParseSpaceSeparatedStringsInBrackets(
- line, [' mP@k['], 0) if pr_rank
- ]
-
-
-def _ParsePrScores(line, num_pr_ranks):
- """Parses PR scores from line in metrics file.
-
- Args:
- line: String, containing line in metrics file with mP@k or mR@k figures.
- num_pr_ranks: Integer, number of scores that should be in output list.
-
- Returns:
- pr_scores: List of floats, containing scores.
-
- Raises:
- ValueError: If input line is malformed.
- """
- pr_scores = [
- float(pr_score) for pr_score in _ParseSpaceSeparatedStringsInBrackets(
- line, (' mP@k[', ' mR@k['), 1) if pr_score
- ]
-
- if len(pr_scores) != num_pr_ranks:
- raise ValueError('Line %s is malformed, expected %d scores but found %d' %
- (line, num_pr_ranks, len(pr_scores)))
-
- return pr_scores
-
-
-def ReadMetricsFile(metrics_path):
- """Reads aggregated retrieval metrics from text file.
-
- Args:
- metrics_path: Full file path, containing aggregated retrieval metrics.
-
- Returns:
- mean_average_precision: Dict mapping each dataset protocol to a float.
- pr_ranks: List of integer ranks used in aggregated recall/precision metrics.
- mean_precisions: Dict mapping each dataset protocol to a NumPy array of
- floats with shape [len(`pr_ranks`)].
- mean_recalls: Dict mapping each dataset protocol to a NumPy array of floats
- with shape [len(`pr_ranks`)].
-
- Raises:
- ValueError: If input file is malformed.
- """
- with tf.io.gfile.GFile(metrics_path, 'r') as f:
- file_contents_stripped = [l.rstrip() for l in f]
-
- if len(file_contents_stripped) % 4:
- raise ValueError(
- 'Malformed input %s: number of lines must be a multiple of 4, '
- 'but it is %d' % (metrics_path, len(file_contents_stripped)))
-
- mean_average_precision = {}
- pr_ranks = []
- mean_precisions = {}
- mean_recalls = {}
- protocols = set()
- for i in range(0, len(file_contents_stripped), 4):
- protocol = file_contents_stripped[i]
- if protocol in protocols:
- raise ValueError(
- 'Malformed input %s: protocol %s is found a second time' %
- (metrics_path, protocol))
- protocols.add(protocol)
-
- # Parse mAP.
- mean_average_precision[protocol] = float(
- file_contents_stripped[i + 1].split('=')[1]) / 100.0
-
- # Parse (or check consistency of) pr_ranks.
- parsed_pr_ranks = _ParsePrRanks(file_contents_stripped[i + 2])
- if not pr_ranks:
- pr_ranks = parsed_pr_ranks
- else:
- if parsed_pr_ranks != pr_ranks:
- raise ValueError('Malformed input %s: inconsistent PR ranks' %
- metrics_path)
-
- # Parse mean precisions.
- mean_precisions[protocol] = np.array(
- _ParsePrScores(file_contents_stripped[i + 2], len(pr_ranks)),
- dtype=float) / 100.0
-
- # Parse mean recalls.
- mean_recalls[protocol] = np.array(
- _ParsePrScores(file_contents_stripped[i + 3], len(pr_ranks)),
- dtype=float) / 100.0
-
- return mean_average_precision, pr_ranks, mean_precisions, mean_recalls
diff --git a/research/delf/delf/python/detect_to_retrieve/extract_aggregation.py b/research/delf/delf/python/detect_to_retrieve/extract_aggregation.py
index f9a0fb3e6c62c0adc583ad3b30b809f36742d586..451c4137d93b9b37cedd487487e98b76854db87d 100644
--- a/research/delf/delf/python/detect_to_retrieve/extract_aggregation.py
+++ b/research/delf/delf/python/detect_to_retrieve/extract_aggregation.py
@@ -25,9 +25,9 @@ from __future__ import print_function
import argparse
import sys
-from tensorflow.python.platform import app
+from absl import app
+from delf.python.datasets.revisited_op import dataset
from delf.python.detect_to_retrieve import aggregation_extraction
-from delf.python.detect_to_retrieve import dataset
cmd_args = None
diff --git a/research/delf/delf/python/detect_to_retrieve/extract_index_boxes_and_features.py b/research/delf/delf/python/detect_to_retrieve/extract_index_boxes_and_features.py
index 2b891de4b0b093aa723c0dce547c2722ee475d7e..80bd721c87423174ce9559f9d9bc161a6adda5da 100644
--- a/research/delf/delf/python/detect_to_retrieve/extract_index_boxes_and_features.py
+++ b/research/delf/delf/python/detect_to_retrieve/extract_index_boxes_and_features.py
@@ -31,9 +31,9 @@ import argparse
import os
import sys
-from tensorflow.python.platform import app
+from absl import app
+from delf.python.datasets.revisited_op import dataset
from delf.python.detect_to_retrieve import boxes_and_features_extraction
-from delf.python.detect_to_retrieve import dataset
cmd_args = None
diff --git a/research/delf/delf/python/detect_to_retrieve/extract_query_features.py b/research/delf/delf/python/detect_to_retrieve/extract_query_features.py
index a0812b191265ec6e5350acf989432747d196a519..2ff4a5a23f50cc6753cff066ca510ad5e639ba1e 100644
--- a/research/delf/delf/python/detect_to_retrieve/extract_query_features.py
+++ b/research/delf/delf/python/detect_to_retrieve/extract_query_features.py
@@ -1,3 +1,4 @@
+# Lint as: python3
# Copyright 2017 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,15 +31,15 @@ import os
import sys
import time
+from absl import app
import numpy as np
import tensorflow as tf
from google.protobuf import text_format
-from tensorflow.python.platform import app
from delf import delf_config_pb2
from delf import feature_io
from delf import utils
-from delf.python.detect_to_retrieve import dataset
+from delf.python.datasets.revisited_op import dataset
from delf import extractor
cmd_args = None
@@ -75,8 +76,8 @@ def main(argv):
query_image_name = query_list[i]
input_image_filename = os.path.join(cmd_args.images_dir,
query_image_name + _IMAGE_EXTENSION)
- output_feature_filename = os.path.join(
- cmd_args.output_features_dir, query_image_name + _DELF_EXTENSION)
+ output_feature_filename = os.path.join(cmd_args.output_features_dir,
+ query_image_name + _DELF_EXTENSION)
if tf.io.gfile.exists(output_feature_filename):
print(f'Skipping {query_image_name}')
continue
@@ -93,8 +94,7 @@ def main(argv):
attention_out = extracted_features['local_features']['attention']
feature_io.WriteToFile(output_feature_filename, locations_out,
- feature_scales_out, descriptors_out,
- attention_out)
+ feature_scales_out, descriptors_out, attention_out)
elapsed = (time.time() - start)
print('Processed %d query images in %f seconds' % (num_images, elapsed))
diff --git a/research/delf/delf/python/detect_to_retrieve/image_reranking.py b/research/delf/delf/python/detect_to_retrieve/image_reranking.py
index 60c29cc18a4436815c721855da0ca4577b06e6c4..8c115835d638e4ec2337233bdfeeebe6e5800f93 100644
--- a/research/delf/delf/python/detect_to_retrieve/image_reranking.py
+++ b/research/delf/delf/python/detect_to_retrieve/image_reranking.py
@@ -47,12 +47,13 @@ def MatchFeatures(query_locations,
index_image_locations,
index_image_descriptors,
ransac_seed=None,
- feature_distance_threshold=0.9,
+ descriptor_matching_threshold=0.9,
ransac_residual_threshold=10.0,
query_im_array=None,
index_im_array=None,
query_im_scale_factors=None,
- index_im_scale_factors=None):
+ index_im_scale_factors=None,
+ use_ratio_test=False):
"""Matches local features using geometric verification.
First, finds putative local feature matches by matching `query_descriptors`
@@ -70,8 +71,10 @@ def MatchFeatures(query_locations,
index_image_descriptors: Descriptors of local features for index image.
NumPy array of shape [#index_image_features, depth].
ransac_seed: Seed used by RANSAC. If None (default), no seed is provided.
- feature_distance_threshold: Distance threshold below which a pair of
- features is considered a potential match, and will be fed into RANSAC.
+ descriptor_matching_threshold: Threshold below which a pair of local
+ descriptors is considered a potential match, and will be fed into RANSAC.
+ If use_ratio_test==False, this is a simple distance threshold. If
+ use_ratio_test==True, this is Lowe's ratio test threshold.
ransac_residual_threshold: Residual error threshold for considering matches
as inliers, used in RANSAC algorithm.
query_im_array: Optional. If not None, contains a NumPy array with the query
@@ -83,6 +86,8 @@ def MatchFeatures(query_locations,
(ie, feature locations are not scaled).
index_im_scale_factors: Optional. Same as `query_im_scale_factors`, but for
index image.
+ use_ratio_test: If True, descriptor matching is performed via ratio test,
+ instead of distance-based threshold.
Returns:
score: Number of inliers of match. If no match is found, returns 0.
@@ -105,22 +110,38 @@ def MatchFeatures(query_locations,
'Local feature dimensionality is not consistent for query and index '
'images.')
- # Find nearest-neighbor matches using a KD tree.
+ # Construct KD-tree used to find nearest neighbors.
index_image_tree = spatial.cKDTree(index_image_descriptors)
- _, indices = index_image_tree.query(
- query_descriptors, distance_upper_bound=feature_distance_threshold)
-
- # Select feature locations for putative matches.
- query_locations_to_use = np.array([
- query_locations[i,]
- for i in range(num_features_query)
- if indices[i] != num_features_index_image
- ])
- index_image_locations_to_use = np.array([
- index_image_locations[indices[i],]
- for i in range(num_features_query)
- if indices[i] != num_features_index_image
- ])
+ if use_ratio_test:
+ distances, indices = index_image_tree.query(
+ query_descriptors, k=2, n_jobs=-1)
+ query_locations_to_use = np.array([
+ query_locations[i,]
+ for i in range(num_features_query)
+ if distances[i][0] < descriptor_matching_threshold * distances[i][1]
+ ])
+ index_image_locations_to_use = np.array([
+ index_image_locations[indices[i][0],]
+ for i in range(num_features_query)
+ if distances[i][0] < descriptor_matching_threshold * distances[i][1]
+ ])
+ else:
+ _, indices = index_image_tree.query(
+ query_descriptors,
+ distance_upper_bound=descriptor_matching_threshold,
+ n_jobs=-1)
+
+ # Select feature locations for putative matches.
+ query_locations_to_use = np.array([
+ query_locations[i,]
+ for i in range(num_features_query)
+ if indices[i] != num_features_index_image
+ ])
+ index_image_locations_to_use = np.array([
+ index_image_locations[indices[i],]
+ for i in range(num_features_query)
+ if indices[i] != num_features_index_image
+ ])
# If there are not enough putative matches, early return 0.
if query_locations_to_use.shape[0] <= _MIN_RANSAC_SAMPLES:
@@ -175,8 +196,9 @@ def RerankByGeometricVerification(input_ranks,
junk_ids,
local_feature_extension=_DELF_EXTENSION,
ransac_seed=None,
- feature_distance_threshold=0.9,
- ransac_residual_threshold=10.0):
+ descriptor_matching_threshold=0.9,
+ ransac_residual_threshold=10.0,
+ use_ratio_test=False):
"""Re-ranks retrieval results using geometric verification.
Args:
@@ -195,10 +217,11 @@ def RerankByGeometricVerification(input_ranks,
local_feature_extension: String, extension to use for loading local feature
files.
ransac_seed: Seed used by RANSAC. If None (default), no seed is provided.
- feature_distance_threshold: Distance threshold below which a pair of local
- features is considered a potential match, and will be fed into RANSAC.
+ descriptor_matching_threshold: Threshold used for local descriptor matching.
ransac_residual_threshold: Residual error threshold for considering matches
as inliers, used in RANSAC algorithm.
+ use_ratio_test: If True, descriptor matching is performed via ratio test,
+ instead of distance-based threshold.
Returns:
output_ranks: 1D NumPy array with index image indices, sorted from the most
@@ -258,8 +281,9 @@ def RerankByGeometricVerification(input_ranks,
index_image_locations,
index_image_descriptors,
ransac_seed=ransac_seed,
- feature_distance_threshold=feature_distance_threshold,
- ransac_residual_threshold=ransac_residual_threshold)
+ descriptor_matching_threshold=descriptor_matching_threshold,
+ ransac_residual_threshold=ransac_residual_threshold,
+ use_ratio_test=use_ratio_test)
# Sort based on (inliers_score, initial_score).
def _InliersInitialScoresSorting(k):
diff --git a/research/delf/delf/python/detect_to_retrieve/perform_retrieval.py b/research/delf/delf/python/detect_to_retrieve/perform_retrieval.py
index c2034dfb285118f4ed8928f996e031365a3ffbbf..2b7a22789259a6ba35ffbe99c0c7725fbc2f7ae5 100644
--- a/research/delf/delf/python/detect_to_retrieve/perform_retrieval.py
+++ b/research/delf/delf/python/detect_to_retrieve/perform_retrieval.py
@@ -1,3 +1,4 @@
+# Lint as: python3
# Copyright 2019 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,15 +24,15 @@ import os
import sys
import time
+from absl import app
import numpy as np
import tensorflow as tf
from google.protobuf import text_format
-from tensorflow.python.platform import app
from delf import aggregation_config_pb2
from delf import datum_io
from delf import feature_aggregation_similarity
-from delf.python.detect_to_retrieve import dataset
+from delf.python.datasets.revisited_op import dataset
from delf.python.detect_to_retrieve import image_reranking
cmd_args = None
diff --git a/research/delf/delf/python/examples/extract_boxes.py b/research/delf/delf/python/examples/extract_boxes.py
index 8851c44fb9a051104adde50a4c869a28cfd513da..1a3b4886a39680ed4d293d3b1116d5a19aac9db6 100644
--- a/research/delf/delf/python/examples/extract_boxes.py
+++ b/research/delf/delf/python/examples/extract_boxes.py
@@ -27,12 +27,12 @@ import os
import sys
import time
+from absl import app
import matplotlib.patches as patches
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
-from tensorflow.python.platform import app
from delf import box_io
from delf import utils
from delf import detector
@@ -153,17 +153,14 @@ def main(argv):
print('Starting to detect objects in images...')
elif i % _STATUS_CHECK_ITERATIONS == 0:
elapsed = (time.time() - start)
- print(
- f'Processing image {i} out of {num_images}, last '
- f'{_STATUS_CHECK_ITERATIONS} images took {elapsed} seconds'
- )
+ print(f'Processing image {i} out of {num_images}, last '
+ f'{_STATUS_CHECK_ITERATIONS} images took {elapsed} seconds')
start = time.time()
# If descriptor already exists, skip its computation.
base_boxes_filename, _ = os.path.splitext(os.path.basename(image_path))
out_boxes_filename = base_boxes_filename + _BOX_EXT
- out_boxes_fullpath = os.path.join(cmd_args.output_dir,
- out_boxes_filename)
+ out_boxes_fullpath = os.path.join(cmd_args.output_dir, out_boxes_filename)
if tf.io.gfile.exists(out_boxes_fullpath):
print(f'Skipping {image_path}')
continue
@@ -173,8 +170,7 @@ def main(argv):
# Extract and save boxes.
(boxes_out, scores_out, class_indices_out) = detector_fn(im)
(selected_boxes, selected_scores,
- selected_class_indices) = _FilterBoxesByScore(boxes_out[0],
- scores_out[0],
+ selected_class_indices) = _FilterBoxesByScore(boxes_out[0], scores_out[0],
class_indices_out[0],
cmd_args.detector_thresh)
@@ -182,8 +178,7 @@ def main(argv):
selected_class_indices)
if cmd_args.output_viz_dir:
out_viz_filename = base_boxes_filename + _VIZ_SUFFIX
- out_viz_fullpath = os.path.join(cmd_args.output_viz_dir,
- out_viz_filename)
+ out_viz_fullpath = os.path.join(cmd_args.output_viz_dir, out_viz_filename)
_PlotBoxesAndSaveImage(im[0], selected_boxes, out_viz_fullpath)
diff --git a/research/delf/delf/python/examples/extract_features.py b/research/delf/delf/python/examples/extract_features.py
index 05fd77316070d39722e133dbd544f5b53791f6d0..1b55cba9fb6f0773a0db5cbe63d5234c02c474cb 100644
--- a/research/delf/delf/python/examples/extract_features.py
+++ b/research/delf/delf/python/examples/extract_features.py
@@ -27,12 +27,12 @@ import os
import sys
import time
+from absl import app
import numpy as np
from six.moves import range
import tensorflow as tf
from google.protobuf import text_format
-from tensorflow.python.platform import app
from delf import delf_config_pb2
from delf import feature_io
from delf import utils
@@ -87,10 +87,8 @@ def main(unused_argv):
print('Starting to extract DELF features from images...')
elif i % _STATUS_CHECK_ITERATIONS == 0:
elapsed = (time.time() - start)
- print(
- f'Processing image {i} out of {num_images}, last '
- f'{_STATUS_CHECK_ITERATIONS} images took {elapsed} seconds'
- )
+ print(f'Processing image {i} out of {num_images}, last '
+ f'{_STATUS_CHECK_ITERATIONS} images took {elapsed} seconds')
start = time.time()
# If descriptor already exists, skip its computation.
diff --git a/research/delf/delf/python/examples/extractor.py b/research/delf/delf/python/examples/extractor.py
index db7b80c9916df6e19006bc00841dabd320c64704..a6932b1de58592bacfef7fd79c03581161560f26 100644
--- a/research/delf/delf/python/examples/extractor.py
+++ b/research/delf/delf/python/examples/extractor.py
@@ -19,81 +19,18 @@ from __future__ import division
from __future__ import print_function
import numpy as np
-from PIL import Image
import tensorflow as tf
from delf import datum_io
from delf import feature_extractor
+from delf import utils
-# Minimum dimensions below which DELF features are not extracted (empty
+# Minimum dimensions below which features are not extracted (empty
# features are returned). This applies after any resizing is performed.
_MIN_HEIGHT = 10
_MIN_WIDTH = 10
-def ResizeImage(image, config, resize_factor=1.0):
- """Resizes image according to config.
-
- Args:
- image: Uint8 array with shape (height, width, 3).
- config: DelfConfig proto containing the model configuration.
- resize_factor: Optional float resize factor for the input image. If given,
- the maximum and minimum allowed image sizes in `config` are scaled by this
- factor. Must be non-negative.
-
- Returns:
- resized_image: Uint8 array with resized image.
- scale_factors: 2D float array, with factors used for resizing along height
- and width (If upscaling, larger than 1; if downscaling, smaller than 1).
-
- Raises:
- ValueError: If `image` has incorrect number of dimensions/channels.
- """
- if resize_factor < 0.0:
- raise ValueError('negative resize_factor is not allowed: %f' %
- resize_factor)
- if image.ndim != 3:
- raise ValueError('image has incorrect number of dimensions: %d' %
- image.ndims)
- height, width, channels = image.shape
-
- # Take into account resize factor.
- max_image_size = resize_factor * config.max_image_size
- min_image_size = resize_factor * config.min_image_size
-
- if channels != 3:
- raise ValueError('image has incorrect number of channels: %d' % channels)
-
- largest_side = max(width, height)
-
- if max_image_size >= 0 and largest_side > max_image_size:
- scale_factor = max_image_size / largest_side
- elif min_image_size >= 0 and largest_side < min_image_size:
- scale_factor = min_image_size / largest_side
- elif config.use_square_images and (height != width):
- scale_factor = 1.0
- else:
- # No resizing needed, early return.
- return image, np.ones(2, dtype=float)
-
- # Note that new_shape is in (width, height) format (PIL convention), while
- # scale_factors are in (height, width) convention (NumPy convention).
- if config.use_square_images:
- new_shape = (int(round(largest_side * scale_factor)),
- int(round(largest_side * scale_factor)))
- else:
- new_shape = (int(round(width * scale_factor)),
- int(round(height * scale_factor)))
-
- scale_factors = np.array([new_shape[1] / height, new_shape[0] / width],
- dtype=float)
-
- pil_image = Image.fromarray(image)
- resized_image = np.array(pil_image.resize(new_shape, resample=Image.BILINEAR))
-
- return resized_image, scale_factors
-
-
def MakeExtractor(config):
"""Creates a function to extract global and/or local features from an image.
@@ -106,18 +43,21 @@ def MakeExtractor(config):
Raises:
ValueError: if config is invalid.
"""
- # Assert the configuration
- if config.use_global_features and hasattr(
- config, 'is_tf2_exported') and config.is_tf2_exported:
- raise ValueError('use_global_features is incompatible with is_tf2_exported')
+ # Assert the configuration.
+ if not config.use_local_features and not config.use_global_features:
+ raise ValueError('Invalid config: at least one of '
+ '{use_local_features, use_global_features} must be True')
# Load model.
model = tf.saved_model.load(config.model_path)
- # Input/output end-points/tensors.
+ # Input image scales to use for extraction.
+ image_scales_tensor = tf.convert_to_tensor(list(config.image_scales))
+
+ # Input (feeds) and output (fetches) end-points. These are only needed when
+ # using a model that was exported using TF1.
feeds = ['input_image:0', 'input_scales:0']
fetches = []
- image_scales_tensor = tf.convert_to_tensor(list(config.image_scales))
# Custom configuration needed when local features are used.
if config.use_local_features:
@@ -159,8 +99,14 @@ def MakeExtractor(config):
# Custom configuration needed when global features are used.
if config.use_global_features:
- # Extra output end-point.
+ # Extra input/output end-points/tensors.
+ feeds.append('input_global_scales_ind:0')
fetches.append('global_descriptors:0')
+ if config.delf_global_config.image_scales_ind:
+ global_scales_ind_tensor = tf.constant(
+ list(config.delf_global_config.image_scales_ind))
+ else:
+ global_scales_ind_tensor = tf.range(len(config.image_scales))
# If using PCA, pre-load required parameters.
global_pca_parameters = {}
@@ -206,7 +152,7 @@ def MakeExtractor(config):
features (key 'local_features' mapping to a dict with keys 'locations',
'descriptors', 'scales', 'attention').
"""
- resized_image, scale_factors = ResizeImage(
+ resized_image, scale_factors = utils.ResizeImage(
image, config, resize_factor=resize_factor)
# If the image is too small, returns empty features.
@@ -231,9 +177,21 @@ def MakeExtractor(config):
extracted_features = {}
output = None
- if config.use_local_features:
- if hasattr(config, 'is_tf2_exported') and config.is_tf2_exported:
- predict = model.signatures['serving_default']
+ if hasattr(config, 'is_tf2_exported') and config.is_tf2_exported:
+ predict = model.signatures['serving_default']
+ if config.use_local_features and config.use_global_features:
+ output_dict = predict(
+ input_image=image_tensor,
+ input_scales=image_scales_tensor,
+ input_max_feature_num=max_feature_num_tensor,
+ input_abs_thres=score_threshold_tensor,
+ input_global_scales_ind=global_scales_ind_tensor)
+ output = [
+ output_dict['boxes'], output_dict['features'],
+ output_dict['scales'], output_dict['scores'],
+ output_dict['global_descriptors']
+ ]
+ elif config.use_local_features:
output_dict = predict(
input_image=image_tensor,
input_scales=image_scales_tensor,
@@ -244,23 +202,29 @@ def MakeExtractor(config):
output_dict['scales'], output_dict['scores']
]
else:
+ output_dict = predict(
+ input_image=image_tensor,
+ input_scales=image_scales_tensor,
+ input_global_scales_ind=global_scales_ind_tensor)
+ output = [output_dict['global_descriptors']]
+ else:
+ if config.use_local_features and config.use_global_features:
+ output = model(image_tensor, image_scales_tensor,
+ score_threshold_tensor, max_feature_num_tensor,
+ global_scales_ind_tensor)
+ elif config.use_local_features:
output = model(image_tensor, image_scales_tensor,
score_threshold_tensor, max_feature_num_tensor)
- else:
- output = model(image_tensor, image_scales_tensor)
+ else:
+ output = model(image_tensor, image_scales_tensor,
+ global_scales_ind_tensor)
# Post-process extracted features: normalize, PCA (optional), pooling.
if config.use_global_features:
raw_global_descriptors = output[-1]
- if config.delf_global_config.image_scales_ind:
- raw_global_descriptors_selected_scales = tf.gather(
- raw_global_descriptors,
- list(config.delf_global_config.image_scales_ind))
- else:
- raw_global_descriptors_selected_scales = raw_global_descriptors
global_descriptors_per_scale = feature_extractor.PostProcessDescriptors(
- raw_global_descriptors_selected_scales,
- config.delf_global_config.use_pca, global_pca_parameters)
+ raw_global_descriptors, config.delf_global_config.use_pca,
+ global_pca_parameters)
unnormalized_global_descriptor = tf.reduce_sum(
global_descriptors_per_scale, axis=0, name='sum_pooling')
global_descriptor = tf.nn.l2_normalize(
@@ -281,7 +245,8 @@ def MakeExtractor(config):
feature_extractor.DelfFeaturePostProcessing(
boxes, raw_local_descriptors, config.delf_local_config.use_pca,
local_pca_parameters))
- locations /= scale_factors
+ if not config.delf_local_config.use_resized_coordinates:
+ locations /= scale_factors
extracted_features.update({
'local_features': {
diff --git a/research/delf/delf/python/examples/extractor_test.py b/research/delf/delf/python/examples/extractor_test.py
deleted file mode 100644
index aa560c75a5ca7f8a48247eb7636643e2369c0e5e..0000000000000000000000000000000000000000
--- a/research/delf/delf/python/examples/extractor_test.py
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright 2019 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for DELF feature extractor."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from absl.testing import parameterized
-import numpy as np
-import tensorflow as tf
-
-from delf import delf_config_pb2
-from delf import extractor
-
-
-class ExtractorTest(tf.test.TestCase, parameterized.TestCase):
-
- @parameterized.named_parameters(
- ('Max-1Min-1', -1, -1, 1.0, False, [4, 2, 3], [1.0, 1.0]),
- ('Max-1Min-1Square', -1, -1, 1.0, True, [4, 4, 3], [1.0, 2.0]),
- ('Max2Min-1', 2, -1, 1.0, False, [2, 1, 3], [0.5, 0.5]),
- ('Max2Min-1Square', 2, -1, 1.0, True, [2, 2, 3], [0.5, 1.0]),
- ('Max8Min-1', 8, -1, 1.0, False, [4, 2, 3], [1.0, 1.0]),
- ('Max8Min-1Square', 8, -1, 1.0, True, [4, 4, 3], [1.0, 2.0]),
- ('Max-1Min1', -1, 1, 1.0, False, [4, 2, 3], [1.0, 1.0]),
- ('Max-1Min1Square', -1, 1, 1.0, True, [4, 4, 3], [1.0, 2.0]),
- ('Max-1Min8', -1, 8, 1.0, False, [8, 4, 3], [2.0, 2.0]),
- ('Max-1Min8Square', -1, 8, 1.0, True, [8, 8, 3], [2.0, 4.0]),
- ('Max16Min8', 16, 8, 1.0, False, [8, 4, 3], [2.0, 2.0]),
- ('Max16Min8Square', 16, 8, 1.0, True, [8, 8, 3], [2.0, 4.0]),
- ('Max2Min2', 2, 2, 1.0, False, [2, 1, 3], [0.5, 0.5]),
- ('Max2Min2Square', 2, 2, 1.0, True, [2, 2, 3], [0.5, 1.0]),
- ('Max-1Min-1Factor0.5', -1, -1, 0.5, False, [4, 2, 3], [1.0, 1.0]),
- ('Max-1Min-1Factor0.5Square', -1, -1, 0.5, True, [4, 4, 3], [1.0, 2.0]),
- ('Max2Min-1Factor2.0', 2, -1, 2.0, False, [4, 2, 3], [1.0, 1.0]),
- ('Max2Min-1Factor2.0Square', 2, -1, 2.0, True, [4, 4, 3], [1.0, 2.0]),
- ('Max-1Min8Factor0.5', -1, 8, 0.5, False, [4, 2, 3], [1.0, 1.0]),
- ('Max-1Min8Factor0.5Square', -1, 8, 0.5, True, [4, 4, 3], [1.0, 2.0]),
- ('Max-1Min8Factor0.25', -1, 8, 0.25, False, [4, 2, 3], [1.0, 1.0]),
- ('Max-1Min8Factor0.25Square', -1, 8, 0.25, True, [4, 4, 3], [1.0, 2.0]),
- ('Max2Min2Factor2.0', 2, 2, 2.0, False, [4, 2, 3], [1.0, 1.0]),
- ('Max2Min2Factor2.0Square', 2, 2, 2.0, True, [4, 4, 3], [1.0, 2.0]),
- ('Max16Min8Factor0.5', 16, 8, 0.5, False, [4, 2, 3], [1.0, 1.0]),
- ('Max16Min8Factor0.5Square', 16, 8, 0.5, True, [4, 4, 3], [1.0, 2.0]),
- )
- def testResizeImageWorks(self, max_image_size, min_image_size, resize_factor,
- square_output, expected_shape,
- expected_scale_factors):
- # Construct image of size 4x2x3.
- image = np.array([[[0, 0, 0], [1, 1, 1]], [[2, 2, 2], [3, 3, 3]],
- [[4, 4, 4], [5, 5, 5]], [[6, 6, 6], [7, 7, 7]]],
- dtype='uint8')
-
- # Set up config.
- config = delf_config_pb2.DelfConfig(
- max_image_size=max_image_size,
- min_image_size=min_image_size,
- use_square_images=square_output)
-
- resized_image, scale_factors = extractor.ResizeImage(
- image, config, resize_factor)
- self.assertAllEqual(resized_image.shape, expected_shape)
- self.assertAllClose(scale_factors, expected_scale_factors)
-
- @parameterized.named_parameters(
- ('Max2Min2', 2, 2, 1.0, False, [2, 1, 3], [0.666666, 0.5]),
- ('Max2Min2Square', 2, 2, 1.0, True, [2, 2, 3], [0.666666, 1.0]),
- )
- def testResizeImageRoundingWorks(self, max_image_size, min_image_size,
- resize_factor, square_output, expected_shape,
- expected_scale_factors):
- # Construct image of size 3x2x3.
- image = np.array([[[0, 0, 0], [1, 1, 1]], [[2, 2, 2], [3, 3, 3]],
- [[4, 4, 4], [5, 5, 5]]],
- dtype='uint8')
-
- # Set up config.
- config = delf_config_pb2.DelfConfig(
- max_image_size=max_image_size,
- min_image_size=min_image_size,
- use_square_images=square_output)
-
- resized_image, scale_factors = extractor.ResizeImage(
- image, config, resize_factor)
- self.assertAllEqual(resized_image.shape, expected_shape)
- self.assertAllClose(scale_factors, expected_scale_factors)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/delf/delf/python/examples/match_images.py b/research/delf/delf/python/examples/match_images.py
index bb030739cb9067bf3be50f999368af622f083b54..f14f93f9eb568b06678e7cfc1162f8e653aa6d91 100644
--- a/research/delf/delf/python/examples/match_images.py
+++ b/research/delf/delf/python/examples/match_images.py
@@ -1,3 +1,4 @@
+# Lint as: python3
# Copyright 2017 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,6 +28,7 @@ from __future__ import print_function
import argparse
import sys
+from absl import app
import matplotlib
# Needed before pyplot import for matplotlib to work properly.
matplotlib.use('Agg')
@@ -38,7 +40,6 @@ from skimage import feature
from skimage import measure
from skimage import transform
-from tensorflow.python.platform import app
from delf import feature_io
cmd_args = None
diff --git a/research/delf/delf/python/google_landmarks_dataset/README.md b/research/delf/delf/python/google_landmarks_dataset/README.md
deleted file mode 100644
index 485c1a946b5b21ddb369cc1bc8645534abbfad1e..0000000000000000000000000000000000000000
--- a/research/delf/delf/python/google_landmarks_dataset/README.md
+++ /dev/null
@@ -1,123 +0,0 @@
-## GLDv2 code/models
-
-[](https://arxiv.org/abs/2004.01804)
-
-These instructions can be used to reproduce results from the
-[GLDv2 paper](https://arxiv.org/abs/2004.01804). We present here results on the
-Revisited Oxford/Paris datasets since they are smaller and quicker to
-reproduce -- but note that a very similar procedure can be used to obtain
-results on the GLDv2 retrieval or recognition datasets.
-
-Note that this directory also contains code to compute GLDv2 metrics: see
-`compute_retrieval_metrics.py`, `compute_recognition_metrics.py` and associated
-file reading / metric computation modules.
-
-For more details on the dataset, please refer to its
-[website](https://github.com/cvdfoundation/google-landmark).
-
-### Install DELF library
-
-To be able to use this code, please follow
-[these instructions](../../../INSTALL_INSTRUCTIONS.md) to properly install the
-DELF library.
-
-### Download Revisited Oxford/Paris datasets
-
-```bash
-mkdir -p ~/revisitop/data && cd ~/revisitop/data
-
-# Oxford dataset.
-wget http://www.robots.ox.ac.uk/~vgg/data/oxbuildings/oxbuild_images.tgz
-mkdir oxford5k_images
-tar -xvzf oxbuild_images.tgz -C oxford5k_images/
-
-# Paris dataset. Download and move all images to same directory.
-wget http://www.robots.ox.ac.uk/~vgg/data/parisbuildings/paris_1.tgz
-wget http://www.robots.ox.ac.uk/~vgg/data/parisbuildings/paris_2.tgz
-mkdir paris6k_images_tmp
-tar -xvzf paris_1.tgz -C paris6k_images_tmp/
-tar -xvzf paris_2.tgz -C paris6k_images_tmp/
-mkdir paris6k_images
-mv paris6k_images_tmp/paris/*/*.jpg paris6k_images/
-
-# Revisited annotations.
-wget http://cmp.felk.cvut.cz/revisitop/data/datasets/roxford5k/gnd_roxford5k.mat
-wget http://cmp.felk.cvut.cz/revisitop/data/datasets/rparis6k/gnd_rparis6k.mat
-```
-
-### Download model
-
-```bash
-# From models/research/delf/delf/python/google_landmarks_dataset
-mkdir parameters && cd parameters
-
-# RN101-ArcFace model trained on GLDv2-clean.
-wget https://storage.googleapis.com/delf/rn101_af_gldv2clean_20200521.tar.gz
-tar -xvzf rn101_af_gldv2clean_20200521.tar.gz
-```
-
-### Feature extraction
-
-We present here commands for extraction on `roxford5k`. To extract on `rparis6k`
-instead, please edit the arguments accordingly (especially the
-`dataset_file_path` argument).
-
-#### Query feature extraction
-
-In the Revisited Oxford/Paris experimental protocol, query images must be the
-cropped before feature extraction (this is done in the `extract_features`
-script, when setting `image_set=query`). Note that this is specific to these
-datasets, and not required for the GLDv2 retrieval/recognition datasets.
-
-Run query feature extraction as follows:
-
-```bash
-# From models/research/delf/delf/python/google_landmarks_dataset
-python3 ../delg/extract_features.py \
- --delf_config_path rn101_af_gldv2clean_config.pbtxt \
- --dataset_file_path ~/revisitop/data/gnd_roxford5k.mat \
- --images_dir ~/revisitop/data/oxford5k_images \
- --image_set query \
- --output_features_dir ~/revisitop/data/oxford5k_features/query
-```
-
-#### Index feature extraction
-
-Run index feature extraction as follows:
-
-```bash
-# From models/research/delf/delf/python/google_landmarks_dataset
-python3 ../delg/extract_features.py \
- --delf_config_path rn101_af_gldv2clean_config.pbtxt \
- --dataset_file_path ~/revisitop/data/gnd_roxford5k.mat \
- --images_dir ~/revisitop/data/oxford5k_images \
- --image_set index \
- --output_features_dir ~/revisitop/data/oxford5k_features/index
-```
-
-### Perform retrieval
-
-To run retrieval on `roxford5k`, the following command can be used:
-
-```bash
-# From models/research/delf/delf/python/google_landmarks_dataset
-python3 ../delg/perform_retrieval.py \
- --dataset_file_path ~/revisitop/data/gnd_roxford5k.mat \
- --query_features_dir ~/revisitop/data/oxford5k_features/query \
- --index_features_dir ~/revisitop/data/oxford5k_features/index \
- --output_dir ~/revisitop/results/oxford5k
-```
-
-A file with named `metrics.txt` will be written to the path given in
-`output_dir`. The contents should look approximately like:
-
-```
-hard
- mAP=55.54
- mP@k[ 1 5 10] [88.57 80.86 70.14]
- mR@k[ 1 5 10] [19.46 33.65 42.44]
-medium
- mAP=76.23
- mP@k[ 1 5 10] [95.71 92.86 90.43]
- mR@k[ 1 5 10] [10.17 25.96 35.29]
-```
diff --git a/research/delf/delf/python/normalization_layers/__init__.py b/research/delf/delf/python/normalization_layers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..9064f503de15c1aa115c5e1ff2f4a5345aadf2af
--- /dev/null
+++ b/research/delf/delf/python/normalization_layers/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
\ No newline at end of file
diff --git a/research/delf/delf/python/normalization_layers/normalization.py b/research/delf/delf/python/normalization_layers/normalization.py
new file mode 100644
index 0000000000000000000000000000000000000000..cfb75da753581bd453fa5168c72dc72bc59eda2b
--- /dev/null
+++ b/research/delf/delf/python/normalization_layers/normalization.py
@@ -0,0 +1,40 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Normalization layer definitions."""
+
+import tensorflow as tf
+
+
+class L2Normalization(tf.keras.layers.Layer):
+ """Normalization layer using L2 norm."""
+
+ def __init__(self):
+ """Initialization of the L2Normalization layer."""
+ super(L2Normalization, self).__init__()
+ # A lower bound value for the norm.
+ self.eps = 1e-6
+
+ def call(self, x, axis=1):
+ """Invokes the L2Normalization instance.
+
+ Args:
+ x: A Tensor.
+ axis: Dimension along which to normalize. A scalar or a vector of
+ integers.
+
+ Returns:
+ norm: A Tensor with the same shape as `x`.
+ """
+ return tf.nn.l2_normalize(x, axis, epsilon=self.eps)
diff --git a/research/delf/delf/python/normalization_layers/normalization_test.py b/research/delf/delf/python/normalization_layers/normalization_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea302566c6969538b5dd6fdea441fd94c2585df9
--- /dev/null
+++ b/research/delf/delf/python/normalization_layers/normalization_test.py
@@ -0,0 +1,36 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for normalization layers."""
+
+import tensorflow as tf
+
+from delf.python.normalization_layers import normalization
+
+
+class NormalizationsTest(tf.test.TestCase):
+
+ def testL2Normalization(self):
+ x = tf.constant([-4.0, 0.0, 4.0])
+ layer = normalization.L2Normalization()
+ # Run tested function.
+ result = layer(x, axis=0)
+ # Define expected result.
+ exp_output = [-0.70710677, 0.0, 0.70710677]
+ # Compare actual and expected.
+ self.assertAllClose(exp_output, result)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/delf/delf/python/pooling_layers/__init__.py b/research/delf/delf/python/pooling_layers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..9064f503de15c1aa115c5e1ff2f4a5345aadf2af
--- /dev/null
+++ b/research/delf/delf/python/pooling_layers/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
\ No newline at end of file
diff --git a/research/delf/delf/python/pooling_layers/pooling.py b/research/delf/delf/python/pooling_layers/pooling.py
new file mode 100644
index 0000000000000000000000000000000000000000..8244a414b31d2a35919fd7836c9bb577eae08fc1
--- /dev/null
+++ b/research/delf/delf/python/pooling_layers/pooling.py
@@ -0,0 +1,194 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Pooling layers definitions."""
+
+import tensorflow as tf
+
+
+class MAC(tf.keras.layers.Layer):
+ """Global max pooling (MAC) layer.
+
+ Maximum Activations of Convolutions (MAC) is simply constructed by
+ max-pooling over all dimensions per feature map. See
+ https://arxiv.org/abs/1511.05879 for a reference.
+ """
+
+ def call(self, x, axis=None):
+ """Invokes the MAC pooling instance.
+
+ Args:
+ x: [B, H, W, D] A float32 Tensor.
+ axis: Dimensions to reduce. By default, dimensions [1, 2] are reduced.
+
+ Returns:
+ output: [B, D] A float32 Tensor.
+ """
+ if axis is None:
+ axis = [1, 2]
+ return mac(x, axis=axis)
+
+
+class SPoC(tf.keras.layers.Layer):
+ """Average pooling (SPoC) layer.
+
+ Sum-pooled convolutional features (SPoC) is based on the sum pooling of the
+ deep features. See https://arxiv.org/pdf/1510.07493.pdf for a reference.
+ """
+
+ def call(self, x, axis=None):
+ """Invokes the SPoC instance.
+
+ Args:
+ x: [B, H, W, D] A float32 Tensor.
+ axis: Dimensions to reduce. By default, dimensions [1, 2] are reduced.
+
+ Returns:
+ output: [B, D] A float32 Tensor.
+ """
+ if axis is None:
+ axis = [1, 2]
+ return spoc(x, axis)
+
+
+class GeM(tf.keras.layers.Layer):
+ """Generalized mean pooling (GeM) layer.
+
+ Generalized Mean Pooling (GeM) computes the generalized mean of each
+ channel in a tensor. See https://arxiv.org/abs/1711.02512 for a reference.
+ """
+
+ def __init__(self, power=3.):
+ """Initialization of the generalized mean pooling (GeM) layer.
+
+ Args:
+ power: Float power > 0 is an inverse exponent parameter, used during the
+ generalized mean pooling computation. Setting this exponent as power > 1
+ increases the contrast of the pooled feature map and focuses on the
+ salient features of the image. GeM is a generalization of the average
+ pooling commonly used in classification networks (power = 1) and of
+ spatial max-pooling layer (power = inf).
+ """
+ super(GeM, self).__init__()
+ self.power = power
+ self.eps = 1e-6
+
+ def call(self, x, axis=None):
+ """Invokes the GeM instance.
+
+ Args:
+ x: [B, H, W, D] A float32 Tensor.
+ axis: Dimensions to reduce. By default, dimensions [1, 2] are reduced.
+
+ Returns:
+ output: [B, D] A float32 Tensor.
+ """
+ if axis is None:
+ axis = [1, 2]
+ return gem(x, power=self.power, eps=self.eps, axis=axis)
+
+
+class GeMPooling2D(tf.keras.layers.Layer):
+ """Generalized mean pooling (GeM) pooling operation for spatial data."""
+
+ def __init__(self,
+ power=20.,
+ pool_size=(2, 2),
+ strides=None,
+ padding='valid',
+ data_format='channels_last'):
+ """Initialization of GeMPooling2D.
+
+ Args:
+ power: Float, power > 0. is an inverse exponent parameter (GeM power).
+ pool_size: Integer or tuple of 2 integers, factors by which to downscale
+ (vertical, horizontal)
+ strides: Integer, tuple of 2 integers, or None. Strides values. If None,
+ it will default to `pool_size`.
+ padding: One of `valid` or `same`. `valid` means no padding. `same`
+ results in padding evenly to the left/right or up/down of the input such
+ that output has the same height/width dimension as the input.
+ data_format: A string, one of `channels_last` (default) or
+ `channels_first`. The ordering of the dimensions in the inputs.
+ `channels_last` corresponds to inputs with shape `(batch, height, width,
+ channels)` while `channels_first` corresponds to inputs with shape
+ `(batch, channels, height, width)`.
+ """
+ super(GeMPooling2D, self).__init__()
+ self.power = power
+ self.eps = 1e-6
+ self.pool_size = pool_size
+ self.strides = strides
+ self.padding = padding.upper()
+ data_format_conv = {
+ 'channels_last': 'NHWC',
+ 'channels_first': 'NCHW',
+ }
+ self.data_format = data_format_conv[data_format]
+
+ def call(self, x):
+ tmp = tf.pow(x, self.power)
+ tmp = tf.nn.avg_pool(tmp, self.pool_size, self.strides, self.padding,
+ self.data_format)
+ out = tf.pow(tmp, 1. / self.power)
+ return out
+
+
+def mac(x, axis=None):
+ """Performs global max pooling (MAC).
+
+ Args:
+ x: [B, H, W, D] A float32 Tensor.
+ axis: Dimensions to reduce. By default, dimensions [1, 2] are reduced.
+
+ Returns:
+ output: [B, D] A float32 Tensor.
+ """
+ if axis is None:
+ axis = [1, 2]
+ return tf.reduce_max(x, axis=axis, keepdims=False)
+
+
+def spoc(x, axis=None):
+ """Performs average pooling (SPoC).
+
+ Args:
+ x: [B, H, W, D] A float32 Tensor.
+ axis: Dimensions to reduce. By default, dimensions [1, 2] are reduced.
+
+ Returns:
+ output: [B, D] A float32 Tensor.
+ """
+ if axis is None:
+ axis = [1, 2]
+ return tf.reduce_mean(x, axis=axis, keepdims=False)
+
+
+def gem(x, axis=None, power=3., eps=1e-6):
+ """Performs generalized mean pooling (GeM).
+
+ Args:
+ x: [B, H, W, D] A float32 Tensor.
+ axis: Dimensions to reduce. By default, dimensions [1, 2] are reduced.
+ power: Float, power > 0 is an inverse exponent parameter (GeM power).
+ eps: Float, parameter for numerical stability.
+
+ Returns:
+ output: [B, D] A float32 Tensor.
+ """
+ if axis is None:
+ axis = [1, 2]
+ tmp = tf.pow(tf.maximum(x, eps), power)
+ out = tf.pow(tf.reduce_mean(tmp, axis=axis, keepdims=False), 1. / power)
+ return out
diff --git a/research/delf/delf/python/pooling_layers/pooling_test.py b/research/delf/delf/python/pooling_layers/pooling_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..78653550e4582c32f59a1caab05fddb45518e2df
--- /dev/null
+++ b/research/delf/delf/python/pooling_layers/pooling_test.py
@@ -0,0 +1,84 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for pooling layers."""
+
+import tensorflow as tf
+
+from delf.python.pooling_layers import pooling
+
+
+class PoolingsTest(tf.test.TestCase):
+
+ def testMac(self):
+ x = tf.constant([[[[0., 1.], [2., 3.]], [[4., 5.], [6., 7.]]]])
+ # Run tested function.
+ result = pooling.mac(x)
+ # Define expected result.
+ exp_output = [[6., 7.]]
+ # Compare actual and expected.
+ self.assertAllClose(exp_output, result)
+
+ def testSpoc(self):
+ x = tf.constant([[[[0., 1.], [2., 3.]], [[4., 5.], [6., 7.]]]])
+ # Run tested function.
+ result = pooling.spoc(x)
+ # Define expected result.
+ exp_output = [[3., 4.]]
+ # Compare actual and expected.
+ self.assertAllClose(exp_output, result)
+
+ def testGem(self):
+ x = tf.constant([[[[0., 1.], [2., 3.]], [[4., 5.], [6., 7.]]]])
+ # Run tested function.
+ result = pooling.gem(x, power=3., eps=1e-6)
+ # Define expected result.
+ exp_output = [[4.1601677, 4.9866314]]
+ # Compare actual and expected.
+ self.assertAllClose(exp_output, result)
+
+ def testGeMPooling2D(self):
+ # Create a testing tensor.
+ x = tf.constant([[[1., 2., 3.],
+ [4., 5., 6.],
+ [7., 8., 9.]]])
+ x = tf.reshape(x, [1, 3, 3, 1])
+
+ # Checking GeMPooling2D relation to MaxPooling2D for the large values of
+ # `p`.
+ max_pool_2d = tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
+ strides=(1, 1), padding='valid')
+ out_max = max_pool_2d(x)
+ gem_pool_2d = pooling.GeMPooling2D(power=30., pool_size=(2, 2),
+ strides=(1, 1), padding='valid')
+ out_gem_max = gem_pool_2d(x)
+
+ # Check that for large `p` GeMPooling2D is close to MaxPooling2D.
+ self.assertAllEqual(out_max, tf.round(out_gem_max))
+
+ # Checking GeMPooling2D relation to AveragePooling2D for the value
+ # of `p` = 1.
+ avg_pool_2d = tf.keras.layers.AveragePooling2D(pool_size=(2, 2),
+ strides=(1, 1),
+ padding='valid')
+ out_avg = avg_pool_2d(x)
+ gem_pool_2d = pooling.GeMPooling2D(power=1., pool_size=(2, 2),
+ strides=(1, 1), padding='valid')
+ out_gem_avg = gem_pool_2d(x)
+ # Check that for `p` equals 1., GeMPooling2D becomes AveragePooling2D.
+ self.assertAllEqual(out_avg, out_gem_avg)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/delf/delf/python/training/README.md b/research/delf/delf/python/training/README.md
index 6712ee2e41116c320bbf6c2c231138c552fa443f..41ea2a0b47f6cbee4be31e6689ba3b77e6aefdd7 100644
--- a/research/delf/delf/python/training/README.md
+++ b/research/delf/delf/python/training/README.md
@@ -1,7 +1,7 @@
-# DELF Training Instructions
+# DELF/DELG Training Instructions
-This README documents the end-to-end process for training a landmark detection
-and retrieval model using the DELF library on the
+This README documents the end-to-end process for training a local and/or global
+image feature model on the
[Google Landmarks Dataset v2](https://github.com/cvdfoundation/google-landmark)
(GLDv2). This can be achieved following these steps:
@@ -143,6 +143,8 @@ curl -Os http://storage.googleapis.com/delf/resnet50_imagenet_weights.tar.gz
tar -xzvf resnet50_imagenet_weights.tar.gz
```
+### Training with Local Features
+
Assuming the TFRecord files were generated in the `gldv2_dataset/tfrecord/`
directory, running the following command should start training a model and
output the results in the `gldv2_training` directory:
@@ -156,26 +158,76 @@ python3 train.py \
--logdir=gldv2_training/
```
-On a multi-GPU machine the batch size can be increased to speed up the training
-using the `--batch_size` parameter. On a 8 Tesla P100 GPUs machine you can set
-the batch size to `256`:
+*NOTE: The `--use_autoencoder` parameter is set by default to `True`, therefore
+the model will be by default trained with the autoencoder.*
+
+### Training with Local and Global Features
+
+It is also possible to train the model with an improved global features head as
+introduced in the [DELG paper](https://arxiv.org/abs/2001.05027). To do this,
+specify the additional parameter `--delg_global_features` when launching the
+training, like in the following example:
```
---batch_size=256
+python3 train.py \
+ --train_file_pattern=gldv2_dataset/tfrecord/train* \
+ --validation_file_pattern=gldv2_dataset/tfrecord/validation* \
+ --imagenet_checkpoint=resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5 \
+ --dataset_version=gld_v2_clean \
+ --logdir=gldv2_training/ \
+ --delg_global_features
```
+*NOTE: The `--use_autoencoder` parameter is set by default to `True`, therefore
+the model will be by default trained with the autoencoder.*
+
+### Hyperparameter Guidelines
+
+In order to improve the convergence of the training, the following
+hyperparameter values have been tested and validated on the following
+infrastructures, the remaining `train.py` flags keeping their **default
+values**:
+* 8 Tesla P100 GPUs: `--batch_size=256`, `--initial_lr=0.01`
+* 4 Tesla P100 GPUs: `--batch_size=128`, `--initial_lr=0.005`
+
## Exporting the Trained Model
Assuming the training output, the TensorFlow checkpoint, is in the
`gldv2_training` directory, running the following commands exports the model.
-### DELF local feature model
+### DELF local feature-only model
+
+This should be used when you are only interested in having a local feature
+model.
+
+```
+python3 model/export_local_model.py \
+ --ckpt_path=gldv2_training/delf_weights \
+ --export_path=gldv2_model_local
+```
+
+### DELG global feature-only model
+
+This should be used when you are only interested in having a global feature
+model.
```
-python3 model/export_model.py \
+python3 model/export_global_model.py \
--ckpt_path=gldv2_training/delf_weights \
- --export_path=gldv2_model_local \
- --block3_strides
+ --export_path=gldv2_model_global \
+ --delg_global_features
+```
+
+### DELG local+global feature model
+
+This should be used when you are interested in jointly extracting local and
+global features.
+
+```
+python3 model/export_local_and_global_model.py \
+ --ckpt_path=gldv2_training/delf_weights \
+ --export_path=gldv2_model_local_and_global \
+ --delg_global_features
```
### Kaggle-compatible global feature model
@@ -184,6 +236,13 @@ To export a global feature model in the format required by the
[2020 Landmark Retrieval challenge](https://www.kaggle.com/c/landmark-retrieval-2020),
you can use the following command:
+*NOTE*: this command is helpful to use the model directly in the above-mentioned
+Kaggle competition; however, this is a different format than the one required in
+this DELF/DELG codebase (ie, if you export the model this way, the commands
+found in the [DELG instructions](../delg/DELG_INSTRUCTIONS.md) would not work).
+To export the model in a manner compatible to this codebase, use a similar
+command as the "DELG global feature-only model" above.
+
```
python3 model/export_global_model.py \
--ckpt_path=gldv2_training/delf_weights \
@@ -193,7 +252,9 @@ python3 model/export_global_model.py \
--normalize_global_descriptor
```
-## Testing the Trained Model
+## Testing the trained model
+
+### Testing the trained local feature model
After the trained model has been exported, it can be used to extract DELF
features from 2 images of the same landmark and to perform a matching test
@@ -266,3 +327,13 @@ python3 ../examples/match_images.py \
The generated image `matched_images.png` should look similar to this one:
+
+### Testing the trained global (or global+local) feature model
+
+Please follow the [DELG instructions](../delg/DELG_INSTRUCTIONS.md). The only
+modification should be to pass a different `delf_config_path` when doing feature
+extraction, which should point to the newly-trained model. As described in the
+[DelfConfig](../../protos/delf_config.proto), you should set the
+`use_local_features` and `use_global_features` in the right way, depending on
+which feature modalities you are using. Note also that you should set
+`is_tf2_exported` to `true`.
diff --git a/research/delf/delf/python/training/datasets/__init__.py b/research/delf/delf/python/training/datasets/__init__.py
deleted file mode 100644
index 7e0a672716945394cce4b2c69ee3d086192da87c..0000000000000000000000000000000000000000
--- a/research/delf/delf/python/training/datasets/__init__.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright 2020 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Module exposing datasets for training."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# pylint: disable=unused-import
-from delf.python.training.datasets import googlelandmarks
-# pylint: enable=unused-import
diff --git a/research/delf/delf/python/training/global_features_utils.py b/research/delf/delf/python/training/global_features_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b35178b1530ac7572bc8f50c3a6fabfff6baca6
--- /dev/null
+++ b/research/delf/delf/python/training/global_features_utils.py
@@ -0,0 +1,229 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for the global model training."""
+
+import os
+
+from absl import logging
+
+import numpy as np
+import tensorflow as tf
+
+from delf.python.datasets.revisited_op import dataset
+
+
+class AverageMeter():
+ """Computes and stores the average and current value of loss."""
+
+ def __init__(self):
+ """Initialization of the AverageMeter."""
+ self.reset()
+
+ def reset(self):
+ """Resets all the values."""
+ self.val = 0
+ self.avg = 0
+ self.sum = 0
+ self.count = 0
+
+ def update(self, val, n=1):
+ """Updates values in the AverageMeter.
+
+ Args:
+ val: Float, loss value.
+ n: Integer, number of instances.
+ """
+ self.val = val
+ self.sum += val * n
+ self.count += n
+ self.avg = self.sum / self.count
+
+
+def compute_metrics_and_print(dataset_name,
+ sorted_index_ids,
+ ground_truth,
+ desired_pr_ranks=None,
+ log=True):
+ """Computes and logs ground-truth metrics for Revisited datasets.
+
+ Args:
+ dataset_name: String, name of the dataset.
+ sorted_index_ids: Integer NumPy array of shape [#queries, #index_images].
+ For each query, contains an array denoting the most relevant index images,
+ sorted from most to least relevant.
+ ground_truth: List containing ground-truth information for dataset. Each
+ entry is a dict corresponding to the ground-truth information for a query.
+ The dict has keys 'ok' and 'junk', mapping to a NumPy array of integers.
+ desired_pr_ranks: List of integers containing the desired precision/recall
+ ranks to be reported. E.g., if precision@1/recall@1 and
+ precision@10/recall@10 are desired, this should be set to [1, 10]. The
+ largest item should be <= #sorted_index_ids. Default: [1, 5, 10].
+ log: Whether to log results using logging.info().
+
+ Returns:
+ mAP: (metricsE, metricsM, metricsH) Tuple of the metrics for different
+ levels of complexity. Each metrics is a list containing:
+ mean_average_precision (float), mean_precisions (NumPy array of
+ floats, with shape [len(desired_pr_ranks)]), mean_recalls (NumPy array
+ of floats, with shape [len(desired_pr_ranks)]), average_precisions
+ (NumPy array of floats, with shape [#queries]), precisions (NumPy array of
+ floats, with shape [#queries, len(desired_pr_ranks)]), recalls (NumPy
+ array of floats, with shape [#queries, len(desired_pr_ranks)]).
+
+ Raises:
+ ValueError: If an unknown dataset name is provided as an argument.
+ """
+ if dataset not in dataset.DATASET_NAMES:
+ raise ValueError('Unknown dataset: {}!'.format(dataset))
+
+ if desired_pr_ranks is None:
+ desired_pr_ranks = [1, 5, 10]
+
+ (easy_ground_truth, medium_ground_truth,
+ hard_ground_truth) = dataset.ParseEasyMediumHardGroundTruth(ground_truth)
+
+ metrics_easy = dataset.ComputeMetrics(sorted_index_ids, easy_ground_truth,
+ desired_pr_ranks)
+ metrics_medium = dataset.ComputeMetrics(sorted_index_ids, medium_ground_truth,
+ desired_pr_ranks)
+ metrics_hard = dataset.ComputeMetrics(sorted_index_ids, hard_ground_truth,
+ desired_pr_ranks)
+
+ debug_and_log(
+ '>> {}: mAP E: {}, M: {}, H: {}'.format(
+ dataset_name, np.around(metrics_easy[0] * 100, decimals=2),
+ np.around(metrics_medium[0] * 100, decimals=2),
+ np.around(metrics_hard[0] * 100, decimals=2)),
+ log=log)
+
+ debug_and_log(
+ '>> {}: mP@k{} E: {}, M: {}, H: {}'.format(
+ dataset_name, desired_pr_ranks,
+ np.around(metrics_easy[1] * 100, decimals=2),
+ np.around(metrics_medium[1] * 100, decimals=2),
+ np.around(metrics_hard[1] * 100, decimals=2)),
+ log=log)
+
+ return metrics_easy, metrics_medium, metrics_hard
+
+
+def htime(time_difference):
+ """Time formatting function.
+
+ Depending on the value of `time_difference` outputs time in an appropriate
+ time format.
+
+ Args:
+ time_difference: Float, time difference between the two events.
+
+ Returns:
+ time: String representing time in an appropriate time format.
+ """
+ time_difference = round(time_difference)
+
+ days = time_difference // 86400
+ hours = time_difference // 3600 % 24
+ minutes = time_difference // 60 % 60
+ seconds = time_difference % 60
+
+ if days > 0:
+ return '{:d}d {:d}h {:d}m {:d}s'.format(days, hours, minutes, seconds)
+ if hours > 0:
+ return '{:d}h {:d}m {:d}s'.format(hours, minutes, seconds)
+ if minutes > 0:
+ return '{:d}m {:d}s'.format(minutes, seconds)
+ return '{:d}s'.format(seconds)
+
+
+def debug_and_log(msg, debug=True, log=True, debug_on_the_same_line=False):
+ """Outputs `msg` to both stdout (if in the debug mode) and the log file.
+
+ Args:
+ msg: String, message to be logged.
+ debug: Bool, if True, will print `msg` to stdout.
+ log: Bool, if True, will redirect `msg` to the logfile.
+ debug_on_the_same_line: Bool, if True, will print `msg` to stdout without a
+ new line. When using this mode, logging to a logfile is disabled.
+ """
+ if debug_on_the_same_line:
+ print(msg, end='')
+ return
+ if debug:
+ print(msg)
+ if log:
+ logging.info(msg)
+
+
+def get_standard_keras_models():
+ """Gets the standard keras model names.
+
+ Returns:
+ model_names: List, names of the standard keras models.
+ """
+ model_names = sorted(
+ name for name in tf.keras.applications.__dict__
+ if not name.startswith('__') and
+ callable(tf.keras.applications.__dict__[name]))
+ return model_names
+
+
+def create_model_directory(training_dataset, arch, pool, whitening, pretrained,
+ loss, loss_margin, optimizer, lr, weight_decay,
+ neg_num, query_size, pool_size, batch_size,
+ update_every, image_size, directory):
+ """Based on the model parameters, creates the model directory.
+
+ If the model directory does not exist, the directory is created.
+
+ Args:
+ training_dataset: String, training dataset name.
+ arch: String, model architecture.
+ pool: String, pooling option.
+ whitening: Bool, whether the model is trained with global whitening.
+ pretrained: Bool, whether the model is initialized with the precomputed
+ weights.
+ loss: String, training loss type.
+ loss_margin: Float, loss margin.
+ optimizer: Sting, used optimizer.
+ lr: Float, initial learning rate.
+ weight_decay: Float, weight decay.
+ neg_num: Integer, Number of negative images per train/val tuple.
+ query_size: Integer, number of queries per one training epoch.
+ pool_size: Integer, size of the pool for hard negative mining.
+ batch_size: Integer, batch size.
+ update_every: Integer, frequency of the model weights update.
+ image_size: Integer, maximum size of longer image side used for training.
+ directory: String, destination where trained network should be saved.
+
+ Returns:
+ folder: String, path to the model folder.
+ """
+ folder = '{}_{}_{}'.format(training_dataset, arch, pool)
+ if whitening:
+ folder += '_whiten'
+ if not pretrained:
+ folder += '_notpretrained'
+ folder += ('_{}_m{:.2f}_{}_lr{:.1e}_wd{:.1e}_nnum{}_qsize{}_psize{}_bsize{}'
+ '_uevery{}_imsize{}').format(loss, loss_margin, optimizer, lr,
+ weight_decay, neg_num, query_size,
+ pool_size, batch_size, update_every,
+ image_size)
+
+ folder = os.path.join(directory, folder)
+ debug_and_log(
+ '>> Creating directory if does not exist:\n>> \'{}\''.format(folder))
+ if not os.path.exists(folder):
+ os.makedirs(folder)
+ return folder
diff --git a/research/delf/delf/python/training/install_delf.sh b/research/delf/delf/python/training/install_delf.sh
index 4feb464aa7def067028e65281d906e006f4533a2..340c83e3419b731920480021f71f223f2176472b 100755
--- a/research/delf/delf/python/training/install_delf.sh
+++ b/research/delf/delf/python/training/install_delf.sh
@@ -37,6 +37,7 @@ install_tensorflow() {
# Install TensorFlow 2.2.
echo "Installing TensorFlow 2.2"
pip3 install --upgrade tensorflow==2.2.0
+ pip3 install tensorflow-addons==0.11.2
local exit_code=$?
handle_exit_code ${exit_code} "Unable to install Tensorflow 2.2."
echo "Installing TensorFlow 2.2 for GPU"
diff --git a/research/delf/delf/python/training/losses/__init__.py b/research/delf/delf/python/training/losses/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..9064f503de15c1aa115c5e1ff2f4a5345aadf2af
--- /dev/null
+++ b/research/delf/delf/python/training/losses/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
\ No newline at end of file
diff --git a/research/delf/delf/python/training/losses/ranking_losses.py b/research/delf/delf/python/training/losses/ranking_losses.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc7c28447905a156b9e326199d218241f1703f3b
--- /dev/null
+++ b/research/delf/delf/python/training/losses/ranking_losses.py
@@ -0,0 +1,175 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ranking loss definitions."""
+
+import tensorflow as tf
+
+
+class ContrastiveLoss(tf.keras.losses.Loss):
+ """Contrastive Loss layer.
+
+ Contrastive Loss layer allows to compute contrastive loss for a batch of
+ images. Implementation based on: https://arxiv.org/abs/1604.02426.
+ """
+
+ def __init__(self, margin=0.7, reduction=tf.keras.losses.Reduction.NONE):
+ """Initialization of Contrastive Loss layer.
+
+ Args:
+ margin: Float contrastive loss margin.
+ reduction: Type of loss reduction.
+ """
+ super(ContrastiveLoss, self).__init__(reduction)
+ self.margin = margin
+ # Parameter for numerical stability.
+ self.eps = 1e-6
+
+ def __call__(self, queries, positives, negatives):
+ """Invokes the Contrastive Loss instance.
+
+ Args:
+ queries: [batch_size, dim] Anchor input tensor.
+ positives: [batch_size, dim] Positive sample input tensor.
+ negatives: [batch_size, num_neg, dim] Negative sample input tensor.
+
+ Returns:
+ loss: Scalar tensor.
+ """
+ return contrastive_loss(
+ queries, positives, negatives, margin=self.margin, eps=self.eps)
+
+
+class TripletLoss(tf.keras.losses.Loss):
+ """Triplet Loss layer.
+
+ Triplet Loss layer computes triplet loss for a batch of images. Triplet
+ loss tries to keep all queries closer to positives than to any negatives.
+ Margin is used to specify when a triplet has become too "easy" and we no
+ longer want to adjust the weights from it. Differently from the Contrastive
+ Loss, Triplet Loss uses squared distances when computing the loss.
+ Implementation based on: https://arxiv.org/abs/1511.07247.
+ """
+
+ def __init__(self, margin=0.1, reduction=tf.keras.losses.Reduction.NONE):
+ """Initialization of Triplet Loss layer.
+
+ Args:
+ margin: Triplet loss margin.
+ reduction: Type of loss reduction.
+ """
+ super(TripletLoss, self).__init__(reduction)
+ self.margin = margin
+
+ def __call__(self, queries, positives, negatives):
+ """Invokes the Triplet Loss instance.
+
+ Args:
+ queries: [batch_size, dim] Anchor input tensor.
+ positives: [batch_size, dim] Positive sample input tensor.
+ negatives: [batch_size, num_neg, dim] Negative sample input tensor.
+
+ Returns:
+ loss: Scalar tensor.
+ """
+ return triplet_loss(queries, positives, negatives, margin=self.margin)
+
+
+def contrastive_loss(queries, positives, negatives, margin=0.7, eps=1e-6):
+ """Calculates Contrastive Loss.
+
+ We expect the `queries`, `positives` and `negatives` to be normalized with
+ unit length for training stability. The contrastive loss directly
+ optimizes this distance by encouraging all positive distances to
+ approach 0, while keeping negative distances above a certain threshold.
+
+ Args:
+ queries: [batch_size, dim] Anchor input tensor.
+ positives: [batch_size, dim] Positive sample input tensor.
+ negatives: [batch_size, num_neg, dim] Negative sample input tensor.
+ margin: Float contrastive loss loss margin.
+ eps: Float parameter for numerical stability.
+
+ Returns:
+ loss: Scalar tensor.
+ """
+ dim = tf.shape(queries)[1]
+ # Number of `queries`.
+ batch_size = tf.shape(queries)[0]
+ # Number of `positives`.
+ np = tf.shape(positives)[0]
+ # Number of `negatives`.
+ num_neg = tf.shape(negatives)[1]
+
+ # Preparing negatives.
+ stacked_negatives = tf.reshape(negatives, [num_neg * batch_size, dim])
+
+ # Preparing queries for further loss calculation.
+ stacked_queries = tf.repeat(queries, num_neg + 1, axis=0)
+ positives_and_negatives = tf.concat([positives, stacked_negatives], axis=0)
+
+ # Calculate an Euclidean norm for each pair of points. For any positive
+ # pair of data points this distance should be small, and for
+ # negative pair it should be large.
+ distances = tf.norm(stacked_queries - positives_and_negatives + eps, axis=1)
+
+ positives_part = 0.5 * tf.pow(distances[:np], 2.0)
+ negatives_part = 0.5 * tf.pow(
+ tf.math.maximum(margin - distances[np:], 0), 2.0)
+
+ # Final contrastive loss calculation.
+ loss = tf.reduce_sum(tf.concat([positives_part, negatives_part], 0))
+ return loss
+
+
+def triplet_loss(queries, positives, negatives, margin=0.1):
+ """Calculates Triplet Loss.
+
+ Triplet loss tries to keep all queries closer to positives than to any
+ negatives. Differently from the Contrastive Loss, Triplet Loss uses squared
+ distances when computing the loss.
+
+ Args:
+ queries: [batch_size, dim] Anchor input tensor.
+ positives: [batch_size, dim] Positive sample input tensor.
+ negatives: [batch_size, num_neg, dim] Negative sample input tensor.
+ margin: Float triplet loss loss margin.
+
+ Returns:
+ loss: Scalar tensor.
+ """
+ dim = tf.shape(queries)[1]
+ # Number of `queries`.
+ batch_size = tf.shape(queries)[0]
+ # Number of `negatives`.
+ num_neg = tf.shape(negatives)[1]
+
+ # Preparing negatives.
+ stacked_negatives = tf.reshape(negatives, [num_neg * batch_size, dim])
+
+ # Preparing queries for further loss calculation.
+ stacked_queries = tf.repeat(queries, num_neg, axis=0)
+
+ # Preparing positives for further loss calculation.
+ stacked_positives = tf.repeat(positives, num_neg, axis=0)
+
+ # Computes *squared* distances.
+ distance_positives = tf.reduce_sum(
+ tf.square(stacked_queries - stacked_positives), axis=1)
+ distance_negatives = tf.reduce_sum(
+ tf.square(stacked_queries - stacked_negatives), axis=1)
+ # Final triplet loss calculation.
+ loss = tf.reduce_sum(
+ tf.maximum(distance_positives - distance_negatives + margin, 0.0))
+ return loss
diff --git a/research/delf/delf/python/training/losses/ranking_losses_test.py b/research/delf/delf/python/training/losses/ranking_losses_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..8e540ca3ce62d59497698ee3a1affcf5757e6362
--- /dev/null
+++ b/research/delf/delf/python/training/losses/ranking_losses_test.py
@@ -0,0 +1,60 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Ranking losses."""
+
+import tensorflow as tf
+from delf.python.training.losses import ranking_losses
+
+
+class RankingLossesTest(tf.test.TestCase):
+
+ def testContrastiveLoss(self):
+ # Testing the correct numeric value.
+ queries = tf.math.l2_normalize(tf.constant([[1.0, 2.0, -2.0]]))
+ positives = tf.math.l2_normalize(tf.constant([[-1.0, 2.0, 0.0]]))
+ negatives = tf.math.l2_normalize(tf.constant([[[-5.0, 0.0, 3.0]]]))
+
+ result = ranking_losses.contrastive_loss(queries, positives, negatives,
+ margin=0.7, eps=1e-6)
+ exp_output = 0.55278635
+ self.assertAllClose(exp_output, result)
+
+ def testTripletLossZeroLoss(self):
+ # Testing the correct numeric value in case if query-positive distance is
+ # smaller than the query-negative distance.
+ queries = tf.math.l2_normalize(tf.constant([[1.0, 2.0, -2.0]]))
+ positives = tf.math.l2_normalize(tf.constant([[-1.0, 2.0, 0.0]]))
+ negatives = tf.math.l2_normalize(tf.constant([[[-5.0, 0.0, 3.0]]]))
+
+ result = ranking_losses.triplet_loss(queries, positives, negatives,
+ margin=0.1)
+ exp_output = 0.0
+ self.assertAllClose(exp_output, result)
+
+ def testTripletLossNonZeroLoss(self):
+ # Testing the correct numeric value in case if query-positive distance is
+ # bigger than the query-negative distance.
+ queries = tf.math.l2_normalize(tf.constant([[1.0, 2.0, -2.0]]))
+ positives = tf.math.l2_normalize(tf.constant([[-5.0, 0.0, 3.0]]))
+ negatives = tf.math.l2_normalize(tf.constant([[[-1.0, 2.0, 0.0]]]))
+
+ result = ranking_losses.triplet_loss(queries, positives, negatives,
+ margin=0.1)
+ exp_output = 2.2520838
+ self.assertAllClose(exp_output, result)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/delf/delf/python/training/model/__init__.py b/research/delf/delf/python/training/model/__init__.py
index dcc888bd8a65e9ba48f15e4082064e7285ac2591..3fd7e87af3539ffd9869485a022cf7df5bffe26a 100644
--- a/research/delf/delf/python/training/model/__init__.py
+++ b/research/delf/delf/python/training/model/__init__.py
@@ -19,6 +19,7 @@ from __future__ import print_function
# pylint: disable=unused-import
from delf.python.training.model import delf_model
+from delf.python.training.model import delg_model
from delf.python.training.model import export_model_utils
from delf.python.training.model import resnet50
# pylint: enable=unused-import
diff --git a/research/delf/delf/python/training/model/delf_model.py b/research/delf/delf/python/training/model/delf_model.py
index f5dda85263fcb3be1de41183e6dad564e607f76f..5cdad73babbada11d3241442da8bbad9f2c5e159 100644
--- a/research/delf/delf/python/training/model/delf_model.py
+++ b/research/delf/delf/python/training/model/delf_model.py
@@ -80,6 +80,38 @@ class AttentionModel(tf.keras.Model):
return feat, prob, score
+class AutoencoderModel(tf.keras.Model):
+ """Instantiates the Keras Autoencoder model."""
+
+ def __init__(self, reduced_dimension, expand_dimension, kernel_size=1,
+ name='autoencoder'):
+ """Initialization of Autoencoder model.
+
+ Args:
+ reduced_dimension: int, the output dimension of the autoencoder layer.
+ expand_dimension: int, the input dimension of the autoencoder layer.
+ kernel_size: int or tuple, height and width of the 2D convolution window.
+ name: str, name to identify model.
+ """
+ super(AutoencoderModel, self).__init__(name=name)
+ self.conv1 = layers.Conv2D(
+ reduced_dimension,
+ kernel_size,
+ padding='same',
+ name='autoenc_conv1')
+ self.conv2 = layers.Conv2D(
+ expand_dimension,
+ kernel_size,
+ activation=tf.keras.activations.relu,
+ padding='same',
+ name='autoenc_conv2')
+
+ def call(self, inputs):
+ dim_reduced_features = self.conv1(inputs)
+ dim_expanded_features = self.conv2(dim_reduced_features)
+ return dim_expanded_features, dim_reduced_features
+
+
class Delf(tf.keras.Model):
"""Instantiates Keras DELF model using ResNet50 as backbone.
@@ -89,12 +121,36 @@ class Delf(tf.keras.Model):
from conv_4 are used to compute an attention map of the same resolution.
"""
- def __init__(self, block3_strides=True, name='DELF'):
+ def __init__(self,
+ block3_strides=True,
+ name='DELF',
+ pooling='avg',
+ gem_power=3.0,
+ embedding_layer=False,
+ embedding_layer_dim=2048,
+ use_dim_reduction=False,
+ reduced_dimension=128,
+ dim_expand_channels=1024):
"""Initialization of DELF model.
Args:
block3_strides: bool, whether to add strides to the output of block3.
name: str, name to identify model.
+ pooling: str, pooling mode for global feature extraction; possible values
+ are 'None', 'avg', 'max', 'gem.'
+ gem_power: float, GeM power for GeM pooling. Only used if pooling ==
+ 'gem'.
+ embedding_layer: bool, whether to create an embedding layer (FC whitening
+ layer).
+ embedding_layer_dim: int, size of the embedding layer.
+ use_dim_reduction: Whether to integrate dimensionality reduction layers.
+ If True, extra layers are added to reduce the dimensionality of the
+ extracted features.
+ reduced_dimension: int, only used if use_dim_reduction is True. The output
+ dimension of the autoencoder layer.
+ dim_expand_channels: int, only used if use_dim_reduction is True. The
+ number of channels of the backbone block used. Default value 1024 is the
+ number of channels of backbone block 'block3'.
"""
super(Delf, self).__init__(name=name)
@@ -103,41 +159,74 @@ class Delf(tf.keras.Model):
'channels_last',
name='backbone',
include_top=False,
- pooling='avg',
+ pooling=pooling,
block3_strides=block3_strides,
- average_pooling=False)
+ average_pooling=False,
+ gem_power=gem_power,
+ embedding_layer=embedding_layer,
+ embedding_layer_dim=embedding_layer_dim)
# Attention model.
self.attention = AttentionModel(name='attention')
- # Define classifiers for training backbone and attention models.
- def init_classifiers(self, num_classes):
- self.num_classes = num_classes
- self.desc_classification = layers.Dense(
- num_classes, activation=None, kernel_regularizer=None, name='desc_fc')
+ # Autoencoder model.
+ self._use_dim_reduction = use_dim_reduction
+ if self._use_dim_reduction:
+ self.autoencoder = AutoencoderModel(reduced_dimension,
+ dim_expand_channels,
+ name='autoencoder')
+ def init_classifiers(self, num_classes, desc_classification=None):
+ """Define classifiers for training backbone and attention models."""
+ self.num_classes = num_classes
+ if desc_classification is None:
+ self.desc_classification = layers.Dense(
+ num_classes, activation=None, kernel_regularizer=None, name='desc_fc')
+ else:
+ self.desc_classification = desc_classification
self.attn_classification = layers.Dense(
num_classes, activation=None, kernel_regularizer=None, name='att_fc')
- # Weights to optimize for descriptor fine tuning.
- @property
- def desc_trainable_weights(self):
- return (self.backbone.trainable_weights +
- self.desc_classification.trainable_weights)
-
- # Weights to optimize for attention model training.
- @property
- def attn_trainable_weights(self):
- return (self.attention.trainable_weights +
- self.attn_classification.trainable_weights)
+ def global_and_local_forward_pass(self, images, training=True):
+ """Run a forward to calculate global descriptor and attention prelogits.
- def call(self, input_image, training=True):
- blocks = {}
-
- self.backbone.build_call(
- input_image, intermediates_dict=blocks, training=training)
+ Args:
+ images: Tensor containing the dataset on which to run the forward pass.
+ training: Indicator of wether the forward pass is running in training mode
+ or not.
- features = blocks['block3'] # pytype: disable=key-error
- _, probs, _ = self.attention(features, training=training)
+ Returns:
+ Global descriptor prelogits, attention prelogits, attention scores,
+ backbone weights.
+ """
+ backbone_blocks = {}
+ desc_prelogits = self.backbone.build_call(
+ images, intermediates_dict=backbone_blocks, training=training)
+ # Prevent gradients from propagating into the backbone. See DELG paper:
+ # https://arxiv.org/abs/2001.05027.
+ block3 = backbone_blocks['block3'] # pytype: disable=key-error
+ block3 = tf.stop_gradient(block3)
+ if self._use_dim_reduction:
+ (dim_expanded_features, dim_reduced_features) = self.autoencoder(block3)
+ attn_prelogits, attn_scores, _ = self.attention(dim_expanded_features,
+ training=training)
+ else:
+ attn_prelogits, attn_scores, _ = self.attention(block3, training=training)
+ dim_expanded_features = None
+ dim_reduced_features = None
+ return (desc_prelogits, attn_prelogits, attn_scores, backbone_blocks,
+ dim_expanded_features, dim_reduced_features)
+
+ def build_call(self, input_image, training=True):
+ (global_feature, _, attn_scores, backbone_blocks, _,
+ dim_reduced_features) = self.global_and_local_forward_pass(input_image,
+ training)
+ if self._use_dim_reduction:
+ features = dim_reduced_features
+ else:
+ features = backbone_blocks['block3'] # pytype: disable=key-error
+ return global_feature, attn_scores, features
+ def call(self, input_image, training=True):
+ _, probs, features = self.build_call(input_image, training=training)
return probs, features
diff --git a/research/delf/delf/python/training/model/delf_model_test.py b/research/delf/delf/python/training/model/delf_model_test.py
index c4cbcef555db3cd6e6395aee69f1479863916bd4..7d5ca44e0c1802e2442fc58f7b7c38b10f99a7f1 100644
--- a/research/delf/delf/python/training/model/delf_model_test.py
+++ b/research/delf/delf/python/training/model/delf_model_test.py
@@ -87,28 +87,21 @@ class DelfTest(tf.test.TestCase, parameterized.TestCase):
return tf.nn.compute_average_loss(
per_example_loss, global_batch_size=batch_size)
- with tf.GradientTape() as desc_tape:
- blocks = {}
- desc_prelogits = model.backbone(
- images, intermediates_dict=blocks, training=False)
- desc_logits = model.desc_classification(desc_prelogits)
+ with tf.GradientTape() as gradient_tape:
+ (desc_prelogits, attn_prelogits, _, _, _,
+ _) = model.global_and_local_forward_pass(images)
+ # Calculate global loss by applying the descriptor classifier.
desc_logits = model.desc_classification(desc_prelogits)
desc_loss = compute_loss(labels, desc_logits)
-
- gradients = desc_tape.gradient(desc_loss, model.desc_trainable_weights)
- clipped, _ = tf.clip_by_global_norm(gradients, clip_norm=clip_val)
- optimizer.apply_gradients(zip(clipped, model.desc_trainable_weights))
-
- with tf.GradientTape() as attn_tape:
- block3 = blocks['block3']
- block3 = tf.stop_gradient(block3)
- attn_prelogits, _, _ = model.attention(block3, training=True)
+ # Calculate attention loss by applying the attention block classifier.
attn_logits = model.attn_classification(attn_prelogits)
attn_loss = compute_loss(labels, attn_logits)
-
- gradients = attn_tape.gradient(attn_loss, model.attn_trainable_weights)
+ # Cumulate global loss and attention loss and backpropagate through the
+ # descriptor layer and attention layer together.
+ total_loss = desc_loss + attn_loss
+ gradients = gradient_tape.gradient(total_loss, model.trainable_weights)
clipped, _ = tf.clip_by_global_norm(gradients, clip_norm=clip_val)
- optimizer.apply_gradients(zip(clipped, model.attn_trainable_weights))
+ optimizer.apply_gradients(zip(clipped, model.trainable_weights))
if __name__ == '__main__':
diff --git a/research/delf/delf/python/training/model/delg_model.py b/research/delf/delf/python/training/model/delg_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..a29161b0581320920fc9d65f2e14fe7772f64aff
--- /dev/null
+++ b/research/delf/delf/python/training/model/delg_model.py
@@ -0,0 +1,178 @@
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""DELG model implementation based on the following paper.
+
+ Unifying Deep Local and Global Features for Image Search
+ https://arxiv.org/abs/2001.05027
+"""
+
+import functools
+import math
+
+from absl import logging
+import tensorflow as tf
+
+from delf.python.training.model import delf_model
+
+layers = tf.keras.layers
+
+
+class Delg(delf_model.Delf):
+ """Instantiates Keras DELG model using ResNet50 as backbone.
+
+ This class implements the [DELG](https://arxiv.org/abs/2001.05027) model for
+ extracting local and global features from images. The same attention layer
+ is trained as in the DELF model. In addition, the extraction of global
+ features is trained using GeMPooling, a FC whitening layer also called
+ "embedding layer" and ArcFace loss.
+ """
+
+ def __init__(self,
+ block3_strides=True,
+ name='DELG',
+ gem_power=3.0,
+ embedding_layer_dim=2048,
+ scale_factor_init=45.25, # sqrt(2048)
+ arcface_margin=0.1,
+ use_dim_reduction=False,
+ reduced_dimension=128,
+ dim_expand_channels=1024):
+ """Initialization of DELG model.
+
+ Args:
+ block3_strides: bool, whether to add strides to the output of block3.
+ name: str, name to identify model.
+ gem_power: float, GeM power parameter.
+ embedding_layer_dim : int, dimension of the embedding layer.
+ scale_factor_init: float.
+ arcface_margin: float, ArcFace margin.
+ use_dim_reduction: Whether to integrate dimensionality reduction layers.
+ If True, extra layers are added to reduce the dimensionality of the
+ extracted features.
+ reduced_dimension: Only used if use_dim_reduction is True, the output
+ dimension of the dim_reduction layer.
+ dim_expand_channels: Only used if use_dim_reduction is True, the
+ number of channels of the backbone block used. Default value 1024 is the
+ number of channels of backbone block 'block3'.
+ """
+ logging.info('Creating Delg model, gem_power %d, embedding_layer_dim %d',
+ gem_power, embedding_layer_dim)
+ super(Delg, self).__init__(block3_strides=block3_strides,
+ name=name,
+ pooling='gem',
+ gem_power=gem_power,
+ embedding_layer=True,
+ embedding_layer_dim=embedding_layer_dim,
+ use_dim_reduction=use_dim_reduction,
+ reduced_dimension=reduced_dimension,
+ dim_expand_channels=dim_expand_channels)
+ self._embedding_layer_dim = embedding_layer_dim
+ self._scale_factor_init = scale_factor_init
+ self._arcface_margin = arcface_margin
+
+ def init_classifiers(self, num_classes):
+ """Define classifiers for training backbone and attention models."""
+ logging.info('Initializing Delg backbone and attention models classifiers')
+ backbone_classifier_func = self._create_backbone_classifier(num_classes)
+ super(Delg, self).init_classifiers(
+ num_classes,
+ desc_classification=backbone_classifier_func)
+
+ def _create_backbone_classifier(self, num_classes):
+ """Define the classifier for training the backbone model."""
+ logging.info('Creating cosine classifier')
+ self.cosine_weights = tf.Variable(
+ initial_value=tf.initializers.GlorotUniform()(
+ shape=[self._embedding_layer_dim, num_classes]),
+ name='cosine_weights',
+ trainable=True)
+ self.scale_factor = tf.Variable(self._scale_factor_init,
+ name='scale_factor',
+ trainable=False)
+ classifier_func = functools.partial(cosine_classifier_logits,
+ num_classes=num_classes,
+ cosine_weights=self.cosine_weights,
+ scale_factor=self.scale_factor,
+ arcface_margin=self._arcface_margin)
+ classifier_func.trainable_weights = [self.cosine_weights]
+ return classifier_func
+
+
+def cosine_classifier_logits(prelogits,
+ labels,
+ num_classes,
+ cosine_weights,
+ scale_factor,
+ arcface_margin,
+ training=True):
+ """Compute cosine classifier logits using ArFace margin.
+
+ Args:
+ prelogits: float tensor of shape [batch_size, embedding_layer_dim].
+ labels: int tensor of shape [batch_size].
+ num_classes: int, number of classes.
+ cosine_weights: float tensor of shape [embedding_layer_dim, num_classes].
+ scale_factor: float.
+ arcface_margin: float. Only used if greater than zero, and training is True.
+ training: bool, True if training, False if eval.
+
+ Returns:
+ logits: Float tensor [batch_size, num_classes].
+ """
+ # L2-normalize prelogits, then obtain cosine similarity.
+ normalized_prelogits = tf.math.l2_normalize(prelogits, axis=1)
+ normalized_weights = tf.math.l2_normalize(cosine_weights, axis=0)
+ cosine_sim = tf.matmul(normalized_prelogits, normalized_weights)
+
+ # Optionally use ArcFace margin.
+ if training and arcface_margin > 0.0:
+ # Reshape labels tensor from [batch_size] to [batch_size, num_classes].
+ one_hot_labels = tf.one_hot(labels, num_classes)
+ cosine_sim = apply_arcface_margin(cosine_sim,
+ one_hot_labels,
+ arcface_margin)
+
+ # Apply the scale factor to logits and return.
+ logits = scale_factor * cosine_sim
+ return logits
+
+
+def apply_arcface_margin(cosine_sim, one_hot_labels, arcface_margin):
+ """Applies ArcFace margin to cosine similarity inputs.
+
+ For a reference, see https://arxiv.org/pdf/1801.07698.pdf. ArFace margin is
+ applied to angles from correct classes (as per the ArcFace paper), and only
+ if they are <= (pi - margin). Otherwise, applying the margin may actually
+ improve their cosine similarity.
+
+ Args:
+ cosine_sim: float tensor with shape [batch_size, num_classes].
+ one_hot_labels: int tensor with shape [batch_size, num_classes].
+ arcface_margin: float.
+
+ Returns:
+ cosine_sim_with_margin: Float tensor with shape [batch_size, num_classes].
+ """
+ theta = tf.acos(cosine_sim, name='acos')
+ selected_labels = tf.where(tf.greater(theta, math.pi - arcface_margin),
+ tf.zeros_like(one_hot_labels),
+ one_hot_labels,
+ name='selected_labels')
+ final_theta = tf.where(tf.cast(selected_labels, dtype=tf.bool),
+ theta + arcface_margin,
+ theta,
+ name='final_theta')
+ return tf.cos(final_theta, name='cosine_sim_with_margin')
diff --git a/research/delf/delf/python/training/model/export_global_model.py b/research/delf/delf/python/training/model/export_global_model.py
index 820cb0e4b93ae75defd21e69ef5edd2e3d9f54a6..e5f9128a0bf72a48ba8bbc4189dc284ec31eba3b 100644
--- a/research/delf/delf/python/training/model/export_global_model.py
+++ b/research/delf/delf/python/training/model/export_global_model.py
@@ -15,7 +15,7 @@
# ==============================================================================
"""Export global feature tensorflow inference model.
-This model includes image pyramids for multi-scale processing.
+The exported model may leverage image pyramids for multi-scale processing.
"""
from __future__ import absolute_import
@@ -29,6 +29,7 @@ from absl import flags
import tensorflow as tf
from delf.python.training.model import delf_model
+from delf.python.training.model import delg_model
from delf.python.training.model import export_model_utils
FLAGS = flags.FLAGS
@@ -50,6 +51,16 @@ flags.DEFINE_enum(
"'global_descriptor'.")
flags.DEFINE_boolean('normalize_global_descriptor', False,
'If True, L2-normalizes global descriptor.')
+flags.DEFINE_boolean('delg_global_features', False,
+ 'Whether the model uses a DELG-like global feature head.')
+flags.DEFINE_float(
+ 'delg_gem_power', 3.0,
+ 'Power for Generalized Mean pooling. Used only if --delg_global_features'
+ 'is present.')
+flags.DEFINE_integer(
+ 'delg_embedding_layer_dim', 2048,
+ 'Size of the FC whitening layer (embedding layer). Used only if'
+ '--delg_global_features is present.')
class _ExtractModule(tf.Module):
@@ -58,7 +69,10 @@ class _ExtractModule(tf.Module):
def __init__(self,
multi_scale_pool_type='None',
normalize_global_descriptor=False,
- input_scales_tensor=None):
+ input_scales_tensor=None,
+ delg_global_features=False,
+ delg_gem_power=3.0,
+ delg_embedding_layer_dim=2048):
"""Initialization of global feature model.
Args:
@@ -69,6 +83,12 @@ class _ExtractModule(tf.Module):
the exported model. If not None, the specified 1D tensor of floats will
be hard-coded as the desired input scales, in conjunction with
ExtractFeaturesFixedScales.
+ delg_global_features: Whether the model uses a DELG-like global feature
+ head.
+ delg_gem_power: Power for Generalized Mean pooling in the DELG model. Used
+ only if 'delg_global_features' is True.
+ delg_embedding_layer_dim: Size of the FC whitening layer (embedding
+ layer). Used only if 'delg_global_features' is True.
"""
self._multi_scale_pool_type = multi_scale_pool_type
self._normalize_global_descriptor = normalize_global_descriptor
@@ -78,7 +98,14 @@ class _ExtractModule(tf.Module):
self._input_scales_tensor = input_scales_tensor
# Setup the DELF model for extraction.
- self._model = delf_model.Delf(block3_strides=False, name='DELF')
+ if delg_global_features:
+ self._model = delg_model.Delg(
+ block3_strides=False,
+ name='DELG',
+ gem_power=delg_gem_power,
+ embedding_layer_dim=delg_embedding_layer_dim)
+ else:
+ self._model = delf_model.Delf(block3_strides=False, name='DELF')
def LoadWeights(self, checkpoint_path):
self._model.load_weights(checkpoint_path)
@@ -134,7 +161,8 @@ def main(argv):
name='input_scales')
module = _ExtractModule(FLAGS.multi_scale_pool_type,
FLAGS.normalize_global_descriptor,
- input_scales_tensor)
+ input_scales_tensor, FLAGS.delg_global_features,
+ FLAGS.delg_gem_power, FLAGS.delg_embedding_layer_dim)
# Load the weights.
checkpoint_path = FLAGS.ckpt_path
diff --git a/research/delf/delf/python/training/model/export_local_and_global_model.py b/research/delf/delf/python/training/model/export_local_and_global_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..a6cee584f87ad0b117b2edbaa095f3cbe9318bb5
--- /dev/null
+++ b/research/delf/delf/python/training/model/export_local_and_global_model.py
@@ -0,0 +1,170 @@
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Export DELG tensorflow inference model.
+
+The exported model can be used to jointly extract local and global features. It
+may use an image pyramid for multi-scale processing, and will include receptive
+field calculation and keypoint selection for the local feature head.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from absl import app
+from absl import flags
+import tensorflow as tf
+
+from delf.python.training.model import delf_model
+from delf.python.training.model import delg_model
+from delf.python.training.model import export_model_utils
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+ 'ckpt_path', '/tmp/delf-logdir/delf-weights', 'Path to saved checkpoint.')
+flags.DEFINE_string('export_path', None, 'Path where model will be exported.')
+flags.DEFINE_boolean(
+ 'delg_global_features', True,
+ 'Whether the model uses a DELG-like global feature head.')
+flags.DEFINE_float(
+ 'delg_gem_power', 3.0,
+ 'Power for Generalized Mean pooling. Used only if --delg_global_features'
+ 'is present.')
+flags.DEFINE_integer(
+ 'delg_embedding_layer_dim', 2048,
+ 'Size of the FC whitening layer (embedding layer). Used only if'
+ '--delg_global_features is present.')
+flags.DEFINE_boolean(
+ 'block3_strides', True,
+ 'Whether to apply strides after block3, used for local feature head.')
+flags.DEFINE_float(
+ 'iou', 1.0, 'IOU for non-max suppression used in local feature head.')
+flags.DEFINE_boolean(
+ 'use_autoencoder', True,
+ 'Whether the exported model should use an autoencoder.')
+flags.DEFINE_float(
+ 'autoencoder_dimensions', 128,
+ 'Number of dimensions of the autoencoder. Used only if'
+ 'use_autoencoder=True.')
+flags.DEFINE_float(
+ 'local_feature_map_channels', 1024,
+ 'Number of channels at backbone layer used for local feature extraction. '
+ 'Default value 1024 is the number of channels of block3. Used only if'
+ 'use_autoencoder=True.')
+
+
+class _ExtractModule(tf.Module):
+ """Helper module to build and save DELG model."""
+
+ def __init__(self,
+ delg_global_features=True,
+ delg_gem_power=3.0,
+ delg_embedding_layer_dim=2048,
+ block3_strides=True,
+ iou=1.0):
+ """Initialization of DELG model.
+
+ Args:
+ delg_global_features: Whether the model uses a DELG-like global feature
+ head.
+ delg_gem_power: Power for Generalized Mean pooling in the DELG model. Used
+ only if 'delg_global_features' is True.
+ delg_embedding_layer_dim: Size of the FC whitening layer (embedding
+ layer). Used only if 'delg_global_features' is True.
+ block3_strides: bool, whether to add strides to the output of block3.
+ iou: IOU for non-max suppression.
+ """
+ self._stride_factor = 2.0 if block3_strides else 1.0
+ self._iou = iou
+
+ # Setup the DELG model for extraction.
+ if delg_global_features:
+ self._model = delg_model.Delg(
+ block3_strides=block3_strides,
+ name='DELG',
+ gem_power=delg_gem_power,
+ embedding_layer_dim=delg_embedding_layer_dim,
+ use_dim_reduction=FLAGS.use_autoencoder,
+ reduced_dimension=FLAGS.autoencoder_dimensions,
+ dim_expand_channels=FLAGS.local_feature_map_channels)
+ else:
+ self._model = delf_model.Delf(
+ block3_strides=block3_strides,
+ name='DELF',
+ use_dim_reduction=FLAGS.use_autoencoder,
+ reduced_dimension=FLAGS.autoencoder_dimensions,
+ dim_expand_channels=FLAGS.local_feature_map_channels)
+
+ def LoadWeights(self, checkpoint_path):
+ self._model.load_weights(checkpoint_path)
+
+ @tf.function(input_signature=[
+ tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8, name='input_image'),
+ tf.TensorSpec(shape=[None], dtype=tf.float32, name='input_scales'),
+ tf.TensorSpec(shape=(), dtype=tf.int32, name='input_max_feature_num'),
+ tf.TensorSpec(shape=(), dtype=tf.float32, name='input_abs_thres'),
+ tf.TensorSpec(
+ shape=[None], dtype=tf.int32, name='input_global_scales_ind')
+ ])
+ def ExtractFeatures(self, input_image, input_scales, input_max_feature_num,
+ input_abs_thres, input_global_scales_ind):
+ extracted_features = export_model_utils.ExtractLocalAndGlobalFeatures(
+ input_image, input_scales, input_max_feature_num, input_abs_thres,
+ input_global_scales_ind, self._iou,
+ lambda x: self._model.build_call(x, training=False),
+ self._stride_factor)
+
+ named_output_tensors = {}
+ named_output_tensors['boxes'] = tf.identity(
+ extracted_features[0], name='boxes')
+ named_output_tensors['features'] = tf.identity(
+ extracted_features[1], name='features')
+ named_output_tensors['scales'] = tf.identity(
+ extracted_features[2], name='scales')
+ named_output_tensors['scores'] = tf.identity(
+ extracted_features[3], name='scores')
+ named_output_tensors['global_descriptors'] = tf.identity(
+ extracted_features[4], name='global_descriptors')
+ return named_output_tensors
+
+
+def main(argv):
+ if len(argv) > 1:
+ raise app.UsageError('Too many command-line arguments.')
+
+ export_path = FLAGS.export_path
+ if os.path.exists(export_path):
+ raise ValueError(f'Export_path {export_path} already exists. Please '
+ 'specify a different path or delete the existing one.')
+
+ module = _ExtractModule(FLAGS.delg_global_features, FLAGS.delg_gem_power,
+ FLAGS.delg_embedding_layer_dim, FLAGS.block3_strides,
+ FLAGS.iou)
+
+ # Load the weights.
+ checkpoint_path = FLAGS.ckpt_path
+ module.LoadWeights(checkpoint_path)
+ print('Checkpoint loaded from ', checkpoint_path)
+
+ # Save the module
+ tf.saved_model.save(module, export_path)
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/research/delf/delf/python/training/model/export_local_model.py b/research/delf/delf/python/training/model/export_local_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..767d363ef7e7868b2caad9b61431dfa99e3beacd
--- /dev/null
+++ b/research/delf/delf/python/training/model/export_local_model.py
@@ -0,0 +1,128 @@
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Export DELF tensorflow inference model.
+
+The exported model may use an image pyramid for multi-scale processing, with
+local feature extraction including receptive field calculation and keypoint
+selection.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from absl import app
+from absl import flags
+import tensorflow as tf
+
+from delf.python.training.model import delf_model
+from delf.python.training.model import export_model_utils
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+ 'ckpt_path', '/tmp/delf-logdir/delf-weights', 'Path to saved checkpoint.')
+flags.DEFINE_string('export_path', None, 'Path where model will be exported.')
+flags.DEFINE_boolean(
+ 'block3_strides', True, 'Whether to apply strides after block3.')
+flags.DEFINE_float('iou', 1.0, 'IOU for non-max suppression.')
+flags.DEFINE_boolean(
+ 'use_autoencoder', True,
+ 'Whether the exported model should use an autoencoder.')
+flags.DEFINE_float(
+ 'autoencoder_dimensions', 128,
+ 'Number of dimensions of the autoencoder. Used only if'
+ 'use_autoencoder=True.')
+flags.DEFINE_float(
+ 'local_feature_map_channels', 1024,
+ 'Number of channels at backbone layer used for local feature extraction. '
+ 'Default value 1024 is the number of channels of block3. Used only if'
+ 'use_autoencoder=True.')
+
+
+class _ExtractModule(tf.Module):
+ """Helper module to build and save DELF model."""
+
+ def __init__(self, block3_strides, iou):
+ """Initialization of DELF model.
+
+ Args:
+ block3_strides: bool, whether to add strides to the output of block3.
+ iou: IOU for non-max suppression.
+ """
+ self._stride_factor = 2.0 if block3_strides else 1.0
+ self._iou = iou
+ # Setup the DELF model for extraction.
+ self._model = delf_model.Delf(
+ block3_strides=block3_strides,
+ name='DELF',
+ use_dim_reduction=FLAGS.use_autoencoder,
+ reduced_dimension=FLAGS.autoencoder_dimensions,
+ dim_expand_channels=FLAGS.local_feature_map_channels)
+
+ def LoadWeights(self, checkpoint_path):
+ self._model.load_weights(checkpoint_path)
+
+ @tf.function(input_signature=[
+ tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8, name='input_image'),
+ tf.TensorSpec(shape=[None], dtype=tf.float32, name='input_scales'),
+ tf.TensorSpec(shape=(), dtype=tf.int32, name='input_max_feature_num'),
+ tf.TensorSpec(shape=(), dtype=tf.float32, name='input_abs_thres')
+ ])
+ def ExtractFeatures(self, input_image, input_scales, input_max_feature_num,
+ input_abs_thres):
+
+ extracted_features = export_model_utils.ExtractLocalFeatures(
+ input_image, input_scales, input_max_feature_num, input_abs_thres,
+ self._iou, lambda x: self._model(x, training=False),
+ self._stride_factor)
+
+ named_output_tensors = {}
+ named_output_tensors['boxes'] = tf.identity(
+ extracted_features[0], name='boxes')
+ named_output_tensors['features'] = tf.identity(
+ extracted_features[1], name='features')
+ named_output_tensors['scales'] = tf.identity(
+ extracted_features[2], name='scales')
+ named_output_tensors['scores'] = tf.identity(
+ extracted_features[3], name='scores')
+ return named_output_tensors
+
+
+def main(argv):
+ if len(argv) > 1:
+ raise app.UsageError('Too many command-line arguments.')
+
+ export_path = FLAGS.export_path
+ if os.path.exists(export_path):
+ raise ValueError(f'Export_path {export_path} already exists. Please '
+ 'specify a different path or delete the existing one.')
+
+ module = _ExtractModule(FLAGS.block3_strides, FLAGS.iou)
+
+ # Load the weights.
+ checkpoint_path = FLAGS.ckpt_path
+ module.LoadWeights(checkpoint_path)
+ print('Checkpoint loaded from ', checkpoint_path)
+
+ # Save the module
+ tf.saved_model.save(module, export_path)
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/research/delf/delf/python/training/model/export_model.py b/research/delf/delf/python/training/model/export_model.py
deleted file mode 100644
index 10fb8905e1e7d6d575c9b0f6480276ca9719662c..0000000000000000000000000000000000000000
--- a/research/delf/delf/python/training/model/export_model.py
+++ /dev/null
@@ -1,111 +0,0 @@
-# Lint as: python3
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Export DELF tensorflow inference model.
-
-This model includes feature extraction, receptive field calculation and
-key-point selection and outputs the selected feature descriptors.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-from absl import app
-from absl import flags
-import tensorflow as tf
-
-from delf.python.training.model import delf_model
-from delf.python.training.model import export_model_utils
-
-FLAGS = flags.FLAGS
-
-flags.DEFINE_string('ckpt_path', '/tmp/delf-logdir/delf-weights',
- 'Path to saved checkpoint.')
-flags.DEFINE_string('export_path', None, 'Path where model will be exported.')
-flags.DEFINE_boolean('block3_strides', False,
- 'Whether to apply strides after block3.')
-flags.DEFINE_float('iou', 1.0, 'IOU for non-max suppression.')
-
-
-class _ExtractModule(tf.Module):
- """Helper module to build and save DELF model."""
-
- def __init__(self, block3_strides, iou):
- """Initialization of DELF model.
-
- Args:
- block3_strides: bool, whether to add strides to the output of block3.
- iou: IOU for non-max suppression.
- """
- self._stride_factor = 2.0 if block3_strides else 1.0
- self._iou = iou
- # Setup the DELF model for extraction.
- self._model = delf_model.Delf(
- block3_strides=block3_strides, name='DELF')
-
- def LoadWeights(self, checkpoint_path):
- self._model.load_weights(checkpoint_path)
-
- @tf.function(input_signature=[
- tf.TensorSpec(shape=[None, None, 3], dtype=tf.uint8, name='input_image'),
- tf.TensorSpec(shape=[None], dtype=tf.float32, name='input_scales'),
- tf.TensorSpec(shape=(), dtype=tf.int32, name='input_max_feature_num'),
- tf.TensorSpec(shape=(), dtype=tf.float32, name='input_abs_thres')
- ])
- def ExtractFeatures(self, input_image, input_scales, input_max_feature_num,
- input_abs_thres):
-
- extracted_features = export_model_utils.ExtractLocalFeatures(
- input_image, input_scales, input_max_feature_num, input_abs_thres,
- self._iou, lambda x: self._model(x, training=False),
- self._stride_factor)
-
- named_output_tensors = {}
- named_output_tensors['boxes'] = tf.identity(
- extracted_features[0], name='boxes')
- named_output_tensors['features'] = tf.identity(
- extracted_features[1], name='features')
- named_output_tensors['scales'] = tf.identity(
- extracted_features[2], name='scales')
- named_output_tensors['scores'] = tf.identity(
- extracted_features[3], name='scores')
- return named_output_tensors
-
-
-def main(argv):
- if len(argv) > 1:
- raise app.UsageError('Too many command-line arguments.')
-
- export_path = FLAGS.export_path
- if os.path.exists(export_path):
- raise ValueError(f'Export_path {export_path} already exists. Please '
- 'specify a different path or delete the existing one.')
-
- module = _ExtractModule(FLAGS.block3_strides, FLAGS.iou)
-
- # Load the weights.
- checkpoint_path = FLAGS.ckpt_path
- module.LoadWeights(checkpoint_path)
- print('Checkpoint loaded from ', checkpoint_path)
-
- # Save the module
- tf.saved_model.save(module, export_path)
-
-
-if __name__ == '__main__':
- app.run(main)
diff --git a/research/delf/delf/python/training/model/export_model_utils.py b/research/delf/delf/python/training/model/export_model_utils.py
index 64d6672569e7d4973c921de8cc8c8c63051589b3..92a616e871544dbb8ec3ac0e5a1d6882b3ab2a7b 100644
--- a/research/delf/delf/python/training/model/export_model_utils.py
+++ b/research/delf/delf/python/training/model/export_model_utils.py
@@ -22,11 +22,14 @@ from __future__ import print_function
import tensorflow as tf
from delf import feature_extractor
-from delf.python.training.datasets import googlelandmarks as gld
+from delf.python.datasets.google_landmarks_dataset import googlelandmarks as gld
from object_detection.core import box_list
from object_detection.core import box_list_ops
+# TODO(andrearaujo): Rewrite this function to be more similar to
+# "ExtractLocalAndGlobalFeatures" below, leveraging autograph to avoid the need
+# for tf.while loop.
def ExtractLocalFeatures(image, image_scales, max_feature_num, abs_thres, iou,
attention_model_fn, stride_factor):
"""Extract local features for input image.
@@ -35,9 +38,9 @@ def ExtractLocalFeatures(image, image_scales, max_feature_num, abs_thres, iou,
image: image tensor of type tf.uint8 with shape [h, w, channels].
image_scales: 1D float tensor which contains float scales used for image
pyramid construction.
- max_feature_num: int tensor denotes the maximum selected feature points.
- abs_thres: float tensor denotes the score threshold for feature selection.
- iou: float scalar denotes the iou threshold for NMS.
+ max_feature_num: int tensor denoting the maximum selected feature points.
+ abs_thres: float tensor denoting the score threshold for feature selection.
+ iou: float scalar denoting the iou threshold for NMS.
attention_model_fn: model function. Follows the signature:
* Args:
* `images`: Image tensor which is re-scaled.
@@ -55,7 +58,7 @@ def ExtractLocalFeatures(image, image_scales, max_feature_num, abs_thres, iou,
scales such that larger image scales correspond to larger image regions,
which is compatible with keypoints detected with other techniques, for
example Congas.
- scores: [N, 1] float tensor denotes the attention score.
+ scores: [N, 1] float tensor denoting the attention score.
"""
original_image_shape_float = tf.gather(
@@ -66,6 +69,8 @@ def ExtractLocalFeatures(image, image_scales, max_feature_num, abs_thres, iou,
image_tensor = tf.expand_dims(image_tensor, 0, name='image/expand_dims')
# Hard code the feature depth and receptive field parameters for now.
+ # We need to revisit this once we change the architecture and selected
+ # convolutional blocks to use as local features.
rf, stride, padding = [291.0, 16.0 * stride_factor, 145.0]
feature_depth = 1024
@@ -189,7 +194,7 @@ def ExtractGlobalFeatures(image,
`image_scales`, those with corresponding indices from this tensor.
model_fn: model function. Follows the signature:
* Args:
- * `images`: Image tensor which is re-scaled.
+ * `images`: Batched image tensor.
* Returns:
* `global_descriptors`: Global descriptors for input images.
multi_scale_pool_type: If set, the global descriptor of each scale is pooled
@@ -266,3 +271,138 @@ def ExtractGlobalFeatures(image,
output_global, axis=normalization_axis, name='l2_normalization')
return output_global
+
+
+@tf.function
+def ExtractLocalAndGlobalFeatures(image, image_scales, max_feature_num,
+ abs_thres, global_scales_ind, iou, model_fn,
+ stride_factor):
+ """Extract local+global features for input image.
+
+ Args:
+ image: image tensor of type tf.uint8 with shape [h, w, channels].
+ image_scales: 1D float tensor which contains float scales used for image
+ pyramid construction.
+ max_feature_num: int tensor denoting the maximum selected feature points.
+ abs_thres: float tensor denoting the score threshold for feature selection.
+ global_scales_ind: Global feature extraction happens only for a subset of
+ `image_scales`, those with corresponding indices from this tensor.
+ iou: float scalar denoting the iou threshold for NMS.
+ model_fn: model function. Follows the signature:
+ * Args:
+ * `images`: Batched image tensor.
+ * Returns:
+ * `global_descriptors`: Global descriptors for input images.
+ * `attention_prob`: Attention map after the non-linearity.
+ * `feature_map`: Feature map after ResNet convolution.
+ stride_factor: integer accounting for striding after block3.
+
+ Returns:
+ boxes: [N, 4] float tensor which denotes the selected receptive boxes. N is
+ the number of final feature points which pass through keypoint selection
+ and NMS steps.
+ local_descriptors: [N, depth] float tensor.
+ feature_scales: [N] float tensor. It is the inverse of the input image
+ scales such that larger image scales correspond to larger image regions,
+ which is compatible with keypoints detected with other techniques, for
+ example Congas.
+ scores: [N, 1] float tensor denoting the attention score.
+ global_descriptors: [S, D] float tensor, with the global descriptors for
+ each scale; S is the number of scales, and D the global descriptor
+ dimensionality.
+ """
+ original_image_shape_float = tf.gather(
+ tf.dtypes.cast(tf.shape(image), tf.float32), [0, 1])
+ image_tensor = gld.NormalizeImages(
+ image, pixel_value_offset=128.0, pixel_value_scale=128.0)
+ image_tensor = tf.expand_dims(image_tensor, 0, name='image/expand_dims')
+
+ # Hard code the receptive field parameters for now.
+ # We need to revisit this once we change the architecture and selected
+ # convolutional blocks to use as local features.
+ rf, stride, padding = [291.0, 16.0 * stride_factor, 145.0]
+
+ def _ResizeAndExtract(scale_index):
+ """Helper function to resize image then extract features.
+
+ Args:
+ scale_index: A valid index in image_scales.
+
+ Returns:
+ global_descriptor: [1,D] tensor denoting the extracted global descriptor.
+ boxes: Box tensor with the shape of [K, 4].
+ local_descriptors: Local descriptor tensor with the shape of [K, depth].
+ scales: Scale tensor with the shape of [K].
+ scores: Score tensor with the shape of [K].
+ """
+ scale = tf.gather(image_scales, scale_index)
+ new_image_size = tf.dtypes.cast(
+ tf.round(original_image_shape_float * scale), tf.int32)
+ resized_image = tf.image.resize(image_tensor, new_image_size)
+ global_descriptor, attention_prob, feature_map = model_fn(resized_image)
+
+ attention_prob = tf.squeeze(attention_prob, axis=[0])
+ feature_map = tf.squeeze(feature_map, axis=[0])
+
+ # Compute RF boxes and re-project them to the original image space.
+ rf_boxes = feature_extractor.CalculateReceptiveBoxes(
+ tf.shape(feature_map)[0],
+ tf.shape(feature_map)[1], rf, stride, padding)
+ rf_boxes = tf.divide(rf_boxes, scale)
+
+ attention_prob = tf.reshape(attention_prob, [-1])
+ feature_map = tf.reshape(feature_map, [-1, tf.shape(feature_map)[2]])
+
+ # Use attention score to select local features.
+ indices = tf.reshape(tf.where(attention_prob >= abs_thres), [-1])
+ boxes = tf.gather(rf_boxes, indices)
+ local_descriptors = tf.gather(feature_map, indices)
+ scores = tf.gather(attention_prob, indices)
+ scales = tf.ones_like(scores, tf.float32) / scale
+
+ return global_descriptor, boxes, local_descriptors, scales, scores
+
+ # TODO(andrearaujo): Currently, a global feature is extracted even for scales
+ # which are not using it. The obtained result is correct, however feature
+ # extraction is slower than expected. We should try to fix this in the future.
+
+ # Run first scale.
+ (output_global_descriptors, output_boxes, output_local_descriptors,
+ output_scales, output_scores) = _ResizeAndExtract(0)
+ if not tf.reduce_any(tf.equal(global_scales_ind, 0)):
+ # If global descriptor is not using the first scale, clear it out.
+ output_global_descriptors = tf.zeros(
+ [0, tf.shape(output_global_descriptors)[1]])
+
+ # Loop over subsequent scales.
+ num_scales = tf.shape(image_scales)[0]
+ for scale_index in tf.range(1, num_scales):
+ # Allow an undefined number of global feature scales to be extracted.
+ tf.autograph.experimental.set_loop_options(
+ shape_invariants=[(output_global_descriptors,
+ tf.TensorShape([None, None]))])
+
+ (global_descriptor, boxes, local_descriptors, scales,
+ scores) = _ResizeAndExtract(scale_index)
+ output_boxes = tf.concat([output_boxes, boxes], 0)
+ output_local_descriptors = tf.concat(
+ [output_local_descriptors, local_descriptors], 0)
+ output_scales = tf.concat([output_scales, scales], 0)
+ output_scores = tf.concat([output_scores, scores], 0)
+ if tf.reduce_any(tf.equal(global_scales_ind, scale_index)):
+ output_global_descriptors = tf.concat(
+ [output_global_descriptors, global_descriptor], 0)
+
+ feature_boxes = box_list.BoxList(output_boxes)
+ feature_boxes.add_field('local_descriptors', output_local_descriptors)
+ feature_boxes.add_field('scales', output_scales)
+ feature_boxes.add_field('scores', output_scores)
+
+ nms_max_boxes = tf.minimum(max_feature_num, feature_boxes.num_boxes())
+ final_boxes = box_list_ops.non_max_suppression(feature_boxes, iou,
+ nms_max_boxes)
+
+ return (final_boxes.get(), final_boxes.get_field('local_descriptors'),
+ final_boxes.get_field('scales'),
+ tf.expand_dims(final_boxes.get_field('scores'),
+ 1), output_global_descriptors)
diff --git a/research/delf/delf/python/training/model/resnet50.py b/research/delf/delf/python/training/model/resnet50.py
index 6daaab67419d99ebcefd7b25f89c284bf00832af..3718ac5b05f85172e18086470855081fa4792751 100644
--- a/research/delf/delf/python/training/model/resnet50.py
+++ b/research/delf/delf/python/training/model/resnet50.py
@@ -29,6 +29,7 @@ from absl import logging
import h5py
import tensorflow as tf
+from delf.python.pooling_layers import pooling as pooling_layers
layers = tf.keras.layers
@@ -183,13 +184,16 @@ class ResNet50(tf.keras.Model):
output of the last convolutional layer. 'avg' means that global average
pooling will be applied to the output of the last convolutional layer, and
thus the output of the model will be a 2D tensor. 'max' means that global
- max pooling will be applied.
+ max pooling will be applied. 'gem' means GeM pooling will be applied.
block3_strides: whether to add a stride of 2 to block3 to make it compatible
with tf.slim ResNet implementation.
average_pooling: whether to do average pooling of block4 features before
global pooling.
classes: optional number of classes to classify images into, only to be
specified if `include_top` is True.
+ gem_power: GeM power for GeM pooling. Only used if pooling == 'gem'.
+ embedding_layer: whether to create an embedding layer (FC whitening layer).
+ embedding_layer_dim: size of the embedding layer.
Raises:
ValueError: in case of invalid argument for data_format.
@@ -202,7 +206,10 @@ class ResNet50(tf.keras.Model):
pooling=None,
block3_strides=False,
average_pooling=True,
- classes=1000):
+ classes=1000,
+ gem_power=3.0,
+ embedding_layer=False,
+ embedding_layer_dim=2048):
super(ResNet50, self).__init__(name=name)
valid_channel_values = ('channels_first', 'channels_last')
@@ -286,8 +293,19 @@ class ResNet50(tf.keras.Model):
elif pooling == 'max':
self.global_pooling = functools.partial(
tf.reduce_max, axis=reduction_indices, keepdims=False)
+ elif pooling == 'gem':
+ logging.info('Adding GeMPooling layer with power %f', gem_power)
+ self.global_pooling = functools.partial(
+ pooling_layers.gem, axis=reduction_indices, power=gem_power)
else:
self.global_pooling = None
+ if embedding_layer:
+ logging.info('Adding embedding layer with dimension %d',
+ embedding_layer_dim)
+ self.embedding_layer = layers.Dense(
+ embedding_layer_dim, name='embedding_layer')
+ else:
+ self.embedding_layer = None
def build_call(self, inputs, training=True, intermediates_dict=None):
"""Building the ResNet50 model.
@@ -358,7 +376,10 @@ class ResNet50(tf.keras.Model):
if self.include_top:
return self.fc1000(self.flatten(x))
elif self.global_pooling:
- return self.global_pooling(x)
+ x = self.global_pooling(x)
+ if self.embedding_layer:
+ x = self.embedding_layer(x)
+ return x
else:
return x
@@ -384,6 +405,7 @@ class ResNet50(tf.keras.Model):
Args:
filepath: String, path to the .h5 file
+
Raises:
ValueError: if the file referenced by `filepath` does not exist.
"""
@@ -417,7 +439,7 @@ class ResNet50(tf.keras.Model):
g = f[inlayer.name]
weight_names = [n.decode('utf8') for n in g.attrs['weight_names']]
weight_values = [g[weight_name] for weight_name in weight_names]
- print('Setting the weights for layer %s' % (inlayer.name))
+ logging.info('Setting the weights for layer %s', inlayer.name)
inlayer.set_weights(weight_values)
finally:
# Clean up the temporary file.
@@ -435,5 +457,4 @@ class ResNet50(tf.keras.Model):
weights = inlayer.get_weights()
logging.info(weights)
else:
- logging.info('Layer %s does not have inner layers.',
- layer.name)
+ logging.info('Layer %s does not have inner layers.', layer.name)
diff --git a/research/delf/delf/python/training/train.py b/research/delf/delf/python/training/train.py
index 12b7a5f9cc3282e59c738f74c7fbd4798021c429..d21decdd49bed9861d10dd6e5f6a1a6432022a18 100644
--- a/research/delf/delf/python/training/train.py
+++ b/research/delf/delf/python/training/train.py
@@ -13,10 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
-"""Training script for DELF on Google Landmarks Dataset.
+"""Training script for DELF/G on Google Landmarks Dataset.
-Script to train DELF using classification loss on Google Landmarks Dataset
-using MirroredStrategy to so it can run on multiple GPUs.
+Uses classification loss, with MirroredStrategy, to support running on multiple
+GPUs.
"""
from __future__ import absolute_import
@@ -24,6 +24,7 @@ from __future__ import division
from __future__ import print_function
import os
+import time
from absl import app
from absl import flags
@@ -32,8 +33,9 @@ import tensorflow as tf
import tensorflow_probability as tfp
# Placeholder for internal import. Do not remove this line.
-from delf.python.training.datasets import googlelandmarks as gld
+from delf.python.datasets.google_landmarks_dataset import googlelandmarks as gld
from delf.python.training.model import delf_model
+from delf.python.training.model import delg_model
FLAGS = flags.FLAGS
@@ -45,8 +47,8 @@ flags.DEFINE_string('validation_file_pattern', '/tmp/data/validation*',
'File pattern of validation dataset files.')
flags.DEFINE_enum(
'dataset_version', 'gld_v1', ['gld_v1', 'gld_v2', 'gld_v2_clean'],
- 'Google Landmarks dataset version, used to determine the'
- 'number of classes.')
+ 'Google Landmarks dataset version, used to determine the number of '
+ 'classes.')
flags.DEFINE_integer('seed', 0, 'Seed to training dataset.')
flags.DEFINE_float('initial_lr', 0.01, 'Initial learning rate.')
flags.DEFINE_integer('batch_size', 32, 'Global batch size.')
@@ -57,6 +59,41 @@ flags.DEFINE_boolean('use_augmentation', True,
flags.DEFINE_string(
'imagenet_checkpoint', None,
'ImageNet checkpoint for ResNet backbone. If None, no checkpoint is used.')
+flags.DEFINE_float(
+ 'attention_loss_weight', 1.0,
+ 'Weight to apply to the attention loss when calculating the '
+ 'total loss of the model.')
+flags.DEFINE_boolean('delg_global_features', False,
+ 'Whether to train a DELG model.')
+flags.DEFINE_float(
+ 'delg_gem_power', 3.0, 'Power for Generalized Mean pooling. Used only if '
+ 'delg_global_features=True.')
+flags.DEFINE_integer(
+ 'delg_embedding_layer_dim', 2048,
+ 'Size of the FC whitening layer (embedding layer). Used only if'
+ 'delg_global_features:True.')
+flags.DEFINE_float(
+ 'delg_scale_factor_init', 45.25,
+ 'Initial value of the scaling factor of the cosine logits. The default '
+ 'value is sqrt(2048). Used only if delg_global_features=True.')
+flags.DEFINE_float('delg_arcface_margin', 0.1,
+ 'ArcFace margin. Used only if delg_global_features=True.')
+flags.DEFINE_integer('image_size', 321, 'Size of each image side to use.')
+flags.DEFINE_boolean('use_autoencoder', True,
+ 'Whether to train an autoencoder.')
+flags.DEFINE_float(
+ 'reconstruction_loss_weight', 10.0,
+ 'Weight to apply to the reconstruction loss from the autoencoder when'
+ 'calculating total loss of the model. Used only if use_autoencoder=True.')
+flags.DEFINE_float(
+ 'autoencoder_dimensions', 128,
+ 'Number of dimensions of the autoencoder. Used only if'
+ 'use_autoencoder=True.')
+flags.DEFINE_float(
+ 'local_feature_map_channels', 1024,
+ 'Number of channels at backbone layer used for local feature extraction. '
+ 'Default value 1024 is the number of channels of block3. Used only if'
+ 'use_autoencoder=True.')
def _record_accuracy(metric, logits, labels):
@@ -90,7 +127,24 @@ def _attention_summaries(scores, global_step):
def create_model(num_classes):
"""Define DELF model, and initialize classifiers."""
- model = delf_model.Delf(block3_strides=FLAGS.block3_strides, name='DELF')
+ if FLAGS.delg_global_features:
+ model = delg_model.Delg(
+ block3_strides=FLAGS.block3_strides,
+ name='DELG',
+ gem_power=FLAGS.delg_gem_power,
+ embedding_layer_dim=FLAGS.delg_embedding_layer_dim,
+ scale_factor_init=FLAGS.delg_scale_factor_init,
+ arcface_margin=FLAGS.delg_arcface_margin,
+ use_dim_reduction=FLAGS.use_autoencoder,
+ reduced_dimension=FLAGS.autoencoder_dimensions,
+ dim_expand_channels=FLAGS.local_feature_map_channels)
+ else:
+ model = delf_model.Delf(
+ block3_strides=FLAGS.block3_strides,
+ name='DELF',
+ use_dim_reduction=FLAGS.use_autoencoder,
+ reduced_dimension=FLAGS.autoencoder_dimensions,
+ dim_expand_channels=FLAGS.local_feature_map_channels)
model.init_classifiers(num_classes)
return model
@@ -130,11 +184,11 @@ def main(argv):
max_iters = FLAGS.max_iters
global_batch_size = FLAGS.batch_size
- image_size = 321
+ image_size = FLAGS.image_size
num_eval_batches = int(50000 / global_batch_size)
report_interval = 100
eval_interval = 1000
- save_interval = 20000
+ save_interval = 1000
initial_lr = FLAGS.initial_lr
@@ -146,7 +200,7 @@ def main(argv):
max_iters = 100
num_eval_batches = 1
save_interval = 1
- report_interval = 1
+ report_interval = 10
# Determine the number of classes based on the version of the dataset.
gld_info = gld.GoogleLandmarksInfo()
@@ -217,7 +271,12 @@ def main(argv):
# Setup checkpoint directory.
checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)
manager = tf.train.CheckpointManager(
- checkpoint, checkpoint_prefix, max_to_keep=3)
+ checkpoint,
+ checkpoint_prefix,
+ max_to_keep=10,
+ keep_checkpoint_every_n_hours=3)
+ # Restores the checkpoint, if existing.
+ checkpoint.restore(manager.latest_checkpoint)
# ------------------------------------------------------------
# Train step to run on one GPU.
@@ -227,15 +286,6 @@ def main(argv):
# Temporary workaround to avoid some corrupted labels.
labels = tf.clip_by_value(labels, 0, model.num_classes)
- global_step = optimizer.iterations
- tf.summary.image('batch_images', (images + 1.0) / 2.0, step=global_step)
- tf.summary.scalar(
- 'image_range/max', tf.reduce_max(images), step=global_step)
- tf.summary.scalar(
- 'image_range/min', tf.reduce_min(images), step=global_step)
-
- # TODO(andrearaujo): we should try to unify the backprop into a single
- # function, instead of applying once to descriptor then to attention.
def _backprop_loss(tape, loss, weights):
"""Backpropogate losses using clipped gradients.
@@ -249,54 +299,70 @@ def main(argv):
optimizer.apply_gradients(zip(clipped, weights))
# Record gradients and loss through backbone.
- with tf.GradientTape() as desc_tape:
-
- blocks = {}
- prelogits = model.backbone(
- images, intermediates_dict=blocks, training=True)
-
- # Report sparsity.
- activations_zero_fractions = {
- 'sparsity/%s' % k: tf.nn.zero_fraction(v)
- for k, v in blocks.items()
- }
- for k, v in activations_zero_fractions.items():
- tf.summary.scalar(k, v, step=global_step)
-
- # Apply descriptor classifier.
- logits = model.desc_classification(prelogits)
-
- desc_loss = compute_loss(labels, logits)
-
- # Backprop only through backbone weights.
- _backprop_loss(desc_tape, desc_loss, model.desc_trainable_weights)
-
- # Record descriptor train accuracy.
- _record_accuracy(desc_train_accuracy, logits, labels)
+ with tf.GradientTape() as gradient_tape:
+ # Make a forward pass to calculate prelogits.
+ (desc_prelogits, attn_prelogits, attn_scores, backbone_blocks,
+ dim_expanded_features, _) = model.global_and_local_forward_pass(images)
+
+ # Calculate global loss by applying the descriptor classifier.
+ if FLAGS.delg_global_features:
+ desc_logits = model.desc_classification(desc_prelogits, labels)
+ else:
+ desc_logits = model.desc_classification(desc_prelogits)
+ desc_loss = compute_loss(labels, desc_logits)
+
+ # Calculate attention loss by applying the attention block classifier.
+ attn_logits = model.attn_classification(attn_prelogits)
+ attn_loss = compute_loss(labels, attn_logits)
+
+ # Calculate reconstruction loss between the attention prelogits and the
+ # backbone.
+ if FLAGS.use_autoencoder:
+ block3 = tf.stop_gradient(backbone_blocks['block3'])
+ reconstruction_loss = tf.math.reduce_mean(
+ tf.keras.losses.MSE(block3, dim_expanded_features))
+ else:
+ reconstruction_loss = 0
- # Record gradients and loss through attention block.
- with tf.GradientTape() as attn_tape:
- block3 = blocks['block3'] # pytype: disable=key-error
+ # Cumulate global loss, attention loss and reconstruction loss.
+ total_loss = (
+ desc_loss + FLAGS.attention_loss_weight * attn_loss +
+ FLAGS.reconstruction_loss_weight * reconstruction_loss)
- # Stopping gradients according to DELG paper:
- # (https://arxiv.org/abs/2001.05027).
- block3 = tf.stop_gradient(block3)
+ # Perform backpropagation through the descriptor and attention layers
+ # together. Note that this will increment the number of iterations of
+ # "optimizer".
+ _backprop_loss(gradient_tape, total_loss, model.trainable_weights)
- prelogits, scores, _ = model.attention(block3, training=True)
- _attention_summaries(scores, global_step)
+ # Step number, for summary purposes.
+ global_step = optimizer.iterations
- # Apply attention block classifier.
- logits = model.attn_classification(prelogits)
+ # Input image-related summaries.
+ tf.summary.image('batch_images', (images + 1.0) / 2.0, step=global_step)
+ tf.summary.scalar(
+ 'image_range/max', tf.reduce_max(images), step=global_step)
+ tf.summary.scalar(
+ 'image_range/min', tf.reduce_min(images), step=global_step)
- attn_loss = compute_loss(labels, logits)
+ # Attention and sparsity summaries.
+ _attention_summaries(attn_scores, global_step)
+ activations_zero_fractions = {
+ 'sparsity/%s' % k: tf.nn.zero_fraction(v)
+ for k, v in backbone_blocks.items()
+ }
+ for k, v in activations_zero_fractions.items():
+ tf.summary.scalar(k, v, step=global_step)
- # Backprop only through attention weights.
- _backprop_loss(attn_tape, attn_loss, model.attn_trainable_weights)
+ # Scaling factor summary for cosine logits for a DELG model.
+ if FLAGS.delg_global_features:
+ tf.summary.scalar(
+ 'desc/scale_factor', model.scale_factor, step=global_step)
- # Record attention train accuracy.
- _record_accuracy(attn_train_accuracy, logits, labels)
+ # Record train accuracies.
+ _record_accuracy(desc_train_accuracy, desc_logits, labels)
+ _record_accuracy(attn_train_accuracy, attn_logits, labels)
- return desc_loss, attn_loss
+ return desc_loss, attn_loss, reconstruction_loss
# ------------------------------------------------------------
def validation_step(inputs):
@@ -308,7 +374,10 @@ def main(argv):
blocks = {}
prelogits = model.backbone(
images, intermediates_dict=blocks, training=False)
- logits = model.desc_classification(prelogits, training=False)
+ if FLAGS.delg_global_features:
+ logits = model.desc_classification(prelogits, labels, training=False)
+ else:
+ logits = model.desc_classification(prelogits, training=False)
softmax_probabilities = tf.keras.layers.Softmax()(logits)
validation_loss = loss_object(labels, logits)
@@ -335,7 +404,7 @@ def main(argv):
def distributed_train_step(dataset_inputs):
"""Get the actual losses."""
# Each (desc, attn) is a list of 3 losses - crossentropy, reg, total.
- desc_per_replica_loss, attn_per_replica_loss = (
+ desc_per_replica_loss, attn_per_replica_loss, recon_per_replica_loss = (
strategy.run(train_step, args=(dataset_inputs,)))
# Reduce over the replicas.
@@ -343,8 +412,10 @@ def main(argv):
tf.distribute.ReduceOp.SUM, desc_per_replica_loss, axis=None)
attn_global_loss = strategy.reduce(
tf.distribute.ReduceOp.SUM, attn_per_replica_loss, axis=None)
+ recon_global_loss = strategy.reduce(
+ tf.distribute.ReduceOp.SUM, recon_per_replica_loss, axis=None)
- return desc_global_loss, attn_global_loss
+ return desc_global_loss, attn_global_loss, recon_global_loss
@tf.function
def distributed_validation_step(dataset_inputs):
@@ -353,15 +424,16 @@ def main(argv):
# ------------------------------------------------------------
# *** TRAIN LOOP ***
with summary_writer.as_default():
- with tf.summary.record_if(
- tf.math.equal(0, optimizer.iterations % report_interval)):
+ record_cond = lambda: tf.equal(optimizer.iterations % report_interval, 0)
+ with tf.summary.record_if(record_cond):
+ global_step_value = optimizer.iterations.numpy()
# TODO(dananghel): try to load pretrained weights at backbone creation.
# Load pretrained weights for ResNet50 trained on ImageNet.
- if FLAGS.imagenet_checkpoint is not None:
+ if (FLAGS.imagenet_checkpoint is not None) and (not global_step_value):
logging.info('Attempting to load ImageNet pretrained weights.')
input_batch = next(train_iter)
- _, _ = distributed_train_step(input_batch)
+ _, _, _ = distributed_train_step(input_batch)
model.backbone.restore_weights(FLAGS.imagenet_checkpoint)
logging.info('Done.')
else:
@@ -369,9 +441,9 @@ def main(argv):
if FLAGS.debug:
model.backbone.log_weights()
- global_step_value = optimizer.iterations.numpy()
+ last_summary_step_value = None
+ last_summary_time = None
while global_step_value < max_iters:
-
# input_batch : images(b, h, w, c), labels(b,).
try:
input_batch = next(train_iter)
@@ -381,24 +453,27 @@ def main(argv):
global_step_value)
break
- # Set learning rate for optimizer to use.
+ # Set learning rate and run the training step over num_gpu gpus.
+ optimizer.learning_rate = _learning_rate_schedule(
+ optimizer.iterations.numpy(), max_iters, initial_lr)
+ desc_dist_loss, attn_dist_loss, recon_dist_loss = (
+ distributed_train_step(input_batch))
+
+ # Step number, to be used for summary/logging.
global_step = optimizer.iterations
global_step_value = global_step.numpy()
- learning_rate = _learning_rate_schedule(global_step_value, max_iters,
- initial_lr)
- optimizer.learning_rate = learning_rate
+ # LR, losses and accuracies summaries.
tf.summary.scalar(
'learning_rate', optimizer.learning_rate, step=global_step)
-
- # Run the training step over num_gpu gpus.
- desc_dist_loss, attn_dist_loss = distributed_train_step(input_batch)
-
- # Log losses and accuracies to tensorboard.
tf.summary.scalar(
'loss/desc/crossentropy', desc_dist_loss, step=global_step)
tf.summary.scalar(
'loss/attn/crossentropy', attn_dist_loss, step=global_step)
+ if FLAGS.use_autoencoder:
+ tf.summary.scalar(
+ 'loss/recon/mse', recon_dist_loss, step=global_step)
+
tf.summary.scalar(
'train_accuracy/desc',
desc_train_accuracy.result(),
@@ -408,6 +483,19 @@ def main(argv):
attn_train_accuracy.result(),
step=global_step)
+ # Summary for number of global steps taken per second.
+ current_time = time.time()
+ if (last_summary_step_value is not None and
+ last_summary_time is not None):
+ tf.summary.scalar(
+ 'global_steps_per_sec',
+ (global_step_value - last_summary_step_value) /
+ (current_time - last_summary_time),
+ step=global_step)
+ if tf.summary.should_record_summaries().numpy():
+ last_summary_step_value = global_step_value
+ last_summary_time = current_time
+
# Print to console if running locally.
if FLAGS.debug:
if global_step_value % report_interval == 0:
@@ -440,12 +528,14 @@ def main(argv):
print('Validation: desc:', desc_validation_result.numpy())
print(' : attn:', attn_validation_result.numpy())
- # Save checkpoint once (each save_interval*n, n \in N) steps.
+ # Save checkpoint once (each save_interval*n, n \in N) steps, or if
+ # this is the last iteration.
# TODO(andrearaujo): save only in one of the two ways. They are
# identical, the only difference is that the manager adds some extra
# prefixes and variables (eg, optimizer variables).
- if global_step_value % save_interval == 0:
- save_path = manager.save()
+ if (global_step_value % save_interval
+ == 0) or (global_step_value >= max_iters):
+ save_path = manager.save(checkpoint_number=global_step_value)
logging.info('Saved (%d) at %s', global_step_value, save_path)
file_path = '%s/delf_weights' % FLAGS.logdir
@@ -461,9 +551,6 @@ def main(argv):
desc_validation_accuracy.reset_states()
attn_validation_accuracy.reset_states()
- if global_step.numpy() > max_iters:
- break
-
logging.info('Finished training for %d steps.', max_iters)
diff --git a/research/delf/delf/python/utils.py b/research/delf/delf/python/utils.py
index dbab2d8c7f1f423991c98851ad509e4684b738b7..46b62cbdf31a3e59e0416fb7caca0e82c78438d5 100644
--- a/research/delf/delf/python/utils.py
+++ b/research/delf/delf/python/utils.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
+import numpy as np
from PIL import Image
from PIL import ImageFile
import tensorflow as tf
@@ -39,3 +40,65 @@ def RgbLoader(path):
img = Image.open(f)
return img.convert('RGB')
+
+def ResizeImage(image, config, resize_factor=1.0):
+ """Resizes image according to config.
+
+ Args:
+ image: Uint8 array with shape (height, width, 3).
+ config: DelfConfig proto containing the model configuration.
+ resize_factor: Optional float resize factor for the input image. If given,
+ the maximum and minimum allowed image sizes in `config` are scaled by this
+ factor. Must be non-negative.
+
+ Returns:
+ resized_image: Uint8 array with resized image.
+ scale_factors: 2D float array, with factors used for resizing along height
+ and width (If upscaling, larger than 1; if downscaling, smaller than 1).
+
+ Raises:
+ ValueError: If `image` has incorrect number of dimensions/channels.
+ """
+ if resize_factor < 0.0:
+ raise ValueError('negative resize_factor is not allowed: %f' %
+ resize_factor)
+ if image.ndim != 3:
+ raise ValueError('image has incorrect number of dimensions: %d' %
+ image.ndims)
+ height, width, channels = image.shape
+
+ # Take into account resize factor.
+ max_image_size = resize_factor * config.max_image_size
+ min_image_size = resize_factor * config.min_image_size
+
+ if channels != 3:
+ raise ValueError('image has incorrect number of channels: %d' % channels)
+
+ largest_side = max(width, height)
+
+ if max_image_size >= 0 and largest_side > max_image_size:
+ scale_factor = max_image_size / largest_side
+ elif min_image_size >= 0 and largest_side < min_image_size:
+ scale_factor = min_image_size / largest_side
+ elif config.use_square_images and (height != width):
+ scale_factor = 1.0
+ else:
+ # No resizing needed, early return.
+ return image, np.ones(2, dtype=float)
+
+ # Note that new_shape is in (width, height) format (PIL convention), while
+ # scale_factors are in (height, width) convention (NumPy convention).
+ if config.use_square_images:
+ new_shape = (int(round(largest_side * scale_factor)),
+ int(round(largest_side * scale_factor)))
+ else:
+ new_shape = (int(round(width * scale_factor)),
+ int(round(height * scale_factor)))
+
+ scale_factors = np.array([new_shape[1] / height, new_shape[0] / width],
+ dtype=float)
+
+ pil_image = Image.fromarray(image)
+ resized_image = np.array(pil_image.resize(new_shape, resample=Image.BILINEAR))
+
+ return resized_image, scale_factors
diff --git a/research/delf/delf/python/utils_test.py b/research/delf/delf/python/utils_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..a07d86d75d8ab5e972d8a8d4c96e2729e92757cb
--- /dev/null
+++ b/research/delf/delf/python/utils_test.py
@@ -0,0 +1,103 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for helper utilities."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+import tensorflow as tf
+
+from delf import delf_config_pb2
+from delf import utils
+
+
+class UtilsTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.named_parameters(
+ ('Max-1Min-1', -1, -1, 1.0, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max-1Min-1Square', -1, -1, 1.0, True, [4, 4, 3], [1.0, 2.0]),
+ ('Max2Min-1', 2, -1, 1.0, False, [2, 1, 3], [0.5, 0.5]),
+ ('Max2Min-1Square', 2, -1, 1.0, True, [2, 2, 3], [0.5, 1.0]),
+ ('Max8Min-1', 8, -1, 1.0, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max8Min-1Square', 8, -1, 1.0, True, [4, 4, 3], [1.0, 2.0]),
+ ('Max-1Min1', -1, 1, 1.0, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max-1Min1Square', -1, 1, 1.0, True, [4, 4, 3], [1.0, 2.0]),
+ ('Max-1Min8', -1, 8, 1.0, False, [8, 4, 3], [2.0, 2.0]),
+ ('Max-1Min8Square', -1, 8, 1.0, True, [8, 8, 3], [2.0, 4.0]),
+ ('Max16Min8', 16, 8, 1.0, False, [8, 4, 3], [2.0, 2.0]),
+ ('Max16Min8Square', 16, 8, 1.0, True, [8, 8, 3], [2.0, 4.0]),
+ ('Max2Min2', 2, 2, 1.0, False, [2, 1, 3], [0.5, 0.5]),
+ ('Max2Min2Square', 2, 2, 1.0, True, [2, 2, 3], [0.5, 1.0]),
+ ('Max-1Min-1Factor0.5', -1, -1, 0.5, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max-1Min-1Factor0.5Square', -1, -1, 0.5, True, [4, 4, 3], [1.0, 2.0]),
+ ('Max2Min-1Factor2.0', 2, -1, 2.0, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max2Min-1Factor2.0Square', 2, -1, 2.0, True, [4, 4, 3], [1.0, 2.0]),
+ ('Max-1Min8Factor0.5', -1, 8, 0.5, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max-1Min8Factor0.5Square', -1, 8, 0.5, True, [4, 4, 3], [1.0, 2.0]),
+ ('Max-1Min8Factor0.25', -1, 8, 0.25, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max-1Min8Factor0.25Square', -1, 8, 0.25, True, [4, 4, 3], [1.0, 2.0]),
+ ('Max2Min2Factor2.0', 2, 2, 2.0, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max2Min2Factor2.0Square', 2, 2, 2.0, True, [4, 4, 3], [1.0, 2.0]),
+ ('Max16Min8Factor0.5', 16, 8, 0.5, False, [4, 2, 3], [1.0, 1.0]),
+ ('Max16Min8Factor0.5Square', 16, 8, 0.5, True, [4, 4, 3], [1.0, 2.0]),
+ )
+ def testResizeImageWorks(self, max_image_size, min_image_size, resize_factor,
+ square_output, expected_shape,
+ expected_scale_factors):
+ # Construct image of size 4x2x3.
+ image = np.array([[[0, 0, 0], [1, 1, 1]], [[2, 2, 2], [3, 3, 3]],
+ [[4, 4, 4], [5, 5, 5]], [[6, 6, 6], [7, 7, 7]]],
+ dtype='uint8')
+
+ # Set up config.
+ config = delf_config_pb2.DelfConfig(
+ max_image_size=max_image_size,
+ min_image_size=min_image_size,
+ use_square_images=square_output)
+
+ resized_image, scale_factors = utils.ResizeImage(image, config,
+ resize_factor)
+ self.assertAllEqual(resized_image.shape, expected_shape)
+ self.assertAllClose(scale_factors, expected_scale_factors)
+
+ @parameterized.named_parameters(
+ ('Max2Min2', 2, 2, 1.0, False, [2, 1, 3], [0.666666, 0.5]),
+ ('Max2Min2Square', 2, 2, 1.0, True, [2, 2, 3], [0.666666, 1.0]),
+ )
+ def testResizeImageRoundingWorks(self, max_image_size, min_image_size,
+ resize_factor, square_output, expected_shape,
+ expected_scale_factors):
+ # Construct image of size 3x2x3.
+ image = np.array([[[0, 0, 0], [1, 1, 1]], [[2, 2, 2], [3, 3, 3]],
+ [[4, 4, 4], [5, 5, 5]]],
+ dtype='uint8')
+
+ # Set up config.
+ config = delf_config_pb2.DelfConfig(
+ max_image_size=max_image_size,
+ min_image_size=min_image_size,
+ use_square_images=square_output)
+
+ resized_image, scale_factors = utils.ResizeImage(image, config,
+ resize_factor)
+ self.assertAllEqual(resized_image.shape, expected_shape)
+ self.assertAllClose(scale_factors, expected_scale_factors)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/delf/delf/python/whiten.py b/research/delf/delf/python/whiten.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2c72d9f17edc72c85f1dbc07b1710290a0bbb43
--- /dev/null
+++ b/research/delf/delf/python/whiten.py
@@ -0,0 +1,125 @@
+# Copyright 2021 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Whitening learning functions."""
+
+import os
+
+import numpy as np
+
+
+def apply_whitening(descriptors,
+ mean_descriptor_vector,
+ projection,
+ output_dim=None):
+ """Applies the whitening to the descriptors as a post-processing step.
+
+ Args:
+ descriptors: [N, D] NumPy array of L2-normalized descriptors to be
+ post-processed.
+ mean_descriptor_vector: Mean descriptor vector.
+ projection: Whitening projection matrix.
+ output_dim: Integer, parameter for the dimensionality reduction. If
+ `output_dim` is None, the dimensionality reduction is not performed.
+
+ Returns:
+ descriptors_whitened: [N, output_dim] NumPy array of L2-normalized
+ descriptors `descriptors` after whitening application.
+ """
+ eps = 1e-6
+ if output_dim is None:
+ output_dim = projection.shape[0]
+
+ descriptors = np.dot(projection[:output_dim, :],
+ descriptors - mean_descriptor_vector)
+ descriptors_whitened = descriptors / (
+ np.linalg.norm(descriptors, ord=2, axis=0, keepdims=True) + eps)
+ return descriptors_whitened
+
+
+def learn_whitening(descriptors, qidxs, pidxs):
+ """Learning the post-processing of fine-tuned descriptor vectors.
+
+ This method of whitening learning leverages the provided labeled data and
+ uses linear discriminant projections. The projection is decomposed into two
+ parts: whitening and rotation. The whitening part is the inverse of the
+ square-root of the intraclass (matching pairs) covariance matrix. The
+ rotation part is the PCA of the interclass (non-matching pairs) covariance
+ matrix in the whitened space. The described approach acts as a
+ post-processing step, equivalently, once the fine-tuning of the CNN is
+ finished. For more information about the method refer to the section 3.4
+ of https://arxiv.org/pdf/1711.02512.pdf.
+
+ Args:
+ descriptors: [N, D] NumPy array of L2-normalized descriptors.
+ qidxs: List of query indexes.
+ pidxs: List of positive pairs indexes.
+
+ Returns:
+ mean_descriptor_vector: [N, 1] NumPy array, mean descriptor vector.
+ projection: [N, N] NumPy array, whitening projection matrix.
+ """
+ # Calculating the mean descriptor vector, which is used to perform centering.
+ mean_descriptor_vector = descriptors[:, qidxs].mean(axis=1, keepdims=True)
+ # Interclass (matching pairs) difference.
+ interclass_difference = descriptors[:, qidxs] - descriptors[:, pidxs]
+ covariance_matrix = (
+ np.dot(interclass_difference, interclass_difference.T) /
+ interclass_difference.shape[1])
+
+ # Whitening part.
+ projection = np.linalg.inv(cholesky(covariance_matrix))
+
+ projected_descriptors = np.dot(projection,
+ descriptors - mean_descriptor_vector)
+ non_matching_covariance_matrix = np.dot(projected_descriptors,
+ projected_descriptors.T)
+ eigval, eigvec = np.linalg.eig(non_matching_covariance_matrix)
+ order = eigval.argsort()[::-1]
+ eigvec = eigvec[:, order]
+
+ # Rotational part.
+ projection = np.dot(eigvec.T, projection)
+ return mean_descriptor_vector, projection
+
+
+def cholesky(matrix):
+ """Cholesky decomposition.
+
+ Cholesky decomposition suitable for non-positive definite matrices: involves
+ adding a small value `alpha` on the matrix diagonal until the matrix
+ becomes positive definite.
+
+ Args:
+ matrix: [K, K] Square matrix to be decomposed.
+
+ Returns:
+ decomposition: [K, K] Upper-triangular Cholesky factor of `matrix`,
+ a matrix with real and positive diagonal entries.
+ """
+ alpha = 0
+ while True:
+ try:
+ # If the input parameter matrix is not positive-definite,
+ # the decomposition fails and we iteratively add a small value `alpha` on
+ # the matrix diagonal.
+ decomposition = np.linalg.cholesky(matrix + alpha * np.eye(*matrix.shape))
+ return decomposition
+ except np.linalg.LinAlgError:
+ if alpha == 0:
+ alpha = 1e-10
+ else:
+ alpha *= 10
+ print(">>>> {}::cholesky: Matrix is not positive definite, adding {:.0e} "
+ "on the diagonal".format(os.path.basename(__file__), alpha))
diff --git a/research/delf/delf/python/whiten_test.py b/research/delf/delf/python/whiten_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..52cc51e65d1087976bee04a1fdbf6fcbaed04217
--- /dev/null
+++ b/research/delf/delf/python/whiten_test.py
@@ -0,0 +1,73 @@
+# Lint as: python3
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for whitening module."""
+
+import numpy as np
+import tensorflow as tf
+
+from delf import whiten
+
+
+class WhitenTest(tf.test.TestCase):
+
+ def testApplyWhitening(self):
+ # Testing the application of the learned whitening.
+ vectors = np.array([[0.14022471, 0.96360618], [0.37601032, 0.25528411]])
+ # Learn whitening for the `vectors`. First element in the `vectors` is
+ # viewed is the example query and the second element is the corresponding
+ # positive.
+ mean_vector, projection = whiten.learn_whitening(vectors, [0], [1])
+ # Apply the computed whitening.
+ whitened_vectors = whiten.apply_whitening(vectors, mean_vector, projection)
+ expected_whitened_vectors = np.array([[0., 9.99999000e-01],
+ [0., -2.81240452e-13]])
+ # Compare the obtained whitened vectors with the expected result.
+ self.assertAllClose(whitened_vectors, expected_whitened_vectors)
+
+ def testLearnWhitening(self):
+ # Testing whitening learning function.
+ descriptors = np.array([[0.14022471, 0.96360618], [0.37601032, 0.25528411]])
+ # Obtain the mean descriptor vector and the projection matrix.
+ mean_vector, projection = whiten.learn_whitening(descriptors, [0], [1])
+ expected_mean_vector = np.array([[0.14022471], [0.37601032]])
+ expected_projection = np.array([[1.18894378e+00, -1.74326044e-01],
+ [1.45071361e+04, 9.89421193e+04]])
+ # Check that the both calculated values are close to the expected values.
+ self.assertAllClose(mean_vector, expected_mean_vector)
+ self.assertAllClose(projection, expected_projection)
+
+ def testCholeskyPositiveDefinite(self):
+ # Testing the Cholesky decomposition for the positive definite matrix.
+ descriptors = np.array([[1, -2j], [2j, 5]])
+ output = whiten.cholesky(descriptors)
+ expected_output = np.array([[1. + 0.j, 0. + 0.j], [0. + 2.j, 1. + 0.j]])
+ # Check that the expected output is obtained.
+ self.assertAllClose(output, expected_output)
+ # Check that the properties of the Cholesky decomposition are satisfied.
+ self.assertAllClose(np.matmul(output, output.T.conj()), descriptors)
+
+ def testCholeskyNonPositiveDefinite(self):
+ # Testing the Cholesky decomposition for a non-positive definite matrix.
+ input_matrix = np.array([[1., 2.], [-2., 1.]])
+ decomposition = whiten.cholesky(input_matrix)
+ expected_output = np.array([[2., -2.], [-2., 2.]])
+ # Check that the properties of the Cholesky decomposition are satisfied.
+ self.assertAllClose(
+ np.matmul(decomposition, decomposition.T), expected_output)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/domain_adaptation/README.md b/research/domain_adaptation/README.md
deleted file mode 100644
index e8a2b83794f11ed3711e6bc26254a90cb5469440..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/README.md
+++ /dev/null
@@ -1,124 +0,0 @@
-
-
-
-
-## Introduction
-This is the code used for two domain adaptation papers.
-
-The `domain_separation` directory contains code for the "Domain Separation
-Networks" paper by Bousmalis K., Trigeorgis G., et al. which was presented at
-NIPS 2016. The paper can be found here: https://arxiv.org/abs/1608.06019.
-
-The `pixel_domain_adaptation` directory contains the code used for the
-"Unsupervised Pixel-Level Domain Adaptation with Generative Adversarial
-Networks" paper by Bousmalis K., et al. (presented at CVPR 2017). The paper can
-be found here: https://arxiv.org/abs/1612.05424. PixelDA aims to perform domain
-adaptation by transfering the visual style of the target domain (which has few
-or no labels) to a source domain (which has many labels). This is accomplished
-using a Generative Adversarial Network (GAN).
-
-### Other implementations
-* [Simplified-DSN](https://github.com/AmirHussein96/Simplified-DSN):
- An unofficial implementation of the [Domain Separation Networks paper](https://arxiv.org/abs/1608.06019).
-
-## Contact
-The domain separation code was open-sourced
-by [Konstantinos Bousmalis](https://github.com/bousmalis)
-(konstantinos@google.com), while the pixel level domain adaptation code was
-open-sourced by [David Dohan](https://github.com/dmrd) (ddohan@google.com).
-
-## Installation
-You will need to have the following installed on your machine before trying out the DSN code.
-
-* TensorFlow 1.x: https://www.tensorflow.org/install/
-* Bazel: https://bazel.build/
-
-## Initial setup
-In order to run the MNIST to MNIST-M experiments, you will need to set the
-data directory:
-
-```
-$ export DSN_DATA_DIR=/your/dir
-```
-
-Add models and models/slim to your `$PYTHONPATH` (assumes $PWD is /models):
-
-```
-$ export PYTHONPATH=$PYTHONPATH:$PWD:$PWD/slim
-```
-
-## Getting the datasets
-
-You can fetch the MNIST data by running
-
-```
- $ bazel run slim:download_and_convert_data -- --dataset_dir $DSN_DATA_DIR --dataset_name=mnist
-```
-
-The MNIST-M dataset is available online [here](http://bit.ly/2nrlUAJ). Once it is downloaded and extracted into your data directory, create TFRecord files by running:
-```
-$ bazel run domain_adaptation/datasets:download_and_convert_mnist_m -- --dataset_dir $DSN_DATA_DIR
-```
-
-# Running PixelDA from MNIST to MNIST-M
-You can run PixelDA as follows (using Tensorboard to examine the results):
-
-```
-$ bazel run domain_adaptation/pixel_domain_adaptation:pixelda_train -- --dataset_dir $DSN_DATA_DIR --source_dataset mnist --target_dataset mnist_m
-```
-
-And evaluation as:
-```
-$ bazel run domain_adaptation/pixel_domain_adaptation:pixelda_eval -- --dataset_dir $DSN_DATA_DIR --source_dataset mnist --target_dataset mnist_m --target_split_name test
-```
-
-The MNIST-M results in the paper were run with the following hparams flag:
-```
---hparams arch=resnet,domain_loss_weight=0.135603587834,num_training_examples=16000000,style_transfer_loss_weight=0.0113173311334,task_loss_in_g_weight=0.0100959947002,task_tower=mnist,task_tower_in_g_step=true
-```
-
-### A note on terminology/language of the code:
-
-The components of the network can be grouped into two parts
-which correspond to elements which are jointly optimized: The generator
-component and the discriminator component.
-
-The generator component takes either an image or noise vector and produces an
-output image.
-
-The discriminator component takes the generated images and the target images
-and attempts to discriminate between them.
-
-## Running DSN code for adapting MNIST to MNIST-M
-
-Then you need to build the binaries with Bazel:
-
-```
-$ bazel build -c opt domain_adaptation/domain_separation/...
-```
-
-You can then train with the following command:
-
-```
-$ ./bazel-bin/domain_adaptation/domain_separation/dsn_train \
- --similarity_loss=dann_loss \
- --basic_tower=dann_mnist \
- --source_dataset=mnist \
- --target_dataset=mnist_m \
- --learning_rate=0.0117249 \
- --gamma_weight=0.251175 \
- --weight_decay=1e-6 \
- --layers_to_regularize=fc3 \
- --nouse_separation \
- --master="" \
- --dataset_dir=${DSN_DATA_DIR} \
- -v --use_logging
-```
-
-Evaluation can be invoked with the following command:
-
-```
-$ ./bazel-bin/domain_adaptation/domain_separation/dsn_eval \
- -v --dataset mnist_m --split test --num_examples=9001 \
- --dataset_dir=${DSN_DATA_DIR}
-```
diff --git a/research/domain_adaptation/WORKSPACE b/research/domain_adaptation/WORKSPACE
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/domain_adaptation/__init__.py b/research/domain_adaptation/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/domain_adaptation/datasets/BUILD b/research/domain_adaptation/datasets/BUILD
deleted file mode 100644
index 067a79374fbcedaa6fcd90293e5365aaad4c18c6..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/datasets/BUILD
+++ /dev/null
@@ -1,45 +0,0 @@
-# Domain Adaptation Scenarios Datasets
-
-package(
- default_visibility = [
- ":internal",
- ],
-)
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-package_group(
- name = "internal",
- packages = [
- "//domain_adaptation/...",
- ],
-)
-
-py_library(
- name = "dataset_factory",
- srcs = ["dataset_factory.py"],
- deps = [
- ":mnist_m",
- "//slim:mnist",
- ],
-)
-
-py_binary(
- name = "download_and_convert_mnist_m",
- srcs = ["download_and_convert_mnist_m.py"],
- deps = [
-
- "//slim:dataset_utils",
- ],
-)
-
-py_binary(
- name = "mnist_m",
- srcs = ["mnist_m.py"],
- deps = [
-
- "//slim:dataset_utils",
- ],
-)
diff --git a/research/domain_adaptation/datasets/__init__.py b/research/domain_adaptation/datasets/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/domain_adaptation/datasets/dataset_factory.py b/research/domain_adaptation/datasets/dataset_factory.py
deleted file mode 100644
index 4ca1b41c412a78d25053fc786c8f81072fe90adb..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/datasets/dataset_factory.py
+++ /dev/null
@@ -1,107 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""A factory-pattern class which returns image/label pairs."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-import tensorflow as tf
-
-from slim.datasets import mnist
-from domain_adaptation.datasets import mnist_m
-
-slim = tf.contrib.slim
-
-
-def get_dataset(dataset_name,
- split_name,
- dataset_dir,
- file_pattern=None,
- reader=None):
- """Given a dataset name and a split_name returns a Dataset.
-
- Args:
- dataset_name: String, the name of the dataset.
- split_name: A train/test split name.
- dataset_dir: The directory where the dataset files are stored.
- file_pattern: The file pattern to use for matching the dataset source files.
- reader: The subclass of tf.ReaderBase. If left as `None`, then the default
- reader defined by each dataset is used.
-
- Returns:
- A tf-slim `Dataset` class.
-
- Raises:
- ValueError: if `dataset_name` isn't recognized.
- """
- dataset_name_to_module = {'mnist': mnist, 'mnist_m': mnist_m}
- if dataset_name not in dataset_name_to_module:
- raise ValueError('Name of dataset unknown %s.' % dataset_name)
-
- return dataset_name_to_module[dataset_name].get_split(split_name, dataset_dir,
- file_pattern, reader)
-
-
-def provide_batch(dataset_name, split_name, dataset_dir, num_readers,
- batch_size, num_preprocessing_threads):
- """Provides a batch of images and corresponding labels.
-
- Args:
- dataset_name: String, the name of the dataset.
- split_name: A train/test split name.
- dataset_dir: The directory where the dataset files are stored.
- num_readers: The number of readers used by DatasetDataProvider.
- batch_size: The size of the batch requested.
- num_preprocessing_threads: The number of preprocessing threads for
- tf.train.batch.
- file_pattern: The file pattern to use for matching the dataset source files.
- reader: The subclass of tf.ReaderBase. If left as `None`, then the default
- reader defined by each dataset is used.
-
- Returns:
- A batch of
- images: tensor of [batch_size, height, width, channels].
- labels: dictionary of labels.
- """
- dataset = get_dataset(dataset_name, split_name, dataset_dir)
- provider = slim.dataset_data_provider.DatasetDataProvider(
- dataset,
- num_readers=num_readers,
- common_queue_capacity=20 * batch_size,
- common_queue_min=10 * batch_size)
- [image, label] = provider.get(['image', 'label'])
-
- # Convert images to float32
- image = tf.image.convert_image_dtype(image, tf.float32)
- image -= 0.5
- image *= 2
-
- # Load the data.
- labels = {}
- images, labels['classes'] = tf.train.batch(
- [image, label],
- batch_size=batch_size,
- num_threads=num_preprocessing_threads,
- capacity=5 * batch_size)
- labels['classes'] = slim.one_hot_encoding(labels['classes'],
- dataset.num_classes)
-
- # Convert mnist to RGB and 32x32 so that it can match mnist_m.
- if dataset_name == 'mnist':
- images = tf.image.grayscale_to_rgb(images)
- images = tf.image.resize_images(images, [32, 32])
- return images, labels
diff --git a/research/domain_adaptation/datasets/download_and_convert_mnist_m.py b/research/domain_adaptation/datasets/download_and_convert_mnist_m.py
deleted file mode 100644
index 3b5004d3d8aaf54656389e517c50f38299714bc7..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/datasets/download_and_convert_mnist_m.py
+++ /dev/null
@@ -1,237 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-r"""Downloads and converts MNIST-M data to TFRecords of TF-Example protos.
-
-This module downloads the MNIST-M data, uncompresses it, reads the files
-that make up the MNIST-M data and creates two TFRecord datasets: one for train
-and one for test. Each TFRecord dataset is comprised of a set of TF-Example
-protocol buffers, each of which contain a single image and label.
-
-The script should take about a minute to run.
-
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import random
-import sys
-
-# Dependency imports
-import numpy as np
-from six.moves import urllib
-import tensorflow as tf
-
-from slim.datasets import dataset_utils
-
-tf.app.flags.DEFINE_string(
- 'dataset_dir', None,
- 'The directory where the output TFRecords and temporary files are saved.')
-
-FLAGS = tf.app.flags.FLAGS
-
-_IMAGE_SIZE = 32
-_NUM_CHANNELS = 3
-
-# The number of images in the training set.
-_NUM_TRAIN_SAMPLES = 59001
-
-# The number of images to be kept from the training set for the validation set.
-_NUM_VALIDATION = 1000
-
-# The number of images in the test set.
-_NUM_TEST_SAMPLES = 9001
-
-# Seed for repeatability.
-_RANDOM_SEED = 0
-
-# The names of the classes.
-_CLASS_NAMES = [
- 'zero',
- 'one',
- 'two',
- 'three',
- 'four',
- 'five',
- 'size',
- 'seven',
- 'eight',
- 'nine',
-]
-
-
-class ImageReader(object):
- """Helper class that provides TensorFlow image coding utilities."""
-
- def __init__(self):
- # Initializes function that decodes RGB PNG data.
- self._decode_png_data = tf.placeholder(dtype=tf.string)
- self._decode_png = tf.image.decode_png(self._decode_png_data, channels=3)
-
- def read_image_dims(self, sess, image_data):
- image = self.decode_png(sess, image_data)
- return image.shape[0], image.shape[1]
-
- def decode_png(self, sess, image_data):
- image = sess.run(
- self._decode_png, feed_dict={self._decode_png_data: image_data})
- assert len(image.shape) == 3
- assert image.shape[2] == 3
- return image
-
-
-def _convert_dataset(split_name, filenames, filename_to_class_id, dataset_dir):
- """Converts the given filenames to a TFRecord dataset.
-
- Args:
- split_name: The name of the dataset, either 'train' or 'valid'.
- filenames: A list of absolute paths to png images.
- filename_to_class_id: A dictionary from filenames (strings) to class ids
- (integers).
- dataset_dir: The directory where the converted datasets are stored.
- """
- print('Converting the {} split.'.format(split_name))
- # Train and validation splits are both in the train directory.
- if split_name in ['train', 'valid']:
- png_directory = os.path.join(dataset_dir, 'mnist_m', 'mnist_m_train')
- elif split_name == 'test':
- png_directory = os.path.join(dataset_dir, 'mnist_m', 'mnist_m_test')
-
- with tf.Graph().as_default():
- image_reader = ImageReader()
-
- with tf.Session('') as sess:
- output_filename = _get_output_filename(dataset_dir, split_name)
-
- with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
- for filename in filenames:
- # Read the filename:
- image_data = tf.gfile.FastGFile(
- os.path.join(png_directory, filename), 'r').read()
- height, width = image_reader.read_image_dims(sess, image_data)
-
- class_id = filename_to_class_id[filename]
- example = dataset_utils.image_to_tfexample(image_data, 'png', height,
- width, class_id)
- tfrecord_writer.write(example.SerializeToString())
-
- sys.stdout.write('\n')
- sys.stdout.flush()
-
-
-def _extract_labels(label_filename):
- """Extract the labels into a dict of filenames to int labels.
-
- Args:
- labels_filename: The filename of the MNIST-M labels.
-
- Returns:
- A dictionary of filenames to int labels.
- """
- print('Extracting labels from: ', label_filename)
- label_file = tf.gfile.FastGFile(label_filename, 'r').readlines()
- label_lines = [line.rstrip('\n').split() for line in label_file]
- labels = {}
- for line in label_lines:
- assert len(line) == 2
- labels[line[0]] = int(line[1])
- return labels
-
-
-def _get_output_filename(dataset_dir, split_name):
- """Creates the output filename.
-
- Args:
- dataset_dir: The directory where the temporary files are stored.
- split_name: The name of the train/test split.
-
- Returns:
- An absolute file path.
- """
- return '%s/mnist_m_%s.tfrecord' % (dataset_dir, split_name)
-
-
-def _get_filenames(dataset_dir):
- """Returns a list of filenames and inferred class names.
-
- Args:
- dataset_dir: A directory containing a set PNG encoded MNIST-M images.
-
- Returns:
- A list of image file paths, relative to `dataset_dir`.
- """
- photo_filenames = []
- for filename in os.listdir(dataset_dir):
- photo_filenames.append(filename)
- return photo_filenames
-
-
-def run(dataset_dir):
- """Runs the download and conversion operation.
-
- Args:
- dataset_dir: The dataset directory where the dataset is stored.
- """
- if not tf.gfile.Exists(dataset_dir):
- tf.gfile.MakeDirs(dataset_dir)
-
- train_filename = _get_output_filename(dataset_dir, 'train')
- testing_filename = _get_output_filename(dataset_dir, 'test')
-
- if tf.gfile.Exists(train_filename) and tf.gfile.Exists(testing_filename):
- print('Dataset files already exist. Exiting without re-creating them.')
- return
-
- # TODO(konstantinos): Add download and cleanup functionality
-
- train_validation_filenames = _get_filenames(
- os.path.join(dataset_dir, 'mnist_m', 'mnist_m_train'))
- test_filenames = _get_filenames(
- os.path.join(dataset_dir, 'mnist_m', 'mnist_m_test'))
-
- # Divide into train and validation:
- random.seed(_RANDOM_SEED)
- random.shuffle(train_validation_filenames)
- train_filenames = train_validation_filenames[_NUM_VALIDATION:]
- validation_filenames = train_validation_filenames[:_NUM_VALIDATION]
-
- train_validation_filenames_to_class_ids = _extract_labels(
- os.path.join(dataset_dir, 'mnist_m', 'mnist_m_train_labels.txt'))
- test_filenames_to_class_ids = _extract_labels(
- os.path.join(dataset_dir, 'mnist_m', 'mnist_m_test_labels.txt'))
-
- # Convert the train, validation, and test sets.
- _convert_dataset('train', train_filenames,
- train_validation_filenames_to_class_ids, dataset_dir)
- _convert_dataset('valid', validation_filenames,
- train_validation_filenames_to_class_ids, dataset_dir)
- _convert_dataset('test', test_filenames, test_filenames_to_class_ids,
- dataset_dir)
-
- # Finally, write the labels file:
- labels_to_class_names = dict(zip(range(len(_CLASS_NAMES)), _CLASS_NAMES))
- dataset_utils.write_label_file(labels_to_class_names, dataset_dir)
-
- print('\nFinished converting the MNIST-M dataset!')
-
-
-def main(_):
- assert FLAGS.dataset_dir
- run(FLAGS.dataset_dir)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/domain_adaptation/datasets/mnist_m.py b/research/domain_adaptation/datasets/mnist_m.py
deleted file mode 100644
index fab6c443cf3d2e9783d19bf52c81b7aa62d56a38..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/datasets/mnist_m.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Provides data for the MNIST-M dataset.
-
-The dataset scripts used to create the dataset can be found at:
-tensorflow_models/domain_adaptation_/datasets/download_and_convert_mnist_m_dataset.py
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-# Dependency imports
-import tensorflow as tf
-
-from slim.datasets import dataset_utils
-
-slim = tf.contrib.slim
-
-_FILE_PATTERN = 'mnist_m_%s.tfrecord'
-
-_SPLITS_TO_SIZES = {'train': 58001, 'valid': 1000, 'test': 9001}
-
-_NUM_CLASSES = 10
-
-_ITEMS_TO_DESCRIPTIONS = {
- 'image': 'A [32 x 32 x 1] RGB image.',
- 'label': 'A single integer between 0 and 9',
-}
-
-
-def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
- """Gets a dataset tuple with instructions for reading MNIST.
-
- Args:
- split_name: A train/test split name.
- dataset_dir: The base directory of the dataset sources.
-
- Returns:
- A `Dataset` namedtuple.
-
- Raises:
- ValueError: if `split_name` is not a valid train/test split.
- """
- if split_name not in _SPLITS_TO_SIZES:
- raise ValueError('split name %s was not recognized.' % split_name)
-
- if not file_pattern:
- file_pattern = _FILE_PATTERN
- file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
-
- # Allowing None in the signature so that dataset_factory can use the default.
- if reader is None:
- reader = tf.TFRecordReader
-
- keys_to_features = {
- 'image/encoded':
- tf.FixedLenFeature((), tf.string, default_value=''),
- 'image/format':
- tf.FixedLenFeature((), tf.string, default_value='png'),
- 'image/class/label':
- tf.FixedLenFeature(
- [1], tf.int64, default_value=tf.zeros([1], dtype=tf.int64)),
- }
-
- items_to_handlers = {
- 'image': slim.tfexample_decoder.Image(shape=[32, 32, 3], channels=3),
- 'label': slim.tfexample_decoder.Tensor('image/class/label', shape=[]),
- }
-
- decoder = slim.tfexample_decoder.TFExampleDecoder(
- keys_to_features, items_to_handlers)
-
- labels_to_names = None
- if dataset_utils.has_labels(dataset_dir):
- labels_to_names = dataset_utils.read_label_file(dataset_dir)
-
- return slim.dataset.Dataset(
- data_sources=file_pattern,
- reader=reader,
- decoder=decoder,
- num_samples=_SPLITS_TO_SIZES[split_name],
- num_classes=_NUM_CLASSES,
- items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
- labels_to_names=labels_to_names)
diff --git a/research/domain_adaptation/domain_separation/BUILD b/research/domain_adaptation/domain_separation/BUILD
deleted file mode 100644
index 14dceda27e49d74eaaaeae21676183b78c72b9c2..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/BUILD
+++ /dev/null
@@ -1,157 +0,0 @@
-# Domain Separation Networks
-
-package(
- default_visibility = [
- ":internal",
- ],
-)
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-package_group(
- name = "internal",
- packages = [
- "//domain_adaptation/...",
- ],
-)
-
-py_library(
- name = "models",
- srcs = [
- "models.py",
- ],
- deps = [
- ":utils",
- ],
-)
-
-py_library(
- name = "losses",
- srcs = [
- "losses.py",
- ],
- deps = [
- ":grl_op_grads_py",
- ":grl_op_shapes_py",
- ":grl_ops",
- ":utils",
- ],
-)
-
-py_test(
- name = "losses_test",
- srcs = [
- "losses_test.py",
- ],
- deps = [
- ":losses",
- ":utils",
- ],
-)
-
-py_library(
- name = "dsn",
- srcs = [
- "dsn.py",
- ],
- deps = [
- ":grl_op_grads_py",
- ":grl_op_shapes_py",
- ":grl_ops",
- ":losses",
- ":models",
- ":utils",
- ],
-)
-
-py_test(
- name = "dsn_test",
- srcs = [
- "dsn_test.py",
- ],
- deps = [
- ":dsn",
- ],
-)
-
-py_binary(
- name = "dsn_train",
- srcs = [
- "dsn_train.py",
- ],
- deps = [
- ":dsn",
- ":models",
- "//domain_adaptation/datasets:dataset_factory",
- ],
-)
-
-py_binary(
- name = "dsn_eval",
- srcs = [
- "dsn_eval.py",
- ],
- deps = [
- ":dsn",
- ":models",
- "//domain_adaptation/datasets:dataset_factory",
- ],
-)
-
-py_test(
- name = "models_test",
- srcs = [
- "models_test.py",
- ],
- deps = [
- ":models",
- "//domain_adaptation/datasets:dataset_factory",
- ],
-)
-
-py_library(
- name = "utils",
- srcs = [
- "utils.py",
- ],
- deps = [
- ],
-)
-
-py_library(
- name = "grl_op_grads_py",
- srcs = [
- "grl_op_grads.py",
- ],
- deps = [
- ":grl_ops",
- ],
-)
-
-py_library(
- name = "grl_op_shapes_py",
- srcs = [
- "grl_op_shapes.py",
- ],
- deps = [
- ],
-)
-
-py_library(
- name = "grl_ops",
- srcs = ["grl_ops.py"],
- data = ["_grl_ops.so"],
-)
-
-py_test(
- name = "grl_ops_test",
- size = "small",
- srcs = ["grl_ops_test.py"],
- deps = [
- ":grl_op_grads_py",
- ":grl_op_shapes_py",
- ":grl_ops",
- ],
-)
diff --git a/research/domain_adaptation/domain_separation/__init__.py b/research/domain_adaptation/domain_separation/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/domain_adaptation/domain_separation/_grl_ops.so b/research/domain_adaptation/domain_separation/_grl_ops.so
deleted file mode 100755
index 4c35473760a76dcb743d58f45eddccecb5f5161e..0000000000000000000000000000000000000000
Binary files a/research/domain_adaptation/domain_separation/_grl_ops.so and /dev/null differ
diff --git a/research/domain_adaptation/domain_separation/dsn.py b/research/domain_adaptation/domain_separation/dsn.py
deleted file mode 100644
index 3018e8a791840ae465bad493913235cc04c31cff..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/dsn.py
+++ /dev/null
@@ -1,355 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Functions to create a DSN model and add the different losses to it.
-
-Specifically, in this file we define the:
- - Shared Encoding Similarity Loss Module, with:
- - The MMD Similarity method
- - The Correlation Similarity method
- - The Gradient Reversal (Domain-Adversarial) method
- - Difference Loss Module
- - Reconstruction Loss Module
- - Task Loss Module
-"""
-from functools import partial
-
-import tensorflow as tf
-
-import losses
-import models
-import utils
-
-slim = tf.contrib.slim
-
-
-################################################################################
-# HELPER FUNCTIONS
-################################################################################
-def dsn_loss_coefficient(params):
- """The global_step-dependent weight that specifies when to kick in DSN losses.
-
- Args:
- params: A dictionary of parameters. Expecting 'domain_separation_startpoint'
-
- Returns:
- A weight to that effectively enables or disables the DSN-related losses,
- i.e. similarity, difference, and reconstruction losses.
- """
- return tf.where(
- tf.less(slim.get_or_create_global_step(),
- params['domain_separation_startpoint']), 1e-10, 1.0)
-
-
-################################################################################
-# MODEL CREATION
-################################################################################
-def create_model(source_images, source_labels, domain_selection_mask,
- target_images, target_labels, similarity_loss, params,
- basic_tower_name):
- """Creates a DSN model.
-
- Args:
- source_images: images from the source domain, a tensor of size
- [batch_size, height, width, channels]
- source_labels: a dictionary with the name, tensor pairs. 'classes' is one-
- hot for the number of classes.
- domain_selection_mask: a boolean tensor of size [batch_size, ] which denotes
- the labeled images that belong to the source domain.
- target_images: images from the target domain, a tensor of size
- [batch_size, height width, channels].
- target_labels: a dictionary with the name, tensor pairs.
- similarity_loss: The type of method to use for encouraging
- the codes from the shared encoder to be similar.
- params: A dictionary of parameters. Expecting 'weight_decay',
- 'layers_to_regularize', 'use_separation', 'domain_separation_startpoint',
- 'alpha_weight', 'beta_weight', 'gamma_weight', 'recon_loss_name',
- 'decoder_name', 'encoder_name'
- basic_tower_name: the name of the tower to use for the shared encoder.
-
- Raises:
- ValueError: if the arch is not one of the available architectures.
- """
- network = getattr(models, basic_tower_name)
- num_classes = source_labels['classes'].get_shape().as_list()[1]
-
- # Make sure we are using the appropriate number of classes.
- network = partial(network, num_classes=num_classes)
-
- # Add the classification/pose estimation loss to the source domain.
- source_endpoints = add_task_loss(source_images, source_labels, network,
- params)
-
- if similarity_loss == 'none':
- # No domain adaptation, we can stop here.
- return
-
- with tf.variable_scope('towers', reuse=True):
- target_logits, target_endpoints = network(
- target_images, weight_decay=params['weight_decay'], prefix='target')
-
- # Plot target accuracy of the train set.
- target_accuracy = utils.accuracy(
- tf.argmax(target_logits, 1), tf.argmax(target_labels['classes'], 1))
-
- if 'quaternions' in target_labels:
- target_quaternion_loss = losses.log_quaternion_loss(
- target_labels['quaternions'], target_endpoints['quaternion_pred'],
- params)
- tf.summary.scalar('eval/Target quaternions', target_quaternion_loss)
-
- tf.summary.scalar('eval/Target accuracy', target_accuracy)
-
- source_shared = source_endpoints[params['layers_to_regularize']]
- target_shared = target_endpoints[params['layers_to_regularize']]
-
- # When using the semisupervised model we include labeled target data in the
- # source classifier. We do not want to include these target domain when
- # we use the similarity loss.
- indices = tf.range(0, source_shared.get_shape().as_list()[0])
- indices = tf.boolean_mask(indices, domain_selection_mask)
- add_similarity_loss(similarity_loss,
- tf.gather(source_shared, indices),
- tf.gather(target_shared, indices), params)
-
- if params['use_separation']:
- add_autoencoders(
- source_images,
- source_shared,
- target_images,
- target_shared,
- params=params,)
-
-
-def add_similarity_loss(method_name,
- source_samples,
- target_samples,
- params,
- scope=None):
- """Adds a loss encouraging the shared encoding from each domain to be similar.
-
- Args:
- method_name: the name of the encoding similarity method to use. Valid
- options include `dann_loss', `mmd_loss' or `correlation_loss'.
- source_samples: a tensor of shape [num_samples, num_features].
- target_samples: a tensor of shape [num_samples, num_features].
- params: a dictionary of parameters. Expecting 'gamma_weight'.
- scope: optional name scope for summary tags.
- Raises:
- ValueError: if `method_name` is not recognized.
- """
- weight = dsn_loss_coefficient(params) * params['gamma_weight']
- method = getattr(losses, method_name)
- method(source_samples, target_samples, weight, scope)
-
-
-def add_reconstruction_loss(recon_loss_name, images, recons, weight, domain):
- """Adds a reconstruction loss.
-
- Args:
- recon_loss_name: The name of the reconstruction loss.
- images: A `Tensor` of size [batch_size, height, width, 3].
- recons: A `Tensor` whose size matches `images`.
- weight: A scalar coefficient for the loss.
- domain: The name of the domain being reconstructed.
-
- Raises:
- ValueError: If `recon_loss_name` is not recognized.
- """
- if recon_loss_name == 'sum_of_pairwise_squares':
- loss_fn = tf.contrib.losses.mean_pairwise_squared_error
- elif recon_loss_name == 'sum_of_squares':
- loss_fn = tf.contrib.losses.mean_squared_error
- else:
- raise ValueError('recon_loss_name value [%s] not recognized.' %
- recon_loss_name)
-
- loss = loss_fn(recons, images, weight)
- assert_op = tf.Assert(tf.is_finite(loss), [loss])
- with tf.control_dependencies([assert_op]):
- tf.summary.scalar('losses/%s Recon Loss' % domain, loss)
-
-
-def add_autoencoders(source_data, source_shared, target_data, target_shared,
- params):
- """Adds the encoders/decoders for our domain separation model w/ incoherence.
-
- Args:
- source_data: images from the source domain, a tensor of size
- [batch_size, height, width, channels]
- source_shared: a tensor with first dimension batch_size
- target_data: images from the target domain, a tensor of size
- [batch_size, height, width, channels]
- target_shared: a tensor with first dimension batch_size
- params: A dictionary of parameters. Expecting 'layers_to_regularize',
- 'beta_weight', 'alpha_weight', 'recon_loss_name', 'decoder_name',
- 'encoder_name', 'weight_decay'
- """
-
- def normalize_images(images):
- images -= tf.reduce_min(images)
- return images / tf.reduce_max(images)
-
- def concat_operation(shared_repr, private_repr):
- return shared_repr + private_repr
-
- mu = dsn_loss_coefficient(params)
-
- # The layer to concatenate the networks at.
- concat_layer = params['layers_to_regularize']
-
- # The coefficient for modulating the private/shared difference loss.
- difference_loss_weight = params['beta_weight'] * mu
-
- # The reconstruction weight.
- recon_loss_weight = params['alpha_weight'] * mu
-
- # The reconstruction loss to use.
- recon_loss_name = params['recon_loss_name']
-
- # The decoder/encoder to use.
- decoder_name = params['decoder_name']
- encoder_name = params['encoder_name']
-
- _, height, width, _ = source_data.get_shape().as_list()
- code_size = source_shared.get_shape().as_list()[-1]
- weight_decay = params['weight_decay']
-
- encoder_fn = getattr(models, encoder_name)
- # Target Auto-encoding.
- with tf.variable_scope('source_encoder'):
- source_endpoints = encoder_fn(
- source_data, code_size, weight_decay=weight_decay)
-
- with tf.variable_scope('target_encoder'):
- target_endpoints = encoder_fn(
- target_data, code_size, weight_decay=weight_decay)
-
- decoder_fn = getattr(models, decoder_name)
-
- decoder = partial(
- decoder_fn,
- height=height,
- width=width,
- channels=source_data.get_shape().as_list()[-1],
- weight_decay=weight_decay)
-
- # Source Auto-encoding.
- source_private = source_endpoints[concat_layer]
- target_private = target_endpoints[concat_layer]
- with tf.variable_scope('decoder'):
- source_recons = decoder(concat_operation(source_shared, source_private))
-
- with tf.variable_scope('decoder', reuse=True):
- source_private_recons = decoder(
- concat_operation(tf.zeros_like(source_private), source_private))
- source_shared_recons = decoder(
- concat_operation(source_shared, tf.zeros_like(source_shared)))
-
- with tf.variable_scope('decoder', reuse=True):
- target_recons = decoder(concat_operation(target_shared, target_private))
- target_shared_recons = decoder(
- concat_operation(target_shared, tf.zeros_like(target_shared)))
- target_private_recons = decoder(
- concat_operation(tf.zeros_like(target_private), target_private))
-
- losses.difference_loss(
- source_private,
- source_shared,
- weight=difference_loss_weight,
- name='Source')
- losses.difference_loss(
- target_private,
- target_shared,
- weight=difference_loss_weight,
- name='Target')
-
- add_reconstruction_loss(recon_loss_name, source_data, source_recons,
- recon_loss_weight, 'source')
- add_reconstruction_loss(recon_loss_name, target_data, target_recons,
- recon_loss_weight, 'target')
-
- # Add summaries
- source_reconstructions = tf.concat(
- axis=2,
- values=map(normalize_images, [
- source_data, source_recons, source_shared_recons,
- source_private_recons
- ]))
- target_reconstructions = tf.concat(
- axis=2,
- values=map(normalize_images, [
- target_data, target_recons, target_shared_recons,
- target_private_recons
- ]))
- tf.summary.image(
- 'Source Images:Recons:RGB',
- source_reconstructions[:, :, :, :3],
- max_outputs=10)
- tf.summary.image(
- 'Target Images:Recons:RGB',
- target_reconstructions[:, :, :, :3],
- max_outputs=10)
-
- if source_reconstructions.get_shape().as_list()[3] == 4:
- tf.summary.image(
- 'Source Images:Recons:Depth',
- source_reconstructions[:, :, :, 3:4],
- max_outputs=10)
- tf.summary.image(
- 'Target Images:Recons:Depth',
- target_reconstructions[:, :, :, 3:4],
- max_outputs=10)
-
-
-def add_task_loss(source_images, source_labels, basic_tower, params):
- """Adds a classification and/or pose estimation loss to the model.
-
- Args:
- source_images: images from the source domain, a tensor of size
- [batch_size, height, width, channels]
- source_labels: labels from the source domain, a tensor of size [batch_size].
- or a tuple of (quaternions, class_labels)
- basic_tower: a function that creates the single tower of the model.
- params: A dictionary of parameters. Expecting 'weight_decay', 'pose_weight'.
- Returns:
- The source endpoints.
-
- Raises:
- RuntimeError: if basic tower does not support pose estimation.
- """
- with tf.variable_scope('towers'):
- source_logits, source_endpoints = basic_tower(
- source_images, weight_decay=params['weight_decay'], prefix='Source')
-
- if 'quaternions' in source_labels: # We have pose estimation as well
- if 'quaternion_pred' not in source_endpoints:
- raise RuntimeError('Please use a model for estimation e.g. pose_mini')
-
- loss = losses.log_quaternion_loss(source_labels['quaternions'],
- source_endpoints['quaternion_pred'],
- params)
-
- assert_op = tf.Assert(tf.is_finite(loss), [loss])
- with tf.control_dependencies([assert_op]):
- quaternion_loss = loss
- tf.summary.histogram('log_quaternion_loss_hist', quaternion_loss)
- slim.losses.add_loss(quaternion_loss * params['pose_weight'])
- tf.summary.scalar('losses/quaternion_loss', quaternion_loss)
-
- classification_loss = tf.losses.softmax_cross_entropy(
- source_labels['classes'], source_logits)
-
- tf.summary.scalar('losses/classification_loss', classification_loss)
- return source_endpoints
diff --git a/research/domain_adaptation/domain_separation/dsn_eval.py b/research/domain_adaptation/domain_separation/dsn_eval.py
deleted file mode 100644
index b6cccdfcc17e8f18e8381530b5c8f41501bda29b..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/dsn_eval.py
+++ /dev/null
@@ -1,161 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# pylint: disable=line-too-long
-"""Evaluation for Domain Separation Networks (DSNs)."""
-# pylint: enable=line-too-long
-import math
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from domain_adaptation.datasets import dataset_factory
-from domain_adaptation.domain_separation import losses
-from domain_adaptation.domain_separation import models
-
-slim = tf.contrib.slim
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_integer('batch_size', 32,
- 'The number of images in each batch.')
-
-tf.app.flags.DEFINE_string('master', '',
- 'BNS name of the TensorFlow master to use.')
-
-tf.app.flags.DEFINE_string('checkpoint_dir', '/tmp/da/',
- 'Directory where the model was written to.')
-
-tf.app.flags.DEFINE_string(
- 'eval_dir', '/tmp/da/',
- 'Directory where we should write the tf summaries to.')
-
-tf.app.flags.DEFINE_string('dataset_dir', None,
- 'The directory where the dataset files are stored.')
-
-tf.app.flags.DEFINE_string('dataset', 'mnist_m',
- 'Which dataset to test on: "mnist", "mnist_m".')
-
-tf.app.flags.DEFINE_string('split', 'valid',
- 'Which portion to test on: "valid", "test".')
-
-tf.app.flags.DEFINE_integer('num_examples', 1000, 'Number of test examples.')
-
-tf.app.flags.DEFINE_string('basic_tower', 'dann_mnist',
- 'The basic tower building block.')
-
-tf.app.flags.DEFINE_bool('enable_precision_recall', False,
- 'If True, precision and recall for each class will '
- 'be added to the metrics.')
-
-tf.app.flags.DEFINE_bool('use_logging', False, 'Debugging messages.')
-
-
-def quaternion_metric(predictions, labels):
- params = {'batch_size': FLAGS.batch_size, 'use_logging': False}
- logcost = losses.log_quaternion_loss_batch(predictions, labels, params)
- return slim.metrics.streaming_mean(logcost)
-
-
-def angle_diff(true_q, pred_q):
- angles = 2 * (
- 180.0 /
- np.pi) * np.arccos(np.abs(np.sum(np.multiply(pred_q, true_q), axis=1)))
- return angles
-
-
-def provide_batch_fn():
- """ The provide_batch function to use. """
- return dataset_factory.provide_batch
-
-
-def main(_):
- g = tf.Graph()
- with g.as_default():
- # Load the data.
- images, labels = provide_batch_fn()(
- FLAGS.dataset, FLAGS.split, FLAGS.dataset_dir, 4, FLAGS.batch_size, 4)
-
- num_classes = labels['classes'].get_shape().as_list()[1]
-
- tf.summary.image('eval_images', images, max_outputs=3)
-
- # Define the model:
- with tf.variable_scope('towers'):
- basic_tower = getattr(models, FLAGS.basic_tower)
- predictions, endpoints = basic_tower(
- images,
- num_classes=num_classes,
- is_training=False,
- batch_norm_params=None)
- metric_names_to_values = {}
-
- # Define the metrics:
- if 'quaternions' in labels: # Also have to evaluate pose estimation!
- quaternion_loss = quaternion_metric(labels['quaternions'],
- endpoints['quaternion_pred'])
-
- angle_errors, = tf.py_func(
- angle_diff, [labels['quaternions'], endpoints['quaternion_pred']],
- [tf.float32])
-
- metric_names_to_values[
- 'Angular mean error'] = slim.metrics.streaming_mean(angle_errors)
- metric_names_to_values['Quaternion Loss'] = quaternion_loss
-
- accuracy = tf.contrib.metrics.streaming_accuracy(
- tf.argmax(predictions, 1), tf.argmax(labels['classes'], 1))
-
- predictions = tf.argmax(predictions, 1)
- labels = tf.argmax(labels['classes'], 1)
- metric_names_to_values['Accuracy'] = accuracy
-
- if FLAGS.enable_precision_recall:
- for i in xrange(num_classes):
- index_map = tf.one_hot(i, depth=num_classes)
- name = 'PR/Precision_{}'.format(i)
- metric_names_to_values[name] = slim.metrics.streaming_precision(
- tf.gather(index_map, predictions), tf.gather(index_map, labels))
- name = 'PR/Recall_{}'.format(i)
- metric_names_to_values[name] = slim.metrics.streaming_recall(
- tf.gather(index_map, predictions), tf.gather(index_map, labels))
-
- names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(
- metric_names_to_values)
-
- # Create the summary ops such that they also print out to std output:
- summary_ops = []
- for metric_name, metric_value in names_to_values.iteritems():
- op = tf.summary.scalar(metric_name, metric_value)
- op = tf.Print(op, [metric_value], metric_name)
- summary_ops.append(op)
-
- # This ensures that we make a single pass over all of the data.
- num_batches = math.ceil(FLAGS.num_examples / float(FLAGS.batch_size))
-
- # Setup the global step.
- slim.get_or_create_global_step()
- slim.evaluation.evaluation_loop(
- FLAGS.master,
- checkpoint_dir=FLAGS.checkpoint_dir,
- logdir=FLAGS.eval_dir,
- num_evals=num_batches,
- eval_op=names_to_updates.values(),
- summary_op=tf.summary.merge(summary_ops))
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/domain_adaptation/domain_separation/dsn_test.py b/research/domain_adaptation/domain_separation/dsn_test.py
deleted file mode 100644
index 3d687398a9b9356455f739417bc96ddb2ca5ad40..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/dsn_test.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for DSN model assembly functions."""
-
-import numpy as np
-import tensorflow as tf
-
-import dsn
-
-
-class HelperFunctionsTest(tf.test.TestCase):
-
- def testBasicDomainSeparationStartPoint(self):
- with self.test_session() as sess:
- # Test for when global_step < domain_separation_startpoint
- step = tf.contrib.slim.get_or_create_global_step()
- sess.run(tf.global_variables_initializer()) # global_step = 0
- params = {'domain_separation_startpoint': 2}
- weight = dsn.dsn_loss_coefficient(params)
- weight_np = sess.run(weight)
- self.assertAlmostEqual(weight_np, 1e-10)
-
- step_op = tf.assign_add(step, 1)
- step_np = sess.run(step_op) # global_step = 1
- weight = dsn.dsn_loss_coefficient(params)
- weight_np = sess.run(weight)
- self.assertAlmostEqual(weight_np, 1e-10)
-
- # Test for when global_step >= domain_separation_startpoint
- step_np = sess.run(step_op) # global_step = 2
- tf.logging.info(step_np)
- weight = dsn.dsn_loss_coefficient(params)
- weight_np = sess.run(weight)
- self.assertAlmostEqual(weight_np, 1.0)
-
-
-class DsnModelAssemblyTest(tf.test.TestCase):
-
- def _testBuildDefaultModel(self):
- images = tf.to_float(np.random.rand(32, 28, 28, 1))
- labels = {}
- labels['classes'] = tf.one_hot(
- tf.to_int32(np.random.randint(0, 9, (32))), 10)
-
- params = {
- 'use_separation': True,
- 'layers_to_regularize': 'fc3',
- 'weight_decay': 0.0,
- 'ps_tasks': 1,
- 'domain_separation_startpoint': 1,
- 'alpha_weight': 1,
- 'beta_weight': 1,
- 'gamma_weight': 1,
- 'recon_loss_name': 'sum_of_squares',
- 'decoder_name': 'small_decoder',
- 'encoder_name': 'default_encoder',
- }
- return images, labels, params
-
- def testBuildModelDann(self):
- images, labels, params = self._testBuildDefaultModel()
-
- with self.test_session():
- dsn.create_model(images, labels,
- tf.cast(tf.ones([32,]), tf.bool), images, labels,
- 'dann_loss', params, 'dann_mnist')
- loss_tensors = tf.contrib.losses.get_losses()
- self.assertEqual(len(loss_tensors), 6)
-
- def testBuildModelDannSumOfPairwiseSquares(self):
- images, labels, params = self._testBuildDefaultModel()
-
- with self.test_session():
- dsn.create_model(images, labels,
- tf.cast(tf.ones([32,]), tf.bool), images, labels,
- 'dann_loss', params, 'dann_mnist')
- loss_tensors = tf.contrib.losses.get_losses()
- self.assertEqual(len(loss_tensors), 6)
-
- def testBuildModelDannMultiPSTasks(self):
- images, labels, params = self._testBuildDefaultModel()
- params['ps_tasks'] = 10
- with self.test_session():
- dsn.create_model(images, labels,
- tf.cast(tf.ones([32,]), tf.bool), images, labels,
- 'dann_loss', params, 'dann_mnist')
- loss_tensors = tf.contrib.losses.get_losses()
- self.assertEqual(len(loss_tensors), 6)
-
- def testBuildModelMmd(self):
- images, labels, params = self._testBuildDefaultModel()
-
- with self.test_session():
- dsn.create_model(images, labels,
- tf.cast(tf.ones([32,]), tf.bool), images, labels,
- 'mmd_loss', params, 'dann_mnist')
- loss_tensors = tf.contrib.losses.get_losses()
- self.assertEqual(len(loss_tensors), 6)
-
- def testBuildModelCorr(self):
- images, labels, params = self._testBuildDefaultModel()
-
- with self.test_session():
- dsn.create_model(images, labels,
- tf.cast(tf.ones([32,]), tf.bool), images, labels,
- 'correlation_loss', params, 'dann_mnist')
- loss_tensors = tf.contrib.losses.get_losses()
- self.assertEqual(len(loss_tensors), 6)
-
- def testBuildModelNoDomainAdaptation(self):
- images, labels, params = self._testBuildDefaultModel()
- params['use_separation'] = False
- with self.test_session():
- dsn.create_model(images, labels,
- tf.cast(tf.ones([32,]), tf.bool), images, labels, 'none',
- params, 'dann_mnist')
- loss_tensors = tf.contrib.losses.get_losses()
- self.assertEqual(len(loss_tensors), 1)
- self.assertEqual(len(tf.contrib.losses.get_regularization_losses()), 0)
-
- def testBuildModelNoAdaptationWeightDecay(self):
- images, labels, params = self._testBuildDefaultModel()
- params['use_separation'] = False
- params['weight_decay'] = 1e-5
- with self.test_session():
- dsn.create_model(images, labels,
- tf.cast(tf.ones([32,]), tf.bool), images, labels, 'none',
- params, 'dann_mnist')
- loss_tensors = tf.contrib.losses.get_losses()
- self.assertEqual(len(loss_tensors), 1)
- self.assertTrue(len(tf.contrib.losses.get_regularization_losses()) >= 1)
-
- def testBuildModelNoSeparation(self):
- images, labels, params = self._testBuildDefaultModel()
- params['use_separation'] = False
- with self.test_session():
- dsn.create_model(images, labels,
- tf.cast(tf.ones([32,]), tf.bool), images, labels,
- 'dann_loss', params, 'dann_mnist')
- loss_tensors = tf.contrib.losses.get_losses()
- self.assertEqual(len(loss_tensors), 2)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/domain_adaptation/domain_separation/dsn_train.py b/research/domain_adaptation/domain_separation/dsn_train.py
deleted file mode 100644
index 5e364ad3037b041125a3523370b3b040478f0d8e..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/dsn_train.py
+++ /dev/null
@@ -1,278 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Training for Domain Separation Networks (DSNs)."""
-from __future__ import division
-
-import tensorflow as tf
-
-from domain_adaptation.datasets import dataset_factory
-import dsn
-
-slim = tf.contrib.slim
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_integer('batch_size', 32,
- 'The number of images in each batch.')
-
-tf.app.flags.DEFINE_string('source_dataset', 'pose_synthetic',
- 'Source dataset to train on.')
-
-tf.app.flags.DEFINE_string('target_dataset', 'pose_real',
- 'Target dataset to train on.')
-
-tf.app.flags.DEFINE_string('target_labeled_dataset', 'none',
- 'Target dataset to train on.')
-
-tf.app.flags.DEFINE_string('dataset_dir', None,
- 'The directory where the dataset files are stored.')
-
-tf.app.flags.DEFINE_string('master', '',
- 'BNS name of the TensorFlow master to use.')
-
-tf.app.flags.DEFINE_string('train_log_dir', '/tmp/da/',
- 'Directory where to write event logs.')
-
-tf.app.flags.DEFINE_string(
- 'layers_to_regularize', 'fc3',
- 'Comma-separated list of layer names to use MMD regularization on.')
-
-tf.app.flags.DEFINE_float('learning_rate', .01, 'The learning rate')
-
-tf.app.flags.DEFINE_float('alpha_weight', 1e-6,
- 'The coefficient for scaling the reconstruction '
- 'loss.')
-
-tf.app.flags.DEFINE_float(
- 'beta_weight', 1e-6,
- 'The coefficient for scaling the private/shared difference loss.')
-
-tf.app.flags.DEFINE_float(
- 'gamma_weight', 1e-6,
- 'The coefficient for scaling the shared encoding similarity loss.')
-
-tf.app.flags.DEFINE_float('pose_weight', 0.125,
- 'The coefficient for scaling the pose loss.')
-
-tf.app.flags.DEFINE_float(
- 'weight_decay', 1e-6,
- 'The coefficient for the L2 regularization applied for all weights.')
-
-tf.app.flags.DEFINE_integer(
- 'save_summaries_secs', 60,
- 'The frequency with which summaries are saved, in seconds.')
-
-tf.app.flags.DEFINE_integer(
- 'save_interval_secs', 60,
- 'The frequency with which the model is saved, in seconds.')
-
-tf.app.flags.DEFINE_integer(
- 'max_number_of_steps', None,
- 'The maximum number of gradient steps. Use None to train indefinitely.')
-
-tf.app.flags.DEFINE_integer(
- 'domain_separation_startpoint', 1,
- 'The global step to add the domain separation losses.')
-
-tf.app.flags.DEFINE_integer(
- 'bipartite_assignment_top_k', 3,
- 'The number of top-k matches to use in bipartite matching adaptation.')
-
-tf.app.flags.DEFINE_float('decay_rate', 0.95, 'Learning rate decay factor.')
-
-tf.app.flags.DEFINE_integer('decay_steps', 20000, 'Learning rate decay steps.')
-
-tf.app.flags.DEFINE_float('momentum', 0.9, 'The momentum value.')
-
-tf.app.flags.DEFINE_bool('use_separation', False,
- 'Use our domain separation model.')
-
-tf.app.flags.DEFINE_bool('use_logging', False, 'Debugging messages.')
-
-tf.app.flags.DEFINE_integer(
- 'ps_tasks', 0,
- 'The number of parameter servers. If the value is 0, then the parameters '
- 'are handled locally by the worker.')
-
-tf.app.flags.DEFINE_integer(
- 'num_readers', 4,
- 'The number of parallel readers that read data from the dataset.')
-
-tf.app.flags.DEFINE_integer('num_preprocessing_threads', 4,
- 'The number of threads used to create the batches.')
-
-tf.app.flags.DEFINE_integer(
- 'task', 0,
- 'The Task ID. This value is used when training with multiple workers to '
- 'identify each worker.')
-
-tf.app.flags.DEFINE_string('decoder_name', 'small_decoder',
- 'The decoder to use.')
-tf.app.flags.DEFINE_string('encoder_name', 'default_encoder',
- 'The encoder to use.')
-
-################################################################################
-# Flags that control the architecture and losses
-################################################################################
-tf.app.flags.DEFINE_string(
- 'similarity_loss', 'grl',
- 'The method to use for encouraging the common encoder codes to be '
- 'similar, one of "grl", "mmd", "corr".')
-
-tf.app.flags.DEFINE_string('recon_loss_name', 'sum_of_pairwise_squares',
- 'The name of the reconstruction loss.')
-
-tf.app.flags.DEFINE_string('basic_tower', 'pose_mini',
- 'The basic tower building block.')
-
-def provide_batch_fn():
- """ The provide_batch function to use. """
- return dataset_factory.provide_batch
-
-def main(_):
- model_params = {
- 'use_separation': FLAGS.use_separation,
- 'domain_separation_startpoint': FLAGS.domain_separation_startpoint,
- 'layers_to_regularize': FLAGS.layers_to_regularize,
- 'alpha_weight': FLAGS.alpha_weight,
- 'beta_weight': FLAGS.beta_weight,
- 'gamma_weight': FLAGS.gamma_weight,
- 'pose_weight': FLAGS.pose_weight,
- 'recon_loss_name': FLAGS.recon_loss_name,
- 'decoder_name': FLAGS.decoder_name,
- 'encoder_name': FLAGS.encoder_name,
- 'weight_decay': FLAGS.weight_decay,
- 'batch_size': FLAGS.batch_size,
- 'use_logging': FLAGS.use_logging,
- 'ps_tasks': FLAGS.ps_tasks,
- 'task': FLAGS.task,
- }
- g = tf.Graph()
- with g.as_default():
- with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
- # Load the data.
- source_images, source_labels = provide_batch_fn()(
- FLAGS.source_dataset, 'train', FLAGS.dataset_dir, FLAGS.num_readers,
- FLAGS.batch_size, FLAGS.num_preprocessing_threads)
- target_images, target_labels = provide_batch_fn()(
- FLAGS.target_dataset, 'train', FLAGS.dataset_dir, FLAGS.num_readers,
- FLAGS.batch_size, FLAGS.num_preprocessing_threads)
-
- # In the unsupervised case all the samples in the labeled
- # domain are from the source domain.
- domain_selection_mask = tf.fill((source_images.get_shape().as_list()[0],),
- True)
-
- # When using the semisupervised model we include labeled target data in
- # the source labelled data.
- if FLAGS.target_labeled_dataset != 'none':
- # 1000 is the maximum number of labelled target samples that exists in
- # the datasets.
- target_semi_images, target_semi_labels = provide_batch_fn()(
- FLAGS.target_labeled_dataset, 'train', FLAGS.batch_size)
-
- # Calculate the proportion of source domain samples in the semi-
- # supervised setting, so that the proportion is set accordingly in the
- # batches.
- proportion = float(source_labels['num_train_samples']) / (
- source_labels['num_train_samples'] +
- target_semi_labels['num_train_samples'])
-
- rnd_tensor = tf.random_uniform(
- (target_semi_images.get_shape().as_list()[0],))
-
- domain_selection_mask = rnd_tensor < proportion
- source_images = tf.where(domain_selection_mask, source_images,
- target_semi_images)
- source_class_labels = tf.where(domain_selection_mask,
- source_labels['classes'],
- target_semi_labels['classes'])
-
- if 'quaternions' in source_labels:
- source_pose_labels = tf.where(domain_selection_mask,
- source_labels['quaternions'],
- target_semi_labels['quaternions'])
- (source_images, source_class_labels, source_pose_labels,
- domain_selection_mask) = tf.train.shuffle_batch(
- [
- source_images, source_class_labels, source_pose_labels,
- domain_selection_mask
- ],
- FLAGS.batch_size,
- 50000,
- 5000,
- num_threads=1,
- enqueue_many=True)
-
- else:
- (source_images, source_class_labels,
- domain_selection_mask) = tf.train.shuffle_batch(
- [source_images, source_class_labels, domain_selection_mask],
- FLAGS.batch_size,
- 50000,
- 5000,
- num_threads=1,
- enqueue_many=True)
- source_labels = {}
- source_labels['classes'] = source_class_labels
- if 'quaternions' in source_labels:
- source_labels['quaternions'] = source_pose_labels
-
- slim.get_or_create_global_step()
- tf.summary.image('source_images', source_images, max_outputs=3)
- tf.summary.image('target_images', target_images, max_outputs=3)
-
- dsn.create_model(
- source_images,
- source_labels,
- domain_selection_mask,
- target_images,
- target_labels,
- FLAGS.similarity_loss,
- model_params,
- basic_tower_name=FLAGS.basic_tower)
-
- # Configure the optimization scheme:
- learning_rate = tf.train.exponential_decay(
- FLAGS.learning_rate,
- slim.get_or_create_global_step(),
- FLAGS.decay_steps,
- FLAGS.decay_rate,
- staircase=True,
- name='learning_rate')
-
- tf.summary.scalar('learning_rate', learning_rate)
- tf.summary.scalar('total_loss', tf.losses.get_total_loss())
-
- opt = tf.train.MomentumOptimizer(learning_rate, FLAGS.momentum)
- tf.logging.set_verbosity(tf.logging.INFO)
- # Run training.
- loss_tensor = slim.learning.create_train_op(
- slim.losses.get_total_loss(),
- opt,
- summarize_gradients=True,
- colocate_gradients_with_ops=True)
- slim.learning.train(
- train_op=loss_tensor,
- logdir=FLAGS.train_log_dir,
- master=FLAGS.master,
- is_chief=FLAGS.task == 0,
- number_of_steps=FLAGS.max_number_of_steps,
- save_summaries_secs=FLAGS.save_summaries_secs,
- save_interval_secs=FLAGS.save_interval_secs)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/domain_adaptation/domain_separation/grl_op_grads.py b/research/domain_adaptation/domain_separation/grl_op_grads.py
deleted file mode 100644
index fcd85ba2b5e7912bffe646a73558af8184812ea6..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/grl_op_grads.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Gradients for operators defined in grl_ops.py."""
-import tensorflow as tf
-
-
-@tf.RegisterGradient("GradientReversal")
-def _GradientReversalGrad(_, grad):
- """The gradients for `gradient_reversal`.
-
- Args:
- _: The `gradient_reversal` `Operation` that we are differentiating,
- which we can use to find the inputs and outputs of the original op.
- grad: Gradient with respect to the output of the `gradient_reversal` op.
-
- Returns:
- Gradient with respect to the input of `gradient_reversal`, which is simply
- the negative of the input gradient.
-
- """
- return tf.negative(grad)
diff --git a/research/domain_adaptation/domain_separation/grl_op_kernels.cc b/research/domain_adaptation/domain_separation/grl_op_kernels.cc
deleted file mode 100644
index ba30128f11e9e88c702d3a80593d930519f346fe..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/grl_op_kernels.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// This file contains the implementations of the ops registered in
-// grl_ops.cc.
-
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/types.pb.h"
-
-namespace tensorflow {
-
-// The gradient reversal op is used in domain adversarial training. It behaves
-// as the identity op during forward propagation, and multiplies its input by -1
-// during backward propagation.
-class GradientReversalOp : public OpKernel {
- public:
- explicit GradientReversalOp(OpKernelConstruction* context)
- : OpKernel(context) {}
-
- // Gradient reversal op behaves as the identity op during forward
- // propagation. Compute() function copied from the IdentityOp::Compute()
- // function here: third_party/tensorflow/core/kernels/identity_op.h.
- void Compute(OpKernelContext* context) override {
- if (IsRefType(context->input_dtype(0))) {
- context->forward_ref_input_to_ref_output(0, 0);
- } else {
- context->set_output(0, context->input(0));
- }
- }
-};
-
-REGISTER_KERNEL_BUILDER(Name("GradientReversal").Device(DEVICE_CPU),
- GradientReversalOp);
-
-} // namespace tensorflow
diff --git a/research/domain_adaptation/domain_separation/grl_op_shapes.py b/research/domain_adaptation/domain_separation/grl_op_shapes.py
deleted file mode 100644
index 52773c680af265beca9125e48bf68152b8a34e56..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/grl_op_shapes.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Shape inference for operators defined in grl_ops.cc."""
diff --git a/research/domain_adaptation/domain_separation/grl_ops.cc b/research/domain_adaptation/domain_separation/grl_ops.cc
deleted file mode 100644
index d441c2b484215605db65a043be6cfa0ab90da2c3..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/grl_ops.cc
+++ /dev/null
@@ -1,36 +0,0 @@
-/* Copyright 2016 The TensorFlow Authors All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Contains custom ops.
-
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/op.h"
-
-namespace tensorflow {
-
-// This custom op is used by adversarial training.
-REGISTER_OP("GradientReversal")
- .Input("input: float")
- .Output("output: float")
- .SetShapeFn(shape_inference::UnchangedShape)
- .Doc(R"doc(
-This op copies the input to the output during forward propagation, and
-negates the input during backward propagation.
-
-input: Tensor.
-output: Tensor, copied from input.
-)doc");
-
-} // namespace tensorflow
diff --git a/research/domain_adaptation/domain_separation/grl_ops.py b/research/domain_adaptation/domain_separation/grl_ops.py
deleted file mode 100644
index 50447247b10caf3e41f3c0fb1c6f943dd3d9de6e..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/grl_ops.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""GradientReversal op Python library."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os.path
-
-import tensorflow as tf
-
-tf.logging.info(tf.resource_loader.get_data_files_path())
-_grl_ops_module = tf.load_op_library(
- os.path.join(tf.resource_loader.get_data_files_path(),
- '_grl_ops.so'))
-gradient_reversal = _grl_ops_module.gradient_reversal
diff --git a/research/domain_adaptation/domain_separation/grl_ops_test.py b/research/domain_adaptation/domain_separation/grl_ops_test.py
deleted file mode 100644
index b431a6c02b60ade92a653d2ee8108c0586c70fbb..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/grl_ops_test.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for grl_ops."""
-
-#from models.domain_adaptation.domain_separation import grl_op_grads # pylint: disable=unused-import
-#from models.domain_adaptation.domain_separation import grl_op_shapes # pylint: disable=unused-import
-import tensorflow as tf
-
-import grl_op_grads
-import grl_ops
-
-FLAGS = tf.app.flags.FLAGS
-
-
-class GRLOpsTest(tf.test.TestCase):
-
- def testGradientReversalOp(self):
- with tf.Graph().as_default():
- with self.test_session():
- # Test that in forward prop, gradient reversal op acts as the
- # identity operation.
- examples = tf.constant([5.0, 4.0, 3.0, 2.0, 1.0])
- output = grl_ops.gradient_reversal(examples)
- expected_output = examples
- self.assertAllEqual(output.eval(), expected_output.eval())
-
- # Test that shape inference works as expected.
- self.assertAllEqual(output.get_shape(), expected_output.get_shape())
-
- # Test that in backward prop, gradient reversal op multiplies
- # gradients by -1.
- examples = tf.constant([[1.0]])
- w = tf.get_variable(name='w', shape=[1, 1])
- b = tf.get_variable(name='b', shape=[1])
- init_op = tf.global_variables_initializer()
- init_op.run()
- features = tf.nn.xw_plus_b(examples, w, b)
- # Construct two outputs: features layer passes directly to output1, but
- # features layer passes through a gradient reversal layer before
- # reaching output2.
- output1 = features
- output2 = grl_ops.gradient_reversal(features)
- gold = tf.constant([1.0])
- loss1 = gold - output1
- loss2 = gold - output2
- opt = tf.train.GradientDescentOptimizer(learning_rate=0.01)
- grads_and_vars_1 = opt.compute_gradients(loss1,
- tf.trainable_variables())
- grads_and_vars_2 = opt.compute_gradients(loss2,
- tf.trainable_variables())
- self.assertAllEqual(len(grads_and_vars_1), len(grads_and_vars_2))
- for i in range(len(grads_and_vars_1)):
- g1 = grads_and_vars_1[i][0]
- g2 = grads_and_vars_2[i][0]
- # Verify that gradients of loss1 are the negative of gradients of
- # loss2.
- self.assertAllEqual(tf.negative(g1).eval(), g2.eval())
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/domain_adaptation/domain_separation/losses.py b/research/domain_adaptation/domain_separation/losses.py
deleted file mode 100644
index 0d882340de10e4dd64d44f9357e8bfc5b1dd4712..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/losses.py
+++ /dev/null
@@ -1,290 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Domain Adaptation Loss Functions.
-
-The following domain adaptation loss functions are defined:
-
-- Maximum Mean Discrepancy (MMD).
- Relevant paper:
- Gretton, Arthur, et al.,
- "A kernel two-sample test."
- The Journal of Machine Learning Research, 2012
-
-- Correlation Loss on a batch.
-"""
-from functools import partial
-import tensorflow as tf
-
-import grl_op_grads # pylint: disable=unused-import
-import grl_op_shapes # pylint: disable=unused-import
-import grl_ops
-import utils
-slim = tf.contrib.slim
-
-
-################################################################################
-# SIMILARITY LOSS
-################################################################################
-def maximum_mean_discrepancy(x, y, kernel=utils.gaussian_kernel_matrix):
- r"""Computes the Maximum Mean Discrepancy (MMD) of two samples: x and y.
-
- Maximum Mean Discrepancy (MMD) is a distance-measure between the samples of
- the distributions of x and y. Here we use the kernel two sample estimate
- using the empirical mean of the two distributions.
-
- MMD^2(P, Q) = || \E{\phi(x)} - \E{\phi(y)} ||^2
- = \E{ K(x, x) } + \E{ K(y, y) } - 2 \E{ K(x, y) },
-
- where K = <\phi(x), \phi(y)>,
- is the desired kernel function, in this case a radial basis kernel.
-
- Args:
- x: a tensor of shape [num_samples, num_features]
- y: a tensor of shape [num_samples, num_features]
- kernel: a function which computes the kernel in MMD. Defaults to the
- GaussianKernelMatrix.
-
- Returns:
- a scalar denoting the squared maximum mean discrepancy loss.
- """
- with tf.name_scope('MaximumMeanDiscrepancy'):
- # \E{ K(x, x) } + \E{ K(y, y) } - 2 \E{ K(x, y) }
- cost = tf.reduce_mean(kernel(x, x))
- cost += tf.reduce_mean(kernel(y, y))
- cost -= 2 * tf.reduce_mean(kernel(x, y))
-
- # We do not allow the loss to become negative.
- cost = tf.where(cost > 0, cost, 0, name='value')
- return cost
-
-
-def mmd_loss(source_samples, target_samples, weight, scope=None):
- """Adds a similarity loss term, the MMD between two representations.
-
- This Maximum Mean Discrepancy (MMD) loss is calculated with a number of
- different Gaussian kernels.
-
- Args:
- source_samples: a tensor of shape [num_samples, num_features].
- target_samples: a tensor of shape [num_samples, num_features].
- weight: the weight of the MMD loss.
- scope: optional name scope for summary tags.
-
- Returns:
- a scalar tensor representing the MMD loss value.
- """
- sigmas = [
- 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 5, 10, 15, 20, 25, 30, 35, 100,
- 1e3, 1e4, 1e5, 1e6
- ]
- gaussian_kernel = partial(
- utils.gaussian_kernel_matrix, sigmas=tf.constant(sigmas))
-
- loss_value = maximum_mean_discrepancy(
- source_samples, target_samples, kernel=gaussian_kernel)
- loss_value = tf.maximum(1e-4, loss_value) * weight
- assert_op = tf.Assert(tf.is_finite(loss_value), [loss_value])
- with tf.control_dependencies([assert_op]):
- tag = 'MMD Loss'
- if scope:
- tag = scope + tag
- tf.summary.scalar(tag, loss_value)
- tf.losses.add_loss(loss_value)
-
- return loss_value
-
-
-def correlation_loss(source_samples, target_samples, weight, scope=None):
- """Adds a similarity loss term, the correlation between two representations.
-
- Args:
- source_samples: a tensor of shape [num_samples, num_features]
- target_samples: a tensor of shape [num_samples, num_features]
- weight: a scalar weight for the loss.
- scope: optional name scope for summary tags.
-
- Returns:
- a scalar tensor representing the correlation loss value.
- """
- with tf.name_scope('corr_loss'):
- source_samples -= tf.reduce_mean(source_samples, 0)
- target_samples -= tf.reduce_mean(target_samples, 0)
-
- source_samples = tf.nn.l2_normalize(source_samples, 1)
- target_samples = tf.nn.l2_normalize(target_samples, 1)
-
- source_cov = tf.matmul(tf.transpose(source_samples), source_samples)
- target_cov = tf.matmul(tf.transpose(target_samples), target_samples)
-
- corr_loss = tf.reduce_mean(tf.square(source_cov - target_cov)) * weight
-
- assert_op = tf.Assert(tf.is_finite(corr_loss), [corr_loss])
- with tf.control_dependencies([assert_op]):
- tag = 'Correlation Loss'
- if scope:
- tag = scope + tag
- tf.summary.scalar(tag, corr_loss)
- tf.losses.add_loss(corr_loss)
-
- return corr_loss
-
-
-def dann_loss(source_samples, target_samples, weight, scope=None):
- """Adds the domain adversarial (DANN) loss.
-
- Args:
- source_samples: a tensor of shape [num_samples, num_features].
- target_samples: a tensor of shape [num_samples, num_features].
- weight: the weight of the loss.
- scope: optional name scope for summary tags.
-
- Returns:
- a scalar tensor representing the correlation loss value.
- """
- with tf.variable_scope('dann'):
- batch_size = tf.shape(source_samples)[0]
- samples = tf.concat(axis=0, values=[source_samples, target_samples])
- samples = slim.flatten(samples)
-
- domain_selection_mask = tf.concat(
- axis=0, values=[tf.zeros((batch_size, 1)), tf.ones((batch_size, 1))])
-
- # Perform the gradient reversal and be careful with the shape.
- grl = grl_ops.gradient_reversal(samples)
- grl = tf.reshape(grl, (-1, samples.get_shape().as_list()[1]))
-
- grl = slim.fully_connected(grl, 100, scope='fc1')
- logits = slim.fully_connected(grl, 1, activation_fn=None, scope='fc2')
-
- domain_predictions = tf.sigmoid(logits)
-
- domain_loss = tf.losses.log_loss(
- domain_selection_mask, domain_predictions, weights=weight)
-
- domain_accuracy = utils.accuracy(
- tf.round(domain_predictions), domain_selection_mask)
-
- assert_op = tf.Assert(tf.is_finite(domain_loss), [domain_loss])
- with tf.control_dependencies([assert_op]):
- tag_loss = 'losses/domain_loss'
- tag_accuracy = 'losses/domain_accuracy'
- if scope:
- tag_loss = scope + tag_loss
- tag_accuracy = scope + tag_accuracy
-
- tf.summary.scalar(tag_loss, domain_loss)
- tf.summary.scalar(tag_accuracy, domain_accuracy)
-
- return domain_loss
-
-
-################################################################################
-# DIFFERENCE LOSS
-################################################################################
-def difference_loss(private_samples, shared_samples, weight=1.0, name=''):
- """Adds the difference loss between the private and shared representations.
-
- Args:
- private_samples: a tensor of shape [num_samples, num_features].
- shared_samples: a tensor of shape [num_samples, num_features].
- weight: the weight of the incoherence loss.
- name: the name of the tf summary.
- """
- private_samples -= tf.reduce_mean(private_samples, 0)
- shared_samples -= tf.reduce_mean(shared_samples, 0)
-
- private_samples = tf.nn.l2_normalize(private_samples, 1)
- shared_samples = tf.nn.l2_normalize(shared_samples, 1)
-
- correlation_matrix = tf.matmul(
- private_samples, shared_samples, transpose_a=True)
-
- cost = tf.reduce_mean(tf.square(correlation_matrix)) * weight
- cost = tf.where(cost > 0, cost, 0, name='value')
-
- tf.summary.scalar('losses/Difference Loss {}'.format(name),
- cost)
- assert_op = tf.Assert(tf.is_finite(cost), [cost])
- with tf.control_dependencies([assert_op]):
- tf.losses.add_loss(cost)
-
-
-################################################################################
-# TASK LOSS
-################################################################################
-def log_quaternion_loss_batch(predictions, labels, params):
- """A helper function to compute the error between quaternions.
-
- Args:
- predictions: A Tensor of size [batch_size, 4].
- labels: A Tensor of size [batch_size, 4].
- params: A dictionary of parameters. Expecting 'use_logging', 'batch_size'.
-
- Returns:
- A Tensor of size [batch_size], denoting the error between the quaternions.
- """
- use_logging = params['use_logging']
- assertions = []
- if use_logging:
- assertions.append(
- tf.Assert(
- tf.reduce_all(
- tf.less(
- tf.abs(tf.reduce_sum(tf.square(predictions), [1]) - 1),
- 1e-4)),
- ['The l2 norm of each prediction quaternion vector should be 1.']))
- assertions.append(
- tf.Assert(
- tf.reduce_all(
- tf.less(
- tf.abs(tf.reduce_sum(tf.square(labels), [1]) - 1), 1e-4)),
- ['The l2 norm of each label quaternion vector should be 1.']))
-
- with tf.control_dependencies(assertions):
- product = tf.multiply(predictions, labels)
- internal_dot_products = tf.reduce_sum(product, [1])
-
- if use_logging:
- internal_dot_products = tf.Print(
- internal_dot_products,
- [internal_dot_products, tf.shape(internal_dot_products)],
- 'internal_dot_products:')
-
- logcost = tf.log(1e-4 + 1 - tf.abs(internal_dot_products))
- return logcost
-
-
-def log_quaternion_loss(predictions, labels, params):
- """A helper function to compute the mean error between batches of quaternions.
-
- The caller is expected to add the loss to the graph.
-
- Args:
- predictions: A Tensor of size [batch_size, 4].
- labels: A Tensor of size [batch_size, 4].
- params: A dictionary of parameters. Expecting 'use_logging', 'batch_size'.
-
- Returns:
- A Tensor of size 1, denoting the mean error between batches of quaternions.
- """
- use_logging = params['use_logging']
- logcost = log_quaternion_loss_batch(predictions, labels, params)
- logcost = tf.reduce_sum(logcost, [0])
- batch_size = params['batch_size']
- logcost = tf.multiply(logcost, 1.0 / batch_size, name='log_quaternion_loss')
- if use_logging:
- logcost = tf.Print(
- logcost, [logcost], '[logcost]', name='log_quaternion_loss_print')
- return logcost
diff --git a/research/domain_adaptation/domain_separation/losses_test.py b/research/domain_adaptation/domain_separation/losses_test.py
deleted file mode 100644
index 46e50301be56f5977adcb3fb00587f076934b785..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/losses_test.py
+++ /dev/null
@@ -1,110 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for DSN losses."""
-from functools import partial
-
-import numpy as np
-import tensorflow as tf
-
-import losses
-import utils
-
-
-def MaximumMeanDiscrepancySlow(x, y, sigmas):
- num_samples = x.get_shape().as_list()[0]
-
- def AverageGaussianKernel(x, y, sigmas):
- result = 0
- for sigma in sigmas:
- dist = tf.reduce_sum(tf.square(x - y))
- result += tf.exp((-1.0 / (2.0 * sigma)) * dist)
- return result / num_samples**2
-
- total = 0
-
- for i in range(num_samples):
- for j in range(num_samples):
- total += AverageGaussianKernel(x[i, :], x[j, :], sigmas)
- total += AverageGaussianKernel(y[i, :], y[j, :], sigmas)
- total += -2 * AverageGaussianKernel(x[i, :], y[j, :], sigmas)
-
- return total
-
-
-class LogQuaternionLossTest(tf.test.TestCase):
-
- def test_log_quaternion_loss_batch(self):
- with self.test_session():
- predictions = tf.random_uniform((10, 4), seed=1)
- predictions = tf.nn.l2_normalize(predictions, 1)
- labels = tf.random_uniform((10, 4), seed=1)
- labels = tf.nn.l2_normalize(labels, 1)
- params = {'batch_size': 10, 'use_logging': False}
- x = losses.log_quaternion_loss_batch(predictions, labels, params)
- self.assertTrue(((10,) == tf.shape(x).eval()).all())
-
-
-class MaximumMeanDiscrepancyTest(tf.test.TestCase):
-
- def test_mmd_name(self):
- with self.test_session():
- x = tf.random_uniform((2, 3), seed=1)
- kernel = partial(utils.gaussian_kernel_matrix, sigmas=tf.constant([1.]))
- loss = losses.maximum_mean_discrepancy(x, x, kernel)
-
- self.assertEquals(loss.op.name, 'MaximumMeanDiscrepancy/value')
-
- def test_mmd_is_zero_when_inputs_are_same(self):
- with self.test_session():
- x = tf.random_uniform((2, 3), seed=1)
- kernel = partial(utils.gaussian_kernel_matrix, sigmas=tf.constant([1.]))
- self.assertEquals(0, losses.maximum_mean_discrepancy(x, x, kernel).eval())
-
- def test_fast_mmd_is_similar_to_slow_mmd(self):
- with self.test_session():
- x = tf.constant(np.random.normal(size=(2, 3)), tf.float32)
- y = tf.constant(np.random.rand(2, 3), tf.float32)
-
- cost_old = MaximumMeanDiscrepancySlow(x, y, [1.]).eval()
- kernel = partial(utils.gaussian_kernel_matrix, sigmas=tf.constant([1.]))
- cost_new = losses.maximum_mean_discrepancy(x, y, kernel).eval()
-
- self.assertAlmostEqual(cost_old, cost_new, delta=1e-5)
-
- def test_multiple_sigmas(self):
- with self.test_session():
- x = tf.constant(np.random.normal(size=(2, 3)), tf.float32)
- y = tf.constant(np.random.rand(2, 3), tf.float32)
-
- sigmas = tf.constant([2., 5., 10, 20, 30])
- kernel = partial(utils.gaussian_kernel_matrix, sigmas=sigmas)
- cost_old = MaximumMeanDiscrepancySlow(x, y, [2., 5., 10, 20, 30]).eval()
- cost_new = losses.maximum_mean_discrepancy(x, y, kernel=kernel).eval()
-
- self.assertAlmostEqual(cost_old, cost_new, delta=1e-5)
-
- def test_mmd_is_zero_when_distributions_are_same(self):
-
- with self.test_session():
- x = tf.random_uniform((1000, 10), seed=1)
- y = tf.random_uniform((1000, 10), seed=3)
-
- kernel = partial(utils.gaussian_kernel_matrix, sigmas=tf.constant([100.]))
- loss = losses.maximum_mean_discrepancy(x, y, kernel=kernel).eval()
-
- self.assertAlmostEqual(0, loss, delta=1e-4)
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/domain_adaptation/domain_separation/models.py b/research/domain_adaptation/domain_separation/models.py
deleted file mode 100644
index 04ccaf82eb9b31a6ea78871204c7df70eca3fbfd..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/models.py
+++ /dev/null
@@ -1,443 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Contains different architectures for the different DSN parts.
-
-We define here the modules that can be used in the different parts of the DSN
-model.
-- shared encoder (dsn_cropped_linemod, dann_xxxx)
-- private encoder (default_encoder)
-- decoder (large_decoder, gtsrb_decoder, small_decoder)
-"""
-import tensorflow as tf
-
-#from models.domain_adaptation.domain_separation
-import utils
-
-slim = tf.contrib.slim
-
-
-def default_batch_norm_params(is_training=False):
- """Returns default batch normalization parameters for DSNs.
-
- Args:
- is_training: whether or not the model is training.
-
- Returns:
- a dictionary that maps batch norm parameter names (strings) to values.
- """
- return {
- # Decay for the moving averages.
- 'decay': 0.5,
- # epsilon to prevent 0s in variance.
- 'epsilon': 0.001,
- 'is_training': is_training
- }
-
-
-################################################################################
-# PRIVATE ENCODERS
-################################################################################
-def default_encoder(images, code_size, batch_norm_params=None,
- weight_decay=0.0):
- """Encodes the given images to codes of the given size.
-
- Args:
- images: a tensor of size [batch_size, height, width, 1].
- code_size: the number of hidden units in the code layer of the classifier.
- batch_norm_params: a dictionary that maps batch norm parameter names to
- values.
- weight_decay: the value for the weight decay coefficient.
-
- Returns:
- end_points: the code of the input.
- """
- end_points = {}
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_params):
- with slim.arg_scope([slim.conv2d], kernel_size=[5, 5], padding='SAME'):
- net = slim.conv2d(images, 32, scope='conv1')
- net = slim.max_pool2d(net, [2, 2], 2, scope='pool1')
- net = slim.conv2d(net, 64, scope='conv2')
- net = slim.max_pool2d(net, [2, 2], 2, scope='pool2')
-
- net = slim.flatten(net)
- end_points['flatten'] = net
- net = slim.fully_connected(net, code_size, scope='fc1')
- end_points['fc3'] = net
- return end_points
-
-
-################################################################################
-# DECODERS
-################################################################################
-def large_decoder(codes,
- height,
- width,
- channels,
- batch_norm_params=None,
- weight_decay=0.0):
- """Decodes the codes to a fixed output size.
-
- Args:
- codes: a tensor of size [batch_size, code_size].
- height: the height of the output images.
- width: the width of the output images.
- channels: the number of the output channels.
- batch_norm_params: a dictionary that maps batch norm parameter names to
- values.
- weight_decay: the value for the weight decay coefficient.
-
- Returns:
- recons: the reconstruction tensor of shape [batch_size, height, width, 3].
- """
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_params):
- net = slim.fully_connected(codes, 600, scope='fc1')
- batch_size = net.get_shape().as_list()[0]
- net = tf.reshape(net, [batch_size, 10, 10, 6])
-
- net = slim.conv2d(net, 32, [5, 5], scope='conv1_1')
-
- net = tf.image.resize_nearest_neighbor(net, (16, 16))
-
- net = slim.conv2d(net, 32, [5, 5], scope='conv2_1')
-
- net = tf.image.resize_nearest_neighbor(net, (32, 32))
-
- net = slim.conv2d(net, 32, [5, 5], scope='conv3_2')
-
- output_size = [height, width]
- net = tf.image.resize_nearest_neighbor(net, output_size)
-
- with slim.arg_scope([slim.conv2d], kernel_size=[3, 3]):
- net = slim.conv2d(net, channels, activation_fn=None, scope='conv4_1')
-
- return net
-
-
-def gtsrb_decoder(codes,
- height,
- width,
- channels,
- batch_norm_params=None,
- weight_decay=0.0):
- """Decodes the codes to a fixed output size. This decoder is specific to GTSRB
-
- Args:
- codes: a tensor of size [batch_size, 100].
- height: the height of the output images.
- width: the width of the output images.
- channels: the number of the output channels.
- batch_norm_params: a dictionary that maps batch norm parameter names to
- values.
- weight_decay: the value for the weight decay coefficient.
-
- Returns:
- recons: the reconstruction tensor of shape [batch_size, height, width, 3].
-
- Raises:
- ValueError: When the input code size is not 100.
- """
- batch_size, code_size = codes.get_shape().as_list()
- if code_size != 100:
- raise ValueError('The code size used as an input to the GTSRB decoder is '
- 'expected to be 100.')
-
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_params):
- net = codes
- net = tf.reshape(net, [batch_size, 10, 10, 1])
- net = slim.conv2d(net, 32, [3, 3], scope='conv1_1')
-
- # First upsampling 20x20
- net = tf.image.resize_nearest_neighbor(net, [20, 20])
-
- net = slim.conv2d(net, 32, [3, 3], scope='conv2_1')
-
- output_size = [height, width]
- # Final upsampling 40 x 40
- net = tf.image.resize_nearest_neighbor(net, output_size)
-
- with slim.arg_scope([slim.conv2d], kernel_size=[3, 3]):
- net = slim.conv2d(net, 16, scope='conv3_1')
- net = slim.conv2d(net, channels, activation_fn=None, scope='conv3_2')
-
- return net
-
-
-def small_decoder(codes,
- height,
- width,
- channels,
- batch_norm_params=None,
- weight_decay=0.0):
- """Decodes the codes to a fixed output size.
-
- Args:
- codes: a tensor of size [batch_size, code_size].
- height: the height of the output images.
- width: the width of the output images.
- channels: the number of the output channels.
- batch_norm_params: a dictionary that maps batch norm parameter names to
- values.
- weight_decay: the value for the weight decay coefficient.
-
- Returns:
- recons: the reconstruction tensor of shape [batch_size, height, width, 3].
- """
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_params):
- net = slim.fully_connected(codes, 300, scope='fc1')
- batch_size = net.get_shape().as_list()[0]
- net = tf.reshape(net, [batch_size, 10, 10, 3])
-
- net = slim.conv2d(net, 16, [3, 3], scope='conv1_1')
- net = slim.conv2d(net, 16, [3, 3], scope='conv1_2')
-
- output_size = [height, width]
- net = tf.image.resize_nearest_neighbor(net, output_size)
-
- with slim.arg_scope([slim.conv2d], kernel_size=[3, 3]):
- net = slim.conv2d(net, 16, scope='conv2_1')
- net = slim.conv2d(net, channels, activation_fn=None, scope='conv2_2')
-
- return net
-
-
-################################################################################
-# SHARED ENCODERS
-################################################################################
-def dann_mnist(images,
- weight_decay=0.0,
- prefix='model',
- num_classes=10,
- **kwargs):
- """Creates a convolution MNIST model.
-
- Note that this model implements the architecture for MNIST proposed in:
- Y. Ganin et al., Domain-Adversarial Training of Neural Networks (DANN),
- JMLR 2015
-
- Args:
- images: the MNIST digits, a tensor of size [batch_size, 28, 28, 1].
- weight_decay: the value for the weight decay coefficient.
- prefix: name of the model to use when prefixing tags.
- num_classes: the number of output classes to use.
- **kwargs: Placeholder for keyword arguments used by other shared encoders.
-
- Returns:
- the output logits, a tensor of size [batch_size, num_classes].
- a dictionary with key/values the layer names and tensors.
- """
- end_points = {}
-
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- activation_fn=tf.nn.relu,):
- with slim.arg_scope([slim.conv2d], padding='SAME'):
- end_points['conv1'] = slim.conv2d(images, 32, [5, 5], scope='conv1')
- end_points['pool1'] = slim.max_pool2d(
- end_points['conv1'], [2, 2], 2, scope='pool1')
- end_points['conv2'] = slim.conv2d(
- end_points['pool1'], 48, [5, 5], scope='conv2')
- end_points['pool2'] = slim.max_pool2d(
- end_points['conv2'], [2, 2], 2, scope='pool2')
- end_points['fc3'] = slim.fully_connected(
- slim.flatten(end_points['pool2']), 100, scope='fc3')
- end_points['fc4'] = slim.fully_connected(
- slim.flatten(end_points['fc3']), 100, scope='fc4')
-
- logits = slim.fully_connected(
- end_points['fc4'], num_classes, activation_fn=None, scope='fc5')
-
- return logits, end_points
-
-
-def dann_svhn(images,
- weight_decay=0.0,
- prefix='model',
- num_classes=10,
- **kwargs):
- """Creates the convolutional SVHN model.
-
- Note that this model implements the architecture for MNIST proposed in:
- Y. Ganin et al., Domain-Adversarial Training of Neural Networks (DANN),
- JMLR 2015
-
- Args:
- images: the SVHN digits, a tensor of size [batch_size, 32, 32, 3].
- weight_decay: the value for the weight decay coefficient.
- prefix: name of the model to use when prefixing tags.
- num_classes: the number of output classes to use.
- **kwargs: Placeholder for keyword arguments used by other shared encoders.
-
- Returns:
- the output logits, a tensor of size [batch_size, num_classes].
- a dictionary with key/values the layer names and tensors.
- """
-
- end_points = {}
-
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- activation_fn=tf.nn.relu,):
- with slim.arg_scope([slim.conv2d], padding='SAME'):
-
- end_points['conv1'] = slim.conv2d(images, 64, [5, 5], scope='conv1')
- end_points['pool1'] = slim.max_pool2d(
- end_points['conv1'], [3, 3], 2, scope='pool1')
- end_points['conv2'] = slim.conv2d(
- end_points['pool1'], 64, [5, 5], scope='conv2')
- end_points['pool2'] = slim.max_pool2d(
- end_points['conv2'], [3, 3], 2, scope='pool2')
- end_points['conv3'] = slim.conv2d(
- end_points['pool2'], 128, [5, 5], scope='conv3')
-
- end_points['fc3'] = slim.fully_connected(
- slim.flatten(end_points['conv3']), 3072, scope='fc3')
- end_points['fc4'] = slim.fully_connected(
- slim.flatten(end_points['fc3']), 2048, scope='fc4')
-
- logits = slim.fully_connected(
- end_points['fc4'], num_classes, activation_fn=None, scope='fc5')
-
- return logits, end_points
-
-
-def dann_gtsrb(images,
- weight_decay=0.0,
- prefix='model',
- num_classes=43,
- **kwargs):
- """Creates the convolutional GTSRB model.
-
- Note that this model implements the architecture for MNIST proposed in:
- Y. Ganin et al., Domain-Adversarial Training of Neural Networks (DANN),
- JMLR 2015
-
- Args:
- images: the GTSRB images, a tensor of size [batch_size, 40, 40, 3].
- weight_decay: the value for the weight decay coefficient.
- prefix: name of the model to use when prefixing tags.
- num_classes: the number of output classes to use.
- **kwargs: Placeholder for keyword arguments used by other shared encoders.
-
- Returns:
- the output logits, a tensor of size [batch_size, num_classes].
- a dictionary with key/values the layer names and tensors.
- """
-
- end_points = {}
-
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- activation_fn=tf.nn.relu,):
- with slim.arg_scope([slim.conv2d], padding='SAME'):
-
- end_points['conv1'] = slim.conv2d(images, 96, [5, 5], scope='conv1')
- end_points['pool1'] = slim.max_pool2d(
- end_points['conv1'], [2, 2], 2, scope='pool1')
- end_points['conv2'] = slim.conv2d(
- end_points['pool1'], 144, [3, 3], scope='conv2')
- end_points['pool2'] = slim.max_pool2d(
- end_points['conv2'], [2, 2], 2, scope='pool2')
- end_points['conv3'] = slim.conv2d(
- end_points['pool2'], 256, [5, 5], scope='conv3')
- end_points['pool3'] = slim.max_pool2d(
- end_points['conv3'], [2, 2], 2, scope='pool3')
-
- end_points['fc3'] = slim.fully_connected(
- slim.flatten(end_points['pool3']), 512, scope='fc3')
-
- logits = slim.fully_connected(
- end_points['fc3'], num_classes, activation_fn=None, scope='fc4')
-
- return logits, end_points
-
-
-def dsn_cropped_linemod(images,
- weight_decay=0.0,
- prefix='model',
- num_classes=11,
- batch_norm_params=None,
- is_training=False):
- """Creates the convolutional pose estimation model for Cropped Linemod.
-
- Args:
- images: the Cropped Linemod samples, a tensor of size
- [batch_size, 64, 64, 4].
- weight_decay: the value for the weight decay coefficient.
- prefix: name of the model to use when prefixing tags.
- num_classes: the number of output classes to use.
- batch_norm_params: a dictionary that maps batch norm parameter names to
- values.
- is_training: specifies whether or not we're currently training the model.
- This variable will determine the behaviour of the dropout layer.
-
- Returns:
- the output logits, a tensor of size [batch_size, num_classes].
- a dictionary with key/values the layer names and tensors.
- """
-
- end_points = {}
-
- tf.summary.image('{}/input_images'.format(prefix), images)
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=slim.l2_regularizer(weight_decay),
- activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm if batch_norm_params else None,
- normalizer_params=batch_norm_params):
- with slim.arg_scope([slim.conv2d], padding='SAME'):
- end_points['conv1'] = slim.conv2d(images, 32, [5, 5], scope='conv1')
- end_points['pool1'] = slim.max_pool2d(
- end_points['conv1'], [2, 2], 2, scope='pool1')
- end_points['conv2'] = slim.conv2d(
- end_points['pool1'], 64, [5, 5], scope='conv2')
- end_points['pool2'] = slim.max_pool2d(
- end_points['conv2'], [2, 2], 2, scope='pool2')
- net = slim.flatten(end_points['pool2'])
- end_points['fc3'] = slim.fully_connected(net, 128, scope='fc3')
- net = slim.dropout(
- end_points['fc3'], 0.5, is_training=is_training, scope='dropout')
-
- with tf.variable_scope('quaternion_prediction'):
- predicted_quaternion = slim.fully_connected(
- net, 4, activation_fn=tf.nn.tanh)
- predicted_quaternion = tf.nn.l2_normalize(predicted_quaternion, 1)
- logits = slim.fully_connected(
- net, num_classes, activation_fn=None, scope='fc4')
- end_points['quaternion_pred'] = predicted_quaternion
-
- return logits, end_points
diff --git a/research/domain_adaptation/domain_separation/models_test.py b/research/domain_adaptation/domain_separation/models_test.py
deleted file mode 100644
index 69d1a27259022569cc5865e49dd6bba5675d834f..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/models_test.py
+++ /dev/null
@@ -1,167 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for DSN components."""
-
-import numpy as np
-import tensorflow as tf
-
-#from models.domain_adaptation.domain_separation
-import models
-
-
-class SharedEncodersTest(tf.test.TestCase):
-
- def _testSharedEncoder(self,
- input_shape=[5, 28, 28, 1],
- model=models.dann_mnist,
- is_training=True):
- images = tf.to_float(np.random.rand(*input_shape))
-
- with self.test_session() as sess:
- logits, _ = model(images)
- sess.run(tf.global_variables_initializer())
- logits_np = sess.run(logits)
- return logits_np
-
- def testBuildGRLMnistModel(self):
- logits = self._testSharedEncoder(model=getattr(models,
- 'dann_mnist'))
- self.assertEqual(logits.shape, (5, 10))
- self.assertTrue(np.any(logits))
-
- def testBuildGRLSvhnModel(self):
- logits = self._testSharedEncoder(model=getattr(models,
- 'dann_svhn'))
- self.assertEqual(logits.shape, (5, 10))
- self.assertTrue(np.any(logits))
-
- def testBuildGRLGtsrbModel(self):
- logits = self._testSharedEncoder([5, 40, 40, 3],
- getattr(models, 'dann_gtsrb'))
- self.assertEqual(logits.shape, (5, 43))
- self.assertTrue(np.any(logits))
-
- def testBuildPoseModel(self):
- logits = self._testSharedEncoder([5, 64, 64, 4],
- getattr(models, 'dsn_cropped_linemod'))
- self.assertEqual(logits.shape, (5, 11))
- self.assertTrue(np.any(logits))
-
- def testBuildPoseModelWithBatchNorm(self):
- images = tf.to_float(np.random.rand(10, 64, 64, 4))
-
- with self.test_session() as sess:
- logits, _ = getattr(models, 'dsn_cropped_linemod')(
- images, batch_norm_params=models.default_batch_norm_params(True))
- sess.run(tf.global_variables_initializer())
- logits_np = sess.run(logits)
- self.assertEqual(logits_np.shape, (10, 11))
- self.assertTrue(np.any(logits_np))
-
-
-class EncoderTest(tf.test.TestCase):
-
- def _testEncoder(self, batch_norm_params=None, channels=1):
- images = tf.to_float(np.random.rand(10, 28, 28, channels))
-
- with self.test_session() as sess:
- end_points = models.default_encoder(
- images, 128, batch_norm_params=batch_norm_params)
- sess.run(tf.global_variables_initializer())
- private_code = sess.run(end_points['fc3'])
- self.assertEqual(private_code.shape, (10, 128))
- self.assertTrue(np.any(private_code))
- self.assertTrue(np.all(np.isfinite(private_code)))
-
- def testEncoder(self):
- self._testEncoder()
-
- def testEncoderMultiChannel(self):
- self._testEncoder(None, 4)
-
- def testEncoderIsTrainingBatchNorm(self):
- self._testEncoder(models.default_batch_norm_params(True))
-
- def testEncoderBatchNorm(self):
- self._testEncoder(models.default_batch_norm_params(False))
-
-
-class DecoderTest(tf.test.TestCase):
-
- def _testDecoder(self,
- height=64,
- width=64,
- channels=4,
- batch_norm_params=None,
- decoder=models.small_decoder):
- codes = tf.to_float(np.random.rand(32, 100))
-
- with self.test_session() as sess:
- output = decoder(
- codes,
- height=height,
- width=width,
- channels=channels,
- batch_norm_params=batch_norm_params)
- sess.run(tf.global_variables_initializer())
- output_np = sess.run(output)
- self.assertEqual(output_np.shape, (32, height, width, channels))
- self.assertTrue(np.any(output_np))
- self.assertTrue(np.all(np.isfinite(output_np)))
-
- def testSmallDecoder(self):
- self._testDecoder(28, 28, 4, None, getattr(models, 'small_decoder'))
-
- def testSmallDecoderThreeChannels(self):
- self._testDecoder(28, 28, 3)
-
- def testSmallDecoderBatchNorm(self):
- self._testDecoder(28, 28, 4, models.default_batch_norm_params(False))
-
- def testSmallDecoderIsTrainingBatchNorm(self):
- self._testDecoder(28, 28, 4, models.default_batch_norm_params(True))
-
- def testLargeDecoder(self):
- self._testDecoder(32, 32, 4, None, getattr(models, 'large_decoder'))
-
- def testLargeDecoderThreeChannels(self):
- self._testDecoder(32, 32, 3, None, getattr(models, 'large_decoder'))
-
- def testLargeDecoderBatchNorm(self):
- self._testDecoder(32, 32, 4,
- models.default_batch_norm_params(False),
- getattr(models, 'large_decoder'))
-
- def testLargeDecoderIsTrainingBatchNorm(self):
- self._testDecoder(32, 32, 4,
- models.default_batch_norm_params(True),
- getattr(models, 'large_decoder'))
-
- def testGtsrbDecoder(self):
- self._testDecoder(40, 40, 3, None, getattr(models, 'large_decoder'))
-
- def testGtsrbDecoderBatchNorm(self):
- self._testDecoder(40, 40, 4,
- models.default_batch_norm_params(False),
- getattr(models, 'gtsrb_decoder'))
-
- def testGtsrbDecoderIsTrainingBatchNorm(self):
- self._testDecoder(40, 40, 4,
- models.default_batch_norm_params(True),
- getattr(models, 'gtsrb_decoder'))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/domain_adaptation/domain_separation/utils.py b/research/domain_adaptation/domain_separation/utils.py
deleted file mode 100644
index e144ee86120bd58eb06b710fb35f3f58b5a05343..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/domain_separation/utils.py
+++ /dev/null
@@ -1,183 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Auxiliary functions for domain adaptation related losses.
-"""
-import math
-import tensorflow as tf
-
-
-def create_summaries(end_points, prefix='', max_images=3, use_op_name=False):
- """Creates a tf summary per endpoint.
-
- If the endpoint is a 4 dimensional tensor it displays it as an image
- otherwise if it is a two dimensional one it creates a histogram summary.
-
- Args:
- end_points: a dictionary of name, tf tensor pairs.
- prefix: an optional string to prefix the summary with.
- max_images: the maximum number of images to display per summary.
- use_op_name: Use the op name as opposed to the shorter end_points key.
- """
- for layer_name in end_points:
- if use_op_name:
- name = end_points[layer_name].op.name
- else:
- name = layer_name
- if len(end_points[layer_name].get_shape().as_list()) == 4:
- # if it's an actual image do not attempt to reshape it
- if end_points[layer_name].get_shape().as_list()[-1] == 1 or end_points[
- layer_name].get_shape().as_list()[-1] == 3:
- visualization_image = end_points[layer_name]
- else:
- visualization_image = reshape_feature_maps(end_points[layer_name])
- tf.summary.image(
- '{}/{}'.format(prefix, name),
- visualization_image,
- max_outputs=max_images)
- elif len(end_points[layer_name].get_shape().as_list()) == 3:
- images = tf.expand_dims(end_points[layer_name], 3)
- tf.summary.image(
- '{}/{}'.format(prefix, name),
- images,
- max_outputs=max_images)
- elif len(end_points[layer_name].get_shape().as_list()) == 2:
- tf.summary.histogram('{}/{}'.format(prefix, name), end_points[layer_name])
-
-
-def reshape_feature_maps(features_tensor):
- """Reshape activations for tf.summary.image visualization.
-
- Arguments:
- features_tensor: a tensor of activations with a square number of feature
- maps, eg 4, 9, 16, etc.
- Returns:
- A composite image with all the feature maps that can be passed as an
- argument to tf.summary.image.
- """
- assert len(features_tensor.get_shape().as_list()) == 4
- num_filters = features_tensor.get_shape().as_list()[-1]
- assert num_filters > 0
- num_filters_sqrt = math.sqrt(num_filters)
- assert num_filters_sqrt.is_integer(
- ), 'Number of filters should be a square number but got {}'.format(
- num_filters)
- num_filters_sqrt = int(num_filters_sqrt)
- conv_summary = tf.unstack(features_tensor, axis=3)
- conv_one_row = tf.concat(axis=2, values=conv_summary[0:num_filters_sqrt])
- ind = 1
- conv_final = conv_one_row
- for ind in range(1, num_filters_sqrt):
- conv_one_row = tf.concat(axis=2,
- values=conv_summary[
- ind * num_filters_sqrt + 0:ind * num_filters_sqrt + num_filters_sqrt])
- conv_final = tf.concat(
- axis=1, values=[tf.squeeze(conv_final), tf.squeeze(conv_one_row)])
- conv_final = tf.expand_dims(conv_final, -1)
- return conv_final
-
-
-def accuracy(predictions, labels):
- """Calculates the classificaton accuracy.
-
- Args:
- predictions: the predicted values, a tensor whose size matches 'labels'.
- labels: the ground truth values, a tensor of any size.
-
- Returns:
- a tensor whose value on evaluation returns the total accuracy.
- """
- return tf.reduce_mean(tf.cast(tf.equal(predictions, labels), tf.float32))
-
-
-def compute_upsample_values(input_tensor, upsample_height, upsample_width):
- """Compute values for an upsampling op (ops.BatchCropAndResize).
-
- Args:
- input_tensor: image tensor with shape [batch, height, width, in_channels]
- upsample_height: integer
- upsample_width: integer
-
- Returns:
- grid_centers: tensor with shape [batch, 1]
- crop_sizes: tensor with shape [batch, 1]
- output_height: integer
- output_width: integer
- """
- batch, input_height, input_width, _ = input_tensor.shape
-
- height_half = input_height / 2.
- width_half = input_width / 2.
- grid_centers = tf.constant(batch * [[height_half, width_half]])
- crop_sizes = tf.constant(batch * [[input_height, input_width]])
- output_height = input_height * upsample_height
- output_width = input_width * upsample_width
-
- return grid_centers, tf.to_float(crop_sizes), output_height, output_width
-
-
-def compute_pairwise_distances(x, y):
- """Computes the squared pairwise Euclidean distances between x and y.
-
- Args:
- x: a tensor of shape [num_x_samples, num_features]
- y: a tensor of shape [num_y_samples, num_features]
-
- Returns:
- a distance matrix of dimensions [num_x_samples, num_y_samples].
-
- Raises:
- ValueError: if the inputs do no matched the specified dimensions.
- """
-
- if not len(x.get_shape()) == len(y.get_shape()) == 2:
- raise ValueError('Both inputs should be matrices.')
-
- if x.get_shape().as_list()[1] != y.get_shape().as_list()[1]:
- raise ValueError('The number of features should be the same.')
-
- norm = lambda x: tf.reduce_sum(tf.square(x), 1)
-
- # By making the `inner' dimensions of the two matrices equal to 1 using
- # broadcasting then we are essentially substracting every pair of rows
- # of x and y.
- # x will be num_samples x num_features x 1,
- # and y will be 1 x num_features x num_samples (after broadcasting).
- # After the substraction we will get a
- # num_x_samples x num_features x num_y_samples matrix.
- # The resulting dist will be of shape num_y_samples x num_x_samples.
- # and thus we need to transpose it again.
- return tf.transpose(norm(tf.expand_dims(x, 2) - tf.transpose(y)))
-
-
-def gaussian_kernel_matrix(x, y, sigmas):
- r"""Computes a Guassian Radial Basis Kernel between the samples of x and y.
-
- We create a sum of multiple gaussian kernels each having a width sigma_i.
-
- Args:
- x: a tensor of shape [num_samples, num_features]
- y: a tensor of shape [num_samples, num_features]
- sigmas: a tensor of floats which denote the widths of each of the
- gaussians in the kernel.
- Returns:
- A tensor of shape [num_samples{x}, num_samples{y}] with the RBF kernel.
- """
- beta = 1. / (2. * (tf.expand_dims(sigmas, 1)))
-
- dist = compute_pairwise_distances(x, y)
-
- s = tf.matmul(beta, tf.reshape(dist, (1, -1)))
-
- return tf.reshape(tf.reduce_sum(tf.exp(-s), 0), tf.shape(dist))
diff --git a/research/domain_adaptation/pixel_domain_adaptation/BUILD b/research/domain_adaptation/pixel_domain_adaptation/BUILD
deleted file mode 100644
index 2bc8d4a49a828f97b8f45166aa2bbc552d4a3b92..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/BUILD
+++ /dev/null
@@ -1,90 +0,0 @@
-# Description:
-# Contains code for domain-adaptation style transfer.
-
-package(
- default_visibility = [
- ":internal",
- ],
-)
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-package_group(
- name = "internal",
- packages = [
- "//domain_adaptation/...",
- ],
-)
-
-py_library(
- name = "pixelda_preprocess",
- srcs = ["pixelda_preprocess.py"],
- deps = [
-
- ],
-)
-
-py_test(
- name = "pixelda_preprocess_test",
- srcs = ["pixelda_preprocess_test.py"],
- deps = [
- ":pixelda_preprocess",
-
- ],
-)
-
-py_library(
- name = "pixelda_model",
- srcs = [
- "pixelda_model.py",
- "pixelda_task_towers.py",
- "hparams.py",
- ],
- deps = [
-
- ],
-)
-
-py_library(
- name = "pixelda_utils",
- srcs = ["pixelda_utils.py"],
- deps = [
-
- ],
-)
-
-py_library(
- name = "pixelda_losses",
- srcs = ["pixelda_losses.py"],
- deps = [
-
- ],
-)
-
-py_binary(
- name = "pixelda_train",
- srcs = ["pixelda_train.py"],
- deps = [
- ":pixelda_losses",
- ":pixelda_model",
- ":pixelda_preprocess",
- ":pixelda_utils",
-
- "//domain_adaptation/datasets:dataset_factory",
- ],
-)
-
-py_binary(
- name = "pixelda_eval",
- srcs = ["pixelda_eval.py"],
- deps = [
- ":pixelda_losses",
- ":pixelda_model",
- ":pixelda_preprocess",
- ":pixelda_utils",
-
- "//domain_adaptation/datasets:dataset_factory",
- ],
-)
diff --git a/research/domain_adaptation/pixel_domain_adaptation/README.md b/research/domain_adaptation/pixel_domain_adaptation/README.md
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/domain_adaptation/pixel_domain_adaptation/baselines/BUILD b/research/domain_adaptation/pixel_domain_adaptation/baselines/BUILD
deleted file mode 100644
index c41a4ffeee80114145c4c3fc32a2191879b1b08a..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/baselines/BUILD
+++ /dev/null
@@ -1,23 +0,0 @@
-licenses(["notice"]) # Apache 2.0
-
-py_binary(
- name = "baseline_train",
- srcs = ["baseline_train.py"],
- deps = [
-
- "//domain_adaptation/datasets:dataset_factory",
- "//domain_adaptation/pixel_domain_adaptation:pixelda_model",
- "//domain_adaptation/pixel_domain_adaptation:pixelda_preprocess",
- ],
-)
-
-py_binary(
- name = "baseline_eval",
- srcs = ["baseline_eval.py"],
- deps = [
-
- "//domain_adaptation/datasets:dataset_factory",
- "//domain_adaptation/pixel_domain_adaptation:pixelda_model",
- "//domain_adaptation/pixel_domain_adaptation:pixelda_preprocess",
- ],
-)
diff --git a/research/domain_adaptation/pixel_domain_adaptation/baselines/README.md b/research/domain_adaptation/pixel_domain_adaptation/baselines/README.md
deleted file mode 100644
index d61195ad2de6867801143aeda906cb5efe30a5e3..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/baselines/README.md
+++ /dev/null
@@ -1,60 +0,0 @@
-The best baselines are obtainable via the following configuration:
-
-
-## MNIST => MNIST_M
-
-Accuracy:
-MNIST-Train: 99.9
-MNIST_M-Train: 63.9
-MNIST_M-Valid: 63.9
-MNIST_M-Test: 63.6
-
-Learning Rate = 0.0001
-Weight Decay = 0.0
-Number of Steps: 105,000
-
-## MNIST => USPS
-
-Accuracy:
-MNIST-Train: 100.0
-USPS-Train: 82.8
-USPS-Valid: 82.8
-USPS-Test: 78.9
-
-Learning Rate = 0.0001
-Weight Decay = 0.0
-Number of Steps: 22,000
-
-## MNIST_M => MNIST
-
-Accuracy:
-MNIST_M-Train: 100
-MNIST-Train: 98.5
-MNIST-Valid: 98.5
-MNIST-Test: 98.1
-
-Learning Rate = 0.001
-Weight Decay = 0.0
-Number of Steps: 604,400
-
-## MNIST_M => MNIST_M
-
-Accuracy:
-MNIST_M-Train: 100.0
-MNIST_M-Valid: 96.6
-MNIST_M-Test: 96.4
-
-Learning Rate = 0.001
-Weight Decay = 0.0
-Number of Steps: 139,400
-
-## USPS => USPS
-
-Accuracy:
-USPS-Train: 100.0
-USPS-Valid: 100.0
-USPS-Test: 96.5
-
-Learning Rate = 0.001
-Weight Decay = 0.0
-Number of Steps: 67,000
diff --git a/research/domain_adaptation/pixel_domain_adaptation/baselines/baseline_eval.py b/research/domain_adaptation/pixel_domain_adaptation/baselines/baseline_eval.py
deleted file mode 100644
index 6b7ef6452b4897b00dc8c977bf40526ad5052ede..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/baselines/baseline_eval.py
+++ /dev/null
@@ -1,141 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-r"""Evals the classification/pose baselines."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from functools import partial
-
-import math
-
-# Dependency imports
-
-import tensorflow as tf
-
-from domain_adaptation.datasets import dataset_factory
-from domain_adaptation.pixel_domain_adaptation import pixelda_preprocess
-from domain_adaptation.pixel_domain_adaptation import pixelda_task_towers
-
-flags = tf.app.flags
-FLAGS = flags.FLAGS
-
-slim = tf.contrib.slim
-
-flags.DEFINE_string('master', '', 'BNS name of the tensorflow server')
-
-flags.DEFINE_string(
- 'checkpoint_dir', None, 'The location of the checkpoint files.')
-
-flags.DEFINE_string(
- 'eval_dir', None, 'The directory where evaluation logs are written.')
-
-flags.DEFINE_integer('batch_size', 32, 'The number of samples per batch.')
-
-flags.DEFINE_string('dataset_name', None, 'The name of the dataset.')
-
-flags.DEFINE_string('dataset_dir', None,
- 'The directory where the data is stored.')
-
-flags.DEFINE_string('split_name', None, 'The name of the train/test split.')
-
-flags.DEFINE_integer('eval_interval_secs', 60 * 5,
- 'How often (in seconds) to run evaluation.')
-
-flags.DEFINE_integer(
- 'num_readers', 4,
- 'The number of parallel readers that read data from the dataset.')
-
-def main(unused_argv):
- tf.logging.set_verbosity(tf.logging.INFO)
- hparams = tf.contrib.training.HParams()
- hparams.weight_decay_task_classifier = 0.0
-
- if FLAGS.dataset_name in ['mnist', 'mnist_m', 'usps']:
- hparams.task_tower = 'mnist'
- else:
- raise ValueError('Unknown dataset %s' % FLAGS.dataset_name)
-
- if not tf.gfile.Exists(FLAGS.eval_dir):
- tf.gfile.MakeDirs(FLAGS.eval_dir)
-
- with tf.Graph().as_default():
- dataset = dataset_factory.get_dataset(FLAGS.dataset_name, FLAGS.split_name,
- FLAGS.dataset_dir)
- num_classes = dataset.num_classes
- num_samples = dataset.num_samples
-
- preprocess_fn = partial(pixelda_preprocess.preprocess_classification,
- is_training=False)
-
- images, labels = dataset_factory.provide_batch(
- FLAGS.dataset_name,
- FLAGS.split_name,
- dataset_dir=FLAGS.dataset_dir,
- num_readers=FLAGS.num_readers,
- batch_size=FLAGS.batch_size,
- num_preprocessing_threads=FLAGS.num_readers)
-
- # Define the model
- logits, _ = pixelda_task_towers.add_task_specific_model(
- images, hparams, num_classes=num_classes, is_training=True)
-
- #####################
- # Define the losses #
- #####################
- if 'classes' in labels:
- one_hot_labels = labels['classes']
- loss = tf.losses.softmax_cross_entropy(
- onehot_labels=one_hot_labels, logits=logits)
- tf.summary.scalar('losses/Classification_Loss', loss)
- else:
- raise ValueError('Only support classification for now.')
-
- total_loss = tf.losses.get_total_loss()
-
- predictions = tf.reshape(tf.argmax(logits, 1), shape=[-1])
- class_labels = tf.argmax(labels['classes'], 1)
-
- metrics_to_values, metrics_to_updates = slim.metrics.aggregate_metric_map({
- 'Mean_Loss':
- tf.contrib.metrics.streaming_mean(total_loss),
- 'Accuracy':
- tf.contrib.metrics.streaming_accuracy(predictions,
- tf.reshape(
- class_labels,
- shape=[-1])),
- 'Recall_at_5':
- tf.contrib.metrics.streaming_recall_at_k(logits, class_labels, 5),
- })
-
- tf.summary.histogram('outputs/Predictions', predictions)
- tf.summary.histogram('outputs/Ground_Truth', class_labels)
-
- for name, value in metrics_to_values.iteritems():
- tf.summary.scalar(name, value)
-
- num_batches = int(math.ceil(num_samples / float(FLAGS.batch_size)))
-
- slim.evaluation.evaluation_loop(
- master=FLAGS.master,
- checkpoint_dir=FLAGS.checkpoint_dir,
- logdir=FLAGS.eval_dir,
- num_evals=num_batches,
- eval_op=metrics_to_updates.values(),
- eval_interval_secs=FLAGS.eval_interval_secs)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/domain_adaptation/pixel_domain_adaptation/baselines/baseline_train.py b/research/domain_adaptation/pixel_domain_adaptation/baselines/baseline_train.py
deleted file mode 100644
index 8c92bd81a7b68879000dd793ba2fd013f395f408..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/baselines/baseline_train.py
+++ /dev/null
@@ -1,161 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-r"""Trains the classification/pose baselines."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from functools import partial
-
-# Dependency imports
-
-import tensorflow as tf
-
-from domain_adaptation.datasets import dataset_factory
-from domain_adaptation.pixel_domain_adaptation import pixelda_preprocess
-from domain_adaptation.pixel_domain_adaptation import pixelda_task_towers
-
-flags = tf.app.flags
-FLAGS = flags.FLAGS
-
-slim = tf.contrib.slim
-
-flags.DEFINE_string('master', '', 'BNS name of the tensorflow server')
-
-flags.DEFINE_integer('task', 0, 'The task ID.')
-
-flags.DEFINE_integer('num_ps_tasks', 0,
- 'The number of parameter servers. If the value is 0, then '
- 'the parameters are handled locally by the worker.')
-
-flags.DEFINE_integer('batch_size', 32, 'The number of samples per batch.')
-
-flags.DEFINE_string('dataset_name', None, 'The name of the dataset.')
-
-flags.DEFINE_string('dataset_dir', None,
- 'The directory where the data is stored.')
-
-flags.DEFINE_string('split_name', None, 'The name of the train/test split.')
-
-flags.DEFINE_float('learning_rate', 0.001, 'The initial learning rate.')
-
-flags.DEFINE_integer(
- 'learning_rate_decay_steps', 20000,
- 'The frequency, in steps, at which the learning rate is decayed.')
-
-flags.DEFINE_float('learning_rate_decay_factor',
- 0.95,
- 'The factor with which the learning rate is decayed.')
-
-flags.DEFINE_float('adam_beta1', 0.5, 'The beta1 value for the AdamOptimizer')
-
-flags.DEFINE_float('weight_decay', 1e-5,
- 'The L2 coefficient on the model weights.')
-
-flags.DEFINE_string(
- 'logdir', None, 'The location of the logs and checkpoints.')
-
-flags.DEFINE_integer('save_interval_secs', 600,
- 'How often, in seconds, we save the model to disk.')
-
-flags.DEFINE_integer('save_summaries_secs', 600,
- 'How often, in seconds, we compute the summaries.')
-
-flags.DEFINE_integer(
- 'num_readers', 4,
- 'The number of parallel readers that read data from the dataset.')
-
-flags.DEFINE_float(
- 'moving_average_decay', 0.9999,
- 'The amount of decay to use for moving averages.')
-
-
-def main(unused_argv):
- tf.logging.set_verbosity(tf.logging.INFO)
- hparams = tf.contrib.training.HParams()
- hparams.weight_decay_task_classifier = FLAGS.weight_decay
-
- if FLAGS.dataset_name in ['mnist', 'mnist_m', 'usps']:
- hparams.task_tower = 'mnist'
- else:
- raise ValueError('Unknown dataset %s' % FLAGS.dataset_name)
-
- with tf.Graph().as_default():
- with tf.device(
- tf.train.replica_device_setter(FLAGS.num_ps_tasks, merge_devices=True)):
- dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
- FLAGS.split_name, FLAGS.dataset_dir)
- num_classes = dataset.num_classes
-
- preprocess_fn = partial(pixelda_preprocess.preprocess_classification,
- is_training=True)
-
- images, labels = dataset_factory.provide_batch(
- FLAGS.dataset_name,
- FLAGS.split_name,
- dataset_dir=FLAGS.dataset_dir,
- num_readers=FLAGS.num_readers,
- batch_size=FLAGS.batch_size,
- num_preprocessing_threads=FLAGS.num_readers)
- # preprocess_fn=preprocess_fn)
-
- # Define the model
- logits, _ = pixelda_task_towers.add_task_specific_model(
- images, hparams, num_classes=num_classes, is_training=True)
-
- # Define the losses
- if 'classes' in labels:
- one_hot_labels = labels['classes']
- loss = tf.losses.softmax_cross_entropy(
- onehot_labels=one_hot_labels, logits=logits)
- tf.summary.scalar('losses/Classification_Loss', loss)
- else:
- raise ValueError('Only support classification for now.')
-
- total_loss = tf.losses.get_total_loss()
- tf.summary.scalar('losses/Total_Loss', total_loss)
-
- # Setup the moving averages
- moving_average_variables = slim.get_model_variables()
- variable_averages = tf.train.ExponentialMovingAverage(
- FLAGS.moving_average_decay, slim.get_or_create_global_step())
- tf.add_to_collection(
- tf.GraphKeys.UPDATE_OPS,
- variable_averages.apply(moving_average_variables))
-
- # Specify the optimization scheme:
- learning_rate = tf.train.exponential_decay(
- FLAGS.learning_rate,
- slim.get_or_create_global_step(),
- FLAGS.learning_rate_decay_steps,
- FLAGS.learning_rate_decay_factor,
- staircase=True)
-
- optimizer = tf.train.AdamOptimizer(learning_rate, beta1=FLAGS.adam_beta1)
-
- train_op = slim.learning.create_train_op(total_loss, optimizer)
-
- slim.learning.train(
- train_op,
- FLAGS.logdir,
- master=FLAGS.master,
- is_chief=(FLAGS.task == 0),
- save_summaries_secs=FLAGS.save_summaries_secs,
- save_interval_secs=FLAGS.save_interval_secs)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/domain_adaptation/pixel_domain_adaptation/hparams.py b/research/domain_adaptation/pixel_domain_adaptation/hparams.py
deleted file mode 100644
index ba9539f7d435c86f9fc92ed3406835bdaf2b50f3..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/hparams.py
+++ /dev/null
@@ -1,201 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Define model HParams."""
-import tensorflow as tf
-
-
-def create_hparams(hparam_string=None):
- """Create model hyperparameters. Parse nondefault from given string."""
- hparams = tf.contrib.training.HParams(
- # The name of the architecture to use.
- arch='resnet',
- lrelu_leakiness=0.2,
- batch_norm_decay=0.9,
- weight_decay=1e-5,
- normal_init_std=0.02,
- generator_kernel_size=3,
- discriminator_kernel_size=3,
-
- # Stop training after this many examples are processed
- # If none, train indefinitely
- num_training_examples=0,
-
- # Apply data augmentation to datasets
- # Applies only in training job
- augment_source_images=False,
- augment_target_images=False,
-
- # Discriminator
- # Number of filters in first layer of discriminator
- num_discriminator_filters=64,
- discriminator_conv_block_size=1, # How many convs to have at each size
- discriminator_filter_factor=2.0, # Multiply # filters by this each layer
- # Add gaussian noise with this stddev to every hidden layer of D
- discriminator_noise_stddev=0.2, # lmetz: Start seeing results at >= 0.1
- # If true, add this gaussian noise to input images to D as well
- discriminator_image_noise=False,
- discriminator_first_stride=1, # Stride in first conv of discriminator
- discriminator_do_pooling=False, # If true, replace stride 2 with avg pool
- discriminator_dropout_keep_prob=0.9, # keep probability for dropout
-
- # DCGAN Generator
- # Number of filters in generator decoder last layer (repeatedly halved
- # from 1st layer)
- num_decoder_filters=64,
- # Number of filters in generator encoder 1st layer (repeatedly doubled
- # after 1st layer)
- num_encoder_filters=64,
-
- # This is the shape to which the noise vector is projected (if we're
- # transferring from noise).
- # Write this way instead of [4, 4, 64] for hparam search flexibility
- projection_shape_size=4,
- projection_shape_channels=64,
-
- # Indicates the method by which we enlarge the spatial representation
- # of an image. Possible values include:
- # - resize_conv: Performs a nearest neighbor resize followed by a conv.
- # - conv2d_transpose: Performs a conv2d_transpose.
- upsample_method='resize_conv',
-
- # Visualization
- summary_steps=500, # Output image summary every N steps
-
- ###################################
- # Task Classifier Hyperparameters #
- ###################################
-
- # Which task-specific prediction tower to use. Possible choices are:
- # none: No task tower.
- # doubling_pose_estimator: classifier + quaternion regressor.
- # [conv + pool]* + FC
- # Classifiers used in DSN paper:
- # gtsrb: Classifier used for GTSRB
- # svhn: Classifier used for SVHN
- # mnist: Classifier used for MNIST
- # pose_mini: Classifier + regressor used for pose_mini
- task_tower='doubling_pose_estimator',
- weight_decay_task_classifier=1e-5,
- source_task_loss_weight=1.0,
- transferred_task_loss_weight=1.0,
-
- # Number of private layers in doubling_pose_estimator task tower
- num_private_layers=2,
-
- # The weight for the log quaternion loss we use for source and transferred
- # samples of the cropped_linemod dataset.
- # In the DSN work, 1/8 of the classifier weight worked well for our log
- # quaternion loss
- source_pose_weight=0.125 * 2.0,
- transferred_pose_weight=0.125 * 1.0,
-
- # If set to True, the style transfer network also attempts to change its
- # weights to maximize the performance of the task tower. If set to False,
- # then the style transfer network only attempts to change its weights to
- # make the transferred images more likely according to the domain
- # classifier.
- task_tower_in_g_step=True,
- task_loss_in_g_weight=1.0, # Weight of task loss in G
-
- #########################################
- # 'simple` generator arch model hparams #
- #########################################
- simple_num_conv_layers=1,
- simple_conv_filters=8,
-
- #########################
- # Resnet Hyperparameters#
- #########################
- resnet_blocks=6, # Number of resnet blocks
- resnet_filters=64, # Number of filters per conv in resnet blocks
- # If true, add original input back to result of convolutions inside the
- # resnet arch. If false, it turns into a simple stack of conv/relu/BN
- # layers.
- resnet_residuals=True,
-
- #######################################
- # The residual / interpretable model. #
- #######################################
- res_int_blocks=2, # The number of residual blocks.
- res_int_convs=2, # The number of conv calls inside each block.
- res_int_filters=64, # The number of filters used by each convolution.
-
- ####################
- # Latent variables #
- ####################
- # if true, then generate random noise and project to input for generator
- noise_channel=True,
- # The number of dimensions in the input noise vector.
- noise_dims=10,
-
- # If true, then one hot encode source image class and project as an
- # additional channel for the input to generator. This gives the generator
- # access to the class, which may help generation performance.
- condition_on_source_class=False,
-
- ########################
- # Loss Hyperparameters #
- ########################
- domain_loss_weight=1.0,
- style_transfer_loss_weight=1.0,
-
- ########################################################################
- # Encourages the transferred images to be similar to the source images #
- # using a configurable metric. #
- ########################################################################
-
- # The weight of the loss function encouraging the source and transferred
- # images to be similar. If set to 0, then the loss function is not used.
- transferred_similarity_loss_weight=0.0,
-
- # The type of loss used to encourage transferred and source image
- # similarity. Valid values include:
- # mpse: Mean Pairwise Squared Error
- # mse: Mean Squared Error
- # hinged_mse: Computes the mean squared error using squared differences
- # greater than hparams.transferred_similarity_max_diff
- # hinged_mae: Computes the mean absolute error using absolute
- # differences greater than hparams.transferred_similarity_max_diff.
- transferred_similarity_loss='mpse',
-
- # The maximum allowable difference between the source and target images.
- # This value is used, in effect, to produce a hinge loss. Note that the
- # range of values should be between 0 and 1.
- transferred_similarity_max_diff=0.4,
-
- ################################
- # Optimization Hyperparameters #
- ################################
- learning_rate=0.001,
- batch_size=32,
- lr_decay_steps=20000,
- lr_decay_rate=0.95,
-
- # Recomendation from the DCGAN paper:
- adam_beta1=0.5,
- clip_gradient_norm=5.0,
-
- # The number of times we run the discriminator train_op in a row.
- discriminator_steps=1,
-
- # The number of times we run the generator train_op in a row.
- generator_steps=1)
-
- if hparam_string:
- tf.logging.info('Parsing command line hparams: %s', hparam_string)
- hparams.parse(hparam_string)
-
- tf.logging.info('Final parsed hparams: %s', hparams.values())
- return hparams
diff --git a/research/domain_adaptation/pixel_domain_adaptation/pixelda_eval.py b/research/domain_adaptation/pixel_domain_adaptation/pixelda_eval.py
deleted file mode 100644
index 23824249a9e95586ed85e40cd89c5f6814977969..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/pixelda_eval.py
+++ /dev/null
@@ -1,298 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-r"""Evaluates the PIXELDA model.
-
--- Compiles the model for CPU.
-$ bazel build -c opt third_party/tensorflow_models/domain_adaptation/pixel_domain_adaptation:pixelda_eval
-
--- Compile the model for GPU.
-$ bazel build -c opt --copt=-mavx --config=cuda \
- third_party/tensorflow_models/domain_adaptation/pixel_domain_adaptation:pixelda_eval
-
--- Runs the training.
-$ ./bazel-bin/third_party/tensorflow_models/domain_adaptation/pixel_domain_adaptation/pixelda_eval \
- --source_dataset=mnist \
- --target_dataset=mnist_m \
- --dataset_dir=/tmp/datasets/ \
- --alsologtostderr
-
--- Visualize the results.
-$ bash learning/brain/tensorboard/tensorboard.sh \
- --port 2222 --logdir=/tmp/pixelda/
-"""
-from functools import partial
-import math
-
-# Dependency imports
-
-import tensorflow as tf
-
-from domain_adaptation.datasets import dataset_factory
-from domain_adaptation.pixel_domain_adaptation import pixelda_model
-from domain_adaptation.pixel_domain_adaptation import pixelda_preprocess
-from domain_adaptation.pixel_domain_adaptation import pixelda_utils
-from domain_adaptation.pixel_domain_adaptation import pixelda_losses
-from domain_adaptation.pixel_domain_adaptation.hparams import create_hparams
-
-slim = tf.contrib.slim
-
-flags = tf.app.flags
-FLAGS = flags.FLAGS
-
-flags.DEFINE_string('master', '', 'BNS name of the TensorFlow master to use.')
-
-flags.DEFINE_string('checkpoint_dir', '/tmp/pixelda/',
- 'Directory where the model was written to.')
-
-flags.DEFINE_string('eval_dir', '/tmp/pixelda/',
- 'Directory where the results are saved to.')
-
-flags.DEFINE_integer('eval_interval_secs', 60,
- 'The frequency, in seconds, with which evaluation is run.')
-
-flags.DEFINE_string('target_split_name', 'test',
- 'The name of the train/test split.')
-flags.DEFINE_string('source_split_name', 'train', 'Split for source dataset.'
- ' Defaults to train.')
-
-flags.DEFINE_string('source_dataset', 'mnist',
- 'The name of the source dataset.')
-
-flags.DEFINE_string('target_dataset', 'mnist_m',
- 'The name of the target dataset.')
-
-flags.DEFINE_string(
- 'dataset_dir',
- '', # None,
- 'The directory where the datasets can be found.')
-
-flags.DEFINE_integer(
- 'num_readers', 4,
- 'The number of parallel readers that read data from the dataset.')
-
-flags.DEFINE_integer('num_preprocessing_threads', 4,
- 'The number of threads used to create the batches.')
-
-# HParams
-
-flags.DEFINE_string('hparams', '', 'Comma separated hyperparameter values')
-
-
-def run_eval(run_dir, checkpoint_dir, hparams):
- """Runs the eval loop.
-
- Args:
- run_dir: The directory where eval specific logs are placed
- checkpoint_dir: The directory where the checkpoints are stored
- hparams: The hyperparameters struct.
-
- Raises:
- ValueError: if hparams.arch is not recognized.
- """
- for checkpoint_path in slim.evaluation.checkpoints_iterator(
- checkpoint_dir, FLAGS.eval_interval_secs):
- with tf.Graph().as_default():
- #########################
- # Preprocess the inputs #
- #########################
- target_dataset = dataset_factory.get_dataset(
- FLAGS.target_dataset,
- split_name=FLAGS.target_split_name,
- dataset_dir=FLAGS.dataset_dir)
- target_images, target_labels = dataset_factory.provide_batch(
- FLAGS.target_dataset, FLAGS.target_split_name, FLAGS.dataset_dir,
- FLAGS.num_readers, hparams.batch_size,
- FLAGS.num_preprocessing_threads)
- num_target_classes = target_dataset.num_classes
- target_labels['class'] = tf.argmax(target_labels['classes'], 1)
- del target_labels['classes']
-
- if hparams.arch not in ['dcgan']:
- source_dataset = dataset_factory.get_dataset(
- FLAGS.source_dataset,
- split_name=FLAGS.source_split_name,
- dataset_dir=FLAGS.dataset_dir)
- num_source_classes = source_dataset.num_classes
- source_images, source_labels = dataset_factory.provide_batch(
- FLAGS.source_dataset, FLAGS.source_split_name, FLAGS.dataset_dir,
- FLAGS.num_readers, hparams.batch_size,
- FLAGS.num_preprocessing_threads)
- source_labels['class'] = tf.argmax(source_labels['classes'], 1)
- del source_labels['classes']
- if num_source_classes != num_target_classes:
- raise ValueError(
- 'Input and output datasets must have same number of classes')
- else:
- source_images = None
- source_labels = None
-
- ####################
- # Define the model #
- ####################
- end_points = pixelda_model.create_model(
- hparams,
- target_images,
- source_images=source_images,
- source_labels=source_labels,
- is_training=False,
- num_classes=num_target_classes)
-
- #######################
- # Metrics & Summaries #
- #######################
- names_to_values, names_to_updates = create_metrics(end_points,
- source_labels,
- target_labels, hparams)
- pixelda_utils.summarize_model(end_points)
- pixelda_utils.summarize_transferred_grid(
- end_points['transferred_images'], source_images, name='Transferred')
- if 'source_images_recon' in end_points:
- pixelda_utils.summarize_transferred_grid(
- end_points['source_images_recon'],
- source_images,
- name='Source Reconstruction')
- pixelda_utils.summarize_images(target_images, 'Target')
-
- for name, value in names_to_values.iteritems():
- tf.summary.scalar(name, value)
-
- # Use the entire split by default
- num_examples = target_dataset.num_samples
-
- num_batches = math.ceil(num_examples / float(hparams.batch_size))
- global_step = slim.get_or_create_global_step()
-
- result = slim.evaluation.evaluate_once(
- master=FLAGS.master,
- checkpoint_path=checkpoint_path,
- logdir=run_dir,
- num_evals=num_batches,
- eval_op=names_to_updates.values(),
- final_op=names_to_values)
-
-
-def to_degrees(log_quaternion_loss):
- """Converts a log quaternion distance to an angle.
-
- Args:
- log_quaternion_loss: The log quaternion distance between two
- unit quaternions (or a batch of pairs of quaternions).
-
- Returns:
- The angle in degrees of the implied angle-axis representation.
- """
- return tf.acos(-(tf.exp(log_quaternion_loss) - 1)) * 2 * 180 / math.pi
-
-
-def create_metrics(end_points, source_labels, target_labels, hparams):
- """Create metrics for the model.
-
- Args:
- end_points: A dictionary of end point name to tensor
- source_labels: Labels for source images. batch_size x 1
- target_labels: Labels for target images. batch_size x 1
- hparams: The hyperparameters struct.
-
- Returns:
- Tuple of (names_to_values, names_to_updates), dictionaries that map a metric
- name to its value and update op, respectively
-
- """
- ###########################################
- # Evaluate the Domain Prediction Accuracy #
- ###########################################
- batch_size = hparams.batch_size
- names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
- ('eval/Domain_Accuracy-Transferred'):
- tf.contrib.metrics.streaming_accuracy(
- tf.to_int32(
- tf.round(tf.sigmoid(end_points[
- 'transferred_domain_logits']))),
- tf.zeros(batch_size, dtype=tf.int32)),
- ('eval/Domain_Accuracy-Target'):
- tf.contrib.metrics.streaming_accuracy(
- tf.to_int32(
- tf.round(tf.sigmoid(end_points['target_domain_logits']))),
- tf.ones(batch_size, dtype=tf.int32))
- })
-
- ################################
- # Evaluate the task classifier #
- ################################
- if 'source_task_logits' in end_points:
- metric_name = 'eval/Task_Accuracy-Source'
- names_to_values[metric_name], names_to_updates[
- metric_name] = tf.contrib.metrics.streaming_accuracy(
- tf.argmax(end_points['source_task_logits'], 1),
- source_labels['class'])
-
- if 'transferred_task_logits' in end_points:
- metric_name = 'eval/Task_Accuracy-Transferred'
- names_to_values[metric_name], names_to_updates[
- metric_name] = tf.contrib.metrics.streaming_accuracy(
- tf.argmax(end_points['transferred_task_logits'], 1),
- source_labels['class'])
-
- if 'target_task_logits' in end_points:
- metric_name = 'eval/Task_Accuracy-Target'
- names_to_values[metric_name], names_to_updates[
- metric_name] = tf.contrib.metrics.streaming_accuracy(
- tf.argmax(end_points['target_task_logits'], 1),
- target_labels['class'])
-
- ##########################################################################
- # Pose data-specific losses.
- ##########################################################################
- if 'quaternion' in source_labels.keys():
- params = {}
- params['use_logging'] = False
- params['batch_size'] = batch_size
-
- angle_loss_source = to_degrees(
- pixelda_losses.log_quaternion_loss_batch(end_points[
- 'source_quaternion'], source_labels['quaternion'], params))
- angle_loss_transferred = to_degrees(
- pixelda_losses.log_quaternion_loss_batch(end_points[
- 'transferred_quaternion'], source_labels['quaternion'], params))
- angle_loss_target = to_degrees(
- pixelda_losses.log_quaternion_loss_batch(end_points[
- 'target_quaternion'], target_labels['quaternion'], params))
-
- metric_name = 'eval/Angle_Loss-Source'
- names_to_values[metric_name], names_to_updates[
- metric_name] = slim.metrics.mean(angle_loss_source)
-
- metric_name = 'eval/Angle_Loss-Transferred'
- names_to_values[metric_name], names_to_updates[
- metric_name] = slim.metrics.mean(angle_loss_transferred)
-
- metric_name = 'eval/Angle_Loss-Target'
- names_to_values[metric_name], names_to_updates[
- metric_name] = slim.metrics.mean(angle_loss_target)
-
- return names_to_values, names_to_updates
-
-
-def main(_):
- tf.logging.set_verbosity(tf.logging.INFO)
- hparams = create_hparams(FLAGS.hparams)
- run_eval(
- run_dir=FLAGS.eval_dir,
- checkpoint_dir=FLAGS.checkpoint_dir,
- hparams=hparams)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/domain_adaptation/pixel_domain_adaptation/pixelda_losses.py b/research/domain_adaptation/pixel_domain_adaptation/pixelda_losses.py
deleted file mode 100644
index cf39765d4d28c5a04cb8868cdc465cdd0129b0df..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/pixelda_losses.py
+++ /dev/null
@@ -1,385 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Defines the various loss functions in use by the PIXELDA model."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-
-def add_domain_classifier_losses(end_points, hparams):
- """Adds losses related to the domain-classifier.
-
- Args:
- end_points: A map of network end point names to `Tensors`.
- hparams: The hyperparameters struct.
-
- Returns:
- loss: A `Tensor` representing the total task-classifier loss.
- """
- if hparams.domain_loss_weight == 0:
- tf.logging.info(
- 'Domain classifier loss weight is 0, so not creating losses.')
- return 0
-
- # The domain prediction loss is minimized with respect to the domain
- # classifier features only. Its aim is to predict the domain of the images.
- # Note: 1 = 'real image' label, 0 = 'fake image' label
- transferred_domain_loss = tf.losses.sigmoid_cross_entropy(
- multi_class_labels=tf.zeros_like(end_points['transferred_domain_logits']),
- logits=end_points['transferred_domain_logits'])
- tf.summary.scalar('Domain_loss_transferred', transferred_domain_loss)
-
- target_domain_loss = tf.losses.sigmoid_cross_entropy(
- multi_class_labels=tf.ones_like(end_points['target_domain_logits']),
- logits=end_points['target_domain_logits'])
- tf.summary.scalar('Domain_loss_target', target_domain_loss)
-
- # Compute the total domain loss:
- total_domain_loss = transferred_domain_loss + target_domain_loss
- total_domain_loss *= hparams.domain_loss_weight
- tf.summary.scalar('Domain_loss_total', total_domain_loss)
-
- return total_domain_loss
-
-def log_quaternion_loss_batch(predictions, labels, params):
- """A helper function to compute the error between quaternions.
-
- Args:
- predictions: A Tensor of size [batch_size, 4].
- labels: A Tensor of size [batch_size, 4].
- params: A dictionary of parameters. Expecting 'use_logging', 'batch_size'.
-
- Returns:
- A Tensor of size [batch_size], denoting the error between the quaternions.
- """
- use_logging = params['use_logging']
- assertions = []
- if use_logging:
- assertions.append(
- tf.Assert(
- tf.reduce_all(
- tf.less(
- tf.abs(tf.reduce_sum(tf.square(predictions), [1]) - 1),
- 1e-4)),
- ['The l2 norm of each prediction quaternion vector should be 1.']))
- assertions.append(
- tf.Assert(
- tf.reduce_all(
- tf.less(
- tf.abs(tf.reduce_sum(tf.square(labels), [1]) - 1), 1e-4)),
- ['The l2 norm of each label quaternion vector should be 1.']))
-
- with tf.control_dependencies(assertions):
- product = tf.multiply(predictions, labels)
- internal_dot_products = tf.reduce_sum(product, [1])
-
- if use_logging:
- internal_dot_products = tf.Print(internal_dot_products, [
- internal_dot_products,
- tf.shape(internal_dot_products)
- ], 'internal_dot_products:')
-
- logcost = tf.log(1e-4 + 1 - tf.abs(internal_dot_products))
- return logcost
-
-
-def log_quaternion_loss(predictions, labels, params):
- """A helper function to compute the mean error between batches of quaternions.
-
- The caller is expected to add the loss to the graph.
-
- Args:
- predictions: A Tensor of size [batch_size, 4].
- labels: A Tensor of size [batch_size, 4].
- params: A dictionary of parameters. Expecting 'use_logging', 'batch_size'.
-
- Returns:
- A Tensor of size 1, denoting the mean error between batches of quaternions.
- """
- use_logging = params['use_logging']
- logcost = log_quaternion_loss_batch(predictions, labels, params)
- logcost = tf.reduce_sum(logcost, [0])
- batch_size = params['batch_size']
- logcost = tf.multiply(logcost, 1.0 / batch_size, name='log_quaternion_loss')
- if use_logging:
- logcost = tf.Print(
- logcost, [logcost], '[logcost]', name='log_quaternion_loss_print')
- return logcost
-
-def _quaternion_loss(labels, predictions, weight, batch_size, domain,
- add_summaries):
- """Creates a Quaternion Loss.
-
- Args:
- labels: The true quaternions.
- predictions: The predicted quaternions.
- weight: A scalar weight.
- batch_size: The size of the batches.
- domain: The name of the domain from which the labels were taken.
- add_summaries: Whether or not to add summaries for the losses.
-
- Returns:
- A `Tensor` representing the loss.
- """
- assert domain in ['Source', 'Transferred']
-
- params = {'use_logging': False, 'batch_size': batch_size}
- loss = weight * log_quaternion_loss(labels, predictions, params)
-
- if add_summaries:
- assert_op = tf.Assert(tf.is_finite(loss), [loss])
- with tf.control_dependencies([assert_op]):
- tf.summary.histogram(
- 'Log_Quaternion_Loss_%s' % domain, loss, collections='losses')
- tf.summary.scalar(
- 'Task_Quaternion_Loss_%s' % domain, loss, collections='losses')
-
- return loss
-
-
-def _add_task_specific_losses(end_points, source_labels, num_classes, hparams,
- add_summaries=False):
- """Adds losses related to the task-classifier.
-
- Args:
- end_points: A map of network end point names to `Tensors`.
- source_labels: A dictionary of output labels to `Tensors`.
- num_classes: The number of classes used by the classifier.
- hparams: The hyperparameters struct.
- add_summaries: Whether or not to add the summaries.
-
- Returns:
- loss: A `Tensor` representing the total task-classifier loss.
- """
- # TODO(ddohan): Make sure the l2 regularization is added to the loss
-
- one_hot_labels = slim.one_hot_encoding(source_labels['class'], num_classes)
- total_loss = 0
-
- if 'source_task_logits' in end_points:
- loss = tf.losses.softmax_cross_entropy(
- onehot_labels=one_hot_labels,
- logits=end_points['source_task_logits'],
- weights=hparams.source_task_loss_weight)
- if add_summaries:
- tf.summary.scalar('Task_Classifier_Loss_Source', loss)
- total_loss += loss
-
- if 'transferred_task_logits' in end_points:
- loss = tf.losses.softmax_cross_entropy(
- onehot_labels=one_hot_labels,
- logits=end_points['transferred_task_logits'],
- weights=hparams.transferred_task_loss_weight)
- if add_summaries:
- tf.summary.scalar('Task_Classifier_Loss_Transferred', loss)
- total_loss += loss
-
- #########################
- # Pose specific losses. #
- #########################
- if 'quaternion' in source_labels:
- total_loss += _quaternion_loss(
- source_labels['quaternion'],
- end_points['source_quaternion'],
- hparams.source_pose_weight,
- hparams.batch_size,
- 'Source',
- add_summaries)
-
- total_loss += _quaternion_loss(
- source_labels['quaternion'],
- end_points['transferred_quaternion'],
- hparams.transferred_pose_weight,
- hparams.batch_size,
- 'Transferred',
- add_summaries)
-
- if add_summaries:
- tf.summary.scalar('Task_Loss_Total', total_loss)
-
- return total_loss
-
-
-def _transferred_similarity_loss(reconstructions,
- source_images,
- weight=1.0,
- method='mse',
- max_diff=0.4,
- name='similarity'):
- """Computes a loss encouraging similarity between source and transferred.
-
- Args:
- reconstructions: A `Tensor` of shape [batch_size, height, width, channels]
- source_images: A `Tensor` of shape [batch_size, height, width, channels].
- weight: Multiple similarity loss by this weight before returning
- method: One of:
- mpse = Mean Pairwise Squared Error
- mse = Mean Squared Error
- hinged_mse = Computes the mean squared error using squared differences
- greater than hparams.transferred_similarity_max_diff
- hinged_mae = Computes the mean absolute error using absolute
- differences greater than hparams.transferred_similarity_max_diff.
- max_diff: Maximum unpenalized difference for hinged losses
- name: Identifying name to use for creating summaries
-
-
- Returns:
- A `Tensor` representing the transferred similarity loss.
-
- Raises:
- ValueError: if `method` is not recognized.
- """
- if weight == 0:
- return 0
-
- source_channels = source_images.shape.as_list()[-1]
- reconstruction_channels = reconstructions.shape.as_list()[-1]
-
- # Convert grayscale source to RGB if target is RGB
- if source_channels == 1 and reconstruction_channels != 1:
- source_images = tf.tile(source_images, [1, 1, 1, reconstruction_channels])
- if reconstruction_channels == 1 and source_channels != 1:
- reconstructions = tf.tile(reconstructions, [1, 1, 1, source_channels])
-
- if method == 'mpse':
- reconstruction_similarity_loss_fn = (
- tf.contrib.losses.mean_pairwise_squared_error)
- elif method == 'masked_mpse':
-
- def masked_mpse(predictions, labels, weight):
- """Masked mpse assuming we have a depth to create a mask from."""
- assert labels.shape.as_list()[-1] == 4
- mask = tf.to_float(tf.less(labels[:, :, :, 3:4], 0.99))
- mask = tf.tile(mask, [1, 1, 1, 4])
- predictions *= mask
- labels *= mask
- tf.image_summary('masked_pred', predictions)
- tf.image_summary('masked_label', labels)
- return tf.contrib.losses.mean_pairwise_squared_error(
- predictions, labels, weight)
-
- reconstruction_similarity_loss_fn = masked_mpse
- elif method == 'mse':
- reconstruction_similarity_loss_fn = tf.contrib.losses.mean_squared_error
- elif method == 'hinged_mse':
-
- def hinged_mse(predictions, labels, weight):
- diffs = tf.square(predictions - labels)
- diffs = tf.maximum(0.0, diffs - max_diff)
- return tf.reduce_mean(diffs) * weight
-
- reconstruction_similarity_loss_fn = hinged_mse
- elif method == 'hinged_mae':
-
- def hinged_mae(predictions, labels, weight):
- diffs = tf.abs(predictions - labels)
- diffs = tf.maximum(0.0, diffs - max_diff)
- return tf.reduce_mean(diffs) * weight
-
- reconstruction_similarity_loss_fn = hinged_mae
- else:
- raise ValueError('Unknown reconstruction loss %s' % method)
-
- reconstruction_similarity_loss = reconstruction_similarity_loss_fn(
- reconstructions, source_images, weight)
-
- name = '%s_Similarity_(%s)' % (name, method)
- tf.summary.scalar(name, reconstruction_similarity_loss)
- return reconstruction_similarity_loss
-
-
-def g_step_loss(source_images, source_labels, end_points, hparams, num_classes):
- """Configures the loss function which runs during the g-step.
-
- Args:
- source_images: A `Tensor` of shape [batch_size, height, width, channels].
- source_labels: A dictionary of `Tensors` of shape [batch_size]. Valid keys
- are 'class' and 'quaternion'.
- end_points: A map of the network end points.
- hparams: The hyperparameters struct.
- num_classes: Number of classes for classifier loss
-
- Returns:
- A `Tensor` representing a loss function.
-
- Raises:
- ValueError: if hparams.transferred_similarity_loss_weight is non-zero but
- hparams.transferred_similarity_loss is invalid.
- """
- generator_loss = 0
-
- ################################################################
- # Adds a loss which encourages the discriminator probabilities #
- # to be high (near one).
- ################################################################
-
- # As per the GAN paper, maximize the log probs, instead of minimizing
- # log(1-probs). Since we're minimizing, we'll minimize -log(probs) which is
- # the same thing.
- style_transfer_loss = tf.losses.sigmoid_cross_entropy(
- logits=end_points['transferred_domain_logits'],
- multi_class_labels=tf.ones_like(end_points['transferred_domain_logits']),
- weights=hparams.style_transfer_loss_weight)
- tf.summary.scalar('Style_transfer_loss', style_transfer_loss)
- generator_loss += style_transfer_loss
-
- # Optimizes the style transfer network to produce transferred images similar
- # to the source images.
- generator_loss += _transferred_similarity_loss(
- end_points['transferred_images'],
- source_images,
- weight=hparams.transferred_similarity_loss_weight,
- method=hparams.transferred_similarity_loss,
- name='transferred_similarity')
-
- # Optimizes the style transfer network to maximize classification accuracy.
- if source_labels is not None and hparams.task_tower_in_g_step:
- generator_loss += _add_task_specific_losses(
- end_points, source_labels, num_classes,
- hparams) * hparams.task_loss_in_g_weight
-
- return generator_loss
-
-
-def d_step_loss(end_points, source_labels, num_classes, hparams):
- """Configures the losses during the D-Step.
-
- Note that during the D-step, the model optimizes both the domain (binary)
- classifier and the task classifier.
-
- Args:
- end_points: A map of the network end points.
- source_labels: A dictionary of output labels to `Tensors`.
- num_classes: The number of classes used by the classifier.
- hparams: The hyperparameters struct.
-
- Returns:
- A `Tensor` representing the value of the D-step loss.
- """
- domain_classifier_loss = add_domain_classifier_losses(end_points, hparams)
-
- task_classifier_loss = 0
- if source_labels is not None:
- task_classifier_loss = _add_task_specific_losses(
- end_points, source_labels, num_classes, hparams, add_summaries=True)
-
- return domain_classifier_loss + task_classifier_loss
diff --git a/research/domain_adaptation/pixel_domain_adaptation/pixelda_model.py b/research/domain_adaptation/pixel_domain_adaptation/pixelda_model.py
deleted file mode 100644
index 16b550a62d88ec2724c91f9dab9e3b34c736ec4f..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/pixelda_model.py
+++ /dev/null
@@ -1,713 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Contains the Domain Adaptation via Style Transfer (PixelDA) model components.
-
-A number of details in the implementation make reference to one of the following
-works:
-
-- "Unsupervised Representation Learning with Deep Convolutional
- Generative Adversarial Networks""
- https://arxiv.org/abs/1511.06434
-
-This paper makes several architecture recommendations:
-1. Use strided convs in discriminator, fractional-strided convs in generator
-2. batchnorm everywhere
-3. remove fully connected layers for deep models
-4. ReLu for all layers in generator, except tanh on output
-5. LeakyReLu for everything in discriminator
-"""
-import functools
-import math
-
-# Dependency imports
-import numpy as np
-
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-from domain_adaptation.pixel_domain_adaptation import pixelda_task_towers
-
-
-def create_model(hparams,
- target_images,
- source_images=None,
- source_labels=None,
- is_training=False,
- noise=None,
- num_classes=None):
- """Create a GAN model.
-
- Arguments:
- hparams: HParam object specifying model params
- target_images: A `Tensor` of size [batch_size, height, width, channels]. It
- is assumed that the images are [-1, 1] normalized.
- source_images: A `Tensor` of size [batch_size, height, width, channels]. It
- is assumed that the images are [-1, 1] normalized.
- source_labels: A `Tensor` of size [batch_size] of categorical labels between
- [0, num_classes]
- is_training: whether model is currently training
- noise: If None, model generates its own noise. Otherwise use provided.
- num_classes: Number of classes for classification
-
- Returns:
- end_points dict with model outputs
-
- Raises:
- ValueError: unknown hparams.arch setting
- """
- if num_classes is None and hparams.arch in ['resnet', 'simple']:
- raise ValueError('Num classes must be provided to create task classifier')
-
- if target_images.dtype != tf.float32:
- raise ValueError('target_images must be tf.float32 and [-1, 1] normalized.')
- if source_images is not None and source_images.dtype != tf.float32:
- raise ValueError('source_images must be tf.float32 and [-1, 1] normalized.')
-
- ###########################
- # Create latent variables #
- ###########################
- latent_vars = dict()
-
- if hparams.noise_channel:
- noise_shape = [hparams.batch_size, hparams.noise_dims]
- if noise is not None:
- assert noise.shape.as_list() == noise_shape
- tf.logging.info('Using provided noise')
- else:
- tf.logging.info('Using random noise')
- noise = tf.random_uniform(
- shape=noise_shape,
- minval=-1,
- maxval=1,
- dtype=tf.float32,
- name='random_noise')
- latent_vars['noise'] = noise
-
- ####################
- # Create generator #
- ####################
-
- with slim.arg_scope(
- [slim.conv2d, slim.conv2d_transpose, slim.fully_connected],
- normalizer_params=batch_norm_params(is_training,
- hparams.batch_norm_decay),
- weights_initializer=tf.random_normal_initializer(
- stddev=hparams.normal_init_std),
- weights_regularizer=tf.contrib.layers.l2_regularizer(
- hparams.weight_decay)):
- with slim.arg_scope([slim.conv2d], padding='SAME'):
- if hparams.arch == 'dcgan':
- end_points = dcgan(
- target_images, latent_vars, hparams, scope='generator')
- elif hparams.arch == 'resnet':
- end_points = resnet_generator(
- source_images,
- target_images.shape.as_list()[1:4],
- hparams=hparams,
- latent_vars=latent_vars)
- elif hparams.arch == 'residual_interpretation':
- end_points = residual_interpretation_generator(
- source_images, is_training=is_training, hparams=hparams)
- elif hparams.arch == 'simple':
- end_points = simple_generator(
- source_images,
- target_images,
- is_training=is_training,
- hparams=hparams,
- latent_vars=latent_vars)
- elif hparams.arch == 'identity':
- # Pass through unmodified, besides changing # channels
- # Used to calculate baseline numbers
- # Also set `generator_steps=0` for baseline
- if hparams.generator_steps:
- raise ValueError('Must set generator_steps=0 for identity arch. Is %s'
- % hparams.generator_steps)
- transferred_images = source_images
- source_channels = source_images.shape.as_list()[-1]
- target_channels = target_images.shape.as_list()[-1]
- if source_channels == 1 and target_channels == 3:
- transferred_images = tf.tile(source_images, [1, 1, 1, 3])
- if source_channels == 3 and target_channels == 1:
- transferred_images = tf.image.rgb_to_grayscale(source_images)
- end_points = {'transferred_images': transferred_images}
- else:
- raise ValueError('Unknown architecture: %s' % hparams.arch)
-
- #####################
- # Domain Classifier #
- #####################
- if hparams.arch in [
- 'dcgan', 'resnet', 'residual_interpretation', 'simple', 'identity',
- ]:
-
- # Add a discriminator for these architectures
- end_points['transferred_domain_logits'] = predict_domain(
- end_points['transferred_images'],
- hparams,
- is_training=is_training,
- reuse=False)
- end_points['target_domain_logits'] = predict_domain(
- target_images,
- hparams,
- is_training=is_training,
- reuse=True)
-
- ###################
- # Task Classifier #
- ###################
- if hparams.task_tower != 'none' and hparams.arch in [
- 'resnet', 'residual_interpretation', 'simple', 'identity',
- ]:
- with tf.variable_scope('discriminator'):
- with tf.variable_scope('task_tower'):
- end_points['source_task_logits'], end_points[
- 'source_quaternion'] = pixelda_task_towers.add_task_specific_model(
- source_images,
- hparams,
- num_classes=num_classes,
- is_training=is_training,
- reuse_private=False,
- private_scope='source_task_classifier',
- reuse_shared=False)
- end_points['transferred_task_logits'], end_points[
- 'transferred_quaternion'] = (
- pixelda_task_towers.add_task_specific_model(
- end_points['transferred_images'],
- hparams,
- num_classes=num_classes,
- is_training=is_training,
- reuse_private=False,
- private_scope='transferred_task_classifier',
- reuse_shared=True))
- end_points['target_task_logits'], end_points[
- 'target_quaternion'] = pixelda_task_towers.add_task_specific_model(
- target_images,
- hparams,
- num_classes=num_classes,
- is_training=is_training,
- reuse_private=True,
- private_scope='transferred_task_classifier',
- reuse_shared=True)
- # Remove any endpoints with None values
- return dict((k, v) for k, v in end_points.iteritems() if v is not None)
-
-
-def batch_norm_params(is_training, batch_norm_decay):
- return {
- 'is_training': is_training,
- # Decay for the moving averages.
- 'decay': batch_norm_decay,
- # epsilon to prevent 0s in variance.
- 'epsilon': 0.001,
- }
-
-
-def lrelu(x, leakiness=0.2):
- """Relu, with optional leaky support."""
- return tf.where(tf.less(x, 0.0), leakiness * x, x, name='leaky_relu')
-
-
-def upsample(net, num_filters, scale=2, method='resize_conv', scope=None):
- """Performs spatial upsampling of the given features.
-
- Args:
- net: A `Tensor` of shape [batch_size, height, width, filters].
- num_filters: The number of output filters.
- scale: The scale of the upsampling. Must be a positive integer greater or
- equal to two.
- method: The method by which the features are upsampled. Valid options
- include 'resize_conv' and 'conv2d_transpose'.
- scope: An optional variable scope.
-
- Returns:
- A new set of features of shape
- [batch_size, height*scale, width*scale, num_filters].
-
- Raises:
- ValueError: if `method` is not valid or
- """
- if scale < 2:
- raise ValueError('scale must be greater or equal to two.')
-
- with tf.variable_scope(scope, 'upsample', [net]):
- if method == 'resize_conv':
- net = tf.image.resize_nearest_neighbor(
- net, [net.shape.as_list()[1] * scale,
- net.shape.as_list()[2] * scale],
- align_corners=True,
- name='resize')
- return slim.conv2d(net, num_filters, stride=1, scope='conv')
- elif method == 'conv2d_transpose':
- return slim.conv2d_transpose(net, num_filters, scope='deconv')
- else:
- raise ValueError('Upsample method [%s] was not recognized.' % method)
-
-
-def project_latent_vars(hparams, proj_shape, latent_vars, combine_method='sum'):
- """Generate noise and project to input volume size.
-
- Args:
- hparams: The hyperparameter HParams struct.
- proj_shape: Shape to project noise (not including batch size).
- latent_vars: dictionary of `'key': Tensor of shape [batch_size, N]`
- combine_method: How to combine the projected values.
- sum = project to volume then sum
- concat = concatenate along last dimension (i.e. channel)
-
- Returns:
- If combine_method=sum, a `Tensor` of size `hparams.projection_shape`
- If combine_method=concat and there are N latent vars, a `Tensor` of size
- `hparams.projection_shape`, with the last channel multiplied by N
-
-
- Raises:
- ValueError: combine_method is not one of sum/concat
- """
- values = []
- for var in latent_vars:
- with tf.variable_scope(var):
- # Project & reshape noise to a HxWxC input
- projected = slim.fully_connected(
- latent_vars[var],
- np.prod(proj_shape),
- activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm)
- values.append(tf.reshape(projected, [hparams.batch_size] + proj_shape))
-
- if combine_method == 'sum':
- result = values[0]
- for value in values[1:]:
- result += value
- elif combine_method == 'concat':
- # Concatenate along last axis
- result = tf.concat(values, len(proj_shape))
- else:
- raise ValueError('Unknown combine_method %s' % combine_method)
-
- tf.logging.info('Latent variables projected to size %s volume', result.shape)
-
- return result
-
-
-def resnet_block(net, hparams):
- """Create a resnet block."""
- net_in = net
- net = slim.conv2d(
- net,
- hparams.resnet_filters,
- stride=1,
- normalizer_fn=slim.batch_norm,
- activation_fn=tf.nn.relu)
- net = slim.conv2d(
- net,
- hparams.resnet_filters,
- stride=1,
- normalizer_fn=slim.batch_norm,
- activation_fn=None)
- if hparams.resnet_residuals:
- net += net_in
- return net
-
-
-def resnet_stack(images, output_shape, hparams, scope=None):
- """Create a resnet style transfer block.
-
- Args:
- images: [batch-size, height, width, channels] image tensor to feed as input
- output_shape: output image shape in form [height, width, channels]
- hparams: hparams objects
- scope: Variable scope
-
- Returns:
- Images after processing with resnet blocks.
- """
- end_points = {}
- if hparams.noise_channel:
- # separate the noise for visualization
- end_points['noise'] = images[:, :, :, -1]
- assert images.shape.as_list()[1:3] == output_shape[0:2]
-
- with tf.variable_scope(scope, 'resnet_style_transfer', [images]):
- with slim.arg_scope(
- [slim.conv2d],
- normalizer_fn=slim.batch_norm,
- kernel_size=[hparams.generator_kernel_size] * 2,
- stride=1):
- net = slim.conv2d(
- images,
- hparams.resnet_filters,
- normalizer_fn=None,
- activation_fn=tf.nn.relu)
- for block in range(hparams.resnet_blocks):
- net = resnet_block(net, hparams)
- end_points['resnet_block_{}'.format(block)] = net
-
- net = slim.conv2d(
- net,
- output_shape[-1],
- kernel_size=[1, 1],
- normalizer_fn=None,
- activation_fn=tf.nn.tanh,
- scope='conv_out')
- end_points['transferred_images'] = net
- return net, end_points
-
-
-def predict_domain(images,
- hparams,
- is_training=False,
- reuse=False,
- scope='discriminator'):
- """Creates a discriminator for a GAN.
-
- Args:
- images: A `Tensor` of size [batch_size, height, width, channels]. It is
- assumed that the images are centered between -1 and 1.
- hparams: hparam object with params for discriminator
- is_training: Specifies whether or not we're training or testing.
- reuse: Whether to reuse variable scope
- scope: An optional variable_scope.
-
- Returns:
- [batch size, 1] - logit output of discriminator.
- """
- with tf.variable_scope(scope, 'discriminator', [images], reuse=reuse):
- lrelu_partial = functools.partial(lrelu, leakiness=hparams.lrelu_leakiness)
- with slim.arg_scope(
- [slim.conv2d],
- kernel_size=[hparams.discriminator_kernel_size] * 2,
- activation_fn=lrelu_partial,
- stride=2,
- normalizer_fn=slim.batch_norm):
-
- def add_noise(hidden, scope_num=None):
- if scope_num:
- hidden = slim.dropout(
- hidden,
- hparams.discriminator_dropout_keep_prob,
- is_training=is_training,
- scope='dropout_%s' % scope_num)
- if hparams.discriminator_noise_stddev == 0:
- return hidden
- return hidden + tf.random_normal(
- hidden.shape.as_list(),
- mean=0.0,
- stddev=hparams.discriminator_noise_stddev)
-
- # As per the recommendation of the DCGAN paper, we don't use batch norm
- # on the discriminator input (https://arxiv.org/pdf/1511.06434v2.pdf).
- if hparams.discriminator_image_noise:
- images = add_noise(images)
- net = slim.conv2d(
- images,
- hparams.num_discriminator_filters,
- normalizer_fn=None,
- stride=hparams.discriminator_first_stride,
- scope='conv1_stride%s' % hparams.discriminator_first_stride)
- net = add_noise(net, 1)
-
- block_id = 2
- # Repeatedly stack
- # discriminator_conv_block_size-1 conv layers with stride 1
- # followed by a stride 2 layer
- # Add (optional) noise at every point
- while net.shape.as_list()[1] > hparams.projection_shape_size:
- num_filters = int(hparams.num_discriminator_filters *
- (hparams.discriminator_filter_factor**(block_id - 1)))
- for conv_id in range(1, hparams.discriminator_conv_block_size):
- net = slim.conv2d(
- net,
- num_filters,
- stride=1,
- scope='conv_%s_%s' % (block_id, conv_id))
- if hparams.discriminator_do_pooling:
- net = slim.conv2d(
- net, num_filters, scope='conv_%s_prepool' % block_id)
- net = slim.avg_pool2d(
- net, kernel_size=[2, 2], stride=2, scope='pool_%s' % block_id)
- else:
- net = slim.conv2d(
- net, num_filters, scope='conv_%s_stride2' % block_id)
- net = add_noise(net, block_id)
- block_id += 1
- net = slim.flatten(net)
- net = slim.fully_connected(
- net,
- 1,
- # Models with BN here generally produce noise
- normalizer_fn=None,
- activation_fn=None,
- scope='fc_logit_out') # Returns logits!
- return net
-
-
-def dcgan_generator(images, output_shape, hparams, scope=None):
- """Transforms the visual style of the input images.
-
- Args:
- images: A `Tensor` of shape [batch_size, height, width, channels].
- output_shape: A list or tuple of 3 elements: the output height, width and
- number of channels.
- hparams: hparams object with generator parameters
- scope: Scope to place generator inside
-
- Returns:
- A `Tensor` of shape [batch_size, height, width, output_channels] which
- represents the result of style transfer.
-
- Raises:
- ValueError: If `output_shape` is not a list or tuple or if it doesn't have
- three elements or if `output_shape` or `images` arent square.
- """
- if not isinstance(output_shape, (tuple, list)):
- raise ValueError('output_shape must be a tuple or list.')
- elif len(output_shape) != 3:
- raise ValueError('output_shape must have three elements.')
-
- if output_shape[0] != output_shape[1]:
- raise ValueError('output_shape must be square')
- if images.shape.as_list()[1] != images.shape.as_list()[2]:
- raise ValueError('images height and width must match.')
-
- outdim = output_shape[0]
- indim = images.shape.as_list()[1]
- num_iterations = int(math.ceil(math.log(float(outdim) / float(indim), 2.0)))
-
- with slim.arg_scope(
- [slim.conv2d, slim.conv2d_transpose],
- kernel_size=[hparams.generator_kernel_size] * 2,
- stride=2):
- with tf.variable_scope(scope or 'generator'):
-
- net = images
-
- # Repeatedly halve # filters until = hparams.decode_filters in last layer
- for i in range(num_iterations):
- num_filters = hparams.num_decoder_filters * 2**(num_iterations - i - 1)
- net = slim.conv2d_transpose(net, num_filters, scope='deconv_%s' % i)
-
- # Crop down to desired size (e.g. 32x32 -> 28x28)
- dif = net.shape.as_list()[1] - outdim
- low = dif / 2
- high = net.shape.as_list()[1] - low
- net = net[:, low:high, low:high, :]
-
- # No batch norm on generator output
- net = slim.conv2d(
- net,
- output_shape[2],
- kernel_size=[1, 1],
- stride=1,
- normalizer_fn=None,
- activation_fn=tf.tanh,
- scope='conv_out')
- return net
-
-
-def dcgan(target_images, latent_vars, hparams, scope='dcgan'):
- """Creates the PixelDA model.
-
- Args:
- target_images: A `Tensor` of shape [batch_size, height, width, 3]
- sampled from the image domain to which we want to transfer.
- latent_vars: dictionary of 'key': Tensor of shape [batch_size, N]
- hparams: The hyperparameter map.
- scope: Surround generator component with this scope
-
- Returns:
- A dictionary of model outputs.
- """
- proj_shape = [
- hparams.projection_shape_size, hparams.projection_shape_size,
- hparams.projection_shape_channels
- ]
- source_volume = project_latent_vars(
- hparams, proj_shape, latent_vars, combine_method='concat')
-
- ###################################################
- # Transfer the source images to the target style. #
- ###################################################
- with tf.variable_scope(scope, 'generator', [target_images]):
- transferred_images = dcgan_generator(
- source_volume,
- output_shape=target_images.shape.as_list()[1:4],
- hparams=hparams)
- assert transferred_images.shape.as_list() == target_images.shape.as_list()
-
- return {'transferred_images': transferred_images}
-
-
-def resnet_generator(images, output_shape, hparams, latent_vars=None):
- """Creates a ResNet-based generator.
-
- Args:
- images: A `Tensor` of shape [batch_size, height, width, num_channels]
- sampled from the image domain from which we want to transfer
- output_shape: A length-3 array indicating the height, width and channels of
- the output.
- hparams: The hyperparameter map.
- latent_vars: dictionary of 'key': Tensor of shape [batch_size, N]
-
- Returns:
- A dictionary of model outputs.
- """
- with tf.variable_scope('generator'):
- if latent_vars:
- noise_channel = project_latent_vars(
- hparams,
- proj_shape=images.shape.as_list()[1:3] + [1],
- latent_vars=latent_vars,
- combine_method='concat')
- images = tf.concat([images, noise_channel], 3)
-
- transferred_images, end_points = resnet_stack(
- images,
- output_shape=output_shape,
- hparams=hparams,
- scope='resnet_stack')
- end_points['transferred_images'] = transferred_images
-
- return end_points
-
-
-def residual_interpretation_block(images, hparams, scope):
- """Learns a residual image which is added to the incoming image.
-
- Args:
- images: A `Tensor` of size [batch_size, height, width, 3]
- hparams: The hyperparameters struct.
- scope: The name of the variable op scope.
-
- Returns:
- The updated images.
- """
- with tf.variable_scope(scope):
- with slim.arg_scope(
- [slim.conv2d],
- normalizer_fn=None,
- kernel_size=[hparams.generator_kernel_size] * 2):
-
- net = images
- for _ in range(hparams.res_int_convs):
- net = slim.conv2d(
- net, hparams.res_int_filters, activation_fn=tf.nn.relu)
- net = slim.conv2d(net, 3, activation_fn=tf.nn.tanh)
-
- # Add the residual
- images += net
-
- # Clip the output
- images = tf.maximum(images, -1.0)
- images = tf.minimum(images, 1.0)
- return images
-
-
-def residual_interpretation_generator(images,
- is_training,
- hparams,
- latent_vars=None):
- """Creates a generator producing purely residual transformations.
-
- A residual generator differs from the resnet generator in that each 'block' of
- the residual generator produces a residual image. Consequently, the 'progress'
- of the model generation process can be directly observed at inference time,
- making it easier to diagnose and understand.
-
- Args:
- images: A `Tensor` of shape [batch_size, height, width, num_channels]
- sampled from the image domain from which we want to transfer. It is
- assumed that the images are centered between -1 and 1.
- is_training: whether or not the model is training.
- hparams: The hyperparameter map.
- latent_vars: dictionary of 'key': Tensor of shape [batch_size, N]
-
- Returns:
- A dictionary of model outputs.
- """
- end_points = {}
-
- with tf.variable_scope('generator'):
- if latent_vars:
- projected_latent = project_latent_vars(
- hparams,
- proj_shape=images.shape.as_list()[1:3] + [images.shape.as_list()[-1]],
- latent_vars=latent_vars,
- combine_method='sum')
- images += projected_latent
- with tf.variable_scope(None, 'residual_style_transfer', [images]):
- for i in range(hparams.res_int_blocks):
- images = residual_interpretation_block(images, hparams,
- 'residual_%d' % i)
- end_points['transferred_images_%d' % i] = images
-
- end_points['transferred_images'] = images
-
- return end_points
-
-
-def simple_generator(source_images, target_images, is_training, hparams,
- latent_vars):
- """Simple generator architecture (stack of convs) for trying small models."""
- end_points = {}
- with tf.variable_scope('generator'):
- feed_source_images = source_images
-
- if latent_vars:
- projected_latent = project_latent_vars(
- hparams,
- proj_shape=source_images.shape.as_list()[1:3] + [1],
- latent_vars=latent_vars,
- combine_method='concat')
- feed_source_images = tf.concat([source_images, projected_latent], 3)
-
- end_points = {}
-
- ###################################################
- # Transfer the source images to the target style. #
- ###################################################
- with slim.arg_scope(
- [slim.conv2d],
- normalizer_fn=slim.batch_norm,
- stride=1,
- kernel_size=[hparams.generator_kernel_size] * 2):
- net = feed_source_images
-
- # N convolutions
- for i in range(1, hparams.simple_num_conv_layers):
- normalizer_fn = None
- if i != 0:
- normalizer_fn = slim.batch_norm
- net = slim.conv2d(
- net,
- hparams.simple_conv_filters,
- normalizer_fn=normalizer_fn,
- activation_fn=tf.nn.relu)
-
- # Project back to right # image channels
- net = slim.conv2d(
- net,
- target_images.shape.as_list()[-1],
- kernel_size=[1, 1],
- stride=1,
- normalizer_fn=None,
- activation_fn=tf.tanh,
- scope='conv_out')
-
- transferred_images = net
- assert transferred_images.shape.as_list() == target_images.shape.as_list()
- end_points['transferred_images'] = transferred_images
-
- return end_points
diff --git a/research/domain_adaptation/pixel_domain_adaptation/pixelda_preprocess.py b/research/domain_adaptation/pixel_domain_adaptation/pixelda_preprocess.py
deleted file mode 100644
index 747c17b18bf007d85e606015da6687a343bf74d2..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/pixelda_preprocess.py
+++ /dev/null
@@ -1,129 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Contains functions for preprocessing the inputs."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-
-import tensorflow as tf
-
-
-def preprocess_classification(image, labels, is_training=False):
- """Preprocesses the image and labels for classification purposes.
-
- Preprocessing includes shifting the images to be 0-centered between -1 and 1.
- This is not only a popular method of preprocessing (inception) but is also
- the mechanism used by DSNs.
-
- Args:
- image: A `Tensor` of size [height, width, 3].
- labels: A dictionary of labels.
- is_training: Whether or not we're training the model.
-
- Returns:
- The preprocessed image and labels.
- """
- # If the image is uint8, this will scale it to 0-1.
- image = tf.image.convert_image_dtype(image, tf.float32)
- image -= 0.5
- image *= 2
-
- return image, labels
-
-
-def preprocess_style_transfer(image,
- labels,
- augment=False,
- size=None,
- is_training=False):
- """Preprocesses the image and labels for style transfer purposes.
-
- Args:
- image: A `Tensor` of size [height, width, 3].
- labels: A dictionary of labels.
- augment: Whether to apply data augmentation to inputs
- size: The height and width to which images should be resized. If left as
- `None`, then no resizing is performed
- is_training: Whether or not we're training the model
-
- Returns:
- The preprocessed image and labels. Scaled to [-1, 1]
- """
- # If the image is uint8, this will scale it to 0-1.
- image = tf.image.convert_image_dtype(image, tf.float32)
- if augment and is_training:
- image = image_augmentation(image)
-
- if size:
- image = resize_image(image, size)
-
- image -= 0.5
- image *= 2
-
- return image, labels
-
-
-def image_augmentation(image):
- """Performs data augmentation by randomly permuting the inputs.
-
- Args:
- image: A float `Tensor` of size [height, width, channels] with values
- in range[0,1].
-
- Returns:
- The mutated batch of images
- """
- # Apply photometric data augmentation (contrast etc.)
- num_channels = image.shape_as_list()[-1]
- if num_channels == 4:
- # Only augment image part
- image, depth = image[:, :, 0:3], image[:, :, 3:4]
- elif num_channels == 1:
- image = tf.image.grayscale_to_rgb(image)
- image = tf.image.random_brightness(image, max_delta=0.1)
- image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
- image = tf.image.random_hue(image, max_delta=0.032)
- image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
- image = tf.clip_by_value(image, 0, 1.0)
- if num_channels == 4:
- image = tf.concat(2, [image, depth])
- elif num_channels == 1:
- image = tf.image.rgb_to_grayscale(image)
- return image
-
-
-def resize_image(image, size=None):
- """Resize image to target size.
-
- Args:
- image: A `Tensor` of size [height, width, 3].
- size: (height, width) to resize image to.
-
- Returns:
- resized image
- """
- if size is None:
- raise ValueError('Must specify size')
-
- if image.shape_as_list()[:2] == size:
- # Don't resize if not necessary
- return image
- image = tf.expand_dims(image, 0)
- image = tf.image.resize_images(image, size)
- image = tf.squeeze(image, 0)
- return image
diff --git a/research/domain_adaptation/pixel_domain_adaptation/pixelda_preprocess_test.py b/research/domain_adaptation/pixel_domain_adaptation/pixelda_preprocess_test.py
deleted file mode 100644
index 73f8c7ff05fc7d2614c419759a02f78ffbcdfec0..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/pixelda_preprocess_test.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Tests for domain_adaptation.pixel_domain_adaptation.pixelda_preprocess."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-
-import tensorflow as tf
-
-from domain_adaptation.pixel_domain_adaptation import pixelda_preprocess
-
-
-class PixelDAPreprocessTest(tf.test.TestCase):
-
- def assert_preprocess_classification_is_centered(self, dtype, is_training):
- tf.set_random_seed(0)
-
- if dtype == tf.uint8:
- image = tf.random_uniform((100, 200, 3), maxval=255, dtype=tf.int64)
- image = tf.cast(image, tf.uint8)
- else:
- image = tf.random_uniform((100, 200, 3), maxval=1.0, dtype=dtype)
-
- labels = {}
- image, labels = pixelda_preprocess.preprocess_classification(
- image, labels, is_training=is_training)
-
- with self.test_session() as sess:
- np_image = sess.run(image)
-
- self.assertTrue(np_image.min() <= -0.95)
- self.assertTrue(np_image.min() >= -1.0)
- self.assertTrue(np_image.max() >= 0.95)
- self.assertTrue(np_image.max() <= 1.0)
-
- def testPreprocessClassificationZeroCentersUint8DuringTrain(self):
- self.assert_preprocess_classification_is_centered(
- tf.uint8, is_training=True)
-
- def testPreprocessClassificationZeroCentersUint8DuringTest(self):
- self.assert_preprocess_classification_is_centered(
- tf.uint8, is_training=False)
-
- def testPreprocessClassificationZeroCentersFloatDuringTrain(self):
- self.assert_preprocess_classification_is_centered(
- tf.float32, is_training=True)
-
- def testPreprocessClassificationZeroCentersFloatDuringTest(self):
- self.assert_preprocess_classification_is_centered(
- tf.float32, is_training=False)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/domain_adaptation/pixel_domain_adaptation/pixelda_task_towers.py b/research/domain_adaptation/pixel_domain_adaptation/pixelda_task_towers.py
deleted file mode 100644
index 1cb42e2d890a7759318cf0981640c0dd1645461e..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/pixelda_task_towers.py
+++ /dev/null
@@ -1,317 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Task towers for PixelDA model."""
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-
-def add_task_specific_model(images,
- hparams,
- num_classes=10,
- is_training=False,
- reuse_private=False,
- private_scope=None,
- reuse_shared=False,
- shared_scope=None):
- """Create a classifier for the given images.
-
- The classifier is composed of a few 'private' layers followed by a few
- 'shared' layers. This lets us account for different image 'style', while
- sharing the last few layers as 'content' layers.
-
- Args:
- images: A `Tensor` of size [batch_size, height, width, 3].
- hparams: model hparams
- num_classes: The number of output classes.
- is_training: whether model is training
- reuse_private: Whether or not to reuse the private weights, which are the
- first few layers in the classifier
- private_scope: The name of the variable_scope for the private (unshared)
- components of the classifier.
- reuse_shared: Whether or not to reuse the shared weights, which are the last
- few layers in the classifier
- shared_scope: The name of the variable_scope for the shared components of
- the classifier.
-
- Returns:
- The logits, a `Tensor` of shape [batch_size, num_classes].
-
- Raises:
- ValueError: If hparams.task_classifier is an unknown value
- """
-
- model = hparams.task_tower
- # Make sure the classifier name shows up in graph
- shared_scope = shared_scope or (model + '_shared')
- kwargs = {
- 'num_classes': num_classes,
- 'is_training': is_training,
- 'reuse_private': reuse_private,
- 'reuse_shared': reuse_shared,
- }
-
- if private_scope:
- kwargs['private_scope'] = private_scope
- if shared_scope:
- kwargs['shared_scope'] = shared_scope
-
- quaternion_pred = None
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- activation_fn=tf.nn.relu,
- weights_regularizer=tf.contrib.layers.l2_regularizer(
- hparams.weight_decay_task_classifier)):
- with slim.arg_scope([slim.conv2d], padding='SAME'):
- if model == 'doubling_pose_estimator':
- logits, quaternion_pred = doubling_cnn_class_and_quaternion(
- images, num_private_layers=hparams.num_private_layers, **kwargs)
- elif model == 'mnist':
- logits, _ = mnist_classifier(images, **kwargs)
- elif model == 'svhn':
- logits, _ = svhn_classifier(images, **kwargs)
- elif model == 'gtsrb':
- logits, _ = gtsrb_classifier(images, **kwargs)
- elif model == 'pose_mini':
- logits, quaternion_pred = pose_mini_tower(images, **kwargs)
- else:
- raise ValueError('Unknown task classifier %s' % model)
-
- return logits, quaternion_pred
-
-
-#####################################
-# Classifiers used in the DSN paper #
-#####################################
-
-
-def mnist_classifier(images,
- is_training=False,
- num_classes=10,
- reuse_private=False,
- private_scope='mnist',
- reuse_shared=False,
- shared_scope='task_model'):
- """Creates the convolutional MNIST model from the gradient reversal paper.
-
- Note that since the output is a set of 'logits', the values fall in the
- interval of (-infinity, infinity). Consequently, to convert the outputs to a
- probability distribution over the characters, one will need to convert them
- using the softmax function:
- logits, endpoints = conv_mnist(images, is_training=False)
- predictions = tf.nn.softmax(logits)
-
- Args:
- images: the MNIST digits, a tensor of size [batch_size, 28, 28, 1].
- is_training: specifies whether or not we're currently training the model.
- This variable will determine the behaviour of the dropout layer.
- num_classes: the number of output classes to use.
-
- Returns:
- the output logits, a tensor of size [batch_size, num_classes].
- a dictionary with key/values the layer names and tensors.
- """
-
- net = {}
-
- with tf.variable_scope(private_scope, reuse=reuse_private):
- net['conv1'] = slim.conv2d(images, 32, [5, 5], scope='conv1')
- net['pool1'] = slim.max_pool2d(net['conv1'], [2, 2], 2, scope='pool1')
-
- with tf.variable_scope(shared_scope, reuse=reuse_shared):
- net['conv2'] = slim.conv2d(net['pool1'], 48, [5, 5], scope='conv2')
- net['pool2'] = slim.max_pool2d(net['conv2'], [2, 2], 2, scope='pool2')
- net['fc3'] = slim.fully_connected(
- slim.flatten(net['pool2']), 100, scope='fc3')
- net['fc4'] = slim.fully_connected(
- slim.flatten(net['fc3']), 100, scope='fc4')
- logits = slim.fully_connected(
- net['fc4'], num_classes, activation_fn=None, scope='fc5')
- return logits, net
-
-
-def svhn_classifier(images,
- is_training=False,
- num_classes=10,
- reuse_private=False,
- private_scope=None,
- reuse_shared=False,
- shared_scope='task_model'):
- """Creates the convolutional SVHN model from the gradient reversal paper.
-
- Note that since the output is a set of 'logits', the values fall in the
- interval of (-infinity, infinity). Consequently, to convert the outputs to a
- probability distribution over the characters, one will need to convert them
- using the softmax function:
- logits = mnist.Mnist(images, is_training=False)
- predictions = tf.nn.softmax(logits)
-
- Args:
- images: the SVHN digits, a tensor of size [batch_size, 40, 40, 3].
- is_training: specifies whether or not we're currently training the model.
- This variable will determine the behaviour of the dropout layer.
- num_classes: the number of output classes to use.
-
- Returns:
- the output logits, a tensor of size [batch_size, num_classes].
- a dictionary with key/values the layer names and tensors.
- """
-
- net = {}
-
- with tf.variable_scope(private_scope, reuse=reuse_private):
- net['conv1'] = slim.conv2d(images, 64, [5, 5], scope='conv1')
- net['pool1'] = slim.max_pool2d(net['conv1'], [3, 3], 2, scope='pool1')
-
- with tf.variable_scope(shared_scope, reuse=reuse_shared):
- net['conv2'] = slim.conv2d(net['pool1'], 64, [5, 5], scope='conv2')
- net['pool2'] = slim.max_pool2d(net['conv2'], [3, 3], 2, scope='pool2')
- net['conv3'] = slim.conv2d(net['pool2'], 128, [5, 5], scope='conv3')
-
- net['fc3'] = slim.fully_connected(
- slim.flatten(net['conv3']), 3072, scope='fc3')
- net['fc4'] = slim.fully_connected(
- slim.flatten(net['fc3']), 2048, scope='fc4')
-
- logits = slim.fully_connected(
- net['fc4'], num_classes, activation_fn=None, scope='fc5')
-
- return logits, net
-
-
-def gtsrb_classifier(images,
- is_training=False,
- num_classes=43,
- reuse_private=False,
- private_scope='gtsrb',
- reuse_shared=False,
- shared_scope='task_model'):
- """Creates the convolutional GTSRB model from the gradient reversal paper.
-
- Note that since the output is a set of 'logits', the values fall in the
- interval of (-infinity, infinity). Consequently, to convert the outputs to a
- probability distribution over the characters, one will need to convert them
- using the softmax function:
- logits = mnist.Mnist(images, is_training=False)
- predictions = tf.nn.softmax(logits)
-
- Args:
- images: the SVHN digits, a tensor of size [batch_size, 40, 40, 3].
- is_training: specifies whether or not we're currently training the model.
- This variable will determine the behaviour of the dropout layer.
- num_classes: the number of output classes to use.
- reuse_private: Whether or not to reuse the private components of the model.
- private_scope: The name of the private scope.
- reuse_shared: Whether or not to reuse the shared components of the model.
- shared_scope: The name of the shared scope.
-
- Returns:
- the output logits, a tensor of size [batch_size, num_classes].
- a dictionary with key/values the layer names and tensors.
- """
-
- net = {}
-
- with tf.variable_scope(private_scope, reuse=reuse_private):
- net['conv1'] = slim.conv2d(images, 96, [5, 5], scope='conv1')
- net['pool1'] = slim.max_pool2d(net['conv1'], [2, 2], 2, scope='pool1')
- with tf.variable_scope(shared_scope, reuse=reuse_shared):
- net['conv2'] = slim.conv2d(net['pool1'], 144, [3, 3], scope='conv2')
- net['pool2'] = slim.max_pool2d(net['conv2'], [2, 2], 2, scope='pool2')
- net['conv3'] = slim.conv2d(net['pool2'], 256, [5, 5], scope='conv3')
- net['pool3'] = slim.max_pool2d(net['conv3'], [2, 2], 2, scope='pool3')
-
- net['fc3'] = slim.fully_connected(
- slim.flatten(net['pool3']), 512, scope='fc3')
- logits = slim.fully_connected(
- net['fc3'], num_classes, activation_fn=None, scope='fc4')
-
- return logits, net
-
-
-#########################
-# pose_mini task towers #
-#########################
-
-
-def pose_mini_tower(images,
- num_classes=11,
- is_training=False,
- reuse_private=False,
- private_scope='pose_mini',
- reuse_shared=False,
- shared_scope='task_model'):
- """Task tower for the pose_mini dataset."""
-
- with tf.variable_scope(private_scope, reuse=reuse_private):
- net = slim.conv2d(images, 32, [5, 5], scope='conv1')
- net = slim.max_pool2d(net, [2, 2], stride=2, scope='pool1')
- with tf.variable_scope(shared_scope, reuse=reuse_shared):
- net = slim.conv2d(net, 64, [5, 5], scope='conv2')
- net = slim.max_pool2d(net, [2, 2], stride=2, scope='pool2')
- net = slim.flatten(net)
-
- net = slim.fully_connected(net, 128, scope='fc3')
- net = slim.dropout(net, 0.5, is_training=is_training, scope='dropout')
- with tf.variable_scope('quaternion_prediction'):
- quaternion_pred = slim.fully_connected(
- net, 4, activation_fn=tf.tanh, scope='fc_q')
- quaternion_pred = tf.nn.l2_normalize(quaternion_pred, 1)
-
- logits = slim.fully_connected(
- net, num_classes, activation_fn=None, scope='fc4')
-
- return logits, quaternion_pred
-
-
-def doubling_cnn_class_and_quaternion(images,
- num_private_layers=1,
- num_classes=10,
- is_training=False,
- reuse_private=False,
- private_scope='doubling_cnn',
- reuse_shared=False,
- shared_scope='task_model'):
- """Alternate conv, pool while doubling filter count."""
- net = images
- depth = 32
- layer_id = 1
-
- with tf.variable_scope(private_scope, reuse=reuse_private):
- while num_private_layers > 0 and net.shape.as_list()[1] > 5:
- net = slim.conv2d(net, depth, [3, 3], scope='conv%s' % layer_id)
- net = slim.max_pool2d(net, [2, 2], stride=2, scope='pool%s' % layer_id)
- depth *= 2
- layer_id += 1
- num_private_layers -= 1
-
- with tf.variable_scope(shared_scope, reuse=reuse_shared):
- while net.shape.as_list()[1] > 5:
- net = slim.conv2d(net, depth, [3, 3], scope='conv%s' % layer_id)
- net = slim.max_pool2d(net, [2, 2], stride=2, scope='pool%s' % layer_id)
- depth *= 2
- layer_id += 1
-
- net = slim.flatten(net)
- net = slim.fully_connected(net, 100, scope='fc1')
- net = slim.dropout(net, 0.5, is_training=is_training, scope='dropout')
- quaternion_pred = slim.fully_connected(
- net, 4, activation_fn=tf.tanh, scope='fc_q')
- quaternion_pred = tf.nn.l2_normalize(quaternion_pred, 1)
-
- logits = slim.fully_connected(
- net, num_classes, activation_fn=None, scope='fc_logits')
-
- return logits, quaternion_pred
diff --git a/research/domain_adaptation/pixel_domain_adaptation/pixelda_train.py b/research/domain_adaptation/pixel_domain_adaptation/pixelda_train.py
deleted file mode 100644
index 4ca072cceafa48769623381b8e564fe650f2a514..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/pixelda_train.py
+++ /dev/null
@@ -1,409 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-r"""Trains the PixelDA model."""
-
-from functools import partial
-import os
-
-# Dependency imports
-
-import tensorflow as tf
-
-from domain_adaptation.datasets import dataset_factory
-from domain_adaptation.pixel_domain_adaptation import pixelda_losses
-from domain_adaptation.pixel_domain_adaptation import pixelda_model
-from domain_adaptation.pixel_domain_adaptation import pixelda_preprocess
-from domain_adaptation.pixel_domain_adaptation import pixelda_utils
-from domain_adaptation.pixel_domain_adaptation.hparams import create_hparams
-
-slim = tf.contrib.slim
-
-flags = tf.app.flags
-FLAGS = flags.FLAGS
-
-flags.DEFINE_string('master', '', 'BNS name of the TensorFlow master to use.')
-
-flags.DEFINE_integer(
- 'ps_tasks', 0,
- 'The number of parameter servers. If the value is 0, then the parameters '
- 'are handled locally by the worker.')
-
-flags.DEFINE_integer(
- 'task', 0,
- 'The Task ID. This value is used when training with multiple workers to '
- 'identify each worker.')
-
-flags.DEFINE_string('train_log_dir', '/tmp/pixelda/',
- 'Directory where to write event logs.')
-
-flags.DEFINE_integer(
- 'save_summaries_steps', 500,
- 'The frequency with which summaries are saved, in seconds.')
-
-flags.DEFINE_integer('save_interval_secs', 300,
- 'The frequency with which the model is saved, in seconds.')
-
-flags.DEFINE_boolean('summarize_gradients', False,
- 'Whether to summarize model gradients')
-
-flags.DEFINE_integer(
- 'print_loss_steps', 100,
- 'The frequency with which the losses are printed, in steps.')
-
-flags.DEFINE_string('source_dataset', 'mnist', 'The name of the source dataset.'
- ' If hparams="arch=dcgan", this flag is ignored.')
-
-flags.DEFINE_string('target_dataset', 'mnist_m',
- 'The name of the target dataset.')
-
-flags.DEFINE_string('source_split_name', 'train',
- 'Name of the train split for the source.')
-
-flags.DEFINE_string('target_split_name', 'train',
- 'Name of the train split for the target.')
-
-flags.DEFINE_string('dataset_dir', '',
- 'The directory where the datasets can be found.')
-
-flags.DEFINE_integer(
- 'num_readers', 4,
- 'The number of parallel readers that read data from the dataset.')
-
-flags.DEFINE_integer('num_preprocessing_threads', 4,
- 'The number of threads used to create the batches.')
-
-# HParams
-
-flags.DEFINE_string('hparams', '', 'Comma separated hyperparameter values')
-
-
-def _get_vars_and_update_ops(hparams, scope):
- """Returns the variables and update ops for a particular variable scope.
-
- Args:
- hparams: The hyperparameters struct.
- scope: The variable scope.
-
- Returns:
- A tuple consisting of trainable variables and update ops.
- """
- is_trainable = lambda x: x in tf.trainable_variables()
- var_list = filter(is_trainable, slim.get_model_variables(scope))
- global_step = slim.get_or_create_global_step()
-
- update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope)
-
- tf.logging.info('All variables for scope: %s',
- slim.get_model_variables(scope))
- tf.logging.info('Trainable variables for scope: %s', var_list)
-
- return var_list, update_ops
-
-
-def _train(discriminator_train_op,
- generator_train_op,
- logdir,
- master='',
- is_chief=True,
- scaffold=None,
- hooks=None,
- chief_only_hooks=None,
- save_checkpoint_secs=600,
- save_summaries_steps=100,
- hparams=None):
- """Runs the training loop.
-
- Args:
- discriminator_train_op: A `Tensor` that, when executed, will apply the
- gradients and return the loss value for the discriminator.
- generator_train_op: A `Tensor` that, when executed, will apply the
- gradients and return the loss value for the generator.
- logdir: The directory where the graph and checkpoints are saved.
- master: The URL of the master.
- is_chief: Specifies whether or not the training is being run by the primary
- replica during replica training.
- scaffold: An tf.train.Scaffold instance.
- hooks: List of `tf.train.SessionRunHook` callbacks which are run inside the
- training loop.
- chief_only_hooks: List of `tf.train.SessionRunHook` instances which are run
- inside the training loop for the chief trainer only.
- save_checkpoint_secs: The frequency, in seconds, that a checkpoint is saved
- using a default checkpoint saver. If `save_checkpoint_secs` is set to
- `None`, then the default checkpoint saver isn't used.
- save_summaries_steps: The frequency, in number of global steps, that the
- summaries are written to disk using a default summary saver. If
- `save_summaries_steps` is set to `None`, then the default summary saver
- isn't used.
- hparams: The hparams struct.
-
- Returns:
- the value of the loss function after training.
-
- Raises:
- ValueError: if `logdir` is `None` and either `save_checkpoint_secs` or
- `save_summaries_steps` are `None.
- """
- global_step = slim.get_or_create_global_step()
-
- scaffold = scaffold or tf.train.Scaffold()
-
- hooks = hooks or []
-
- if is_chief:
- session_creator = tf.train.ChiefSessionCreator(
- scaffold=scaffold, checkpoint_dir=logdir, master=master)
-
- if chief_only_hooks:
- hooks.extend(chief_only_hooks)
- hooks.append(tf.train.StepCounterHook(output_dir=logdir))
-
- if save_summaries_steps:
- if logdir is None:
- raise ValueError(
- 'logdir cannot be None when save_summaries_steps is None')
- hooks.append(
- tf.train.SummarySaverHook(
- scaffold=scaffold,
- save_steps=save_summaries_steps,
- output_dir=logdir))
-
- if save_checkpoint_secs:
- if logdir is None:
- raise ValueError(
- 'logdir cannot be None when save_checkpoint_secs is None')
- hooks.append(
- tf.train.CheckpointSaverHook(
- logdir, save_secs=save_checkpoint_secs, scaffold=scaffold))
- else:
- session_creator = tf.train.WorkerSessionCreator(
- scaffold=scaffold, master=master)
-
- with tf.train.MonitoredSession(
- session_creator=session_creator, hooks=hooks) as session:
- loss = None
- while not session.should_stop():
- # Run the domain classifier op X times.
- for _ in range(hparams.discriminator_steps):
- if session.should_stop():
- return loss
- loss, np_global_step = session.run(
- [discriminator_train_op, global_step])
- if np_global_step % FLAGS.print_loss_steps == 0:
- tf.logging.info('Step %d: Discriminator Loss = %.2f', np_global_step,
- loss)
-
- # Run the generator op X times.
- for _ in range(hparams.generator_steps):
- if session.should_stop():
- return loss
- loss, np_global_step = session.run([generator_train_op, global_step])
- if np_global_step % FLAGS.print_loss_steps == 0:
- tf.logging.info('Step %d: Generator Loss = %.2f', np_global_step,
- loss)
- return loss
-
-
-def run_training(run_dir, checkpoint_dir, hparams):
- """Runs the training loop.
-
- Args:
- run_dir: The directory where training specific logs are placed
- checkpoint_dir: The directory where the checkpoints and log files are
- stored.
- hparams: The hyperparameters struct.
-
- Raises:
- ValueError: if hparams.arch is not recognized.
- """
- for path in [run_dir, checkpoint_dir]:
- if not tf.gfile.Exists(path):
- tf.gfile.MakeDirs(path)
-
- # Serialize hparams to log dir
- hparams_filename = os.path.join(checkpoint_dir, 'hparams.json')
- with tf.gfile.FastGFile(hparams_filename, 'w') as f:
- f.write(hparams.to_json())
-
- with tf.Graph().as_default():
- with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
- global_step = slim.get_or_create_global_step()
-
- #########################
- # Preprocess the inputs #
- #########################
- target_dataset = dataset_factory.get_dataset(
- FLAGS.target_dataset,
- split_name='train',
- dataset_dir=FLAGS.dataset_dir)
- target_images, _ = dataset_factory.provide_batch(
- FLAGS.target_dataset, 'train', FLAGS.dataset_dir, FLAGS.num_readers,
- hparams.batch_size, FLAGS.num_preprocessing_threads)
- num_target_classes = target_dataset.num_classes
-
- if hparams.arch not in ['dcgan']:
- source_dataset = dataset_factory.get_dataset(
- FLAGS.source_dataset,
- split_name='train',
- dataset_dir=FLAGS.dataset_dir)
- num_source_classes = source_dataset.num_classes
- source_images, source_labels = dataset_factory.provide_batch(
- FLAGS.source_dataset, 'train', FLAGS.dataset_dir, FLAGS.num_readers,
- hparams.batch_size, FLAGS.num_preprocessing_threads)
- # Data provider provides 1 hot labels, but we expect categorical.
- source_labels['class'] = tf.argmax(source_labels['classes'], 1)
- del source_labels['classes']
- if num_source_classes != num_target_classes:
- raise ValueError(
- 'Source and Target datasets must have same number of classes. '
- 'Are %d and %d' % (num_source_classes, num_target_classes))
- else:
- source_images = None
- source_labels = None
-
- ####################
- # Define the model #
- ####################
- end_points = pixelda_model.create_model(
- hparams,
- target_images,
- source_images=source_images,
- source_labels=source_labels,
- is_training=True,
- num_classes=num_target_classes)
-
- #################################
- # Get the variables to optimize #
- #################################
- generator_vars, generator_update_ops = _get_vars_and_update_ops(
- hparams, 'generator')
- discriminator_vars, discriminator_update_ops = _get_vars_and_update_ops(
- hparams, 'discriminator')
-
- ########################
- # Configure the losses #
- ########################
- generator_loss = pixelda_losses.g_step_loss(
- source_images,
- source_labels,
- end_points,
- hparams,
- num_classes=num_target_classes)
- discriminator_loss = pixelda_losses.d_step_loss(
- end_points, source_labels, num_target_classes, hparams)
-
- ###########################
- # Create the training ops #
- ###########################
- learning_rate = hparams.learning_rate
- if hparams.lr_decay_steps:
- learning_rate = tf.train.exponential_decay(
- learning_rate,
- slim.get_or_create_global_step(),
- decay_steps=hparams.lr_decay_steps,
- decay_rate=hparams.lr_decay_rate,
- staircase=True)
- tf.summary.scalar('Learning_rate', learning_rate)
-
-
- if hparams.discriminator_steps == 0:
- discriminator_train_op = tf.no_op()
- else:
- discriminator_optimizer = tf.train.AdamOptimizer(
- learning_rate, beta1=hparams.adam_beta1)
-
- discriminator_train_op = slim.learning.create_train_op(
- discriminator_loss,
- discriminator_optimizer,
- update_ops=discriminator_update_ops,
- variables_to_train=discriminator_vars,
- clip_gradient_norm=hparams.clip_gradient_norm,
- summarize_gradients=FLAGS.summarize_gradients)
-
- if hparams.generator_steps == 0:
- generator_train_op = tf.no_op()
- else:
- generator_optimizer = tf.train.AdamOptimizer(
- learning_rate, beta1=hparams.adam_beta1)
- generator_train_op = slim.learning.create_train_op(
- generator_loss,
- generator_optimizer,
- update_ops=generator_update_ops,
- variables_to_train=generator_vars,
- clip_gradient_norm=hparams.clip_gradient_norm,
- summarize_gradients=FLAGS.summarize_gradients)
-
- #############
- # Summaries #
- #############
- pixelda_utils.summarize_model(end_points)
- pixelda_utils.summarize_transferred_grid(
- end_points['transferred_images'], source_images, name='Transferred')
- if 'source_images_recon' in end_points:
- pixelda_utils.summarize_transferred_grid(
- end_points['source_images_recon'],
- source_images,
- name='Source Reconstruction')
- pixelda_utils.summaries_color_distributions(end_points['transferred_images'],
- 'Transferred')
- pixelda_utils.summaries_color_distributions(target_images, 'Target')
-
- if source_images is not None:
- pixelda_utils.summarize_transferred(source_images,
- end_points['transferred_images'])
- pixelda_utils.summaries_color_distributions(source_images, 'Source')
- pixelda_utils.summaries_color_distributions(
- tf.abs(source_images - end_points['transferred_images']),
- 'Abs(Source_minus_Transferred)')
-
- number_of_steps = None
- if hparams.num_training_examples:
- # Want to control by amount of data seen, not # steps
- number_of_steps = hparams.num_training_examples / hparams.batch_size
-
- hooks = [tf.train.StepCounterHook(),]
-
- chief_only_hooks = [
- tf.train.CheckpointSaverHook(
- saver=tf.train.Saver(),
- checkpoint_dir=run_dir,
- save_secs=FLAGS.save_interval_secs)
- ]
-
- if number_of_steps:
- hooks.append(tf.train.StopAtStepHook(last_step=number_of_steps))
-
- _train(
- discriminator_train_op,
- generator_train_op,
- logdir=run_dir,
- master=FLAGS.master,
- is_chief=FLAGS.task == 0,
- hooks=hooks,
- chief_only_hooks=chief_only_hooks,
- save_checkpoint_secs=None,
- save_summaries_steps=FLAGS.save_summaries_steps,
- hparams=hparams)
-
-def main(_):
- tf.logging.set_verbosity(tf.logging.INFO)
- hparams = create_hparams(FLAGS.hparams)
- run_training(
- run_dir=FLAGS.train_log_dir,
- checkpoint_dir=FLAGS.train_log_dir,
- hparams=hparams)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/domain_adaptation/pixel_domain_adaptation/pixelda_utils.py b/research/domain_adaptation/pixel_domain_adaptation/pixelda_utils.py
deleted file mode 100644
index 28e8006f267f9bf7f13c3dff78625cc4cbd00185..0000000000000000000000000000000000000000
--- a/research/domain_adaptation/pixel_domain_adaptation/pixelda_utils.py
+++ /dev/null
@@ -1,195 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Utilities for PixelDA model."""
-import math
-
-# Dependency imports
-
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-flags = tf.app.flags
-FLAGS = flags.FLAGS
-
-
-def remove_depth(images):
- """Takes a batch of images and remove depth channel if present."""
- if images.shape.as_list()[-1] == 4:
- return images[:, :, :, 0:3]
- return images
-
-
-def image_grid(images, max_grid_size=4):
- """Given images and N, return first N^2 images as an NxN image grid.
-
- Args:
- images: a `Tensor` of size [batch_size, height, width, channels]
- max_grid_size: Maximum image grid height/width
-
- Returns:
- Single image batch, of dim [1, h*n, w*n, c]
- """
- images = remove_depth(images)
- batch_size = images.shape.as_list()[0]
- grid_size = min(int(math.sqrt(batch_size)), max_grid_size)
- assert images.shape.as_list()[0] >= grid_size * grid_size
-
- # If we have a depth channel
- if images.shape.as_list()[-1] == 4:
- images = images[:grid_size * grid_size, :, :, 0:3]
- depth = tf.image.grayscale_to_rgb(images[:grid_size * grid_size, :, :, 3:4])
-
- images = tf.reshape(images, [-1, images.shape.as_list()[2], 3])
- split = tf.split(0, grid_size, images)
- depth = tf.reshape(depth, [-1, images.shape.as_list()[2], 3])
- depth_split = tf.split(0, grid_size, depth)
- grid = tf.concat(split + depth_split, 1)
- return tf.expand_dims(grid, 0)
- else:
- images = images[:grid_size * grid_size, :, :, :]
- images = tf.reshape(
- images, [-1, images.shape.as_list()[2],
- images.shape.as_list()[3]])
- split = tf.split(images, grid_size, 0)
- grid = tf.concat(split, 1)
- return tf.expand_dims(grid, 0)
-
-
-def source_and_output_image_grid(output_images,
- source_images=None,
- max_grid_size=4):
- """Create NxN image grid for output, concatenate source grid if given.
-
- Makes grid out of output_images and, if provided, source_images, and
- concatenates them.
-
- Args:
- output_images: [batch_size, h, w, c] tensor of images
- source_images: optional[batch_size, h, w, c] tensor of images
- max_grid_size: Image grid height/width
-
- Returns:
- Single image batch, of dim [1, h*n, w*n, c]
-
-
- """
- output_grid = image_grid(output_images, max_grid_size=max_grid_size)
- if source_images is not None:
- source_grid = image_grid(source_images, max_grid_size=max_grid_size)
- # Make sure they have the same # of channels before concat
- # Assumes either 1 or 3 channels
- if output_grid.shape.as_list()[-1] != source_grid.shape.as_list()[-1]:
- if output_grid.shape.as_list()[-1] == 1:
- output_grid = tf.tile(output_grid, [1, 1, 1, 3])
- if source_grid.shape.as_list()[-1] == 1:
- source_grid = tf.tile(source_grid, [1, 1, 1, 3])
- output_grid = tf.concat([output_grid, source_grid], 1)
- return output_grid
-
-
-def summarize_model(end_points):
- """Summarizes the given model via its end_points.
-
- Args:
- end_points: A dictionary of end_point names to `Tensor`.
- """
- tf.summary.histogram('domain_logits_transferred',
- tf.sigmoid(end_points['transferred_domain_logits']))
-
- tf.summary.histogram('domain_logits_target',
- tf.sigmoid(end_points['target_domain_logits']))
-
-
-def summarize_transferred_grid(transferred_images,
- source_images=None,
- name='Transferred'):
- """Produces a visual grid summarization of the image transferrence.
-
- Args:
- transferred_images: A `Tensor` of size [batch_size, height, width, c].
- source_images: A `Tensor` of size [batch_size, height, width, c].
- name: Name to use in summary name
- """
- if source_images is not None:
- grid = source_and_output_image_grid(transferred_images, source_images)
- else:
- grid = image_grid(transferred_images)
- tf.summary.image('%s_Images_Grid' % name, grid, max_outputs=1)
-
-
-def summarize_transferred(source_images,
- transferred_images,
- max_images=20,
- name='Transferred'):
- """Produces a visual summary of the image transferrence.
-
- This summary displays the source image, transferred image, and a grayscale
- difference image which highlights the differences between input and output.
-
- Args:
- source_images: A `Tensor` of size [batch_size, height, width, channels].
- transferred_images: A `Tensor` of size [batch_size, height, width, channels]
- max_images: The number of images to show.
- name: Name to use in summary name
-
- Raises:
- ValueError: If number of channels in source and target are incompatible
- """
- source_channels = source_images.shape.as_list()[-1]
- transferred_channels = transferred_images.shape.as_list()[-1]
- if source_channels < transferred_channels:
- if source_channels != 1:
- raise ValueError(
- 'Source must be 1 channel or same # of channels as target')
- source_images = tf.tile(source_images, [1, 1, 1, transferred_channels])
- if transferred_channels < source_channels:
- if transferred_channels != 1:
- raise ValueError(
- 'Target must be 1 channel or same # of channels as source')
- transferred_images = tf.tile(transferred_images, [1, 1, 1, source_channels])
- diffs = tf.abs(source_images - transferred_images)
- diffs = tf.reduce_max(diffs, reduction_indices=[3], keep_dims=True)
- diffs = tf.tile(diffs, [1, 1, 1, max(source_channels, transferred_channels)])
-
- transition_images = tf.concat([
- source_images,
- transferred_images,
- diffs,
- ], 2)
-
- tf.summary.image(
- '%s_difference' % name, transition_images, max_outputs=max_images)
-
-
-def summaries_color_distributions(images, name):
- """Produces a histogram of the color distributions of the images.
-
- Args:
- images: A `Tensor` of size [batch_size, height, width, 3].
- name: The name of the images being summarized.
- """
- tf.summary.histogram('color_values/%s' % name, images)
-
-
-def summarize_images(images, name):
- """Produces a visual summary of the given images.
-
- Args:
- images: A `Tensor` of size [batch_size, height, width, 3].
- name: The name of the images being summarized.
- """
- grid = image_grid(images)
- tf.summary.image('%s_Images' % name, grid, max_outputs=1)
diff --git a/research/feelvos/CONTRIBUTING.md b/research/feelvos/CONTRIBUTING.md
deleted file mode 100644
index 939e5341e74dc2371c8b47f0e27b50581bed5f63..0000000000000000000000000000000000000000
--- a/research/feelvos/CONTRIBUTING.md
+++ /dev/null
@@ -1,28 +0,0 @@
-# How to Contribute
-
-We'd love to accept your patches and contributions to this project. There are
-just a few small guidelines you need to follow.
-
-## Contributor License Agreement
-
-Contributions to this project must be accompanied by a Contributor License
-Agreement. You (or your employer) retain the copyright to your contribution;
-this simply gives us permission to use and redistribute your contributions as
-part of the project. Head over to to see
-your current agreements on file or to sign a new one.
-
-You generally only need to submit a CLA once, so if you've already submitted one
-(even if it was for a different project), you probably don't need to do it
-again.
-
-## Code reviews
-
-All submissions, including submissions by project members, require review. We
-use GitHub pull requests for this purpose. Consult
-[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
-information on using pull requests.
-
-## Community Guidelines
-
-This project follows [Google's Open Source Community
-Guidelines](https://opensource.google.com/conduct/).
diff --git a/research/feelvos/LICENSE b/research/feelvos/LICENSE
deleted file mode 100644
index d645695673349e3947e8e5ae42332d0ac3164cd7..0000000000000000000000000000000000000000
--- a/research/feelvos/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/research/feelvos/README.md b/research/feelvos/README.md
deleted file mode 100644
index 69017c8b19fc1427c47cbdfbdce408ffa92ec32c..0000000000000000000000000000000000000000
--- a/research/feelvos/README.md
+++ /dev/null
@@ -1,102 +0,0 @@
-
-
-
-
-# FEELVOS: Fast End-to-End Embedding Learning for Video Object Segmentation
-
-FEELVOS is a fast model for video object segmentation which does not rely on fine-tuning on the
-first frame.
-
-For details, please refer to our paper. If you find the code useful, please
-also consider citing it.
-
-* FEELVOS:
-
-```
-@inproceedings{feelvos2019,
- title={FEELVOS: Fast End-to-End Embedding Learning for Video Object Segmentation},
- author={Paul Voigtlaender and Yuning Chai and Florian Schroff and Hartwig Adam and Bastian Leibe and Liang-Chieh Chen},
- booktitle={CVPR},
- year={2019}
-}
-```
-
-## Dependencies
-
-FEELVOS requires a good GPU with around 12 GB of memory and depends on the following libraries
-
-* TensorFlow
-* Pillow
-* Numpy
-* Scipy
-* Scikit Learn Image
-* tf Slim (which is included in the "tensorflow/models/research/" checkout)
-* DeepLab (which is included in the "tensorflow/models/research/" checkout)
-* correlation_cost (optional, see below)
-
-For detailed steps to install Tensorflow, follow the [Tensorflow installation
-instructions](https://www.tensorflow.org/install/). A typical user can install
-Tensorflow using the following command:
-
-```bash
-pip install tensorflow-gpu
-```
-
-The remaining libraries can also be installed with pip using:
-
-```bash
-pip install pillow scipy scikit-image
-```
-
-## Dependency on correlation_cost
-
-For fast cross-correlation, we use correlation cost as an external dependency. By default FEELVOS
-will use a slow and memory hungry fallback implementation without correlation_cost. If you care for
-performance, you should set up correlation_cost by following the instructions in
-correlation_cost/README and afterwards setting ```USE_CORRELATION_COST = True``` in
-utils/embedding_utils.py.
-
-## Pre-trained Models
-
-We provide 2 pre-trained FEELVOS models, both are based on Xception-65:
-
-* [Trained on DAVIS 2017](http://download.tensorflow.org/models/feelvos_davis17_trained.tar.gz)
-* [Trained on DAVIS 2017 and YouTube-VOS](http://download.tensorflow.org/models/feelvos_davis17_and_youtubevos_trained.tar.gz)
-
-Additionally, we provide a [DeepLab checkpoint for Xception-65 pre-trained on ImageNet and COCO](http://download.tensorflow.org/models/xception_65_coco_pretrained_2018_10_02.tar.gz),
-which can be used as an initialization for training FEELVOS.
-
-## Pre-computed Segmentation Masks
-
-We provide [pre-computed segmentation masks](http://download.tensorflow.org/models/feelvos_precomputed_masks.zip)
-for FEELVOS both for training with and without YouTube-VOS data for the following datasets:
-
-* DAVIS 2017 validation set
-* DAVIS 2017 test-dev set
-* YouTube-Objects dataset
-
-## Local Inference
-For a demo of local inference on DAVIS 2017 run
-
-```bash
-# From tensorflow/models/research/feelvos
-sh eval.sh
-```
-
-## Local Training
-For a demo of local training on DAVIS 2017 run
-
-```bash
-# From tensorflow/models/research/feelvos
-sh train.sh
-```
-
-## Contacts (Maintainers)
-* Paul Voigtlaender, github: [pvoigtlaender](https://github.com/pvoigtlaender)
-* Yuning Chai, github: [yuningchai](https://github.com/yuningchai)
-* Liang-Chieh Chen, github: [aquariusjay](https://github.com/aquariusjay)
-
-## License
-
-All the codes in feelvos folder is covered by the [LICENSE](https://github.com/tensorflow/models/blob/master/LICENSE)
-under tensorflow/models. Please refer to the LICENSE for details.
diff --git a/research/feelvos/__init__.py b/research/feelvos/__init__.py
deleted file mode 100644
index 6f1373443d0ff84fd90714e41dade400ab41a22c..0000000000000000000000000000000000000000
--- a/research/feelvos/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
diff --git a/research/feelvos/common.py b/research/feelvos/common.py
deleted file mode 100644
index 98f5a9ce348aea36efa4b3cc57048d3659f18895..0000000000000000000000000000000000000000
--- a/research/feelvos/common.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Provides flags that are common to scripts.
-
-Common flags from train/vis_video.py are collected in this script.
-"""
-import tensorflow as tf
-
-from deeplab import common
-
-flags = tf.app.flags
-
-flags.DEFINE_enum(
- 'classification_loss', 'softmax_with_attention',
- ['softmax', 'triplet', 'softmax_with_attention'],
- 'Type of loss function used for classifying pixels, can be either softmax, '
- 'softmax_with_attention, or triplet.')
-
-flags.DEFINE_integer('k_nearest_neighbors', 1,
- 'The number of nearest neighbors to use.')
-
-flags.DEFINE_integer('embedding_dimension', 100, 'The dimension used for the '
- 'learned embedding')
-
-flags.DEFINE_boolean('use_softmax_feedback', True,
- 'Whether to give the softmax predictions of the last '
- 'frame as additional input to the segmentation head.')
-
-flags.DEFINE_boolean('sample_adjacent_and_consistent_query_frames', True,
- 'If true, the query frames (all but the first frame '
- 'which is the reference frame) will be sampled such '
- 'that they are adjacent video frames and have the same '
- 'crop coordinates and flip augmentation. Note that if '
- 'use_softmax_feedback is True, this option will '
- 'automatically be activated.')
-
-flags.DEFINE_integer('embedding_seg_feature_dimension', 256,
- 'The dimensionality used in the segmentation head layers.')
-
-flags.DEFINE_integer('embedding_seg_n_layers', 4, 'The number of layers in the '
- 'segmentation head.')
-
-flags.DEFINE_integer('embedding_seg_kernel_size', 7, 'The kernel size used in '
- 'the segmentation head.')
-
-flags.DEFINE_multi_integer('embedding_seg_atrous_rates', [],
- 'The atrous rates to use for the segmentation head.')
-
-flags.DEFINE_boolean('normalize_nearest_neighbor_distances', True,
- 'Whether to normalize the nearest neighbor distances '
- 'to [0,1] using sigmoid, scale and shift.')
-
-flags.DEFINE_boolean('also_attend_to_previous_frame', True, 'Whether to also '
- 'use nearest neighbor attention with respect to the '
- 'previous frame.')
-
-flags.DEFINE_bool('use_local_previous_frame_attention', True,
- 'Whether to restrict the previous frame attention to a local '
- 'search window. Only has an effect, if '
- 'also_attend_to_previous_frame is True.')
-
-flags.DEFINE_integer('previous_frame_attention_window_size', 15,
- 'The window size used for local previous frame attention,'
- ' if use_local_previous_frame_attention is True.')
-
-flags.DEFINE_boolean('use_first_frame_matching', True, 'Whether to extract '
- 'features by matching to the reference frame. This should '
- 'always be true except for ablation experiments.')
-
-FLAGS = flags.FLAGS
-
-# Constants
-
-# Perform semantic segmentation predictions.
-OUTPUT_TYPE = common.OUTPUT_TYPE
-
-# Semantic segmentation item names.
-LABELS_CLASS = common.LABELS_CLASS
-IMAGE = common.IMAGE
-HEIGHT = common.HEIGHT
-WIDTH = common.WIDTH
-IMAGE_NAME = common.IMAGE_NAME
-SOURCE_ID = 'source_id'
-VIDEO_ID = 'video_id'
-LABEL = common.LABEL
-ORIGINAL_IMAGE = common.ORIGINAL_IMAGE
-PRECEDING_FRAME_LABEL = 'preceding_frame_label'
-
-# Test set name.
-TEST_SET = common.TEST_SET
-
-# Internal constants.
-OBJECT_LABEL = 'object_label'
-
-
-class VideoModelOptions(common.ModelOptions):
- """Internal version of immutable class to hold model options."""
-
- def __new__(cls,
- outputs_to_num_classes,
- crop_size=None,
- atrous_rates=None,
- output_stride=8):
- """Constructor to set default values.
-
- Args:
- outputs_to_num_classes: A dictionary from output type to the number of
- classes. For example, for the task of semantic segmentation with 21
- semantic classes, we would have outputs_to_num_classes['semantic'] = 21.
- crop_size: A tuple [crop_height, crop_width].
- atrous_rates: A list of atrous convolution rates for ASPP.
- output_stride: The ratio of input to output spatial resolution.
-
- Returns:
- A new VideoModelOptions instance.
- """
- self = super(VideoModelOptions, cls).__new__(
- cls,
- outputs_to_num_classes,
- crop_size,
- atrous_rates,
- output_stride)
- # Add internal flags.
- self.classification_loss = FLAGS.classification_loss
-
- return self
-
-
-def parse_decoder_output_stride():
- """Parses decoder output stride.
-
- FEELVOS assumes decoder_output_stride = 4. Thus, this function is created for
- this particular purpose.
-
- Returns:
- An integer specifying the decoder_output_stride.
-
- Raises:
- ValueError: If decoder_output_stride is None or contains more than one
- element.
- """
- if FLAGS.decoder_output_stride:
- decoder_output_stride = [
- int(x) for x in FLAGS.decoder_output_stride]
- if len(decoder_output_stride) != 1:
- raise ValueError('Expect decoder output stride has only one element.')
- decoder_output_stride = decoder_output_stride[0]
- else:
- raise ValueError('Expect flag decoder output stride not to be None.')
- return decoder_output_stride
diff --git a/research/feelvos/correlation_cost/README.md b/research/feelvos/correlation_cost/README.md
deleted file mode 100644
index 6cdbe550c7fcf63191f6967dd99c72cf341302bc..0000000000000000000000000000000000000000
--- a/research/feelvos/correlation_cost/README.md
+++ /dev/null
@@ -1,36 +0,0 @@
-# correlation_cost
-
-FEELVOS uses correlation_cost as an optional dependency to improve the speed and memory consumption
-of cross-correlation.
-
-## Installation
-
-Unfortunately we cannot provide the code for correlation_cost directly, so you
-will have to copy some files from this pull request
-https://github.com/tensorflow/tensorflow/pull/21392/. For your convenience we
-prepared scripts to download and adjust the code automatically.
-
-In the best case, all you need to do is run compile.sh with the path to your
-CUDA installation (tested only with CUDA 9).
-Note that the path should be to a folder containing the cuda folder, not to the
-cuda folder itself, e.g. if your cuda is in /usr/local/cuda-9.0, you can create
-a symlink /usr/local/cuda pointing to /usr/local/cuda-9.0 and then run
-
-```bash
-sh build.sh /usr/local/
-```
-
-This will
-
-* Download the code via ```sh get_code.sh ```
-* Apply minor adjustments to the code via ```sh fix_code.sh```
-* Clone the dependencies cub and thrust from github via ```sh clone_dependencies.sh```
-* Compile a shared library correlation_cost.so for correlation_cost via
-```sh compile.sh "${CUDA_DIR}"```
-
-Please review the licenses of correlation_cost, cub, and thrust.
-
-## Enabling correlation_cost
-If you managed to create the correlation_cost.so file, then set
-```USE_CORRELATION_COST = True``` in feelvos/utils/embedding_utils.py and try to run
-```sh eval.sh```.
diff --git a/research/feelvos/correlation_cost/build.sh b/research/feelvos/correlation_cost/build.sh
deleted file mode 100755
index 37d9adb3147df07646a462fd170772393abf5642..0000000000000000000000000000000000000000
--- a/research/feelvos/correlation_cost/build.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-#
-# This script is used to download and build the code for correlation_cost.
-#
-# Usage:
-# sh ./build.sh cuda_dir
-# Where cuda_dir points to a directory containing the cuda folder (not the cuda folder itself).
-#
-#
-
-if [ "$#" -ne 1 ]; then
- echo "Illegal number of parameters, usage: ./build.sh cuda_dir"
- echo "Where cuda_dir points to a directory containing the cuda folder (not the cuda folder itself)"
- exit 1
-fi
-
-set -e
-set -x
-
-sh ./get_code.sh
-sh ./fix_code.sh
-sh ./clone_dependencies.sh
-sh ./compile.sh $1
diff --git a/research/feelvos/correlation_cost/clone_dependencies.sh b/research/feelvos/correlation_cost/clone_dependencies.sh
deleted file mode 100755
index 9174313f58a833a5ab547e21c63cdc87681cbc5d..0000000000000000000000000000000000000000
--- a/research/feelvos/correlation_cost/clone_dependencies.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-#
-# This script is used to clone the dependencies, i.e. cub and thrust, of correlation_cost from github.
-#
-# Usage:
-# sh ./clone_dependencies.sh
-#
-#
-
-# Clone cub.
-if [ ! -d cub ] ; then
- git clone https://github.com/dmlc/cub.git
-fi
-# Clone thrust.
-if [ ! -d thrust ] ; then
- git clone https://github.com/thrust/thrust.git
-fi
diff --git a/research/feelvos/correlation_cost/compile.sh b/research/feelvos/correlation_cost/compile.sh
deleted file mode 100755
index 6025292dfa78b44dd6fcf2f1b349af936a43fcc7..0000000000000000000000000000000000000000
--- a/research/feelvos/correlation_cost/compile.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-#
-# This script is used to compile the code for correlation_cost and create correlation_cost.so.
-#
-# Usage:
-# sh ./compile.sh cuda_dir
-# Where cuda_dir points to a directory containing the cuda folder (not the cuda folder itself).
-#
-#
-
-if [ "$#" -ne 1 ]; then
- echo "Illegal number of parameters, usage: ./compile.sh cuda_dir"
- exit 1
-fi
-CUDA_DIR=$1
-
-if [ ! -d "${CUDA_DIR}/cuda" ]; then
- echo "cuda_dir must point to a directory containing the cuda folder, not to the cuda folder itself"
- exit 1
-fi
-
-TF_CFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )
-TF_LFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') )
-CUB_DIR=cub
-THRUST_DIR=thrust
-
-# Depending on the versions of your nvcc and gcc, the flag --expt-relaxed-constexpr might be required or should be removed.
-# If nvcc complains about a too new gcc version, you can point it to another gcc
-# version by using something like nvcc -ccbin /path/to/your/gcc6
-nvcc -std=c++11 --expt-relaxed-constexpr -I ./ -I ${CUB_DIR}/../ -I ${THRUST_DIR} -I ${CUDA_DIR}/ -c -o correlation_cost_op_gpu.o kernels/correlation_cost_op_gpu.cu.cc ${TF_CFLAGS[@]} -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC
-
-g++ -std=c++11 -I ./ -L ${CUDA_DIR}/cuda/lib64 -shared -o correlation_cost.so ops/correlation_cost_op.cc kernels/correlation_cost_op.cc correlation_cost_op_gpu.o ${TF_CFLAGS[@]} -fPIC -lcudart ${TF_LFLAGS[@]} -D GOOGLE_CUDA=1
diff --git a/research/feelvos/correlation_cost/fix_code.sh b/research/feelvos/correlation_cost/fix_code.sh
deleted file mode 100755
index d4f285db3d745fc55a20bac57f97c6ca2fd8a5c4..0000000000000000000000000000000000000000
--- a/research/feelvos/correlation_cost/fix_code.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-#
-# This script is used to modify the downloaded code.
-#
-# Usage:
-# sh ./fix_code.sh
-#
-#
-
-sed -i "s/tensorflow\/contrib\/correlation_cost\///g" kernels/correlation_cost_op_gpu.cu.cc
-sed -i "s/tensorflow\/contrib\/correlation_cost\///g" kernels/correlation_cost_op.cc
-sed -i "s/external\/cub_archive\//cub\//g" kernels/correlation_cost_op_gpu.cu.cc
-
-sed -i "s/from tensorflow.contrib.util import loader/import tensorflow as tf/g" python/ops/correlation_cost_op.py
-grep -v "from tensorflow" python/ops/correlation_cost_op.py | grep -v resource_loader.get_path_to_datafile > correlation_cost_op.py.tmp && mv correlation_cost_op.py.tmp python/ops/correlation_cost_op.py
-sed -i "s/array_ops/tf/g" python/ops/correlation_cost_op.py
-sed -i "s/ops/tf/g" python/ops/correlation_cost_op.py
-sed -i "s/loader.load_op_library(/tf.load_op_library('feelvos\/correlation_cost\/correlation_cost.so')/g" python/ops/correlation_cost_op.py
-sed -i "s/gen_correlation_cost_op/_correlation_cost_op_so/g" python/ops/correlation_cost_op.py
diff --git a/research/feelvos/correlation_cost/get_code.sh b/research/feelvos/correlation_cost/get_code.sh
deleted file mode 100755
index 337142166ac4b61835417e807ef0a495532d749c..0000000000000000000000000000000000000000
--- a/research/feelvos/correlation_cost/get_code.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-#
-# This script is used to download the code for correlation_cost.
-#
-# Usage:
-# sh ./get_code.sh
-#
-#
-
-mkdir -p kernels ops python/ops
-touch __init__.py
-touch python/__init__.py
-touch python/ops/__init__.py
-wget https://raw.githubusercontent.com/tensorflow/tensorflow/91b163b9bd8dd0f8c2631b4245a67dfd387536a6/tensorflow/contrib/correlation_cost/ops/correlation_cost_op.cc -O ops/correlation_cost_op.cc
-wget https://raw.githubusercontent.com/tensorflow/tensorflow/91b163b9bd8dd0f8c2631b4245a67dfd387536a6/tensorflow/contrib/correlation_cost/python/ops/correlation_cost_op.py -O python/ops/correlation_cost_op.py
-wget https://raw.githubusercontent.com/tensorflow/tensorflow/91b163b9bd8dd0f8c2631b4245a67dfd387536a6/tensorflow/contrib/correlation_cost/kernels/correlation_cost_op.cc -O kernels/correlation_cost_op.cc
-wget https://raw.githubusercontent.com/tensorflow/tensorflow/91b163b9bd8dd0f8c2631b4245a67dfd387536a6/tensorflow/contrib/correlation_cost/kernels/correlation_cost_op.h -O kernels/correlation_cost_op.h
-wget https://raw.githubusercontent.com/tensorflow/tensorflow/91b163b9bd8dd0f8c2631b4245a67dfd387536a6/tensorflow/contrib/correlation_cost/kernels/correlation_cost_op_gpu.cu.cc -O kernels/correlation_cost_op_gpu.cu.cc
diff --git a/research/feelvos/datasets/__init__.py b/research/feelvos/datasets/__init__.py
deleted file mode 100644
index 6f1373443d0ff84fd90714e41dade400ab41a22c..0000000000000000000000000000000000000000
--- a/research/feelvos/datasets/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
diff --git a/research/feelvos/datasets/build_davis2017_data.py b/research/feelvos/datasets/build_davis2017_data.py
deleted file mode 100644
index 5e093fc3b4531f5439957ea3608770441bd5ce4a..0000000000000000000000000000000000000000
--- a/research/feelvos/datasets/build_davis2017_data.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Converts DAVIS 2017 data to TFRecord file format with SequenceExample protos.
-"""
-
-import io
-import math
-import os
-from StringIO import StringIO
-import numpy as np
-import PIL
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_string('data_folder', 'DAVIS2017/',
- 'Folder containing the DAVIS 2017 data')
-
-tf.app.flags.DEFINE_string('imageset', 'val',
- 'Which subset to use, either train or val')
-
-tf.app.flags.DEFINE_string(
- 'output_dir', './tfrecord',
- 'Path to save converted TFRecords of TensorFlow examples.')
-
-_NUM_SHARDS_TRAIN = 10
-_NUM_SHARDS_VAL = 1
-
-
-def read_image(path):
- with open(path) as fid:
- image_str = fid.read()
- image = PIL.Image.open(io.BytesIO(image_str))
- w, h = image.size
- return image_str, (h, w)
-
-
-def read_annotation(path):
- """Reads a single image annotation from a png image.
-
- Args:
- path: Path to the png image.
-
- Returns:
- png_string: The png encoded as string.
- size: Tuple of (height, width).
- """
- with open(path) as fid:
- x = np.array(PIL.Image.open(fid))
- h, w = x.shape
- im = PIL.Image.fromarray(x)
-
- output = StringIO()
- im.save(output, format='png')
- png_string = output.getvalue()
- output.close()
-
- return png_string, (h, w)
-
-
-def process_video(key, input_dir, anno_dir):
- """Creates a SequenceExample for the video.
-
- Args:
- key: Name of the video.
- input_dir: Directory which contains the image files.
- anno_dir: Directory which contains the annotation files.
-
- Returns:
- The created SequenceExample.
- """
- frame_names = sorted(tf.gfile.ListDirectory(input_dir))
- anno_files = sorted(tf.gfile.ListDirectory(anno_dir))
- assert len(frame_names) == len(anno_files)
-
- sequence = tf.train.SequenceExample()
- context = sequence.context.feature
- features = sequence.feature_lists.feature_list
-
- for i, name in enumerate(frame_names):
- image_str, image_shape = read_image(
- os.path.join(input_dir, name))
- anno_str, anno_shape = read_annotation(
- os.path.join(anno_dir, name[:-4] + '.png'))
- image_encoded = features['image/encoded'].feature.add()
- image_encoded.bytes_list.value.append(image_str)
- segmentation_encoded = features['segmentation/object/encoded'].feature.add()
- segmentation_encoded.bytes_list.value.append(anno_str)
-
- np.testing.assert_array_equal(np.array(image_shape), np.array(anno_shape))
-
- if i == 0:
- first_shape = np.array(image_shape)
- else:
- np.testing.assert_array_equal(np.array(image_shape), first_shape)
-
- context['video_id'].bytes_list.value.append(key.encode('ascii'))
- context['clip/frames'].int64_list.value.append(len(frame_names))
- context['image/format'].bytes_list.value.append('JPEG')
- context['image/channels'].int64_list.value.append(3)
- context['image/height'].int64_list.value.append(first_shape[0])
- context['image/width'].int64_list.value.append(first_shape[1])
- context['segmentation/object/format'].bytes_list.value.append('PNG')
- context['segmentation/object/height'].int64_list.value.append(first_shape[0])
- context['segmentation/object/width'].int64_list.value.append(first_shape[1])
-
- return sequence
-
-
-def convert(data_folder, imageset, output_dir, num_shards):
- """Converts the specified subset of DAVIS 2017 to TFRecord format.
-
- Args:
- data_folder: The path to the DAVIS 2017 data.
- imageset: The subset to use, either train or val.
- output_dir: Where to store the TFRecords.
- num_shards: The number of shards used for storing the data.
- """
- sets_file = os.path.join(data_folder, 'ImageSets', '2017', imageset + '.txt')
- vids = [x.strip() for x in open(sets_file).readlines()]
- num_vids = len(vids)
- num_vids_per_shard = int(math.ceil(num_vids) / float(num_shards))
- for shard_id in range(num_shards):
- output_filename = os.path.join(
- output_dir,
- '%s-%05d-of-%05d.tfrecord' % (imageset, shard_id, num_shards))
- with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
- start_idx = shard_id * num_vids_per_shard
- end_idx = min((shard_id + 1) * num_vids_per_shard, num_vids)
- for i in range(start_idx, end_idx):
- print('Converting video %d/%d shard %d video %s' % (
- i + 1, num_vids, shard_id, vids[i]))
- img_dir = os.path.join(data_folder, 'JPEGImages', '480p', vids[i])
- anno_dir = os.path.join(data_folder, 'Annotations', '480p', vids[i])
- example = process_video(vids[i], img_dir, anno_dir)
- tfrecord_writer.write(example.SerializeToString())
-
-
-def main(unused_argv):
- imageset = FLAGS.imageset
- assert imageset in ('train', 'val')
- if imageset == 'train':
- num_shards = _NUM_SHARDS_TRAIN
- else:
- num_shards = _NUM_SHARDS_VAL
- convert(FLAGS.data_folder, FLAGS.imageset, FLAGS.output_dir, num_shards)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/feelvos/datasets/download_and_convert_davis17.sh b/research/feelvos/datasets/download_and_convert_davis17.sh
deleted file mode 100644
index 011be61ba7586c8f3d141ccc00194d1c7ae56c3a..0000000000000000000000000000000000000000
--- a/research/feelvos/datasets/download_and_convert_davis17.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-#
-# Script to download and preprocess the DAVIS 2017 dataset.
-#
-# Usage:
-# bash ./download_and_convert_davis17.sh
-
-# Exit immediately if a command exits with a non-zero status.
-set -e
-
-CURRENT_DIR=$(pwd)
-WORK_DIR="./davis17"
-mkdir -p "${WORK_DIR}"
-cd "${WORK_DIR}"
-
-# Helper function to download and unpack the DAVIS 2017 dataset.
-download_and_uncompress() {
- local BASE_URL=${1}
- local FILENAME=${2}
-
- if [ ! -f "${FILENAME}" ]; then
- echo "Downloading ${FILENAME} to ${WORK_DIR}"
- wget -nd -c "${BASE_URL}/${FILENAME}"
- echo "Uncompressing ${FILENAME}"
- unzip "${FILENAME}"
- fi
-}
-
-BASE_URL="https://data.vision.ee.ethz.ch/csergi/share/davis/"
-FILENAME="DAVIS-2017-trainval-480p.zip"
-
-download_and_uncompress "${BASE_URL}" "${FILENAME}"
-
-cd "${CURRENT_DIR}"
-
-# Root path for DAVIS 2017 dataset.
-DAVIS_ROOT="${WORK_DIR}/DAVIS"
-
-# Build TFRecords of the dataset.
-# First, create output directory for storing TFRecords.
-OUTPUT_DIR="${WORK_DIR}/tfrecord"
-mkdir -p "${OUTPUT_DIR}"
-
-IMAGE_FOLDER="${DAVIS_ROOT}/JPEGImages"
-LIST_FOLDER="${DAVIS_ROOT}/ImageSets/Segmentation"
-
-# Convert validation set.
-if [ ! -f "${OUTPUT_DIR}/val-00000-of-00001.tfrecord" ]; then
- echo "Converting DAVIS 2017 dataset (val)..."
- python ./build_davis2017_data.py \
- --data_folder="${DAVIS_ROOT}" \
- --imageset=val \
- --output_dir="${OUTPUT_DIR}"
-fi
-
-# Convert training set.
-if [ ! -f "${OUTPUT_DIR}/train-00009-of-00010.tfrecord" ]; then
- echo "Converting DAVIS 2017 dataset (train)..."
- python ./build_davis2017_data.py \
- --data_folder="${DAVIS_ROOT}" \
- --imageset=train \
- --output_dir="${OUTPUT_DIR}"
-fi
diff --git a/research/feelvos/datasets/tfsequence_example_decoder.py b/research/feelvos/datasets/tfsequence_example_decoder.py
deleted file mode 100644
index 2fa3e95d5b98eb00aa485371037b4ad6b0e7ece3..0000000000000000000000000000000000000000
--- a/research/feelvos/datasets/tfsequence_example_decoder.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contains the TFExampleDecoder.
-
-The TFExampleDecode is a DataDecoder used to decode TensorFlow Example protos.
-In order to do so each requested item must be paired with one or more Example
-features that are parsed to produce the Tensor-based manifestation of the item.
-"""
-
-import tensorflow as tf
-slim = tf.contrib.slim
-data_decoder = slim.data_decoder
-
-
-class TFSequenceExampleDecoder(data_decoder.DataDecoder):
- """A decoder for TensorFlow SequenceExamples.
-
- Decoding SequenceExample proto buffers is comprised of two stages:
- (1) Example parsing and (2) tensor manipulation.
-
- In the first stage, the tf.parse_single_sequence_example function is called
- with a list of FixedLenFeatures and SparseLenFeatures. These instances tell TF
- how to parse the example. The output of this stage is a set of tensors.
-
- In the second stage, the resulting tensors are manipulated to provide the
- requested 'item' tensors.
-
- To perform this decoding operation, a SequenceExampleDecoder is given a list
- of ItemHandlers. Each ItemHandler indicates the set of features for stage 1
- and contains the instructions for post_processing its tensors for stage 2.
- """
-
- def __init__(self, keys_to_context_features, keys_to_sequence_features,
- items_to_handlers):
- """Constructs the decoder.
-
- Args:
- keys_to_context_features: a dictionary from TF-SequenceExample context
- keys to either tf.VarLenFeature or tf.FixedLenFeature instances.
- See tensorflow's parsing_ops.py.
- keys_to_sequence_features: a dictionary from TF-SequenceExample sequence
- keys to either tf.VarLenFeature or tf.FixedLenSequenceFeature instances.
- See tensorflow's parsing_ops.py.
- items_to_handlers: a dictionary from items (strings) to ItemHandler
- instances. Note that the ItemHandler's are provided the keys that they
- use to return the final item Tensors.
-
- Raises:
- ValueError: if the same key is present for context features and sequence
- features.
- """
- unique_keys = set()
- unique_keys.update(keys_to_context_features)
- unique_keys.update(keys_to_sequence_features)
- if len(unique_keys) != (
- len(keys_to_context_features) + len(keys_to_sequence_features)):
- # This situation is ambiguous in the decoder's keys_to_tensors variable.
- raise ValueError('Context and sequence keys are not unique. \n'
- ' Context keys: %s \n Sequence keys: %s' %
- (list(keys_to_context_features.keys()),
- list(keys_to_sequence_features.keys())))
-
- self._keys_to_context_features = keys_to_context_features
- self._keys_to_sequence_features = keys_to_sequence_features
- self._items_to_handlers = items_to_handlers
-
- def list_items(self):
- """See base class."""
- return self._items_to_handlers.keys()
-
- def decode(self, serialized_example, items=None):
- """Decodes the given serialized TF-SequenceExample.
-
- Args:
- serialized_example: a serialized TF-SequenceExample tensor.
- items: the list of items to decode. These must be a subset of the item
- keys in self._items_to_handlers. If `items` is left as None, then all
- of the items in self._items_to_handlers are decoded.
-
- Returns:
- the decoded items, a list of tensor.
- """
-
- context, feature_list = tf.parse_single_sequence_example(
- serialized_example, self._keys_to_context_features,
- self._keys_to_sequence_features)
-
- # Reshape non-sparse elements just once:
- for k in self._keys_to_context_features:
- v = self._keys_to_context_features[k]
- if isinstance(v, tf.FixedLenFeature):
- context[k] = tf.reshape(context[k], v.shape)
-
- if not items:
- items = self._items_to_handlers.keys()
-
- outputs = []
- for item in items:
- handler = self._items_to_handlers[item]
- keys_to_tensors = {
- key: context[key] if key in context else feature_list[key]
- for key in handler.keys
- }
- outputs.append(handler.tensors_to_item(keys_to_tensors))
- return outputs
diff --git a/research/feelvos/datasets/video_dataset.py b/research/feelvos/datasets/video_dataset.py
deleted file mode 100644
index 17b62e989af866df0232a0e6d921faee84fe1fa7..0000000000000000000000000000000000000000
--- a/research/feelvos/datasets/video_dataset.py
+++ /dev/null
@@ -1,196 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Provides data from video object segmentation datasets.
-
-This file provides both images and annotations (instance segmentations) for
-TensorFlow. Currently, we support the following datasets:
-
-1. DAVIS 2017 (https://davischallenge.org/davis2017/code.html).
-
-2. DAVIS 2016 (https://davischallenge.org/davis2016/code.html).
-
-3. YouTube-VOS (https://youtube-vos.org/dataset/download).
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import os.path
-import tensorflow as tf
-from feelvos.datasets import tfsequence_example_decoder
-
-slim = tf.contrib.slim
-dataset = slim.dataset
-tfexample_decoder = slim.tfexample_decoder
-
-
-_ITEMS_TO_DESCRIPTIONS = {
- 'image': 'A color image of varying height and width.',
- 'labels_class': ('A semantic segmentation label whose size matches image.'
- 'Its values range from 0 (background) to num_classes.'),
-}
-
-# Named tuple to describe the dataset properties.
-DatasetDescriptor = collections.namedtuple(
- 'DatasetDescriptor',
- ['splits_to_sizes', # Splits of the dataset into training, val, and test.
- 'num_classes', # Number of semantic classes.
- 'ignore_label', # Ignore label value.
- ]
-)
-
-_DAVIS_2016_INFORMATION = DatasetDescriptor(
- splits_to_sizes={'train': [30, 1830],
- 'val': [20, 1376]},
- num_classes=2,
- ignore_label=255,
-)
-
-_DAVIS_2017_INFORMATION = DatasetDescriptor(
- splits_to_sizes={'train': [60, 4219],
- 'val': [30, 2023],
- 'test-dev': [30, 2037]},
- num_classes=None, # Number of instances per videos differ.
- ignore_label=255,
-)
-
-_YOUTUBE_VOS_2018_INFORMATION = DatasetDescriptor(
- # Leave these sizes as None to allow for different splits into
- # training and validation sets.
- splits_to_sizes={'train': [None, None],
- 'val': [None, None]},
- num_classes=None, # Number of instances per video differs.
- ignore_label=255,
-)
-
-_DATASETS_INFORMATION = {
- 'davis_2016': _DAVIS_2016_INFORMATION,
- 'davis_2017': _DAVIS_2017_INFORMATION,
- 'youtube_vos_2018': _YOUTUBE_VOS_2018_INFORMATION,
-}
-
-# Default file pattern of SSTable. Note we include '-' to avoid the confusion
-# between `train-` and `trainval-` sets.
-_FILE_PATTERN = '%s-*'
-
-
-def get_dataset(dataset_name,
- split_name,
- dataset_dir,
- file_pattern=None,
- data_type='tf_sequence_example',
- decode_video_frames=False):
- """Gets an instance of slim Dataset.
-
- Args:
- dataset_name: String, dataset name.
- split_name: String, the train/val Split name.
- dataset_dir: String, the directory of the dataset sources.
- file_pattern: String, file pattern of SSTable.
- data_type: String, data type. Currently supports 'tf_example' and
- 'annotated_image'.
- decode_video_frames: Boolean, decode the images or not. Not decoding it here
- is useful if we subsample later
-
- Returns:
- An instance of slim Dataset.
-
- Raises:
- ValueError: If the dataset_name or split_name is not recognized, or if
- the dataset_type is not supported.
- """
- if dataset_name not in _DATASETS_INFORMATION:
- raise ValueError('The specified dataset is not supported yet.')
-
- splits_to_sizes = _DATASETS_INFORMATION[dataset_name].splits_to_sizes
-
- if split_name not in splits_to_sizes:
- raise ValueError('data split name %s not recognized' % split_name)
-
- # Prepare the variables for different datasets.
- num_classes = _DATASETS_INFORMATION[dataset_name].num_classes
- ignore_label = _DATASETS_INFORMATION[dataset_name].ignore_label
-
- if file_pattern is None:
- file_pattern = _FILE_PATTERN
- file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
- if data_type == 'tf_sequence_example':
- keys_to_context_features = {
- 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'),
- 'image/height': tf.FixedLenFeature((), tf.int64, default_value=0),
- 'image/width': tf.FixedLenFeature((), tf.int64, default_value=0),
- 'segmentation/object/format': tf.FixedLenFeature(
- (), tf.string, default_value='png'),
- 'video_id': tf.FixedLenFeature((), tf.string, default_value='unknown')
- }
- label_name = 'class' if dataset_name == 'davis_2016' else 'object'
- keys_to_sequence_features = {
- 'image/encoded': tf.FixedLenSequenceFeature((), dtype=tf.string),
- 'segmentation/{}/encoded'.format(label_name):
- tf.FixedLenSequenceFeature((), tf.string),
- 'segmentation/{}/encoded'.format(label_name):
- tf.FixedLenSequenceFeature((), tf.string),
- }
- items_to_handlers = {
- 'height': tfexample_decoder.Tensor('image/height'),
- 'width': tfexample_decoder.Tensor('image/width'),
- 'video_id': tfexample_decoder.Tensor('video_id')
- }
- if decode_video_frames:
- decode_image_handler = tfexample_decoder.Image(
- image_key='image/encoded',
- format_key='image/format',
- channels=3,
- repeated=True)
- items_to_handlers['image'] = decode_image_handler
- decode_label_handler = tfexample_decoder.Image(
- image_key='segmentation/{}/encoded'.format(label_name),
- format_key='segmentation/{}/format'.format(label_name),
- channels=1,
- repeated=True)
- items_to_handlers['labels_class'] = decode_label_handler
- else:
- items_to_handlers['image/encoded'] = tfexample_decoder.Tensor(
- 'image/encoded')
- items_to_handlers[
- 'segmentation/object/encoded'] = tfexample_decoder.Tensor(
- 'segmentation/{}/encoded'.format(label_name))
- decoder = tfsequence_example_decoder.TFSequenceExampleDecoder(
- keys_to_context_features, keys_to_sequence_features, items_to_handlers)
- else:
- raise ValueError('Unknown data type.')
-
- size = splits_to_sizes[split_name]
- if isinstance(size, collections.Sequence):
- num_videos = size[0]
- num_samples = size[1]
- else:
- num_videos = 0
- num_samples = size
-
- return dataset.Dataset(
- data_sources=file_pattern,
- reader=tf.TFRecordReader,
- decoder=decoder,
- num_samples=num_samples,
- num_videos=num_videos,
- items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
- ignore_label=ignore_label,
- num_classes=num_classes,
- name=dataset_name,
- multi_label=True)
diff --git a/research/feelvos/eval.sh b/research/feelvos/eval.sh
deleted file mode 100755
index 96cb7f409a1e652ba8263f35c3786cb0cb77f5d1..0000000000000000000000000000000000000000
--- a/research/feelvos/eval.sh
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-#
-# This script is used to locally run inference on DAVIS 2017. Users could also
-# modify from this script for their use case. See train.sh for an example of
-# local training.
-#
-# Usage:
-# # From the tensorflow/models/research/feelvos directory.
-# sh ./eval.sh
-#
-#
-
-# Exit immediately if a command exits with a non-zero status.
-set -e
-
-# Move one-level up to tensorflow/models/research directory.
-cd ..
-
-# Update PYTHONPATH.
-export PYTHONPATH=$PYTHONPATH:`pwd`:`pwd`/slim:`pwd`/feelvos
-
-# Set up the working environment.
-CURRENT_DIR=$(pwd)
-WORK_DIR="${CURRENT_DIR}/feelvos"
-
-# Run embedding_utils_test first to make sure the PYTHONPATH is correctly set.
-python "${WORK_DIR}"/utils/embedding_utils_test.py -v
-
-# Go to datasets folder and download and convert the DAVIS 2017 dataset.
-DATASET_DIR="datasets"
-cd "${WORK_DIR}/${DATASET_DIR}"
-sh download_and_convert_davis17.sh
-
-# Go to models folder and download and unpack the DAVIS 2017 trained model.
-MODELS_DIR="models"
-mkdir -p "${WORK_DIR}/${MODELS_DIR}"
-cd "${WORK_DIR}/${MODELS_DIR}"
-if [ ! -d "feelvos_davis17_trained" ]; then
- wget http://download.tensorflow.org/models/feelvos_davis17_trained.tar.gz
- tar -xvf feelvos_davis17_trained.tar.gz
- echo "model_checkpoint_path: \"model.ckpt-200004\"" > feelvos_davis17_trained/checkpoint
- rm feelvos_davis17_trained.tar.gz
-fi
-CHECKPOINT_DIR="${WORK_DIR}/${MODELS_DIR}/feelvos_davis17_trained/"
-
-# Go back to orignal directory.
-cd "${CURRENT_DIR}"
-
-# Set up the working directories.
-DAVIS_FOLDER="davis17"
-EXP_FOLDER="exp/eval_on_val_set"
-VIS_LOGDIR="${WORK_DIR}/${DATASET_DIR}/${DAVIS_FOLDER}/${EXP_FOLDER}/eval"
-mkdir -p ${VIS_LOGDIR}
-
-DAVIS_DATASET="${WORK_DIR}/${DATASET_DIR}/${DAVIS_FOLDER}/tfrecord"
-
-python "${WORK_DIR}"/vis_video.py \
- --dataset=davis_2017 \
- --dataset_dir="${DAVIS_DATASET}" \
- --vis_logdir="${VIS_LOGDIR}" \
- --checkpoint_dir="${CHECKPOINT_DIR}" \
- --logtostderr \
- --atrous_rates=12 \
- --atrous_rates=24 \
- --atrous_rates=36 \
- --decoder_output_stride=4 \
- --model_variant=xception_65 \
- --multi_grid=1 \
- --multi_grid=1 \
- --multi_grid=1 \
- --output_stride=8 \
- --save_segmentations
diff --git a/research/feelvos/input_preprocess.py b/research/feelvos/input_preprocess.py
deleted file mode 100644
index 954c0b42ef2650b1c25ec8071933beee57e9bd69..0000000000000000000000000000000000000000
--- a/research/feelvos/input_preprocess.py
+++ /dev/null
@@ -1,280 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Prepare the data used for FEELVOS training/evaluation."""
-import tensorflow as tf
-
-from deeplab.core import feature_extractor
-from deeplab.core import preprocess_utils
-
-# The probability of flipping the images and labels
-# left-right during training
-_PROB_OF_FLIP = 0.5
-
-get_random_scale = preprocess_utils.get_random_scale
-randomly_scale_image_and_label = (
- preprocess_utils.randomly_scale_image_and_label)
-
-
-def preprocess_image_and_label(image,
- label,
- crop_height,
- crop_width,
- min_resize_value=None,
- max_resize_value=None,
- resize_factor=None,
- min_scale_factor=1.,
- max_scale_factor=1.,
- scale_factor_step_size=0,
- ignore_label=255,
- is_training=True,
- model_variant=None):
- """Preprocesses the image and label.
-
- Args:
- image: Input image.
- label: Ground truth annotation label.
- crop_height: The height value used to crop the image and label.
- crop_width: The width value used to crop the image and label.
- min_resize_value: Desired size of the smaller image side.
- max_resize_value: Maximum allowed size of the larger image side.
- resize_factor: Resized dimensions are multiple of factor plus one.
- min_scale_factor: Minimum scale factor value.
- max_scale_factor: Maximum scale factor value.
- scale_factor_step_size: The step size from min scale factor to max scale
- factor. The input is randomly scaled based on the value of
- (min_scale_factor, max_scale_factor, scale_factor_step_size).
- ignore_label: The label value which will be ignored for training and
- evaluation.
- is_training: If the preprocessing is used for training or not.
- model_variant: Model variant (string) for choosing how to mean-subtract the
- images. See feature_extractor.network_map for supported model variants.
-
- Returns:
- original_image: Original image (could be resized).
- processed_image: Preprocessed image.
- label: Preprocessed ground truth segmentation label.
-
- Raises:
- ValueError: Ground truth label not provided during training.
- """
- if is_training and label is None:
- raise ValueError('During training, label must be provided.')
- if model_variant is None:
- tf.logging.warning('Default mean-subtraction is performed. Please specify '
- 'a model_variant. See feature_extractor.network_map for '
- 'supported model variants.')
-
- # Keep reference to original image.
- original_image = image
-
- processed_image = tf.cast(image, tf.float32)
-
- if label is not None:
- label = tf.cast(label, tf.int32)
-
- # Resize image and label to the desired range.
- if min_resize_value is not None or max_resize_value is not None:
- [processed_image, label] = (
- preprocess_utils.resize_to_range(
- image=processed_image,
- label=label,
- min_size=min_resize_value,
- max_size=max_resize_value,
- factor=resize_factor,
- align_corners=True))
- # The `original_image` becomes the resized image.
- original_image = tf.identity(processed_image)
-
- # Data augmentation by randomly scaling the inputs.
- scale = get_random_scale(
- min_scale_factor, max_scale_factor, scale_factor_step_size)
- processed_image, label = randomly_scale_image_and_label(
- processed_image, label, scale)
-
- processed_image.set_shape([None, None, 3])
-
- if crop_height is not None and crop_width is not None:
- # Pad image and label to have dimensions >= [crop_height, crop_width].
- image_shape = tf.shape(processed_image)
- image_height = image_shape[0]
- image_width = image_shape[1]
-
- target_height = image_height + tf.maximum(crop_height - image_height, 0)
- target_width = image_width + tf.maximum(crop_width - image_width, 0)
-
- # Pad image with mean pixel value.
- mean_pixel = tf.reshape(
- feature_extractor.mean_pixel(model_variant), [1, 1, 3])
- processed_image = preprocess_utils.pad_to_bounding_box(
- processed_image, 0, 0, target_height, target_width, mean_pixel)
-
- if label is not None:
- label = preprocess_utils.pad_to_bounding_box(
- label, 0, 0, target_height, target_width, ignore_label)
-
- # Randomly crop the image and label.
- if is_training and label is not None:
- processed_image, label = preprocess_utils.random_crop(
- [processed_image, label], crop_height, crop_width)
-
- processed_image.set_shape([crop_height, crop_width, 3])
-
- if label is not None:
- label.set_shape([crop_height, crop_width, 1])
-
- if is_training:
- # Randomly left-right flip the image and label.
- processed_image, label, _ = preprocess_utils.flip_dim(
- [processed_image, label], _PROB_OF_FLIP, dim=1)
-
- return original_image, processed_image, label
-
-
-def preprocess_images_and_labels_consistently(images,
- labels,
- crop_height,
- crop_width,
- min_resize_value=None,
- max_resize_value=None,
- resize_factor=None,
- min_scale_factor=1.,
- max_scale_factor=1.,
- scale_factor_step_size=0,
- ignore_label=255,
- is_training=True,
- model_variant=None):
- """Preprocesses images and labels in a consistent way.
-
- Similar to preprocess_image_and_label, but works on a list of images
- and a list of labels and uses the same crop coordinates and either flips
- all images and labels or none of them.
-
- Args:
- images: List of input images.
- labels: List of ground truth annotation labels.
- crop_height: The height value used to crop the image and label.
- crop_width: The width value used to crop the image and label.
- min_resize_value: Desired size of the smaller image side.
- max_resize_value: Maximum allowed size of the larger image side.
- resize_factor: Resized dimensions are multiple of factor plus one.
- min_scale_factor: Minimum scale factor value.
- max_scale_factor: Maximum scale factor value.
- scale_factor_step_size: The step size from min scale factor to max scale
- factor. The input is randomly scaled based on the value of
- (min_scale_factor, max_scale_factor, scale_factor_step_size).
- ignore_label: The label value which will be ignored for training and
- evaluation.
- is_training: If the preprocessing is used for training or not.
- model_variant: Model variant (string) for choosing how to mean-subtract the
- images. See feature_extractor.network_map for supported model variants.
-
- Returns:
- original_images: Original images (could be resized).
- processed_images: Preprocessed images.
- labels: Preprocessed ground truth segmentation labels.
-
- Raises:
- ValueError: Ground truth label not provided during training.
- """
- if is_training and labels is None:
- raise ValueError('During training, labels must be provided.')
- if model_variant is None:
- tf.logging.warning('Default mean-subtraction is performed. Please specify '
- 'a model_variant. See feature_extractor.network_map for '
- 'supported model variants.')
- if labels is not None:
- assert len(images) == len(labels)
- num_imgs = len(images)
-
- # Keep reference to original images.
- original_images = images
-
- processed_images = [tf.cast(image, tf.float32) for image in images]
-
- if labels is not None:
- labels = [tf.cast(label, tf.int32) for label in labels]
-
- # Resize images and labels to the desired range.
- if min_resize_value is not None or max_resize_value is not None:
- processed_images, labels = zip(*[
- preprocess_utils.resize_to_range(
- image=processed_image,
- label=label,
- min_size=min_resize_value,
- max_size=max_resize_value,
- factor=resize_factor,
- align_corners=True) for processed_image, label
- in zip(processed_images, labels)])
- # The `original_images` becomes the resized images.
- original_images = [tf.identity(processed_image)
- for processed_image in processed_images]
-
- # Data augmentation by randomly scaling the inputs.
- scale = get_random_scale(
- min_scale_factor, max_scale_factor, scale_factor_step_size)
- processed_images, labels = zip(
- *[randomly_scale_image_and_label(processed_image, label, scale)
- for processed_image, label in zip(processed_images, labels)])
-
- for processed_image in processed_images:
- processed_image.set_shape([None, None, 3])
-
- if crop_height is not None and crop_width is not None:
- # Pad image and label to have dimensions >= [crop_height, crop_width].
- image_shape = tf.shape(processed_images[0])
- image_height = image_shape[0]
- image_width = image_shape[1]
-
- target_height = image_height + tf.maximum(crop_height - image_height, 0)
- target_width = image_width + tf.maximum(crop_width - image_width, 0)
-
- # Pad image with mean pixel value.
- mean_pixel = tf.reshape(
- feature_extractor.mean_pixel(model_variant), [1, 1, 3])
- processed_images = [preprocess_utils.pad_to_bounding_box(
- processed_image, 0, 0, target_height, target_width, mean_pixel)
- for processed_image in processed_images]
-
- if labels is not None:
- labels = [preprocess_utils.pad_to_bounding_box(
- label, 0, 0, target_height, target_width, ignore_label)
- for label in labels]
-
- # Randomly crop the images and labels.
- if is_training and labels is not None:
- cropped = preprocess_utils.random_crop(
- processed_images + labels, crop_height, crop_width)
- assert len(cropped) == 2 * num_imgs
- processed_images = cropped[:num_imgs]
- labels = cropped[num_imgs:]
-
- for processed_image in processed_images:
- processed_image.set_shape([crop_height, crop_width, 3])
-
- if labels is not None:
- for label in labels:
- label.set_shape([crop_height, crop_width, 1])
-
- if is_training:
- # Randomly left-right flip the image and label.
- res = preprocess_utils.flip_dim(
- list(processed_images + labels), _PROB_OF_FLIP, dim=1)
- maybe_flipped = res[:-1]
- assert len(maybe_flipped) == 2 * num_imgs
- processed_images = maybe_flipped[:num_imgs]
- labels = maybe_flipped[num_imgs:]
-
- return original_images, processed_images, labels
diff --git a/research/feelvos/model.py b/research/feelvos/model.py
deleted file mode 100644
index f145f91616958b7327d99bb55efb1b7b5016a223..0000000000000000000000000000000000000000
--- a/research/feelvos/model.py
+++ /dev/null
@@ -1,480 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""Provides DeepLab model definition and helper functions.
-
-DeepLab is a deep learning system for semantic image segmentation with
-the following features:
-
-(1) Atrous convolution to explicitly control the resolution at which
-feature responses are computed within Deep Convolutional Neural Networks.
-
-(2) Atrous spatial pyramid pooling (ASPP) to robustly segment objects at
-multiple scales with filters at multiple sampling rates and effective
-fields-of-views.
-
-(3) ASPP module augmented with image-level feature and batch normalization.
-
-(4) A simple yet effective decoder module to recover the object boundaries.
-
-See the following papers for more details:
-
-"Encoder-Decoder with Atrous Separable Convolution for Semantic Image
-Segmentation"
-Liang-Chieh Chen, Yukun Zhu, George Papandreou, Florian Schroff, Hartwig Adam.
-(https://arxiv.org/abs1802.02611)
-
-"Rethinking Atrous Convolution for Semantic Image Segmentation,"
-Liang-Chieh Chen, George Papandreou, Florian Schroff, Hartwig Adam
-(https://arxiv.org/abs/1706.05587)
-
-"DeepLab: Semantic Image Segmentation with Deep Convolutional Nets,
-Atrous Convolution, and Fully Connected CRFs",
-Liang-Chieh Chen*, George Papandreou*, Iasonas Kokkinos, Kevin Murphy,
-Alan L Yuille (* equal contribution)
-(https://arxiv.org/abs/1606.00915)
-
-"Semantic Image Segmentation with Deep Convolutional Nets and Fully Connected
-CRFs"
-Liang-Chieh Chen*, George Papandreou*, Iasonas Kokkinos, Kevin Murphy,
-Alan L. Yuille (* equal contribution)
-(https://arxiv.org/abs/1412.7062)
-"""
-import collections
-import tensorflow as tf
-
-from deeplab import model
-from feelvos import common
-from feelvos.utils import embedding_utils
-from feelvos.utils import train_utils
-
-slim = tf.contrib.slim
-
-
-get_branch_logits = model.get_branch_logits
-get_extra_layer_scopes = model.get_extra_layer_scopes
-multi_scale_logits_v2 = model.multi_scale_logits
-refine_by_decoder = model.refine_by_decoder
-scale_dimension = model.scale_dimension
-split_separable_conv2d = model.split_separable_conv2d
-
-MERGED_LOGITS_SCOPE = model.MERGED_LOGITS_SCOPE
-IMAGE_POOLING_SCOPE = model.IMAGE_POOLING_SCOPE
-ASPP_SCOPE = model.ASPP_SCOPE
-CONCAT_PROJECTION_SCOPE = model.CONCAT_PROJECTION_SCOPE
-
-
-def predict_labels(images,
- model_options,
- image_pyramid=None,
- reference_labels=None,
- k_nearest_neighbors=1,
- embedding_dimension=None,
- use_softmax_feedback=False,
- initial_softmax_feedback=None,
- embedding_seg_feature_dimension=256,
- embedding_seg_n_layers=4,
- embedding_seg_kernel_size=7,
- embedding_seg_atrous_rates=None,
- also_return_softmax_probabilities=False,
- num_frames_per_video=None,
- normalize_nearest_neighbor_distances=False,
- also_attend_to_previous_frame=False,
- use_local_previous_frame_attention=False,
- previous_frame_attention_window_size=9,
- use_first_frame_matching=True,
- also_return_embeddings=False,
- ref_embeddings=None):
- """Predicts segmentation labels.
-
- Args:
- images: A tensor of size [batch, height, width, channels].
- model_options: An InternalModelOptions instance to configure models.
- image_pyramid: Input image scales for multi-scale feature extraction.
- reference_labels: A tensor of size [batch, height, width, 1].
- ground truth labels used to perform a nearest neighbor query
- k_nearest_neighbors: Integer, the number of neighbors to use for nearest
- neighbor queries.
- embedding_dimension: Integer, the dimension used for the learned embedding.
- use_softmax_feedback: Boolean, whether to give the softmax predictions of
- the last frame as additional input to the segmentation head.
- initial_softmax_feedback: Float32 tensor, or None. Can be used to
- initialize the softmax predictions used for the feedback loop.
- Typically only useful for inference. Only has an effect if
- use_softmax_feedback is True.
- embedding_seg_feature_dimension: Integer, the dimensionality used in the
- segmentation head layers.
- embedding_seg_n_layers: Integer, the number of layers in the segmentation
- head.
- embedding_seg_kernel_size: Integer, the kernel size used in the
- segmentation head.
- embedding_seg_atrous_rates: List of integers of length
- embedding_seg_n_layers, the atrous rates to use for the segmentation head.
- also_return_softmax_probabilities: Boolean, if true, additionally return
- the softmax probabilities as second return value.
- num_frames_per_video: Integer, the number of frames per video.
- normalize_nearest_neighbor_distances: Boolean, whether to normalize the
- nearest neighbor distances to [0,1] using sigmoid, scale and shift.
- also_attend_to_previous_frame: Boolean, whether to also use nearest
- neighbor attention with respect to the previous frame.
- use_local_previous_frame_attention: Boolean, whether to restrict the
- previous frame attention to a local search window.
- Only has an effect, if also_attend_to_previous_frame is True.
- previous_frame_attention_window_size: Integer, the window size used for
- local previous frame attention, if use_local_previous_frame_attention
- is True.
- use_first_frame_matching: Boolean, whether to extract features by matching
- to the reference frame. This should always be true except for ablation
- experiments.
- also_return_embeddings: Boolean, whether to return the embeddings as well.
- ref_embeddings: Tuple of
- (first_frame_embeddings, previous_frame_embeddings),
- each of shape [batch, height, width, embedding_dimension], or None.
-
- Returns:
- A dictionary with keys specifying the output_type (e.g., semantic
- prediction) and values storing Tensors representing predictions (argmax
- over channels). Each prediction has size [batch, height, width].
- If also_return_softmax_probabilities is True, the second return value are
- the softmax probabilities.
- If also_return_embeddings is True, it will also return an embeddings
- tensor of shape [batch, height, width, embedding_dimension].
-
- Raises:
- ValueError: If classification_loss is not softmax, softmax_with_attention,
- nor triplet.
- """
- if (model_options.classification_loss == 'triplet' and
- reference_labels is None):
- raise ValueError('Need reference_labels for triplet loss')
-
- if model_options.classification_loss == 'softmax_with_attention':
- if embedding_dimension is None:
- raise ValueError('Need embedding_dimension for softmax_with_attention '
- 'loss')
- if reference_labels is None:
- raise ValueError('Need reference_labels for softmax_with_attention loss')
- res = (
- multi_scale_logits_with_nearest_neighbor_matching(
- images,
- model_options=model_options,
- image_pyramid=image_pyramid,
- is_training=False,
- reference_labels=reference_labels,
- clone_batch_size=1,
- num_frames_per_video=num_frames_per_video,
- embedding_dimension=embedding_dimension,
- max_neighbors_per_object=0,
- k_nearest_neighbors=k_nearest_neighbors,
- use_softmax_feedback=use_softmax_feedback,
- initial_softmax_feedback=initial_softmax_feedback,
- embedding_seg_feature_dimension=embedding_seg_feature_dimension,
- embedding_seg_n_layers=embedding_seg_n_layers,
- embedding_seg_kernel_size=embedding_seg_kernel_size,
- embedding_seg_atrous_rates=embedding_seg_atrous_rates,
- normalize_nearest_neighbor_distances=
- normalize_nearest_neighbor_distances,
- also_attend_to_previous_frame=also_attend_to_previous_frame,
- use_local_previous_frame_attention=
- use_local_previous_frame_attention,
- previous_frame_attention_window_size=
- previous_frame_attention_window_size,
- use_first_frame_matching=use_first_frame_matching,
- also_return_embeddings=also_return_embeddings,
- ref_embeddings=ref_embeddings
- ))
- if also_return_embeddings:
- outputs_to_scales_to_logits, embeddings = res
- else:
- outputs_to_scales_to_logits = res
- embeddings = None
- else:
- outputs_to_scales_to_logits = multi_scale_logits_v2(
- images,
- model_options=model_options,
- image_pyramid=image_pyramid,
- is_training=False,
- fine_tune_batch_norm=False)
-
- predictions = {}
- for output in sorted(outputs_to_scales_to_logits):
- scales_to_logits = outputs_to_scales_to_logits[output]
- original_logits = scales_to_logits[MERGED_LOGITS_SCOPE]
- if isinstance(original_logits, list):
- assert len(original_logits) == 1
- original_logits = original_logits[0]
- logits = tf.image.resize_bilinear(original_logits, tf.shape(images)[1:3],
- align_corners=True)
- if model_options.classification_loss in ('softmax',
- 'softmax_with_attention'):
- predictions[output] = tf.argmax(logits, 3)
- elif model_options.classification_loss == 'triplet':
- # to keep this fast, we do the nearest neighbor assignment on the
- # resolution at which the embedding is extracted and scale the result up
- # afterwards
- embeddings = original_logits
- reference_labels_logits_size = tf.squeeze(
- tf.image.resize_nearest_neighbor(
- reference_labels[tf.newaxis],
- train_utils.resolve_shape(embeddings)[1:3],
- align_corners=True), axis=0)
- nn_labels = embedding_utils.assign_labels_by_nearest_neighbors(
- embeddings[0], embeddings[1:], reference_labels_logits_size,
- k_nearest_neighbors)
- predictions[common.OUTPUT_TYPE] = tf.image.resize_nearest_neighbor(
- nn_labels, tf.shape(images)[1:3], align_corners=True)
- else:
- raise ValueError(
- 'Only support softmax, triplet, or softmax_with_attention for '
- 'classification_loss.')
-
- if also_return_embeddings:
- assert also_return_softmax_probabilities
- return predictions, tf.nn.softmax(original_logits, axis=-1), embeddings
- elif also_return_softmax_probabilities:
- return predictions, tf.nn.softmax(original_logits, axis=-1)
- else:
- return predictions
-
-
-def multi_scale_logits_with_nearest_neighbor_matching(
- images,
- model_options,
- image_pyramid,
- clone_batch_size,
- reference_labels,
- num_frames_per_video,
- embedding_dimension,
- max_neighbors_per_object,
- weight_decay=0.0001,
- is_training=False,
- fine_tune_batch_norm=False,
- k_nearest_neighbors=1,
- use_softmax_feedback=False,
- initial_softmax_feedback=None,
- embedding_seg_feature_dimension=256,
- embedding_seg_n_layers=4,
- embedding_seg_kernel_size=7,
- embedding_seg_atrous_rates=None,
- normalize_nearest_neighbor_distances=False,
- also_attend_to_previous_frame=False,
- damage_initial_previous_frame_mask=False,
- use_local_previous_frame_attention=False,
- previous_frame_attention_window_size=9,
- use_first_frame_matching=True,
- also_return_embeddings=False,
- ref_embeddings=None):
- """Gets the logits for multi-scale inputs using nearest neighbor attention.
-
- Adjusted version of multi_scale_logits_v2 to support nearest neighbor
- attention and a variable number of classes for each element of the batch.
- The returned logits are all downsampled (due to max-pooling layers)
- for both training and evaluation.
-
- Args:
- images: A tensor of size [batch, height, width, channels].
- model_options: A ModelOptions instance to configure models.
- image_pyramid: Input image scales for multi-scale feature extraction.
- clone_batch_size: Integer, the number of videos on a batch.
- reference_labels: The segmentation labels of the reference frame on which
- attention is applied.
- num_frames_per_video: Integer, the number of frames per video.
- embedding_dimension: Integer, the dimension of the embedding.
- max_neighbors_per_object: Integer, the maximum number of candidates
- for the nearest neighbor query per object after subsampling.
- Can be 0 for no subsampling.
- weight_decay: The weight decay for model variables.
- is_training: Is training or not.
- fine_tune_batch_norm: Fine-tune the batch norm parameters or not.
- k_nearest_neighbors: Integer, the number of nearest neighbors to use.
- use_softmax_feedback: Boolean, whether to give the softmax predictions of
- the last frame as additional input to the segmentation head.
- initial_softmax_feedback: List of Float32 tensors, or None.
- Can be used to initialize the softmax predictions used for the feedback
- loop. Only has an effect if use_softmax_feedback is True.
- embedding_seg_feature_dimension: Integer, the dimensionality used in the
- segmentation head layers.
- embedding_seg_n_layers: Integer, the number of layers in the segmentation
- head.
- embedding_seg_kernel_size: Integer, the kernel size used in the
- segmentation head.
- embedding_seg_atrous_rates: List of integers of length
- embedding_seg_n_layers, the atrous rates to use for the segmentation head.
- normalize_nearest_neighbor_distances: Boolean, whether to normalize the
- nearest neighbor distances to [0,1] using sigmoid, scale and shift.
- also_attend_to_previous_frame: Boolean, whether to also use nearest
- neighbor attention with respect to the previous frame.
- damage_initial_previous_frame_mask: Boolean, whether to artificially damage
- the initial previous frame mask. Only has an effect if
- also_attend_to_previous_frame is True.
- use_local_previous_frame_attention: Boolean, whether to restrict the
- previous frame attention to a local search window.
- Only has an effect, if also_attend_to_previous_frame is True.
- previous_frame_attention_window_size: Integer, the window size used for
- local previous frame attention, if use_local_previous_frame_attention
- is True.
- use_first_frame_matching: Boolean, whether to extract features by matching
- to the reference frame. This should always be true except for ablation
- experiments.
- also_return_embeddings: Boolean, whether to return the embeddings as well.
- ref_embeddings: Tuple of
- (first_frame_embeddings, previous_frame_embeddings),
- each of shape [batch, height, width, embedding_dimension], or None.
-
- Returns:
- outputs_to_scales_to_logits: A map of maps from output_type (e.g.,
- semantic prediction) to a dictionary of multi-scale logits names to
- logits. For each output_type, the dictionary has keys which
- correspond to the scales and values which correspond to the logits.
- For example, if `scales` equals [1.0, 1.5], then the keys would
- include 'merged_logits', 'logits_1.00' and 'logits_1.50'.
- If also_return_embeddings is True, it will also return an embeddings
- tensor of shape [batch, height, width, embedding_dimension].
-
- Raises:
- ValueError: If model_options doesn't specify crop_size and its
- add_image_level_feature = True, since add_image_level_feature requires
- crop_size information.
- """
- # Setup default values.
- if not image_pyramid:
- image_pyramid = [1.0]
- crop_height = (
- model_options.crop_size[0]
- if model_options.crop_size else tf.shape(images)[1])
- crop_width = (
- model_options.crop_size[1]
- if model_options.crop_size else tf.shape(images)[2])
-
- # Compute the height, width for the output logits.
- if model_options.decoder_output_stride:
- logits_output_stride = min(model_options.decoder_output_stride)
- else:
- logits_output_stride = model_options.output_stride
- logits_height = scale_dimension(
- crop_height,
- max(1.0, max(image_pyramid)) / logits_output_stride)
- logits_width = scale_dimension(
- crop_width,
- max(1.0, max(image_pyramid)) / logits_output_stride)
-
- # Compute the logits for each scale in the image pyramid.
- outputs_to_scales_to_logits = {
- k: {}
- for k in model_options.outputs_to_num_classes
- }
-
- for image_scale in image_pyramid:
- if image_scale != 1.0:
- scaled_height = scale_dimension(crop_height, image_scale)
- scaled_width = scale_dimension(crop_width, image_scale)
- scaled_crop_size = [scaled_height, scaled_width]
- scaled_images = tf.image.resize_bilinear(
- images, scaled_crop_size, align_corners=True)
- scaled_reference_labels = tf.image.resize_nearest_neighbor(
- reference_labels, scaled_crop_size, align_corners=True
- )
- if model_options.crop_size is None:
- scaled_crop_size = None
- if model_options.crop_size:
- scaled_images.set_shape([None, scaled_height, scaled_width, 3])
- else:
- scaled_crop_size = model_options.crop_size
- scaled_images = images
- scaled_reference_labels = reference_labels
-
- updated_options = model_options._replace(crop_size=scaled_crop_size)
- res = embedding_utils.get_logits_with_matching(
- scaled_images,
- updated_options,
- weight_decay=weight_decay,
- reuse=tf.AUTO_REUSE,
- is_training=is_training,
- fine_tune_batch_norm=fine_tune_batch_norm,
- reference_labels=scaled_reference_labels,
- batch_size=clone_batch_size,
- num_frames_per_video=num_frames_per_video,
- embedding_dimension=embedding_dimension,
- max_neighbors_per_object=max_neighbors_per_object,
- k_nearest_neighbors=k_nearest_neighbors,
- use_softmax_feedback=use_softmax_feedback,
- initial_softmax_feedback=initial_softmax_feedback,
- embedding_seg_feature_dimension=embedding_seg_feature_dimension,
- embedding_seg_n_layers=embedding_seg_n_layers,
- embedding_seg_kernel_size=embedding_seg_kernel_size,
- embedding_seg_atrous_rates=embedding_seg_atrous_rates,
- normalize_nearest_neighbor_distances=
- normalize_nearest_neighbor_distances,
- also_attend_to_previous_frame=also_attend_to_previous_frame,
- damage_initial_previous_frame_mask=damage_initial_previous_frame_mask,
- use_local_previous_frame_attention=use_local_previous_frame_attention,
- previous_frame_attention_window_size=
- previous_frame_attention_window_size,
- use_first_frame_matching=use_first_frame_matching,
- also_return_embeddings=also_return_embeddings,
- ref_embeddings=ref_embeddings
- )
- if also_return_embeddings:
- outputs_to_logits, embeddings = res
- else:
- outputs_to_logits = res
- embeddings = None
-
- # Resize the logits to have the same dimension before merging.
- for output in sorted(outputs_to_logits):
- if isinstance(outputs_to_logits[output], collections.Sequence):
- outputs_to_logits[output] = [tf.image.resize_bilinear(
- x, [logits_height, logits_width], align_corners=True)
- for x in outputs_to_logits[output]]
- else:
- outputs_to_logits[output] = tf.image.resize_bilinear(
- outputs_to_logits[output], [logits_height, logits_width],
- align_corners=True)
-
- # Return when only one input scale.
- if len(image_pyramid) == 1:
- for output in sorted(model_options.outputs_to_num_classes):
- outputs_to_scales_to_logits[output][
- MERGED_LOGITS_SCOPE] = outputs_to_logits[output]
- if also_return_embeddings:
- return outputs_to_scales_to_logits, embeddings
- else:
- return outputs_to_scales_to_logits
-
- # Save logits to the output map.
- for output in sorted(model_options.outputs_to_num_classes):
- outputs_to_scales_to_logits[output][
- 'logits_%.2f' % image_scale] = outputs_to_logits[output]
-
- # Merge the logits from all the multi-scale inputs.
- for output in sorted(model_options.outputs_to_num_classes):
- # Concatenate the multi-scale logits for each output type.
- all_logits = [
- [tf.expand_dims(l, axis=4)]
- for logits in outputs_to_scales_to_logits[output].values()
- for l in logits
- ]
- transposed = map(list, zip(*all_logits))
- all_logits = [tf.concat(t, 4) for t in transposed]
- merge_fn = (
- tf.reduce_max
- if model_options.merge_method == 'max' else tf.reduce_mean)
- outputs_to_scales_to_logits[output][MERGED_LOGITS_SCOPE] = [merge_fn(
- l, axis=4) for l in all_logits]
-
- if also_return_embeddings:
- return outputs_to_scales_to_logits, embeddings
- else:
- return outputs_to_scales_to_logits
diff --git a/research/feelvos/train.py b/research/feelvos/train.py
deleted file mode 100644
index 16c085722749bcfde5aeff15cdbec336e5efe451..0000000000000000000000000000000000000000
--- a/research/feelvos/train.py
+++ /dev/null
@@ -1,630 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Training script for the FEELVOS model.
-
-See model.py for more details and usage.
-"""
-import six
-import tensorflow as tf
-
-from feelvos import common
-from feelvos import model
-from feelvos.datasets import video_dataset
-from feelvos.utils import embedding_utils
-from feelvos.utils import train_utils
-from feelvos.utils import video_input_generator
-from deployment import model_deploy
-
-slim = tf.contrib.slim
-prefetch_queue = slim.prefetch_queue
-flags = tf.app.flags
-FLAGS = flags.FLAGS
-
-# Settings for multi-GPUs/multi-replicas training.
-
-flags.DEFINE_integer('num_clones', 1, 'Number of clones to deploy.')
-
-flags.DEFINE_boolean('clone_on_cpu', False, 'Use CPUs to deploy clones.')
-
-flags.DEFINE_integer('num_replicas', 1, 'Number of worker replicas.')
-
-flags.DEFINE_integer('startup_delay_steps', 15,
- 'Number of training steps between replicas startup.')
-
-flags.DEFINE_integer('num_ps_tasks', 0,
- 'The number of parameter servers. If the value is 0, then '
- 'the parameters are handled locally by the worker.')
-
-flags.DEFINE_string('master', '', 'BNS name of the tensorflow server')
-
-flags.DEFINE_integer('task', 0, 'The task ID.')
-
-# Settings for logging.
-
-flags.DEFINE_string('train_logdir', None,
- 'Where the checkpoint and logs are stored.')
-
-flags.DEFINE_integer('log_steps', 10,
- 'Display logging information at every log_steps.')
-
-flags.DEFINE_integer('save_interval_secs', 1200,
- 'How often, in seconds, we save the model to disk.')
-
-flags.DEFINE_integer('save_summaries_secs', 600,
- 'How often, in seconds, we compute the summaries.')
-
-# Settings for training strategy.
-
-flags.DEFINE_enum('learning_policy', 'poly', ['poly', 'step'],
- 'Learning rate policy for training.')
-
-flags.DEFINE_float('base_learning_rate', 0.0007,
- 'The base learning rate for model training.')
-
-flags.DEFINE_float('learning_rate_decay_factor', 0.1,
- 'The rate to decay the base learning rate.')
-
-flags.DEFINE_integer('learning_rate_decay_step', 2000,
- 'Decay the base learning rate at a fixed step.')
-
-flags.DEFINE_float('learning_power', 0.9,
- 'The power value used in the poly learning policy.')
-
-flags.DEFINE_integer('training_number_of_steps', 200000,
- 'The number of steps used for training')
-
-flags.DEFINE_float('momentum', 0.9, 'The momentum value to use')
-
-flags.DEFINE_integer('train_batch_size', 6,
- 'The number of images in each batch during training.')
-
-flags.DEFINE_integer('train_num_frames_per_video', 3,
- 'The number of frames used per video during training')
-
-flags.DEFINE_float('weight_decay', 0.00004,
- 'The value of the weight decay for training.')
-
-flags.DEFINE_multi_integer('train_crop_size', [465, 465],
- 'Image crop size [height, width] during training.')
-
-flags.DEFINE_float('last_layer_gradient_multiplier', 1.0,
- 'The gradient multiplier for last layers, which is used to '
- 'boost the gradient of last layers if the value > 1.')
-
-flags.DEFINE_boolean('upsample_logits', True,
- 'Upsample logits during training.')
-
-flags.DEFINE_integer('batch_capacity_factor', 16, 'Batch capacity factor.')
-
-flags.DEFINE_integer('num_readers', 1, 'Number of readers for data provider.')
-
-flags.DEFINE_integer('batch_num_threads', 1, 'Batch number of threads.')
-
-flags.DEFINE_integer('prefetch_queue_capacity_factor', 32,
- 'Prefetch queue capacity factor.')
-
-flags.DEFINE_integer('prefetch_queue_num_threads', 1,
- 'Prefetch queue number of threads.')
-
-flags.DEFINE_integer('train_max_neighbors_per_object', 1024,
- 'The maximum number of candidates for the nearest '
- 'neighbor query per object after subsampling')
-
-# Settings for fine-tuning the network.
-
-flags.DEFINE_string('tf_initial_checkpoint', None,
- 'The initial checkpoint in tensorflow format.')
-
-flags.DEFINE_boolean('initialize_last_layer', False,
- 'Initialize the last layer.')
-
-flags.DEFINE_boolean('last_layers_contain_logits_only', False,
- 'Only consider logits as last layers or not.')
-
-flags.DEFINE_integer('slow_start_step', 0,
- 'Training model with small learning rate for few steps.')
-
-flags.DEFINE_float('slow_start_learning_rate', 1e-4,
- 'Learning rate employed during slow start.')
-
-flags.DEFINE_boolean('fine_tune_batch_norm', False,
- 'Fine tune the batch norm parameters or not.')
-
-flags.DEFINE_float('min_scale_factor', 1.,
- 'Mininum scale factor for data augmentation.')
-
-flags.DEFINE_float('max_scale_factor', 1.3,
- 'Maximum scale factor for data augmentation.')
-
-flags.DEFINE_float('scale_factor_step_size', 0,
- 'Scale factor step size for data augmentation.')
-
-flags.DEFINE_multi_integer('atrous_rates', None,
- 'Atrous rates for atrous spatial pyramid pooling.')
-
-flags.DEFINE_integer('output_stride', 8,
- 'The ratio of input to output spatial resolution.')
-
-flags.DEFINE_boolean('sample_only_first_frame_for_finetuning', False,
- 'Whether to only sample the first frame during '
- 'fine-tuning. This should be False when using lucid data, '
- 'but True when fine-tuning on the first frame only. Only '
- 'has an effect if first_frame_finetuning is True.')
-
-flags.DEFINE_multi_integer('first_frame_finetuning', [0],
- 'Whether to only sample the first frame for '
- 'fine-tuning.')
-
-# Dataset settings.
-
-flags.DEFINE_multi_string('dataset', [], 'Name of the segmentation datasets.')
-
-flags.DEFINE_multi_float('dataset_sampling_probabilities', [],
- 'A list of probabilities to sample each of the '
- 'datasets.')
-
-flags.DEFINE_string('train_split', 'train',
- 'Which split of the dataset to be used for training')
-
-flags.DEFINE_multi_string('dataset_dir', [], 'Where the datasets reside.')
-
-flags.DEFINE_multi_integer('three_frame_dataset', [0],
- 'Whether the dataset has exactly three frames per '
- 'video of which the first is to be used as reference'
- ' and the two others are consecutive frames to be '
- 'used as query frames.'
- 'Set true for pascal lucid data.')
-
-flags.DEFINE_boolean('damage_initial_previous_frame_mask', False,
- 'Whether to artificially damage the initial previous '
- 'frame mask. Only has an effect if '
- 'also_attend_to_previous_frame is True.')
-
-flags.DEFINE_float('top_k_percent_pixels', 0.15, 'Float in [0.0, 1.0].'
- 'When its value < 1.0, only compute the loss for the top k'
- 'percent pixels (e.g., the top 20% pixels). This is useful'
- 'for hard pixel mining.')
-
-flags.DEFINE_integer('hard_example_mining_step', 100000,
- 'The training step in which the hard exampling mining '
- 'kicks off. Note that we gradually reduce the mining '
- 'percent to the top_k_percent_pixels. For example, if '
- 'hard_example_mining_step=100K and '
- 'top_k_percent_pixels=0.25, then mining percent will '
- 'gradually reduce from 100% to 25% until 100K steps '
- 'after which we only mine top 25% pixels. Only has an '
- 'effect if top_k_percent_pixels < 1.0')
-
-
-def _build_deeplab(inputs_queue_or_samples, outputs_to_num_classes,
- ignore_label):
- """Builds a clone of DeepLab.
-
- Args:
- inputs_queue_or_samples: A prefetch queue for images and labels, or
- directly a dict of the samples.
- outputs_to_num_classes: A map from output type to the number of classes.
- For example, for the task of semantic segmentation with 21 semantic
- classes, we would have outputs_to_num_classes['semantic'] = 21.
- ignore_label: Ignore label.
-
- Returns:
- A map of maps from output_type (e.g., semantic prediction) to a
- dictionary of multi-scale logits names to logits. For each output_type,
- the dictionary has keys which correspond to the scales and values which
- correspond to the logits. For example, if `scales` equals [1.0, 1.5],
- then the keys would include 'merged_logits', 'logits_1.00' and
- 'logits_1.50'.
-
- Raises:
- ValueError: If classification_loss is not softmax, softmax_with_attention,
- or triplet.
- """
- if hasattr(inputs_queue_or_samples, 'dequeue'):
- samples = inputs_queue_or_samples.dequeue()
- else:
- samples = inputs_queue_or_samples
- train_crop_size = (None if 0 in FLAGS.train_crop_size else
- FLAGS.train_crop_size)
-
- model_options = common.VideoModelOptions(
- outputs_to_num_classes=outputs_to_num_classes,
- crop_size=train_crop_size,
- atrous_rates=FLAGS.atrous_rates,
- output_stride=FLAGS.output_stride)
-
- if model_options.classification_loss == 'softmax_with_attention':
- clone_batch_size = FLAGS.train_batch_size // FLAGS.num_clones
-
- # Create summaries of ground truth labels.
- for n in range(clone_batch_size):
- tf.summary.image(
- 'gt_label_%d' % n,
- tf.cast(samples[common.LABEL][
- n * FLAGS.train_num_frames_per_video:
- (n + 1) * FLAGS.train_num_frames_per_video],
- tf.uint8) * 32, max_outputs=FLAGS.train_num_frames_per_video)
-
- if common.PRECEDING_FRAME_LABEL in samples:
- preceding_frame_label = samples[common.PRECEDING_FRAME_LABEL]
- init_softmax = []
- for n in range(clone_batch_size):
- init_softmax_n = embedding_utils.create_initial_softmax_from_labels(
- preceding_frame_label[n, tf.newaxis],
- samples[common.LABEL][n * FLAGS.train_num_frames_per_video,
- tf.newaxis],
- common.parse_decoder_output_stride(),
- reduce_labels=True)
- init_softmax_n = tf.squeeze(init_softmax_n, axis=0)
- init_softmax.append(init_softmax_n)
- tf.summary.image('preceding_frame_label',
- tf.cast(preceding_frame_label[n, tf.newaxis],
- tf.uint8) * 32)
- else:
- init_softmax = None
-
- outputs_to_scales_to_logits = (
- model.multi_scale_logits_with_nearest_neighbor_matching(
- samples[common.IMAGE],
- model_options=model_options,
- image_pyramid=FLAGS.image_pyramid,
- weight_decay=FLAGS.weight_decay,
- is_training=True,
- fine_tune_batch_norm=FLAGS.fine_tune_batch_norm,
- reference_labels=samples[common.LABEL],
- clone_batch_size=FLAGS.train_batch_size // FLAGS.num_clones,
- num_frames_per_video=FLAGS.train_num_frames_per_video,
- embedding_dimension=FLAGS.embedding_dimension,
- max_neighbors_per_object=FLAGS.train_max_neighbors_per_object,
- k_nearest_neighbors=FLAGS.k_nearest_neighbors,
- use_softmax_feedback=FLAGS.use_softmax_feedback,
- initial_softmax_feedback=init_softmax,
- embedding_seg_feature_dimension=
- FLAGS.embedding_seg_feature_dimension,
- embedding_seg_n_layers=FLAGS.embedding_seg_n_layers,
- embedding_seg_kernel_size=FLAGS.embedding_seg_kernel_size,
- embedding_seg_atrous_rates=FLAGS.embedding_seg_atrous_rates,
- normalize_nearest_neighbor_distances=
- FLAGS.normalize_nearest_neighbor_distances,
- also_attend_to_previous_frame=FLAGS.also_attend_to_previous_frame,
- damage_initial_previous_frame_mask=
- FLAGS.damage_initial_previous_frame_mask,
- use_local_previous_frame_attention=
- FLAGS.use_local_previous_frame_attention,
- previous_frame_attention_window_size=
- FLAGS.previous_frame_attention_window_size,
- use_first_frame_matching=FLAGS.use_first_frame_matching
- ))
- else:
- outputs_to_scales_to_logits = model.multi_scale_logits_v2(
- samples[common.IMAGE],
- model_options=model_options,
- image_pyramid=FLAGS.image_pyramid,
- weight_decay=FLAGS.weight_decay,
- is_training=True,
- fine_tune_batch_norm=FLAGS.fine_tune_batch_norm)
-
- if model_options.classification_loss == 'softmax':
- for output, num_classes in six.iteritems(outputs_to_num_classes):
- train_utils.add_softmax_cross_entropy_loss_for_each_scale(
- outputs_to_scales_to_logits[output],
- samples[common.LABEL],
- num_classes,
- ignore_label,
- loss_weight=1.0,
- upsample_logits=FLAGS.upsample_logits,
- scope=output)
- elif model_options.classification_loss == 'triplet':
- for output, _ in six.iteritems(outputs_to_num_classes):
- train_utils.add_triplet_loss_for_each_scale(
- FLAGS.train_batch_size // FLAGS.num_clones,
- FLAGS.train_num_frames_per_video,
- FLAGS.embedding_dimension, outputs_to_scales_to_logits[output],
- samples[common.LABEL], scope=output)
- elif model_options.classification_loss == 'softmax_with_attention':
- labels = samples[common.LABEL]
- batch_size = FLAGS.train_batch_size // FLAGS.num_clones
- num_frames_per_video = FLAGS.train_num_frames_per_video
- h, w = train_utils.resolve_shape(labels)[1:3]
- labels = tf.reshape(labels, tf.stack(
- [batch_size, num_frames_per_video, h, w, 1]))
- # Strip the reference labels off.
- if FLAGS.also_attend_to_previous_frame or FLAGS.use_softmax_feedback:
- n_ref_frames = 2
- else:
- n_ref_frames = 1
- labels = labels[:, n_ref_frames:]
- # Merge batch and time dimensions.
- labels = tf.reshape(labels, tf.stack(
- [batch_size * (num_frames_per_video - n_ref_frames), h, w, 1]))
-
- for output, num_classes in six.iteritems(outputs_to_num_classes):
- train_utils.add_dynamic_softmax_cross_entropy_loss_for_each_scale(
- outputs_to_scales_to_logits[output],
- labels,
- ignore_label,
- loss_weight=1.0,
- upsample_logits=FLAGS.upsample_logits,
- scope=output,
- top_k_percent_pixels=FLAGS.top_k_percent_pixels,
- hard_example_mining_step=FLAGS.hard_example_mining_step)
- else:
- raise ValueError('Only support softmax, softmax_with_attention'
- ' or triplet for classification_loss.')
-
- return outputs_to_scales_to_logits
-
-
-def main(unused_argv):
- # Set up deployment (i.e., multi-GPUs and/or multi-replicas).
- config = model_deploy.DeploymentConfig(
- num_clones=FLAGS.num_clones,
- clone_on_cpu=FLAGS.clone_on_cpu,
- replica_id=FLAGS.task,
- num_replicas=FLAGS.num_replicas,
- num_ps_tasks=FLAGS.num_ps_tasks)
-
- with tf.Graph().as_default():
- with tf.device(config.inputs_device()):
- train_crop_size = (None if 0 in FLAGS.train_crop_size else
- FLAGS.train_crop_size)
- assert FLAGS.dataset
- assert len(FLAGS.dataset) == len(FLAGS.dataset_dir)
- if len(FLAGS.first_frame_finetuning) == 1:
- first_frame_finetuning = (list(FLAGS.first_frame_finetuning)
- * len(FLAGS.dataset))
- else:
- first_frame_finetuning = FLAGS.first_frame_finetuning
- if len(FLAGS.three_frame_dataset) == 1:
- three_frame_dataset = (list(FLAGS.three_frame_dataset)
- * len(FLAGS.dataset))
- else:
- three_frame_dataset = FLAGS.three_frame_dataset
- assert len(FLAGS.dataset) == len(first_frame_finetuning)
- assert len(FLAGS.dataset) == len(three_frame_dataset)
- datasets, samples_list = zip(
- *[_get_dataset_and_samples(config, train_crop_size, dataset,
- dataset_dir, bool(first_frame_finetuning_),
- bool(three_frame_dataset_))
- for dataset, dataset_dir, first_frame_finetuning_,
- three_frame_dataset_ in zip(FLAGS.dataset, FLAGS.dataset_dir,
- first_frame_finetuning,
- three_frame_dataset)])
- # Note that this way of doing things is wasteful since it will evaluate
- # all branches but just use one of them. But let's do it anyway for now,
- # since it's easy and will probably be fast enough.
- dataset = datasets[0]
- if len(samples_list) == 1:
- samples = samples_list[0]
- else:
- probabilities = FLAGS.dataset_sampling_probabilities
- if probabilities:
- assert len(probabilities) == len(samples_list)
- else:
- # Default to uniform probabilities.
- probabilities = [1.0 / len(samples_list) for _ in samples_list]
- probabilities = tf.constant(probabilities)
- logits = tf.log(probabilities[tf.newaxis])
- rand_idx = tf.squeeze(tf.multinomial(logits, 1, output_dtype=tf.int32),
- axis=[0, 1])
-
- def wrap(x):
- def f():
- return x
- return f
-
- samples = tf.case({tf.equal(rand_idx, idx): wrap(s)
- for idx, s in enumerate(samples_list)},
- exclusive=True)
-
- # Prefetch_queue requires the shape to be known at graph creation time.
- # So we only use it if we crop to a fixed size.
- if train_crop_size is None:
- inputs_queue = samples
- else:
- inputs_queue = prefetch_queue.prefetch_queue(
- samples,
- capacity=FLAGS.prefetch_queue_capacity_factor*config.num_clones,
- num_threads=FLAGS.prefetch_queue_num_threads)
-
- # Create the global step on the device storing the variables.
- with tf.device(config.variables_device()):
- global_step = tf.train.get_or_create_global_step()
-
- # Define the model and create clones.
- model_fn = _build_deeplab
- if FLAGS.classification_loss == 'triplet':
- embedding_dim = FLAGS.embedding_dimension
- output_type_to_dim = {'embedding': embedding_dim}
- else:
- output_type_to_dim = {common.OUTPUT_TYPE: dataset.num_classes}
- model_args = (inputs_queue, output_type_to_dim, dataset.ignore_label)
- clones = model_deploy.create_clones(config, model_fn, args=model_args)
-
- # Gather update_ops from the first clone. These contain, for example,
- # the updates for the batch_norm variables created by model_fn.
- first_clone_scope = config.clone_scope(0)
- update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope)
-
- # Gather initial summaries.
- summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
-
- # Add summaries for model variables.
- for model_var in tf.contrib.framework.get_model_variables():
- summaries.add(tf.summary.histogram(model_var.op.name, model_var))
-
- # Add summaries for losses.
- for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
- summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss))
-
- # Build the optimizer based on the device specification.
- with tf.device(config.optimizer_device()):
- learning_rate = train_utils.get_model_learning_rate(
- FLAGS.learning_policy,
- FLAGS.base_learning_rate,
- FLAGS.learning_rate_decay_step,
- FLAGS.learning_rate_decay_factor,
- FLAGS.training_number_of_steps,
- FLAGS.learning_power,
- FLAGS.slow_start_step,
- FLAGS.slow_start_learning_rate)
- optimizer = tf.train.MomentumOptimizer(learning_rate, FLAGS.momentum)
- summaries.add(tf.summary.scalar('learning_rate', learning_rate))
-
- startup_delay_steps = FLAGS.task * FLAGS.startup_delay_steps
-
- with tf.device(config.variables_device()):
- total_loss, grads_and_vars = model_deploy.optimize_clones(
- clones, optimizer)
- total_loss = tf.check_numerics(total_loss, 'Loss is inf or nan.')
- summaries.add(tf.summary.scalar('total_loss', total_loss))
-
- # Modify the gradients for biases and last layer variables.
- last_layers = model.get_extra_layer_scopes(
- FLAGS.last_layers_contain_logits_only)
- grad_mult = train_utils.get_model_gradient_multipliers(
- last_layers, FLAGS.last_layer_gradient_multiplier)
- if grad_mult:
- grads_and_vars = slim.learning.multiply_gradients(grads_and_vars,
- grad_mult)
-
- with tf.name_scope('grad_clipping'):
- grads_and_vars = slim.learning.clip_gradient_norms(grads_and_vars, 5.0)
-
- # Create histogram summaries for the gradients.
- # We have too many summaries for mldash, so disable this one for now.
- # for grad, var in grads_and_vars:
- # summaries.add(tf.summary.histogram(
- # var.name.replace(':0', '_0') + '/gradient', grad))
-
- # Create gradient update op.
- grad_updates = optimizer.apply_gradients(grads_and_vars,
- global_step=global_step)
- update_ops.append(grad_updates)
- update_op = tf.group(*update_ops)
- with tf.control_dependencies([update_op]):
- train_tensor = tf.identity(total_loss, name='train_op')
-
- # Add the summaries from the first clone. These contain the summaries
- # created by model_fn and either optimize_clones() or _gather_clone_loss().
- summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
- first_clone_scope))
-
- # Merge all summaries together.
- summary_op = tf.summary.merge(list(summaries))
-
- # Soft placement allows placing on CPU ops without GPU implementation.
- session_config = tf.ConfigProto(allow_soft_placement=True,
- log_device_placement=False)
-
- # Start the training.
- slim.learning.train(
- train_tensor,
- logdir=FLAGS.train_logdir,
- log_every_n_steps=FLAGS.log_steps,
- master=FLAGS.master,
- number_of_steps=FLAGS.training_number_of_steps,
- is_chief=(FLAGS.task == 0),
- session_config=session_config,
- startup_delay_steps=startup_delay_steps,
- init_fn=train_utils.get_model_init_fn(FLAGS.train_logdir,
- FLAGS.tf_initial_checkpoint,
- FLAGS.initialize_last_layer,
- last_layers,
- ignore_missing_vars=True),
- summary_op=summary_op,
- save_summaries_secs=FLAGS.save_summaries_secs,
- save_interval_secs=FLAGS.save_interval_secs)
-
-
-def _get_dataset_and_samples(config, train_crop_size, dataset_name,
- dataset_dir, first_frame_finetuning,
- three_frame_dataset):
- """Creates dataset object and samples dict of tensor.
-
- Args:
- config: A DeploymentConfig.
- train_crop_size: Integer, the crop size used for training.
- dataset_name: String, the name of the dataset.
- dataset_dir: String, the directory of the dataset.
- first_frame_finetuning: Boolean, whether the used dataset is a dataset
- for first frame fine-tuning.
- three_frame_dataset: Boolean, whether the dataset has exactly three frames
- per video of which the first is to be used as reference and the two
- others are consecutive frames to be used as query frames.
-
- Returns:
- dataset: An instance of slim Dataset.
- samples: A dictionary of tensors for semantic segmentation.
- """
-
- # Split the batch across GPUs.
- assert FLAGS.train_batch_size % config.num_clones == 0, (
- 'Training batch size not divisble by number of clones (GPUs).')
-
- clone_batch_size = FLAGS.train_batch_size / config.num_clones
-
- if first_frame_finetuning:
- train_split = 'val'
- else:
- train_split = FLAGS.train_split
-
- data_type = 'tf_sequence_example'
- # Get dataset-dependent information.
- dataset = video_dataset.get_dataset(
- dataset_name,
- train_split,
- dataset_dir=dataset_dir,
- data_type=data_type)
-
- tf.gfile.MakeDirs(FLAGS.train_logdir)
- tf.logging.info('Training on %s set', train_split)
-
- samples = video_input_generator.get(
- dataset,
- FLAGS.train_num_frames_per_video,
- train_crop_size,
- clone_batch_size,
- num_readers=FLAGS.num_readers,
- num_threads=FLAGS.batch_num_threads,
- min_resize_value=FLAGS.min_resize_value,
- max_resize_value=FLAGS.max_resize_value,
- resize_factor=FLAGS.resize_factor,
- min_scale_factor=FLAGS.min_scale_factor,
- max_scale_factor=FLAGS.max_scale_factor,
- scale_factor_step_size=FLAGS.scale_factor_step_size,
- dataset_split=FLAGS.train_split,
- is_training=True,
- model_variant=FLAGS.model_variant,
- batch_capacity_factor=FLAGS.batch_capacity_factor,
- decoder_output_stride=common.parse_decoder_output_stride(),
- first_frame_finetuning=first_frame_finetuning,
- sample_only_first_frame_for_finetuning=
- FLAGS.sample_only_first_frame_for_finetuning,
- sample_adjacent_and_consistent_query_frames=
- FLAGS.sample_adjacent_and_consistent_query_frames or
- FLAGS.use_softmax_feedback,
- remap_labels_to_reference_frame=True,
- three_frame_dataset=three_frame_dataset,
- add_prev_frame_label=not FLAGS.also_attend_to_previous_frame
- )
- return dataset, samples
-
-
-if __name__ == '__main__':
- flags.mark_flag_as_required('train_logdir')
- tf.logging.set_verbosity(tf.logging.INFO)
- tf.app.run()
diff --git a/research/feelvos/train.sh b/research/feelvos/train.sh
deleted file mode 100755
index 63b7ea19d4c53dea932322c3885abb9a95237e0c..0000000000000000000000000000000000000000
--- a/research/feelvos/train.sh
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-#
-# This script is used to run local training on DAVIS 2017. Users could also
-# modify from this script for their use case. See eval.sh for an example of
-# local inference with a pre-trained model.
-#
-# Note that this script runs local training with a single GPU and a smaller crop
-# and batch size, while in the paper, we trained our models with 16 GPUS with
-# --num_clones=2, --train_batch_size=6, --num_replicas=8,
-# --training_number_of_steps=200000, --train_crop_size=465,
-# --train_crop_size=465.
-#
-# Usage:
-# # From the tensorflow/models/research/feelvos directory.
-# sh ./train.sh
-#
-#
-
-# Exit immediately if a command exits with a non-zero status.
-set -e
-
-# Move one-level up to tensorflow/models/research directory.
-cd ..
-
-# Update PYTHONPATH.
-export PYTHONPATH=$PYTHONPATH:`pwd`:`pwd`/slim:`pwd`/feelvos
-
-# Set up the working environment.
-CURRENT_DIR=$(pwd)
-WORK_DIR="${CURRENT_DIR}/feelvos"
-
-# Set up the working directories.
-DATASET_DIR="datasets"
-DAVIS_FOLDER="davis17"
-DAVIS_DATASET="${WORK_DIR}/${DATASET_DIR}/${DAVIS_FOLDER}/tfrecord"
-EXP_FOLDER="exp/train"
-TRAIN_LOGDIR="${WORK_DIR}/${DATASET_DIR}/${DAVIS_FOLDER}/${EXP_FOLDER}/train"
-mkdir -p ${TRAIN_LOGDIR}
-
-# Go to datasets folder and download and convert the DAVIS 2017 dataset.
-DATASET_DIR="datasets"
-cd "${WORK_DIR}/${DATASET_DIR}"
-sh download_and_convert_davis17.sh
-
-# Go to models folder and download and unpack the COCO pre-trained model.
-MODELS_DIR="models"
-mkdir -p "${WORK_DIR}/${MODELS_DIR}"
-cd "${WORK_DIR}/${MODELS_DIR}"
-if [ ! -d "xception_65_coco_pretrained" ]; then
- wget http://download.tensorflow.org/models/xception_65_coco_pretrained_2018_10_02.tar.gz
- tar -xvf xception_65_coco_pretrained_2018_10_02.tar.gz
- rm xception_65_coco_pretrained_2018_10_02.tar.gz
-fi
-INIT_CKPT="${WORK_DIR}/${MODELS_DIR}/xception_65_coco_pretrained/x65-b2u1s2p-d48-2-3x256-sc-cr300k_init.ckpt"
-
-# Go back to orignal directory.
-cd "${CURRENT_DIR}"
-
-python "${WORK_DIR}"/train.py \
- --dataset=davis_2017 \
- --dataset_dir="${DAVIS_DATASET}" \
- --train_logdir="${TRAIN_LOGDIR}" \
- --tf_initial_checkpoint="${INIT_CKPT}" \
- --logtostderr \
- --atrous_rates=6 \
- --atrous_rates=12 \
- --atrous_rates=18 \
- --decoder_output_stride=4 \
- --model_variant=xception_65 \
- --multi_grid=1 \
- --multi_grid=1 \
- --multi_grid=1 \
- --output_stride=16 \
- --weight_decay=0.00004 \
- --num_clones=1 \
- --train_batch_size=1 \
- --train_crop_size=300 \
- --train_crop_size=300
diff --git a/research/feelvos/utils/__init__.py b/research/feelvos/utils/__init__.py
deleted file mode 100644
index 6f1373443d0ff84fd90714e41dade400ab41a22c..0000000000000000000000000000000000000000
--- a/research/feelvos/utils/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
diff --git a/research/feelvos/utils/embedding_utils.py b/research/feelvos/utils/embedding_utils.py
deleted file mode 100644
index 233c70d9327d08251537c58821dd8405b42f0fe7..0000000000000000000000000000000000000000
--- a/research/feelvos/utils/embedding_utils.py
+++ /dev/null
@@ -1,1082 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities for the instance embedding for segmentation."""
-
-import numpy as np
-import tensorflow as tf
-from deeplab import model
-from deeplab.core import preprocess_utils
-from feelvos.utils import mask_damaging
-
-slim = tf.contrib.slim
-resolve_shape = preprocess_utils.resolve_shape
-WRONG_LABEL_PADDING_DISTANCE = 1e20
-
-# With correlation_cost local matching will be much faster. But we provide a
-# slow fallback for convenience.
-USE_CORRELATION_COST = False
-if USE_CORRELATION_COST:
- # pylint: disable=g-import-not-at-top
- from correlation_cost.python.ops import correlation_cost_op
-
-
-def pairwise_distances(x, y):
- """Computes pairwise squared l2 distances between tensors x and y.
-
- Args:
- x: Tensor of shape [n, feature_dim].
- y: Tensor of shape [m, feature_dim].
-
- Returns:
- Float32 distances tensor of shape [n, m].
- """
- # d[i,j] = (x[i] - y[j]) * (x[i] - y[j])'
- # = sum(x[i]^2, 1) + sum(y[j]^2, 1) - 2 * x[i] * y[j]'
- xs = tf.reduce_sum(x * x, axis=1)[:, tf.newaxis]
- ys = tf.reduce_sum(y * y, axis=1)[tf.newaxis, :]
- d = xs + ys - 2 * tf.matmul(x, y, transpose_b=True)
- return d
-
-
-def pairwise_distances2(x, y):
- """Computes pairwise squared l2 distances between tensors x and y.
-
- Naive implementation, high memory use. Could be useful to test the more
- efficient implementation.
-
- Args:
- x: Tensor of shape [n, feature_dim].
- y: Tensor of shape [m, feature_dim].
-
- Returns:
- distances of shape [n, m].
- """
- return tf.reduce_sum(tf.squared_difference(
- x[:, tf.newaxis], y[tf.newaxis, :]), axis=-1)
-
-
-def cross_correlate(x, y, max_distance=9):
- """Efficiently computes the cross correlation of x and y.
-
- Optimized implementation using correlation_cost.
- Note that we do not normalize by the feature dimension.
-
- Args:
- x: Float32 tensor of shape [height, width, feature_dim].
- y: Float32 tensor of shape [height, width, feature_dim].
- max_distance: Integer, the maximum distance in pixel coordinates
- per dimension which is considered to be in the search window.
-
- Returns:
- Float32 tensor of shape [height, width, (2 * max_distance + 1) ** 2].
- """
- with tf.name_scope('cross_correlation'):
- corr = correlation_cost_op.correlation_cost(
- x[tf.newaxis], y[tf.newaxis], kernel_size=1,
- max_displacement=max_distance, stride_1=1, stride_2=1,
- pad=max_distance)
- corr = tf.squeeze(corr, axis=0)
- # This correlation implementation takes the mean over the feature_dim,
- # but we want sum here, so multiply by feature_dim.
- feature_dim = resolve_shape(x)[-1]
- corr *= feature_dim
- return corr
-
-
-def local_pairwise_distances(x, y, max_distance=9):
- """Computes pairwise squared l2 distances using a local search window.
-
- Optimized implementation using correlation_cost.
-
- Args:
- x: Float32 tensor of shape [height, width, feature_dim].
- y: Float32 tensor of shape [height, width, feature_dim].
- max_distance: Integer, the maximum distance in pixel coordinates
- per dimension which is considered to be in the search window.
-
- Returns:
- Float32 distances tensor of shape
- [height, width, (2 * max_distance + 1) ** 2].
- """
- with tf.name_scope('local_pairwise_distances'):
- # d[i,j] = (x[i] - y[j]) * (x[i] - y[j])'
- # = sum(x[i]^2, -1) + sum(y[j]^2, -1) - 2 * x[i] * y[j]'
- corr = cross_correlate(x, y, max_distance=max_distance)
- xs = tf.reduce_sum(x * x, axis=2)[..., tf.newaxis]
- ys = tf.reduce_sum(y * y, axis=2)[..., tf.newaxis]
- ones_ys = tf.ones_like(ys)
- ys = cross_correlate(ones_ys, ys, max_distance=max_distance)
- d = xs + ys - 2 * corr
- # Boundary should be set to Inf.
- boundary = tf.equal(
- cross_correlate(ones_ys, ones_ys, max_distance=max_distance), 0)
- d = tf.where(boundary, tf.fill(tf.shape(d), tf.constant(np.float('inf'))),
- d)
- return d
-
-
-def local_pairwise_distances2(x, y, max_distance=9):
- """Computes pairwise squared l2 distances using a local search window.
-
- Naive implementation using map_fn.
- Used as a slow fallback for when correlation_cost is not available.
-
- Args:
- x: Float32 tensor of shape [height, width, feature_dim].
- y: Float32 tensor of shape [height, width, feature_dim].
- max_distance: Integer, the maximum distance in pixel coordinates
- per dimension which is considered to be in the search window.
-
- Returns:
- Float32 distances tensor of shape
- [height, width, (2 * max_distance + 1) ** 2].
- """
- with tf.name_scope('local_pairwise_distances2'):
- padding_val = 1e20
- padded_y = tf.pad(y, [[max_distance, max_distance],
- [max_distance, max_distance], [0, 0]],
- constant_values=padding_val)
- height, width, _ = resolve_shape(x)
- dists = []
- for y_start in range(2 * max_distance + 1):
- y_end = y_start + height
- y_slice = padded_y[y_start:y_end]
- for x_start in range(2 * max_distance + 1):
- x_end = x_start + width
- offset_y = y_slice[:, x_start:x_end]
- dist = tf.reduce_sum(tf.squared_difference(x, offset_y), axis=2)
- dists.append(dist)
- dists = tf.stack(dists, axis=2)
- return dists
-
-
-def majority_vote(labels):
- """Performs a label majority vote along axis 1.
-
- Second try, hopefully this time more efficient.
- We assume that the labels are contiguous starting from 0.
- It will also work for non-contiguous labels, but be inefficient.
-
- Args:
- labels: Int tensor of shape [n, k]
-
- Returns:
- The majority of labels along axis 1
- """
- max_label = tf.reduce_max(labels)
- one_hot = tf.one_hot(labels, depth=max_label + 1)
- summed = tf.reduce_sum(one_hot, axis=1)
- majority = tf.argmax(summed, axis=1)
- return majority
-
-
-def assign_labels_by_nearest_neighbors(reference_embeddings, query_embeddings,
- reference_labels, k=1):
- """Segments by nearest neighbor query wrt the reference frame.
-
- Args:
- reference_embeddings: Tensor of shape [height, width, embedding_dim],
- the embedding vectors for the reference frame
- query_embeddings: Tensor of shape [n_query_images, height, width,
- embedding_dim], the embedding vectors for the query frames
- reference_labels: Tensor of shape [height, width, 1], the class labels of
- the reference frame
- k: Integer, the number of nearest neighbors to use
-
- Returns:
- The labels of the nearest neighbors as [n_query_frames, height, width, 1]
- tensor
-
- Raises:
- ValueError: If k < 1.
- """
- if k < 1:
- raise ValueError('k must be at least 1')
- dists = flattened_pairwise_distances(reference_embeddings, query_embeddings)
- if k == 1:
- nn_indices = tf.argmin(dists, axis=1)[..., tf.newaxis]
- else:
- _, nn_indices = tf.nn.top_k(-dists, k, sorted=False)
- reference_labels = tf.reshape(reference_labels, [-1])
- nn_labels = tf.gather(reference_labels, nn_indices)
- if k == 1:
- nn_labels = tf.squeeze(nn_labels, axis=1)
- else:
- nn_labels = majority_vote(nn_labels)
- height = tf.shape(reference_embeddings)[0]
- width = tf.shape(reference_embeddings)[1]
- n_query_frames = query_embeddings.shape[0]
- nn_labels = tf.reshape(nn_labels, [n_query_frames, height, width, 1])
- return nn_labels
-
-
-def flattened_pairwise_distances(reference_embeddings, query_embeddings):
- """Calculates flattened tensor of pairwise distances between ref and query.
-
- Args:
- reference_embeddings: Tensor of shape [..., embedding_dim],
- the embedding vectors for the reference frame
- query_embeddings: Tensor of shape [n_query_images, height, width,
- embedding_dim], the embedding vectors for the query frames.
-
- Returns:
- A distance tensor of shape [reference_embeddings.size / embedding_dim,
- query_embeddings.size / embedding_dim]
- """
- embedding_dim = resolve_shape(query_embeddings)[-1]
- reference_embeddings = tf.reshape(reference_embeddings, [-1, embedding_dim])
- first_dim = -1
- query_embeddings = tf.reshape(query_embeddings, [first_dim, embedding_dim])
- dists = pairwise_distances(query_embeddings, reference_embeddings)
- return dists
-
-
-def nearest_neighbor_features_per_object(
- reference_embeddings, query_embeddings, reference_labels,
- max_neighbors_per_object, k_nearest_neighbors, gt_ids=None, n_chunks=100):
- """Calculates the distance to the nearest neighbor per object.
-
- For every pixel of query_embeddings calculate the distance to the
- nearest neighbor in the (possibly subsampled) reference_embeddings per object.
-
- Args:
- reference_embeddings: Tensor of shape [height, width, embedding_dim],
- the embedding vectors for the reference frame.
- query_embeddings: Tensor of shape [n_query_images, height, width,
- embedding_dim], the embedding vectors for the query frames.
- reference_labels: Tensor of shape [height, width, 1], the class labels of
- the reference frame.
- max_neighbors_per_object: Integer, the maximum number of candidates
- for the nearest neighbor query per object after subsampling,
- or 0 for no subsampling.
- k_nearest_neighbors: Integer, the number of nearest neighbors to use.
- gt_ids: Int tensor of shape [n_objs] of the sorted unique ground truth
- ids in the first frame. If None, it will be derived from
- reference_labels.
- n_chunks: Integer, the number of chunks to use to save memory
- (set to 1 for no chunking).
-
- Returns:
- nn_features: A float32 tensor of nearest neighbor features of shape
- [n_query_images, height, width, n_objects, feature_dim].
- gt_ids: An int32 tensor of the unique sorted object ids present
- in the reference labels.
- """
- with tf.name_scope('nn_features_per_object'):
- reference_labels_flat = tf.reshape(reference_labels, [-1])
- if gt_ids is None:
- ref_obj_ids, _ = tf.unique(reference_labels_flat)
- ref_obj_ids = tf.contrib.framework.sort(ref_obj_ids)
- gt_ids = ref_obj_ids
- embedding_dim = resolve_shape(reference_embeddings)[-1]
- reference_embeddings_flat = tf.reshape(reference_embeddings,
- [-1, embedding_dim])
-
- reference_embeddings_flat, reference_labels_flat = (
- subsample_reference_embeddings_and_labels(reference_embeddings_flat,
- reference_labels_flat,
- gt_ids,
- max_neighbors_per_object))
- shape = resolve_shape(query_embeddings)
- query_embeddings_flat = tf.reshape(query_embeddings, [-1, embedding_dim])
- nn_features = _nearest_neighbor_features_per_object_in_chunks(
- reference_embeddings_flat, query_embeddings_flat, reference_labels_flat,
- gt_ids, k_nearest_neighbors, n_chunks)
- nn_features_dim = resolve_shape(nn_features)[-1]
- nn_features_reshaped = tf.reshape(nn_features,
- tf.stack(shape[:3] + [tf.size(gt_ids),
- nn_features_dim]))
- return nn_features_reshaped, gt_ids
-
-
-def _nearest_neighbor_features_per_object_in_chunks(
- reference_embeddings_flat, query_embeddings_flat, reference_labels_flat,
- ref_obj_ids, k_nearest_neighbors, n_chunks):
- """Calculates the nearest neighbor features per object in chunks to save mem.
-
- Uses chunking to bound the memory use.
-
- Args:
- reference_embeddings_flat: Tensor of shape [n, embedding_dim],
- the embedding vectors for the reference frame.
- query_embeddings_flat: Tensor of shape [m, embedding_dim], the embedding
- vectors for the query frames.
- reference_labels_flat: Tensor of shape [n], the class labels of the
- reference frame.
- ref_obj_ids: int tensor of unique object ids in the reference labels.
- k_nearest_neighbors: Integer, the number of nearest neighbors to use.
- n_chunks: Integer, the number of chunks to use to save memory
- (set to 1 for no chunking).
-
- Returns:
- nn_features: A float32 tensor of nearest neighbor features of shape
- [m, n_objects, feature_dim].
- """
- chunk_size = tf.cast(tf.ceil(tf.cast(tf.shape(query_embeddings_flat)[0],
- tf.float32) / n_chunks), tf.int32)
- wrong_label_mask = tf.not_equal(reference_labels_flat,
- ref_obj_ids[:, tf.newaxis])
- all_features = []
- for n in range(n_chunks):
- if n_chunks == 1:
- query_embeddings_flat_chunk = query_embeddings_flat
- else:
- chunk_start = n * chunk_size
- chunk_end = (n + 1) * chunk_size
- query_embeddings_flat_chunk = query_embeddings_flat[chunk_start:chunk_end]
- # Use control dependencies to make sure that the chunks are not processed
- # in parallel which would prevent any peak memory savings.
- with tf.control_dependencies(all_features):
- features = _nn_features_per_object_for_chunk(
- reference_embeddings_flat, query_embeddings_flat_chunk,
- wrong_label_mask, k_nearest_neighbors
- )
- all_features.append(features)
- if n_chunks == 1:
- nn_features = all_features[0]
- else:
- nn_features = tf.concat(all_features, axis=0)
- return nn_features
-
-
-def _nn_features_per_object_for_chunk(
- reference_embeddings, query_embeddings, wrong_label_mask,
- k_nearest_neighbors):
- """Extracts features for each object using nearest neighbor attention.
-
- Args:
- reference_embeddings: Tensor of shape [n_chunk, embedding_dim],
- the embedding vectors for the reference frame.
- query_embeddings: Tensor of shape [m_chunk, embedding_dim], the embedding
- vectors for the query frames.
- wrong_label_mask:
- k_nearest_neighbors: Integer, the number of nearest neighbors to use.
-
- Returns:
- nn_features: A float32 tensor of nearest neighbor features of shape
- [m_chunk, n_objects, feature_dim].
- """
- reference_embeddings_key = reference_embeddings
- query_embeddings_key = query_embeddings
- dists = flattened_pairwise_distances(reference_embeddings_key,
- query_embeddings_key)
- dists = (dists[:, tf.newaxis, :] +
- tf.cast(wrong_label_mask[tf.newaxis, :, :], tf.float32) *
- WRONG_LABEL_PADDING_DISTANCE)
- if k_nearest_neighbors == 1:
- features = tf.reduce_min(dists, axis=2, keepdims=True)
- else:
- # Find the closest k and combine them according to attention_feature_type
- dists, _ = tf.nn.top_k(-dists, k=k_nearest_neighbors)
- dists = -dists
- # If not enough real neighbors were found, pad with the farthest real
- # neighbor.
- valid_mask = tf.less(dists, WRONG_LABEL_PADDING_DISTANCE)
- masked_dists = dists * tf.cast(valid_mask, tf.float32)
- pad_dist = tf.tile(tf.reduce_max(masked_dists, axis=2)[..., tf.newaxis],
- multiples=[1, 1, k_nearest_neighbors])
- dists = tf.where(valid_mask, dists, pad_dist)
- # take mean of distances
- features = tf.reduce_mean(dists, axis=2, keepdims=True)
- return features
-
-
-def create_embedding_segmentation_features(features, feature_dimension,
- n_layers, kernel_size, reuse,
- atrous_rates=None):
- """Extracts features which can be used to estimate the final segmentation.
-
- Args:
- features: input features of shape [batch, height, width, features]
- feature_dimension: Integer, the dimensionality used in the segmentation
- head layers.
- n_layers: Integer, the number of layers in the segmentation head.
- kernel_size: Integer, the kernel size used in the segmentation head.
- reuse: reuse mode for the variable_scope.
- atrous_rates: List of integers of length n_layers, the atrous rates to use.
-
- Returns:
- Features to be used to estimate the segmentation labels of shape
- [batch, height, width, embedding_seg_feat_dim].
- """
- if atrous_rates is None or not atrous_rates:
- atrous_rates = [1 for _ in range(n_layers)]
- assert len(atrous_rates) == n_layers
- with tf.variable_scope('embedding_seg', reuse=reuse):
- for n in range(n_layers):
- features = model.split_separable_conv2d(
- features, feature_dimension, kernel_size=kernel_size,
- rate=atrous_rates[n], scope='split_separable_conv2d_{}'.format(n))
- return features
-
-
-def add_image_summaries(images, nn_features, logits, batch_size,
- prev_frame_nn_features=None):
- """Adds image summaries of input images, attention features and logits.
-
- Args:
- images: Image tensor of shape [batch, height, width, channels].
- nn_features: Nearest neighbor attention features of shape
- [batch_size, height, width, n_objects, 1].
- logits: Float32 tensor of logits.
- batch_size: Integer, the number of videos per clone per mini-batch.
- prev_frame_nn_features: Nearest neighbor attention features wrt. the
- last frame of shape [batch_size, height, width, n_objects, 1].
- Can be None.
- """
- # Separate reference and query images.
- reshaped_images = tf.reshape(images, tf.stack(
- [batch_size, -1] + resolve_shape(images)[1:]))
- reference_images = reshaped_images[:, 0]
- query_images = reshaped_images[:, 1:]
- query_images_reshaped = tf.reshape(query_images, tf.stack(
- [-1] + resolve_shape(images)[1:]))
- tf.summary.image('ref_images', reference_images, max_outputs=batch_size)
- tf.summary.image('query_images', query_images_reshaped, max_outputs=10)
- predictions = tf.cast(
- tf.argmax(logits, axis=-1), tf.uint8)[..., tf.newaxis]
- # Scale up so that we can actually see something.
- tf.summary.image('predictions', predictions * 32, max_outputs=10)
- # We currently only show the first dimension of the features for background
- # and the first foreground object.
- tf.summary.image('nn_fg_features', nn_features[..., 0:1, 0],
- max_outputs=batch_size)
- if prev_frame_nn_features is not None:
- tf.summary.image('nn_fg_features_prev', prev_frame_nn_features[..., 0:1, 0],
- max_outputs=batch_size)
- tf.summary.image('nn_bg_features', nn_features[..., 1:2, 0],
- max_outputs=batch_size)
- if prev_frame_nn_features is not None:
- tf.summary.image('nn_bg_features_prev',
- prev_frame_nn_features[..., 1:2, 0],
- max_outputs=batch_size)
-
-
-def get_embeddings(images, model_options, embedding_dimension):
- """Extracts embedding vectors for images. Should only be used for inference.
-
- Args:
- images: A tensor of shape [batch, height, width, channels].
- model_options: A ModelOptions instance to configure models.
- embedding_dimension: Integer, the dimension of the embedding.
-
- Returns:
- embeddings: A tensor of shape [batch, height, width, embedding_dimension].
- """
- features, end_points = model.extract_features(
- images,
- model_options,
- is_training=False)
-
- if model_options.decoder_output_stride is not None:
- decoder_output_stride = min(model_options.decoder_output_stride)
- if model_options.crop_size is None:
- height = tf.shape(images)[1]
- width = tf.shape(images)[2]
- else:
- height, width = model_options.crop_size
- features = model.refine_by_decoder(
- features,
- end_points,
- crop_size=[height, width],
- decoder_output_stride=[decoder_output_stride],
- decoder_use_separable_conv=model_options.decoder_use_separable_conv,
- model_variant=model_options.model_variant,
- is_training=False)
-
- with tf.variable_scope('embedding'):
- embeddings = split_separable_conv2d_with_identity_initializer(
- features, embedding_dimension, scope='split_separable_conv2d')
- return embeddings
-
-
-def get_logits_with_matching(images,
- model_options,
- weight_decay=0.0001,
- reuse=None,
- is_training=False,
- fine_tune_batch_norm=False,
- reference_labels=None,
- batch_size=None,
- num_frames_per_video=None,
- embedding_dimension=None,
- max_neighbors_per_object=0,
- k_nearest_neighbors=1,
- use_softmax_feedback=True,
- initial_softmax_feedback=None,
- embedding_seg_feature_dimension=256,
- embedding_seg_n_layers=4,
- embedding_seg_kernel_size=7,
- embedding_seg_atrous_rates=None,
- normalize_nearest_neighbor_distances=True,
- also_attend_to_previous_frame=True,
- damage_initial_previous_frame_mask=False,
- use_local_previous_frame_attention=True,
- previous_frame_attention_window_size=15,
- use_first_frame_matching=True,
- also_return_embeddings=False,
- ref_embeddings=None):
- """Gets the logits by atrous/image spatial pyramid pooling using attention.
-
- Args:
- images: A tensor of size [batch, height, width, channels].
- model_options: A ModelOptions instance to configure models.
- weight_decay: The weight decay for model variables.
- reuse: Reuse the model variables or not.
- is_training: Is training or not.
- fine_tune_batch_norm: Fine-tune the batch norm parameters or not.
- reference_labels: The segmentation labels of the reference frame on which
- attention is applied.
- batch_size: Integer, the number of videos on a batch
- num_frames_per_video: Integer, the number of frames per video
- embedding_dimension: Integer, the dimension of the embedding
- max_neighbors_per_object: Integer, the maximum number of candidates
- for the nearest neighbor query per object after subsampling.
- Can be 0 for no subsampling.
- k_nearest_neighbors: Integer, the number of nearest neighbors to use.
- use_softmax_feedback: Boolean, whether to give the softmax predictions of
- the last frame as additional input to the segmentation head.
- initial_softmax_feedback: List of Float32 tensors, or None. Can be used to
- initialize the softmax predictions used for the feedback loop.
- Only has an effect if use_softmax_feedback is True.
- embedding_seg_feature_dimension: Integer, the dimensionality used in the
- segmentation head layers.
- embedding_seg_n_layers: Integer, the number of layers in the segmentation
- head.
- embedding_seg_kernel_size: Integer, the kernel size used in the
- segmentation head.
- embedding_seg_atrous_rates: List of integers of length
- embedding_seg_n_layers, the atrous rates to use for the segmentation head.
- normalize_nearest_neighbor_distances: Boolean, whether to normalize the
- nearest neighbor distances to [0,1] using sigmoid, scale and shift.
- also_attend_to_previous_frame: Boolean, whether to also use nearest
- neighbor attention with respect to the previous frame.
- damage_initial_previous_frame_mask: Boolean, whether to artificially damage
- the initial previous frame mask. Only has an effect if
- also_attend_to_previous_frame is True.
- use_local_previous_frame_attention: Boolean, whether to restrict the
- previous frame attention to a local search window.
- Only has an effect, if also_attend_to_previous_frame is True.
- previous_frame_attention_window_size: Integer, the window size used for
- local previous frame attention, if use_local_previous_frame_attention
- is True.
- use_first_frame_matching: Boolean, whether to extract features by matching
- to the reference frame. This should always be true except for ablation
- experiments.
- also_return_embeddings: Boolean, whether to return the embeddings as well.
- ref_embeddings: Tuple of
- (first_frame_embeddings, previous_frame_embeddings),
- each of shape [batch, height, width, embedding_dimension], or None.
- Returns:
- outputs_to_logits: A map from output_type to logits.
- If also_return_embeddings is True, it will also return an embeddings
- tensor of shape [batch, height, width, embedding_dimension].
- """
- features, end_points = model.extract_features(
- images,
- model_options,
- weight_decay=weight_decay,
- reuse=reuse,
- is_training=is_training,
- fine_tune_batch_norm=fine_tune_batch_norm)
-
- if model_options.decoder_output_stride:
- decoder_output_stride = min(model_options.decoder_output_stride)
- if model_options.crop_size is None:
- height = tf.shape(images)[1]
- width = tf.shape(images)[2]
- else:
- height, width = model_options.crop_size
- decoder_height = model.scale_dimension(height, 1.0 / decoder_output_stride)
- decoder_width = model.scale_dimension(width, 1.0 / decoder_output_stride)
- features = model.refine_by_decoder(
- features,
- end_points,
- crop_size=[height, width],
- decoder_output_stride=[decoder_output_stride],
- decoder_use_separable_conv=model_options.decoder_use_separable_conv,
- model_variant=model_options.model_variant,
- weight_decay=weight_decay,
- reuse=reuse,
- is_training=is_training,
- fine_tune_batch_norm=fine_tune_batch_norm)
-
- with tf.variable_scope('embedding', reuse=reuse):
- embeddings = split_separable_conv2d_with_identity_initializer(
- features, embedding_dimension, scope='split_separable_conv2d')
- embeddings = tf.identity(embeddings, name='embeddings')
- scaled_reference_labels = tf.image.resize_nearest_neighbor(
- reference_labels,
- resolve_shape(embeddings, 4)[1:3],
- align_corners=True)
- h, w = decoder_height, decoder_width
- if num_frames_per_video is None:
- num_frames_per_video = tf.size(embeddings) // (
- batch_size * h * w * embedding_dimension)
- new_labels_shape = tf.stack([batch_size, -1, h, w, 1])
- reshaped_reference_labels = tf.reshape(scaled_reference_labels,
- new_labels_shape)
- new_embeddings_shape = tf.stack([batch_size,
- num_frames_per_video, h, w,
- embedding_dimension])
- reshaped_embeddings = tf.reshape(embeddings, new_embeddings_shape)
- all_nn_features = []
- all_ref_obj_ids = []
- # To keep things simple, we do all this separate for each sequence for now.
- for n in range(batch_size):
- embedding = reshaped_embeddings[n]
- if ref_embeddings is None:
- n_chunks = 100
- reference_embedding = embedding[0]
- if also_attend_to_previous_frame or use_softmax_feedback:
- queries_embedding = embedding[2:]
- else:
- queries_embedding = embedding[1:]
- else:
- if USE_CORRELATION_COST:
- n_chunks = 20
- else:
- n_chunks = 500
- reference_embedding = ref_embeddings[0][n]
- queries_embedding = embedding
- reference_labels = reshaped_reference_labels[n][0]
- nn_features_n, ref_obj_ids = nearest_neighbor_features_per_object(
- reference_embedding, queries_embedding, reference_labels,
- max_neighbors_per_object, k_nearest_neighbors, n_chunks=n_chunks)
- if normalize_nearest_neighbor_distances:
- nn_features_n = (tf.nn.sigmoid(nn_features_n) - 0.5) * 2
- all_nn_features.append(nn_features_n)
- all_ref_obj_ids.append(ref_obj_ids)
-
- feat_dim = resolve_shape(features)[-1]
- features = tf.reshape(features, tf.stack(
- [batch_size, num_frames_per_video, h, w, feat_dim]))
- if ref_embeddings is None:
- # Strip the features for the reference frame.
- if also_attend_to_previous_frame or use_softmax_feedback:
- features = features[:, 2:]
- else:
- features = features[:, 1:]
-
- # To keep things simple, we do all this separate for each sequence for now.
- outputs_to_logits = {output: [] for
- output in model_options.outputs_to_num_classes}
- for n in range(batch_size):
- features_n = features[n]
- nn_features_n = all_nn_features[n]
- nn_features_n_tr = tf.transpose(nn_features_n, [3, 0, 1, 2, 4])
- n_objs = tf.shape(nn_features_n_tr)[0]
- # Repeat features for every object.
- features_n_tiled = tf.tile(features_n[tf.newaxis],
- multiples=[n_objs, 1, 1, 1, 1])
- prev_frame_labels = None
- if also_attend_to_previous_frame:
- prev_frame_labels = reshaped_reference_labels[n, 1]
- if is_training and damage_initial_previous_frame_mask:
- # Damage the previous frame masks.
- prev_frame_labels = mask_damaging.damage_masks(prev_frame_labels,
- dilate=False)
- tf.summary.image('prev_frame_labels',
- tf.cast(prev_frame_labels[tf.newaxis],
- tf.uint8) * 32)
- initial_softmax_feedback_n = create_initial_softmax_from_labels(
- prev_frame_labels, reshaped_reference_labels[n][0],
- decoder_output_stride=None, reduce_labels=True)
- elif initial_softmax_feedback is not None:
- initial_softmax_feedback_n = initial_softmax_feedback[n]
- else:
- initial_softmax_feedback_n = None
- if initial_softmax_feedback_n is None:
- last_softmax = tf.zeros((n_objs, h, w, 1), dtype=tf.float32)
- else:
- last_softmax = tf.transpose(initial_softmax_feedback_n, [2, 0, 1])[
- ..., tf.newaxis]
- assert len(model_options.outputs_to_num_classes) == 1
- output = model_options.outputs_to_num_classes.keys()[0]
- logits = []
- n_ref_frames = 1
- prev_frame_nn_features_n = None
- if also_attend_to_previous_frame or use_softmax_feedback:
- n_ref_frames += 1
- if ref_embeddings is not None:
- n_ref_frames = 0
- for t in range(num_frames_per_video - n_ref_frames):
- to_concat = [features_n_tiled[:, t]]
- if use_first_frame_matching:
- to_concat.append(nn_features_n_tr[:, t])
- if use_softmax_feedback:
- to_concat.append(last_softmax)
- if also_attend_to_previous_frame:
- assert normalize_nearest_neighbor_distances, (
- 'previous frame attention currently only works when normalized '
- 'distances are used')
- embedding = reshaped_embeddings[n]
- if ref_embeddings is None:
- last_frame_embedding = embedding[t + 1]
- query_embeddings = embedding[t + 2, tf.newaxis]
- else:
- last_frame_embedding = ref_embeddings[1][0]
- query_embeddings = embedding
- if use_local_previous_frame_attention:
- assert query_embeddings.shape[0] == 1
- prev_frame_nn_features_n = (
- local_previous_frame_nearest_neighbor_features_per_object(
- last_frame_embedding,
- query_embeddings[0],
- prev_frame_labels,
- all_ref_obj_ids[n],
- max_distance=previous_frame_attention_window_size)
- )
- else:
- prev_frame_nn_features_n, _ = (
- nearest_neighbor_features_per_object(
- last_frame_embedding, query_embeddings, prev_frame_labels,
- max_neighbors_per_object, k_nearest_neighbors,
- gt_ids=all_ref_obj_ids[n]))
- prev_frame_nn_features_n = (tf.nn.sigmoid(
- prev_frame_nn_features_n) - 0.5) * 2
- prev_frame_nn_features_n_sq = tf.squeeze(prev_frame_nn_features_n,
- axis=0)
- prev_frame_nn_features_n_tr = tf.transpose(
- prev_frame_nn_features_n_sq, [2, 0, 1, 3])
- to_concat.append(prev_frame_nn_features_n_tr)
- features_n_concat_t = tf.concat(to_concat, axis=-1)
- embedding_seg_features_n_t = (
- create_embedding_segmentation_features(
- features_n_concat_t, embedding_seg_feature_dimension,
- embedding_seg_n_layers, embedding_seg_kernel_size,
- reuse or n > 0, atrous_rates=embedding_seg_atrous_rates))
- logits_t = model.get_branch_logits(
- embedding_seg_features_n_t,
- 1,
- model_options.atrous_rates,
- aspp_with_batch_norm=model_options.aspp_with_batch_norm,
- kernel_size=model_options.logits_kernel_size,
- weight_decay=weight_decay,
- reuse=reuse or n > 0 or t > 0,
- scope_suffix=output
- )
- logits.append(logits_t)
- prev_frame_labels = tf.transpose(tf.argmax(logits_t, axis=0),
- [2, 0, 1])
- last_softmax = tf.nn.softmax(logits_t, axis=0)
- logits = tf.stack(logits, axis=1)
- logits_shape = tf.stack(
- [n_objs, num_frames_per_video - n_ref_frames] +
- resolve_shape(logits)[2:-1])
- logits_reshaped = tf.reshape(logits, logits_shape)
- logits_transposed = tf.transpose(logits_reshaped, [1, 2, 3, 0])
- outputs_to_logits[output].append(logits_transposed)
-
- add_image_summaries(
- images[n * num_frames_per_video: (n+1) * num_frames_per_video],
- nn_features_n,
- logits_transposed,
- batch_size=1,
- prev_frame_nn_features=prev_frame_nn_features_n)
- if also_return_embeddings:
- return outputs_to_logits, embeddings
- else:
- return outputs_to_logits
-
-
-def subsample_reference_embeddings_and_labels(
- reference_embeddings_flat, reference_labels_flat, ref_obj_ids,
- max_neighbors_per_object):
- """Subsamples the reference embedding vectors and labels.
-
- After subsampling, at most max_neighbors_per_object items will remain per
- class.
-
- Args:
- reference_embeddings_flat: Tensor of shape [n, embedding_dim],
- the embedding vectors for the reference frame.
- reference_labels_flat: Tensor of shape [n, 1],
- the class labels of the reference frame.
- ref_obj_ids: An int32 tensor of the unique object ids present
- in the reference labels.
- max_neighbors_per_object: Integer, the maximum number of candidates
- for the nearest neighbor query per object after subsampling,
- or 0 for no subsampling.
-
- Returns:
- reference_embeddings_flat: Tensor of shape [n_sub, embedding_dim],
- the subsampled embedding vectors for the reference frame.
- reference_labels_flat: Tensor of shape [n_sub, 1],
- the class labels of the reference frame.
- """
- if max_neighbors_per_object == 0:
- return reference_embeddings_flat, reference_labels_flat
- same_label_mask = tf.equal(reference_labels_flat[tf.newaxis, :],
- ref_obj_ids[:, tf.newaxis])
- max_neighbors_per_object_repeated = tf.tile(
- tf.constant(max_neighbors_per_object)[tf.newaxis],
- multiples=[tf.size(ref_obj_ids)])
- # Somehow map_fn on GPU caused trouble sometimes, so let's put it on CPU
- # for now.
- with tf.device('cpu:0'):
- subsampled_indices = tf.map_fn(_create_subsampling_mask,
- (same_label_mask,
- max_neighbors_per_object_repeated),
- dtype=tf.int64,
- name='subsample_labels_map_fn',
- parallel_iterations=1)
- mask = tf.not_equal(subsampled_indices, tf.constant(-1, dtype=tf.int64))
- masked_indices = tf.boolean_mask(subsampled_indices, mask)
- reference_embeddings_flat = tf.gather(reference_embeddings_flat,
- masked_indices)
- reference_labels_flat = tf.gather(reference_labels_flat, masked_indices)
- return reference_embeddings_flat, reference_labels_flat
-
-
-def _create_subsampling_mask(args):
- """Creates boolean mask which can be used to subsample the labels.
-
- Args:
- args: tuple of (label_mask, max_neighbors_per_object), where label_mask
- is the mask to be subsampled and max_neighbors_per_object is a int scalar,
- the maximum number of neighbors to be retained after subsampling.
-
- Returns:
- The boolean mask for subsampling the labels.
- """
- label_mask, max_neighbors_per_object = args
- indices = tf.squeeze(tf.where(label_mask), axis=1)
- shuffled_indices = tf.random_shuffle(indices)
- subsampled_indices = shuffled_indices[:max_neighbors_per_object]
- n_pad = max_neighbors_per_object - tf.size(subsampled_indices)
- padded_label = -1
- padding = tf.fill((n_pad,), tf.constant(padded_label, dtype=tf.int64))
- padded = tf.concat([subsampled_indices, padding], axis=0)
- return padded
-
-
-def conv2d_identity_initializer(scale=1.0, mean=0, stddev=3e-2):
- """Creates an identity initializer for TensorFlow conv2d.
-
- We add a small amount of normal noise to the initialization matrix.
- Code copied from lcchen@.
-
- Args:
- scale: The scale coefficient for the identity weight matrix.
- mean: A 0-D Tensor or Python value of type `dtype`. The mean of the
- truncated normal distribution.
- stddev: A 0-D Tensor or Python value of type `dtype`. The standard deviation
- of the truncated normal distribution.
-
- Returns:
- An identity initializer function for TensorFlow conv2d.
- """
- def _initializer(shape, dtype=tf.float32, partition_info=None):
- """Returns the identity matrix scaled by `scale`.
-
- Args:
- shape: A tuple of int32 numbers indicating the shape of the initializing
- matrix.
- dtype: The data type of the initializing matrix.
- partition_info: (Optional) variable_scope._PartitionInfo object holding
- additional information about how the variable is partitioned. This input
- is not used in our case, but is required by TensorFlow.
-
- Returns:
- A identity matrix.
-
- Raises:
- ValueError: If len(shape) != 4, or shape[0] != shape[1], or shape[0] is
- not odd, or shape[1] is not odd..
- """
- del partition_info
- if len(shape) != 4:
- raise ValueError('Expect shape length to be 4.')
- if shape[0] != shape[1]:
- raise ValueError('Expect shape[0] = shape[1].')
- if shape[0] % 2 != 1:
- raise ValueError('Expect shape[0] to be odd value.')
- if shape[1] % 2 != 1:
- raise ValueError('Expect shape[1] to be odd value.')
- weights = np.zeros(shape, dtype=np.float32)
- center_y = shape[0] / 2
- center_x = shape[1] / 2
- min_channel = min(shape[2], shape[3])
- for i in range(min_channel):
- weights[center_y, center_x, i, i] = scale
- return tf.constant(weights, dtype=dtype) + tf.truncated_normal(
- shape, mean=mean, stddev=stddev, dtype=dtype)
-
- return _initializer
-
-
-def split_separable_conv2d_with_identity_initializer(
- inputs,
- filters,
- kernel_size=3,
- rate=1,
- weight_decay=0.00004,
- scope=None):
- """Splits a separable conv2d into depthwise and pointwise conv2d.
-
- This operation differs from `tf.layers.separable_conv2d` as this operation
- applies activation function between depthwise and pointwise conv2d.
-
- Args:
- inputs: Input tensor with shape [batch, height, width, channels].
- filters: Number of filters in the 1x1 pointwise convolution.
- kernel_size: A list of length 2: [kernel_height, kernel_width] of
- of the filters. Can be an int if both values are the same.
- rate: Atrous convolution rate for the depthwise convolution.
- weight_decay: The weight decay to use for regularizing the model.
- scope: Optional scope for the operation.
-
- Returns:
- Computed features after split separable conv2d.
- """
- initializer = conv2d_identity_initializer()
- outputs = slim.separable_conv2d(
- inputs,
- None,
- kernel_size=kernel_size,
- depth_multiplier=1,
- rate=rate,
- weights_initializer=initializer,
- weights_regularizer=None,
- scope=scope + '_depthwise')
- return slim.conv2d(
- outputs,
- filters,
- 1,
- weights_initializer=initializer,
- weights_regularizer=slim.l2_regularizer(weight_decay),
- scope=scope + '_pointwise')
-
-
-def create_initial_softmax_from_labels(last_frame_labels, reference_labels,
- decoder_output_stride, reduce_labels):
- """Creates initial softmax predictions from last frame labels.
-
- Args:
- last_frame_labels: last frame labels of shape [1, height, width, 1].
- reference_labels: reference frame labels of shape [1, height, width, 1].
- decoder_output_stride: Integer, the stride of the decoder. Can be None, in
- this case it's assumed that the last_frame_labels and reference_labels
- are already scaled to the decoder output resolution.
- reduce_labels: Boolean, whether to reduce the depth of the softmax one_hot
- encoding to the actual number of labels present in the reference frame
- (otherwise the depth will be the highest label index + 1).
-
- Returns:
- init_softmax: the initial softmax predictions.
- """
- if decoder_output_stride is None:
- labels_output_size = last_frame_labels
- reference_labels_output_size = reference_labels
- else:
- h = tf.shape(last_frame_labels)[1]
- w = tf.shape(last_frame_labels)[2]
- h_sub = model.scale_dimension(h, 1.0 / decoder_output_stride)
- w_sub = model.scale_dimension(w, 1.0 / decoder_output_stride)
- labels_output_size = tf.image.resize_nearest_neighbor(
- last_frame_labels, [h_sub, w_sub], align_corners=True)
- reference_labels_output_size = tf.image.resize_nearest_neighbor(
- reference_labels, [h_sub, w_sub], align_corners=True)
- if reduce_labels:
- unique_labels, _ = tf.unique(tf.reshape(reference_labels_output_size, [-1]))
- depth = tf.size(unique_labels)
- else:
- depth = tf.reduce_max(reference_labels_output_size) + 1
- one_hot_assertion = tf.assert_less(tf.reduce_max(labels_output_size), depth)
- with tf.control_dependencies([one_hot_assertion]):
- init_softmax = tf.one_hot(tf.squeeze(labels_output_size,
- axis=-1),
- depth=depth,
- dtype=tf.float32)
- return init_softmax
-
-
-def local_previous_frame_nearest_neighbor_features_per_object(
- prev_frame_embedding, query_embedding, prev_frame_labels,
- gt_ids, max_distance=9):
- """Computes nearest neighbor features while only allowing local matches.
-
- Args:
- prev_frame_embedding: Tensor of shape [height, width, embedding_dim],
- the embedding vectors for the last frame.
- query_embedding: Tensor of shape [height, width, embedding_dim],
- the embedding vectors for the query frames.
- prev_frame_labels: Tensor of shape [height, width, 1], the class labels of
- the previous frame.
- gt_ids: Int Tensor of shape [n_objs] of the sorted unique ground truth
- ids in the first frame.
- max_distance: Integer, the maximum distance allowed for local matching.
-
- Returns:
- nn_features: A float32 np.array of nearest neighbor features of shape
- [1, height, width, n_objects, 1].
- """
- with tf.name_scope(
- 'local_previous_frame_nearest_neighbor_features_per_object'):
- if USE_CORRELATION_COST:
- tf.logging.info('Using correlation_cost.')
- d = local_pairwise_distances(query_embedding, prev_frame_embedding,
- max_distance=max_distance)
- else:
- # Slow fallback in case correlation_cost is not available.
- tf.logging.warn('correlation cost is not available, using slow fallback '
- 'implementation.')
- d = local_pairwise_distances2(query_embedding, prev_frame_embedding,
- max_distance=max_distance)
- d = (tf.nn.sigmoid(d) - 0.5) * 2
- height = tf.shape(prev_frame_embedding)[0]
- width = tf.shape(prev_frame_embedding)[1]
-
- # Create offset versions of the mask.
- if USE_CORRELATION_COST:
- # New, faster code with cross-correlation via correlation_cost.
- # Due to padding we have to add 1 to the labels.
- offset_labels = correlation_cost_op.correlation_cost(
- tf.ones((1, height, width, 1)),
- tf.cast(prev_frame_labels + 1, tf.float32)[tf.newaxis],
- kernel_size=1,
- max_displacement=max_distance, stride_1=1, stride_2=1,
- pad=max_distance)
- offset_labels = tf.squeeze(offset_labels, axis=0)[..., tf.newaxis]
- # Subtract the 1 again and round.
- offset_labels = tf.round(offset_labels - 1)
- offset_masks = tf.equal(
- offset_labels,
- tf.cast(gt_ids, tf.float32)[tf.newaxis, tf.newaxis, tf.newaxis, :])
- else:
- # Slower code, without dependency to correlation_cost
- masks = tf.equal(prev_frame_labels, gt_ids[tf.newaxis, tf.newaxis])
- padded_masks = tf.pad(masks,
- [[max_distance, max_distance],
- [max_distance, max_distance],
- [0, 0]])
- offset_masks = []
- for y_start in range(2 * max_distance + 1):
- y_end = y_start + height
- masks_slice = padded_masks[y_start:y_end]
- for x_start in range(2 * max_distance + 1):
- x_end = x_start + width
- offset_mask = masks_slice[:, x_start:x_end]
- offset_masks.append(offset_mask)
- offset_masks = tf.stack(offset_masks, axis=2)
-
- pad = tf.ones((height, width, (2 * max_distance + 1) ** 2, tf.size(gt_ids)))
- d_tiled = tf.tile(d[..., tf.newaxis], multiples=(1, 1, 1, tf.size(gt_ids)))
- d_masked = tf.where(offset_masks, d_tiled, pad)
- dists = tf.reduce_min(d_masked, axis=2)
- dists = tf.reshape(dists, (1, height, width, tf.size(gt_ids), 1))
- return dists
diff --git a/research/feelvos/utils/embedding_utils_test.py b/research/feelvos/utils/embedding_utils_test.py
deleted file mode 100644
index ddebd7b4e7fcc9402887ebf59d247fea815d6cda..0000000000000000000000000000000000000000
--- a/research/feelvos/utils/embedding_utils_test.py
+++ /dev/null
@@ -1,213 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for embedding utils."""
-
-import unittest
-import numpy as np
-import tensorflow as tf
-from feelvos.utils import embedding_utils
-
-if embedding_utils.USE_CORRELATION_COST:
- # pylint: disable=g-import-not-at-top
- from correlation_cost.python.ops import correlation_cost_op
-
-
-class EmbeddingUtilsTest(tf.test.TestCase):
-
- def test_pairwise_distances(self):
- x = np.arange(100, dtype=np.float32).reshape(20, 5)
- y = np.arange(100, 200, dtype=np.float32).reshape(20, 5)
- g = tf.Graph()
- with g.as_default():
- with self.test_session(graph=g) as sess:
- x = tf.constant(x)
- y = tf.constant(y)
- d1 = embedding_utils.pairwise_distances(x, y)
- d2 = embedding_utils.pairwise_distances2(x, y)
- d1_val, d2_val = sess.run([d1, d2])
- self.assertAllClose(d1_val, d2_val)
-
- @unittest.skipIf(not embedding_utils.USE_CORRELATION_COST,
- 'depends on correlation_cost')
- def test_correlation_cost_one_dimensional(self):
- a = np.array([[[[1.0], [2.0]], [[3.0], [4.0]]]])
- b = np.array([[[[2.0], [1.0]], [[4.0], [3.0]]]])
- g = tf.Graph()
- with g.as_default():
- with self.test_session(graph=g) as sess:
- c = correlation_cost_op.correlation_cost(
- a, b, kernel_size=1, max_displacement=1, stride_1=1, stride_2=1,
- pad=1)
- c = tf.squeeze(c, axis=0)
- c_val = sess.run(c)
- self.assertAllEqual(c_val.shape, (2, 2, 9))
- for y in range(2):
- for x in range(2):
- for dy in range(-1, 2):
- for dx in range(-1, 2):
- a_slice = a[0, y, x, 0]
- if y + dy < 0 or y + dy > 1 or x + dx < 0 or x + dx > 1:
- b_slice = 0
- else:
- b_slice = b[0, y + dy, x + dx, 0]
- expected = a_slice * b_slice
- dy0 = dy + 1
- dx0 = dx + 1
- self.assertAlmostEqual(c_val[y, x, 3 * dy0 + dx0], expected)
-
- @unittest.skipIf(not embedding_utils.USE_CORRELATION_COST,
- 'depends on correlation_cost')
- def test_correlation_cost_two_dimensional(self):
- a = np.array([[[[1.0, -5.0], [7.0, 2.0]], [[1.0, 3.0], [3.0, 4.0]]]])
- b = np.array([[[[2.0, 1.0], [0.0, -9.0]], [[4.0, 3.0], [3.0, 1.0]]]])
- g = tf.Graph()
- with g.as_default():
- with self.test_session(graph=g) as sess:
- c = correlation_cost_op.correlation_cost(
- a, b, kernel_size=1, max_displacement=1, stride_1=1, stride_2=1,
- pad=1)
- c = tf.squeeze(c, axis=0)
- c_val = sess.run(c)
- self.assertAllEqual(c_val.shape, (2, 2, 9))
- for y in range(2):
- for x in range(2):
- for dy in range(-1, 2):
- for dx in range(-1, 2):
- a_slice = a[0, y, x, :]
- if y + dy < 0 or y + dy > 1 or x + dx < 0 or x + dx > 1:
- b_slice = 0
- else:
- b_slice = b[0, y + dy, x + dx, :]
- expected = (a_slice * b_slice).mean()
- dy0 = dy + 1
- dx0 = dx + 1
- self.assertAlmostEqual(c_val[y, x, 3 * dy0 + dx0], expected)
-
- @unittest.skipIf(not embedding_utils.USE_CORRELATION_COST,
- 'depends on correlation_cost')
- def test_local_pairwise_distances_one_dimensional(self):
- a = np.array([[[1.0], [2.0]], [[3.0], [4.0]]])
- b = np.array([[[2.0], [1.0]], [[4.0], [3.0]]])
- g = tf.Graph()
- with g.as_default():
- with self.test_session(graph=g) as sess:
- a_tf = tf.constant(a, dtype=tf.float32)
- b_tf = tf.constant(b, dtype=tf.float32)
- d = embedding_utils.local_pairwise_distances(a_tf, b_tf,
- max_distance=1)
- d_val = sess.run(d)
- for y in range(2):
- for x in range(2):
- for dy in range(-1, 2):
- for dx in range(-1, 2):
- a_slice = a[y, x, 0]
- if y + dy < 0 or y + dy > 1 or x + dx < 0 or x + dx > 1:
- expected = np.float('inf')
- else:
- b_slice = b[y + dy, x + dx, 0]
- expected = (a_slice - b_slice) ** 2
- dy0 = dy + 1
- dx0 = dx + 1
- self.assertAlmostEqual(d_val[y, x, 3 * dy0 + dx0], expected)
-
- @unittest.skipIf(not embedding_utils.USE_CORRELATION_COST,
- 'depends on correlation_cost')
- def test_local_pairwise_distances_shape(self):
- a = np.zeros((4, 5, 2))
- b = np.zeros((4, 5, 2))
- g = tf.Graph()
- with g.as_default():
- with self.test_session(graph=g) as sess:
- a_tf = tf.constant(a, dtype=tf.float32)
- b_tf = tf.constant(b, dtype=tf.float32)
- d = embedding_utils.local_pairwise_distances(a_tf, b_tf, max_distance=4)
- d_val = sess.run(d)
- self.assertAllEqual(d_val.shape, (4, 5, 81))
-
- @unittest.skipIf(not embedding_utils.USE_CORRELATION_COST,
- 'depends on correlation_cost')
- def test_local_pairwise_distances_two_dimensional(self):
- a = np.array([[[1.0, -5.0], [7.0, 2.0]], [[1.0, 3.0], [3.0, 4.0]]])
- b = np.array([[[2.0, 1.0], [0.0, -9.0]], [[4.0, 3.0], [3.0, 1.0]]])
- g = tf.Graph()
- with g.as_default():
- with self.test_session(graph=g) as sess:
- a_tf = tf.constant(a, dtype=tf.float32)
- b_tf = tf.constant(b, dtype=tf.float32)
- d = embedding_utils.local_pairwise_distances(a_tf, b_tf,
- max_distance=1)
- d_val = sess.run(d)
- for y in range(2):
- for x in range(2):
- for dy in range(-1, 2):
- for dx in range(-1, 2):
- a_slice = a[y, x, :]
- if y + dy < 0 or y + dy > 1 or x + dx < 0 or x + dx > 1:
- expected = np.float('inf')
- else:
- b_slice = b[y + dy, x + dx, :]
- expected = ((a_slice - b_slice) ** 2).sum()
- dy0 = dy + 1
- dx0 = dx + 1
- self.assertAlmostEqual(d_val[y, x, 3 * dy0 + dx0], expected)
-
- @unittest.skipIf(not embedding_utils.USE_CORRELATION_COST,
- 'depends on correlation_cost')
- def test_local_previous_frame_nearest_neighbor_features_per_object(self):
- prev_frame_embedding = np.array([[[1.0, -5.0], [7.0, 2.0]],
- [[1.0, 3.0], [3.0, 4.0]]]) / 10
- query_embedding = np.array([[[2.0, 1.0], [0.0, -9.0]],
- [[4.0, 3.0], [3.0, 1.0]]]) / 10
- prev_frame_labels = np.array([[[0], [1]], [[1], [0]]])
- gt_ids = np.array([0, 1])
- g = tf.Graph()
- with g.as_default():
- with self.test_session(graph=g) as sess:
- prev_frame_embedding_tf = tf.constant(prev_frame_embedding,
- dtype=tf.float32)
- query_embedding_tf = tf.constant(query_embedding, dtype=tf.float32)
- embu = embedding_utils
- dists = (
- embu.local_previous_frame_nearest_neighbor_features_per_object(
- prev_frame_embedding_tf, query_embedding_tf,
- prev_frame_labels, gt_ids, max_distance=1))
- dists = tf.squeeze(dists, axis=4)
- dists = tf.squeeze(dists, axis=0)
- dists_val = sess.run(dists)
- for obj_id in gt_ids:
- for y in range(2):
- for x in range(2):
- curr_min = 1.0
- for dy in range(-1, 2):
- for dx in range(-1, 2):
- # Attention: here we shift the prev frame embedding,
- # not the query.
- if y + dy < 0 or y + dy > 1 or x + dx < 0 or x + dx > 1:
- continue
- if prev_frame_labels[y + dy, x + dx, 0] != obj_id:
- continue
- prev_frame_slice = prev_frame_embedding[y + dy, x + dx, :]
- query_frame_slice = query_embedding[y, x, :]
- v_unnorm = ((prev_frame_slice - query_frame_slice) ** 2).sum()
- v = ((1.0 / (1.0 + np.exp(-v_unnorm))) - 0.5) * 2
- curr_min = min(curr_min, v)
- expected = curr_min
- self.assertAlmostEqual(dists_val[y, x, obj_id], expected,
- places=5)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/feelvos/utils/eval_utils.py b/research/feelvos/utils/eval_utils.py
deleted file mode 100644
index 517ec0d788eb3a6ec48246e10920dd4b55332bf5..0000000000000000000000000000000000000000
--- a/research/feelvos/utils/eval_utils.py
+++ /dev/null
@@ -1,153 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utility functions for evaluations."""
-
-import numpy as np
-import PIL
-import tensorflow as tf
-
-pascal_colormap = [
- 0, 0, 0,
- 0.5020, 0, 0,
- 0, 0.5020, 0,
- 0.5020, 0.5020, 0,
- 0, 0, 0.5020,
- 0.5020, 0, 0.5020,
- 0, 0.5020, 0.5020,
- 0.5020, 0.5020, 0.5020,
- 0.2510, 0, 0,
- 0.7529, 0, 0,
- 0.2510, 0.5020, 0,
- 0.7529, 0.5020, 0,
- 0.2510, 0, 0.5020,
- 0.7529, 0, 0.5020,
- 0.2510, 0.5020, 0.5020,
- 0.7529, 0.5020, 0.5020,
- 0, 0.2510, 0,
- 0.5020, 0.2510, 0,
- 0, 0.7529, 0,
- 0.5020, 0.7529, 0,
- 0, 0.2510, 0.5020,
- 0.5020, 0.2510, 0.5020,
- 0, 0.7529, 0.5020,
- 0.5020, 0.7529, 0.5020,
- 0.2510, 0.2510, 0]
-
-
-def save_segmentation_with_colormap(filename, img):
- """Saves a segmentation with the pascal colormap as expected for DAVIS eval.
-
- Args:
- filename: Where to store the segmentation.
- img: A numpy array of the segmentation to be saved.
- """
- if img.shape[-1] == 1:
- img = img[..., 0]
-
- # Save with colormap.
- colormap = (np.array(pascal_colormap) * 255).round().astype('uint8')
- colormap_image = PIL.Image.new('P', (16, 16))
- colormap_image.putpalette(colormap)
- pil_image = PIL.Image.fromarray(img.astype('uint8'))
- pil_image_with_colormap = pil_image.quantize(palette=colormap_image)
- with tf.gfile.GFile(filename, 'w') as f:
- pil_image_with_colormap.save(f)
-
-
-def save_embeddings(filename, embeddings):
- with tf.gfile.GFile(filename, 'w') as f:
- np.save(f, embeddings)
-
-
-def calculate_iou(pred_labels, ref_labels):
- """Calculates the intersection over union for binary segmentation.
-
- Args:
- pred_labels: predicted segmentation labels.
- ref_labels: reference segmentation labels.
-
- Returns:
- The IoU between pred_labels and ref_labels
- """
- if ref_labels.any():
- i = np.logical_and(pred_labels, ref_labels).sum()
- u = np.logical_or(pred_labels, ref_labels).sum()
- return i.astype('float') / u
- else:
- if pred_labels.any():
- return 0.0
- else:
- return 1.0
-
-
-def calculate_multi_object_miou_tf(pred_labels, ref_labels):
- """Calculates the mIoU for a batch of predicted and reference labels.
-
- Args:
- pred_labels: Int32 tensor of shape [batch, height, width, 1].
- ref_labels: Int32 tensor of shape [batch, height, width, 1].
-
- Returns:
- The mIoU between pred_labels and ref_labels as float32 scalar tensor.
- """
-
- def calculate_multi_object_miou(pred_labels_, ref_labels_):
- """Calculates the mIoU for predicted and reference labels in numpy.
-
- Args:
- pred_labels_: int32 np.array of shape [batch, height, width, 1].
- ref_labels_: int32 np.array of shape [batch, height, width, 1].
-
- Returns:
- The mIoU between pred_labels_ and ref_labels_.
- """
- assert len(pred_labels_.shape) == 4
- assert pred_labels_.shape[3] == 1
- assert pred_labels_.shape == ref_labels_.shape
- ious = []
- for pred_label, ref_label in zip(pred_labels_, ref_labels_):
- ids = np.setdiff1d(np.unique(ref_label), [0])
- if ids.size == 0:
- continue
- for id_ in ids:
- iou = calculate_iou(pred_label == id_, ref_label == id_)
- ious.append(iou)
- if ious:
- return np.cast['float32'](np.mean(ious))
- else:
- return np.cast['float32'](1.0)
-
- miou = tf.py_func(calculate_multi_object_miou, [pred_labels, ref_labels],
- tf.float32, name='calculate_multi_object_miou')
- miou.set_shape(())
- return miou
-
-
-def calculate_multi_object_ious(pred_labels, ref_labels, label_set):
- """Calculates the intersection over union for binary segmentation.
-
- Args:
- pred_labels: predicted segmentation labels.
- ref_labels: reference segmentation labels.
- label_set: int np.array of object ids.
-
- Returns:
- float np.array of IoUs between pred_labels and ref_labels
- for each object in label_set.
- """
- # Background should not be included as object label.
- return np.array([calculate_iou(pred_labels == label, ref_labels == label)
- for label in label_set if label != 0])
diff --git a/research/feelvos/utils/mask_damaging.py b/research/feelvos/utils/mask_damaging.py
deleted file mode 100644
index 74f3cdab5a0e4374f0cd238544a9a582fd0eef92..0000000000000000000000000000000000000000
--- a/research/feelvos/utils/mask_damaging.py
+++ /dev/null
@@ -1,176 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities for artificially damaging segmentation masks."""
-
-import numpy as np
-from scipy.ndimage import interpolation
-from skimage import morphology
-from skimage import transform
-import tensorflow as tf
-
-
-def damage_masks(labels, shift=True, scale=True, rotate=True, dilate=True):
- """Damages segmentation masks by random transformations.
-
- Args:
- labels: Int32 labels tensor of shape (height, width, 1).
- shift: Boolean, whether to damage the masks by shifting.
- scale: Boolean, whether to damage the masks by scaling.
- rotate: Boolean, whether to damage the masks by rotation.
- dilate: Boolean, whether to damage the masks by dilation.
-
- Returns:
- The damaged version of labels.
- """
- def _damage_masks_np(labels_):
- return damage_masks_np(labels_, shift, scale, rotate, dilate)
- damaged_masks = tf.py_func(_damage_masks_np, [labels], tf.int32,
- name='damage_masks')
- damaged_masks.set_shape(labels.get_shape())
- return damaged_masks
-
-
-def damage_masks_np(labels, shift=True, scale=True, rotate=True, dilate=True):
- """Performs the actual mask damaging in numpy.
-
- Args:
- labels: Int32 numpy array of shape (height, width, 1).
- shift: Boolean, whether to damage the masks by shifting.
- scale: Boolean, whether to damage the masks by scaling.
- rotate: Boolean, whether to damage the masks by rotation.
- dilate: Boolean, whether to damage the masks by dilation.
-
- Returns:
- The damaged version of labels.
- """
- unique_labels = np.unique(labels)
- unique_labels = np.setdiff1d(unique_labels, [0])
- # Shuffle to get random depth ordering when combining together.
- np.random.shuffle(unique_labels)
- damaged_labels = np.zeros_like(labels)
- for l in unique_labels:
- obj_mask = (labels == l)
- damaged_obj_mask = _damage_single_object_mask(obj_mask, shift, scale,
- rotate, dilate)
- damaged_labels[damaged_obj_mask] = l
- return damaged_labels
-
-
-def _damage_single_object_mask(mask, shift, scale, rotate, dilate):
- """Performs mask damaging in numpy for a single object.
-
- Args:
- mask: Boolean numpy array of shape(height, width, 1).
- shift: Boolean, whether to damage the masks by shifting.
- scale: Boolean, whether to damage the masks by scaling.
- rotate: Boolean, whether to damage the masks by rotation.
- dilate: Boolean, whether to damage the masks by dilation.
-
- Returns:
- The damaged version of mask.
- """
- # For now we just do shifting and scaling. Better would be Affine or thin
- # spline plate transformations.
- if shift:
- mask = _shift_mask(mask)
- if scale:
- mask = _scale_mask(mask)
- if rotate:
- mask = _rotate_mask(mask)
- if dilate:
- mask = _dilate_mask(mask)
- return mask
-
-
-def _shift_mask(mask, max_shift_factor=0.05):
- """Damages a mask for a single object by randomly shifting it in numpy.
-
- Args:
- mask: Boolean numpy array of shape(height, width, 1).
- max_shift_factor: Float scalar, the maximum factor for random shifting.
-
- Returns:
- The shifted version of mask.
- """
- nzy, nzx, _ = mask.nonzero()
- h = nzy.max() - nzy.min()
- w = nzx.max() - nzx.min()
- size = np.sqrt(h * w)
- offset = np.random.uniform(-size * max_shift_factor, size * max_shift_factor,
- 2)
- shifted_mask = interpolation.shift(np.squeeze(mask, axis=2),
- offset, order=0).astype('bool')[...,
- np.newaxis]
- return shifted_mask
-
-
-def _scale_mask(mask, scale_amount=0.025):
- """Damages a mask for a single object by randomly scaling it in numpy.
-
- Args:
- mask: Boolean numpy array of shape(height, width, 1).
- scale_amount: Float scalar, the maximum factor for random scaling.
-
- Returns:
- The scaled version of mask.
- """
- nzy, nzx, _ = mask.nonzero()
- cy = 0.5 * (nzy.max() - nzy.min())
- cx = 0.5 * (nzx.max() - nzx.min())
- scale_factor = np.random.uniform(1.0 - scale_amount, 1.0 + scale_amount)
- shift = transform.SimilarityTransform(translation=[-cx, -cy])
- inv_shift = transform.SimilarityTransform(translation=[cx, cy])
- s = transform.SimilarityTransform(scale=[scale_factor, scale_factor])
- m = (shift + (s + inv_shift)).inverse
- scaled_mask = transform.warp(mask, m) > 0.5
- return scaled_mask
-
-
-def _rotate_mask(mask, max_rot_degrees=3.0):
- """Damages a mask for a single object by randomly rotating it in numpy.
-
- Args:
- mask: Boolean numpy array of shape(height, width, 1).
- max_rot_degrees: Float scalar, the maximum number of degrees to rotate.
-
- Returns:
- The scaled version of mask.
- """
- cy = 0.5 * mask.shape[0]
- cx = 0.5 * mask.shape[1]
- rot_degrees = np.random.uniform(-max_rot_degrees, max_rot_degrees)
- shift = transform.SimilarityTransform(translation=[-cx, -cy])
- inv_shift = transform.SimilarityTransform(translation=[cx, cy])
- r = transform.SimilarityTransform(rotation=np.deg2rad(rot_degrees))
- m = (shift + (r + inv_shift)).inverse
- scaled_mask = transform.warp(mask, m) > 0.5
- return scaled_mask
-
-
-def _dilate_mask(mask, dilation_radius=5):
- """Damages a mask for a single object by dilating it in numpy.
-
- Args:
- mask: Boolean numpy array of shape(height, width, 1).
- dilation_radius: Integer, the radius of the used disk structure element.
-
- Returns:
- The dilated version of mask.
- """
- disk = morphology.disk(dilation_radius, dtype=np.bool)
- dilated_mask = morphology.binary_dilation(
- np.squeeze(mask, axis=2), selem=disk)[..., np.newaxis]
- return dilated_mask
diff --git a/research/feelvos/utils/train_utils.py b/research/feelvos/utils/train_utils.py
deleted file mode 100644
index 02a04cd33645931c5c795bef8559c0d3f5c4c23c..0000000000000000000000000000000000000000
--- a/research/feelvos/utils/train_utils.py
+++ /dev/null
@@ -1,269 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utility functions for training."""
-import collections
-import six
-import tensorflow as tf
-
-from deeplab.core import preprocess_utils
-from deeplab.utils import train_utils
-from feelvos.utils import embedding_utils
-from feelvos.utils import eval_utils
-
-slim = tf.contrib.slim
-add_softmax_cross_entropy_loss_for_each_scale = (
- train_utils.add_softmax_cross_entropy_loss_for_each_scale)
-get_model_gradient_multipliers = train_utils.get_model_gradient_multipliers
-get_model_learning_rate = train_utils.get_model_learning_rate
-resolve_shape = preprocess_utils.resolve_shape
-
-
-def add_triplet_loss_for_each_scale(batch_size, num_frames_per_video,
- embedding_dim, scales_to_embeddings,
- labels, scope):
- """Adds triplet loss for logits of each scale.
-
- Args:
- batch_size: Int, the number of video chunks sampled per batch
- num_frames_per_video: Int, the number of frames per video.
- embedding_dim: Int, the dimension of the learned embedding
- scales_to_embeddings: A map from embedding names for different scales to
- embeddings. The embeddings have shape [batch, embeddings_height,
- embeddings_width, embedding_dim].
- labels: Groundtruth labels with shape [batch, image_height, image_width, 1].
- scope: String, the scope for the loss.
-
- Raises:
- ValueError: labels is None.
- """
- if labels is None:
- raise ValueError('No label for triplet loss.')
- for scale, embeddings in scales_to_embeddings.iteritems():
- loss_scope = None
- if scope:
- loss_scope = '%s_%s' % (scope, scale)
- # Label is downsampled to the same size as logits.
- scaled_labels = tf.image.resize_nearest_neighbor(
- labels,
- resolve_shape(embeddings, 4)[1:3],
- align_corners=True)
- # Reshape from [batch * num_frames, ...] to [batch, num_frames, ...].
- h = tf.shape(embeddings)[1]
- w = tf.shape(embeddings)[2]
- new_labels_shape = tf.stack([batch_size, num_frames_per_video, h, w, 1])
- reshaped_labels = tf.reshape(scaled_labels, new_labels_shape)
- new_embeddings_shape = tf.stack([batch_size, num_frames_per_video, h, w,
- -1])
- reshaped_embeddings = tf.reshape(embeddings, new_embeddings_shape)
-
- with tf.name_scope(loss_scope):
- total_loss = tf.constant(0, dtype=tf.float32)
- for n in range(batch_size):
- embedding = reshaped_embeddings[n]
- label = reshaped_labels[n]
- n_pixels = h * w
- n_anchors_used = 256
- sampled_anchor_indices = tf.random_shuffle(tf.range(n_pixels))[
- :n_anchors_used]
- anchors_pool = tf.reshape(embedding[0], [-1, embedding_dim])
- anchors_pool_classes = tf.reshape(label[0], [-1])
- anchors = tf.gather(anchors_pool, sampled_anchor_indices)
- anchor_classes = tf.gather(anchors_pool_classes, sampled_anchor_indices)
-
- pos_neg_pool = tf.reshape(embedding[1:], [-1, embedding_dim])
- pos_neg_pool_classes = tf.reshape(label[1:], [-1])
- dists = embedding_utils.pairwise_distances(anchors, pos_neg_pool)
- pos_mask = tf.equal(anchor_classes[:, tf.newaxis],
- pos_neg_pool_classes[tf.newaxis, :])
- neg_mask = tf.logical_not(pos_mask)
- pos_mask_f = tf.cast(pos_mask, tf.float32)
- neg_mask_f = tf.cast(neg_mask, tf.float32)
- pos_dists = pos_mask_f * dists + 1e20 * neg_mask_f
- neg_dists = neg_mask_f * dists + 1e20 * pos_mask_f
- pos_dists_min = tf.reduce_min(pos_dists, axis=1)
- neg_dists_min = tf.reduce_min(neg_dists, axis=1)
- margin = 1.0
- loss = tf.nn.relu(pos_dists_min - neg_dists_min + margin)
- # Handle case that no positive is present (per anchor).
- any_pos = tf.reduce_any(pos_mask, axis=1)
- loss *= tf.cast(any_pos, tf.float32)
- # Average over anchors
- loss = tf.reduce_mean(loss, axis=0)
- total_loss += loss
- total_loss /= batch_size
- # Scale the loss up a bit.
- total_loss *= 3.0
- tf.add_to_collection(tf.GraphKeys.LOSSES, total_loss)
-
-
-def add_dynamic_softmax_cross_entropy_loss_for_each_scale(
- scales_to_logits, labels, ignore_label, loss_weight=1.0,
- upsample_logits=True, scope=None, top_k_percent_pixels=1.0,
- hard_example_mining_step=100000):
- """Adds softmax cross entropy loss per scale for logits with varying classes.
-
- Also adds summaries for mIoU.
-
- Args:
- scales_to_logits: A map from logits names for different scales to logits.
- The logits are a list of length batch_size of tensors of shape
- [time, logits_height, logits_width, num_classes].
- labels: Groundtruth labels with shape [batch_size * time, image_height,
- image_width, 1].
- ignore_label: Integer, label to ignore.
- loss_weight: Float, loss weight.
- upsample_logits: Boolean, upsample logits or not.
- scope: String, the scope for the loss.
- top_k_percent_pixels: A float, the value lies in [0.0, 1.0]. When its
- value < 1.0, only compute the loss for the top k percent pixels (e.g.,
- the top 20% pixels). This is useful for hard pixel mining.
- hard_example_mining_step: An integer, the training step in which the
- hard exampling mining kicks off. Note that we gradually reduce the
- mining percent to the top_k_percent_pixels. For example, if
- hard_example_mining_step=100K and top_k_percent_pixels=0.25, then
- mining percent will gradually reduce from 100% to 25% until 100K steps
- after which we only mine top 25% pixels.
-
- Raises:
- ValueError: Label or logits is None.
- """
- if labels is None:
- raise ValueError('No label for softmax cross entropy loss.')
-
- if top_k_percent_pixels < 0 or top_k_percent_pixels > 1:
- raise ValueError('Unexpected value of top_k_percent_pixels.')
-
- for scale, logits in six.iteritems(scales_to_logits):
- loss_scope = None
- if scope:
- loss_scope = '%s_%s' % (scope, scale)
-
- if upsample_logits:
- # Label is not downsampled, and instead we upsample logits.
- assert isinstance(logits, collections.Sequence)
- logits = [tf.image.resize_bilinear(
- x,
- preprocess_utils.resolve_shape(labels, 4)[1:3],
- align_corners=True) for x in logits]
- scaled_labels = labels
- else:
- # Label is downsampled to the same size as logits.
- assert isinstance(logits, collections.Sequence)
- scaled_labels = tf.image.resize_nearest_neighbor(
- labels,
- preprocess_utils.resolve_shape(logits[0], 4)[1:3],
- align_corners=True)
-
- batch_size = len(logits)
- num_time = preprocess_utils.resolve_shape(logits[0])[0]
- reshaped_labels = tf.reshape(
- scaled_labels, ([batch_size, num_time] +
- preprocess_utils.resolve_shape(scaled_labels)[1:]))
- for n, logits_n in enumerate(logits):
- labels_n = reshaped_labels[n]
- labels_n = tf.reshape(labels_n, shape=[-1])
- not_ignore_mask = tf.to_float(tf.not_equal(labels_n,
- ignore_label)) * loss_weight
- num_classes_n = tf.shape(logits_n)[-1]
- one_hot_labels = slim.one_hot_encoding(
- labels_n, num_classes_n, on_value=1.0, off_value=0.0)
- logits_n_flat = tf.reshape(logits_n, shape=[-1, num_classes_n])
- if top_k_percent_pixels == 1.0:
- tf.losses.softmax_cross_entropy(
- one_hot_labels,
- logits_n_flat,
- weights=not_ignore_mask,
- scope=loss_scope)
- else:
- # Only compute the loss for top k percent pixels.
- # First, compute the loss for all pixels. Note we do not put the loss
- # to loss_collection and set reduction = None to keep the shape.
- num_pixels = tf.to_float(tf.shape(logits_n_flat)[0])
- pixel_losses = tf.losses.softmax_cross_entropy(
- one_hot_labels,
- logits_n_flat,
- weights=not_ignore_mask,
- scope='pixel_losses',
- loss_collection=None,
- reduction=tf.losses.Reduction.NONE)
- # Compute the top_k_percent pixels based on current training step.
- if hard_example_mining_step == 0:
- # Directly focus on the top_k pixels.
- top_k_pixels = tf.to_int32(top_k_percent_pixels * num_pixels)
- else:
- # Gradually reduce the mining percent to top_k_percent_pixels.
- global_step = tf.to_float(tf.train.get_or_create_global_step())
- ratio = tf.minimum(1.0, global_step / hard_example_mining_step)
- top_k_pixels = tf.to_int32(
- (ratio * top_k_percent_pixels + (1.0 - ratio)) * num_pixels)
- _, top_k_indices = tf.nn.top_k(pixel_losses,
- k=top_k_pixels,
- sorted=True,
- name='top_k_percent_pixels')
- # Compute the loss for the top k percent pixels.
- tf.losses.softmax_cross_entropy(
- tf.gather(one_hot_labels, top_k_indices),
- tf.gather(logits_n_flat, top_k_indices),
- weights=tf.gather(not_ignore_mask, top_k_indices),
- scope=loss_scope)
-
- pred_n = tf.argmax(logits_n, axis=-1, output_type=tf.int32)[
- ..., tf.newaxis]
- labels_n = labels[n * num_time: (n + 1) * num_time]
- miou = eval_utils.calculate_multi_object_miou_tf(pred_n, labels_n)
- tf.summary.scalar('miou', miou)
-
-
-def get_model_init_fn(train_logdir,
- tf_initial_checkpoint,
- initialize_last_layer,
- last_layers,
- ignore_missing_vars=False):
- """Gets the function initializing model variables from a checkpoint.
-
- Args:
- train_logdir: Log directory for training.
- tf_initial_checkpoint: TensorFlow checkpoint for initialization.
- initialize_last_layer: Initialize last layer or not.
- last_layers: Last layers of the model.
- ignore_missing_vars: Ignore missing variables in the checkpoint.
-
- Returns:
- Initialization function.
- """
- if tf_initial_checkpoint is None:
- tf.logging.info('Not initializing the model from a checkpoint.')
- return None
-
- if tf.train.latest_checkpoint(train_logdir):
- tf.logging.info('Ignoring initialization; other checkpoint exists')
- return None
-
- tf.logging.info('Initializing model from path: %s', tf_initial_checkpoint)
-
- # Variables that will not be restored.
- exclude_list = ['global_step']
- if not initialize_last_layer:
- exclude_list.extend(last_layers)
-
- variables_to_restore = slim.get_variables_to_restore(exclude=exclude_list)
-
- if variables_to_restore:
- return slim.assign_from_checkpoint_fn(
- tf_initial_checkpoint,
- variables_to_restore,
- ignore_missing_vars=ignore_missing_vars)
- return None
diff --git a/research/feelvos/utils/video_input_generator.py b/research/feelvos/utils/video_input_generator.py
deleted file mode 100644
index c0135e50110c677865217c8a3f13d1d1d891f0b2..0000000000000000000000000000000000000000
--- a/research/feelvos/utils/video_input_generator.py
+++ /dev/null
@@ -1,558 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Wrapper for providing semantic segmentation video data."""
-
-import tensorflow as tf
-from feelvos import input_preprocess
-from feelvos import model
-from feelvos.utils import mask_damaging
-from feelvos.utils import train_utils
-
-slim = tf.contrib.slim
-dataset_data_provider = slim.dataset_data_provider
-
-
-MIN_LABEL_COUNT = 10
-
-
-def decode_image_sequence(tensor, image_format='jpeg', shape=None,
- channels=3, raw_dtype=tf.uint8):
- """Decodes a sequence of images.
-
- Args:
- tensor: the tensor of strings to decode, shape: [num_images]
- image_format: a string (possibly tensor) with the format of the image.
- Options include 'jpeg', 'png', and 'raw'.
- shape: a list or tensor of the decoded image shape for a single image.
- channels: if 'shape' is None, the third dimension of the image is set to
- this value.
- raw_dtype: if the image is encoded as raw bytes, this is the method of
- decoding the bytes into values.
- Returns:
- The decoded images with shape [time, height, width, channels].
- """
- handler = slim.tfexample_decoder.Image(
- shape=shape, channels=channels, dtype=raw_dtype, repeated=True)
- return handler.tensors_to_item({'image/encoded': tensor,
- 'image/format': image_format})
-
-
-def _get_data(data_provider, dataset_split, video_frames_are_decoded):
- """Gets data from data provider.
-
- Args:
- data_provider: An object of slim.data_provider.
- dataset_split: Dataset split.
- video_frames_are_decoded: Boolean, whether the video frames are already
- decoded
-
- Returns:
- image: Image Tensor.
- label: Label Tensor storing segmentation annotations.
- object_label: An integer refers to object_label according to labelmap. If
- the example has more than one object_label, take the first one.
- image_name: Image name.
- height: Image height.
- width: Image width.
- video_id: String tensor representing the name of the video.
-
- Raises:
- ValueError: Failed to find label.
- """
-
- if video_frames_are_decoded:
- image, = data_provider.get(['image'])
- else:
- image, = data_provider.get(['image/encoded'])
-
- # Some datasets do not contain image_name.
- if 'image_name' in data_provider.list_items():
- image_name, = data_provider.get(['image_name'])
- else:
- image_name = tf.constant('')
-
- height, width = data_provider.get(['height', 'width'])
-
- label = None
- if dataset_split != 'test':
- if video_frames_are_decoded:
- if 'labels_class' not in data_provider.list_items():
- raise ValueError('Failed to find labels.')
- label, = data_provider.get(['labels_class'])
- else:
- key = 'segmentation/object/encoded'
- if key not in data_provider.list_items():
- raise ValueError('Failed to find labels.')
- label, = data_provider.get([key])
-
- object_label = None
- video_id, = data_provider.get(['video_id'])
-
- return image, label, object_label, image_name, height, width, video_id
-
-
-def _has_foreground_and_background_in_first_frame(label, subsampling_factor):
- """Checks if the labels have foreground and background in the first frame.
-
- Args:
- label: Label tensor of shape [num_frames, height, width, 1].
- subsampling_factor: Integer, the subsampling factor.
-
- Returns:
- Boolean, whether the labels have foreground and background in the first
- frame.
- """
- h, w = train_utils.resolve_shape(label)[1:3]
- label_downscaled = tf.squeeze(
- tf.image.resize_nearest_neighbor(label[0, tf.newaxis],
- [h // subsampling_factor,
- w // subsampling_factor],
- align_corners=True),
- axis=0)
- is_bg = tf.equal(label_downscaled, 0)
- is_fg = tf.logical_not(is_bg)
- # Just using reduce_any was not robust enough, so lets make sure the count
- # is above MIN_LABEL_COUNT.
- fg_count = tf.reduce_sum(tf.cast(is_fg, tf.int32))
- bg_count = tf.reduce_sum(tf.cast(is_bg, tf.int32))
- has_bg = tf.greater_equal(fg_count, MIN_LABEL_COUNT)
- has_fg = tf.greater_equal(bg_count, MIN_LABEL_COUNT)
- return tf.logical_and(has_bg, has_fg)
-
-
-def _has_foreground_and_background_in_first_frame_2(label,
- decoder_output_stride):
- """Checks if the labels have foreground and background in the first frame.
-
- Second attempt, this time we use the actual output dimension for resizing.
-
- Args:
- label: Label tensor of shape [num_frames, height, width, 1].
- decoder_output_stride: Integer, the stride of the decoder output.
-
- Returns:
- Boolean, whether the labels have foreground and background in the first
- frame.
- """
- h, w = train_utils.resolve_shape(label)[1:3]
- h_sub = model.scale_dimension(h, 1.0 / decoder_output_stride)
- w_sub = model.scale_dimension(w, 1.0 / decoder_output_stride)
- label_downscaled = tf.squeeze(
- tf.image.resize_nearest_neighbor(label[0, tf.newaxis], [h_sub, w_sub],
- align_corners=True), axis=0)
- is_bg = tf.equal(label_downscaled, 0)
- is_fg = tf.logical_not(is_bg)
- # Just using reduce_any was not robust enough, so lets make sure the count
- # is above MIN_LABEL_COUNT.
- fg_count = tf.reduce_sum(tf.cast(is_fg, tf.int32))
- bg_count = tf.reduce_sum(tf.cast(is_bg, tf.int32))
- has_bg = tf.greater_equal(fg_count, MIN_LABEL_COUNT)
- has_fg = tf.greater_equal(bg_count, MIN_LABEL_COUNT)
- return tf.logical_and(has_bg, has_fg)
-
-
-def _has_enough_pixels_of_each_object_in_first_frame(
- label, decoder_output_stride):
- """Checks if for each object (incl. background) enough pixels are visible.
-
- During test time, we will usually not see a reference frame in which only
- very few pixels of one object are visible. These cases can be problematic
- during training, especially if more than the 1-nearest neighbor is used.
- That's why this function can be used to detect and filter these cases.
-
- Args:
- label: Label tensor of shape [num_frames, height, width, 1].
- decoder_output_stride: Integer, the stride of the decoder output.
-
- Returns:
- Boolean, whether the labels have enough pixels of each object in the first
- frame.
- """
- h, w = train_utils.resolve_shape(label)[1:3]
- h_sub = model.scale_dimension(h, 1.0 / decoder_output_stride)
- w_sub = model.scale_dimension(w, 1.0 / decoder_output_stride)
- label_downscaled = tf.squeeze(
- tf.image.resize_nearest_neighbor(label[0, tf.newaxis], [h_sub, w_sub],
- align_corners=True), axis=0)
- _, _, counts = tf.unique_with_counts(
- tf.reshape(label_downscaled, [-1]))
- has_enough_pixels_per_object = tf.reduce_all(
- tf.greater_equal(counts, MIN_LABEL_COUNT))
- return has_enough_pixels_per_object
-
-
-def get(dataset,
- num_frames_per_video,
- crop_size,
- batch_size,
- min_resize_value=None,
- max_resize_value=None,
- resize_factor=None,
- min_scale_factor=1.,
- max_scale_factor=1.,
- scale_factor_step_size=0,
- preprocess_image_and_label=True,
- num_readers=1,
- num_threads=1,
- dataset_split=None,
- is_training=True,
- model_variant=None,
- batch_capacity_factor=32,
- video_frames_are_decoded=False,
- decoder_output_stride=None,
- first_frame_finetuning=False,
- sample_only_first_frame_for_finetuning=False,
- sample_adjacent_and_consistent_query_frames=False,
- remap_labels_to_reference_frame=True,
- generate_prev_frame_mask_by_mask_damaging=False,
- three_frame_dataset=False,
- add_prev_frame_label=True):
- """Gets the dataset split for semantic segmentation.
-
- This functions gets the dataset split for semantic segmentation. In
- particular, it is a wrapper of (1) dataset_data_provider which returns the raw
- dataset split, (2) input_preprcess which preprocess the raw data, and (3) the
- Tensorflow operation of batching the preprocessed data. Then, the output could
- be directly used by training, evaluation or visualization.
-
- Args:
- dataset: An instance of slim Dataset.
- num_frames_per_video: The number of frames used per video
- crop_size: Image crop size [height, width].
- batch_size: Batch size.
- min_resize_value: Desired size of the smaller image side.
- max_resize_value: Maximum allowed size of the larger image side.
- resize_factor: Resized dimensions are multiple of factor plus one.
- min_scale_factor: Minimum scale factor value.
- max_scale_factor: Maximum scale factor value.
- scale_factor_step_size: The step size from min scale factor to max scale
- factor. The input is randomly scaled based on the value of
- (min_scale_factor, max_scale_factor, scale_factor_step_size).
- preprocess_image_and_label: Boolean variable specifies if preprocessing of
- image and label will be performed or not.
- num_readers: Number of readers for data provider.
- num_threads: Number of threads for batching data.
- dataset_split: Dataset split.
- is_training: Is training or not.
- model_variant: Model variant (string) for choosing how to mean-subtract the
- images. See feature_extractor.network_map for supported model variants.
- batch_capacity_factor: Batch capacity factor affecting the training queue
- batch capacity.
- video_frames_are_decoded: Boolean, whether the video frames are already
- decoded
- decoder_output_stride: Integer, the stride of the decoder output.
- first_frame_finetuning: Boolean, whether to only sample the first frame
- for fine-tuning.
- sample_only_first_frame_for_finetuning: Boolean, whether to only sample the
- first frame during fine-tuning. This should be False when using lucid or
- wonderland data, but true when fine-tuning on the first frame only.
- Only has an effect if first_frame_finetuning is True.
- sample_adjacent_and_consistent_query_frames: Boolean, if true, the query
- frames (all but the first frame which is the reference frame) will be
- sampled such that they are adjacent video frames and have the same
- crop coordinates and flip augmentation.
- remap_labels_to_reference_frame: Boolean, whether to remap the labels of
- the query frames to match the labels of the (downscaled) reference frame.
- If a query frame contains a label which is not present in the reference,
- it will be mapped to background.
- generate_prev_frame_mask_by_mask_damaging: Boolean, whether to generate
- the masks used as guidance from the previous frame by damaging the
- ground truth mask.
- three_frame_dataset: Boolean, whether the dataset has exactly three frames
- per video of which the first is to be used as reference and the two
- others are consecutive frames to be used as query frames.
- add_prev_frame_label: Boolean, whether to sample one more frame before the
- first query frame to obtain a previous frame label. Only has an effect,
- if sample_adjacent_and_consistent_query_frames is True and
- generate_prev_frame_mask_by_mask_damaging is False.
-
- Returns:
- A dictionary of batched Tensors for semantic segmentation.
-
- Raises:
- ValueError: dataset_split is None, or Failed to find labels.
- """
- if dataset_split is None:
- raise ValueError('Unknown dataset split.')
- if model_variant is None:
- tf.logging.warning('Please specify a model_variant. See '
- 'feature_extractor.network_map for supported model '
- 'variants.')
-
- data_provider = dataset_data_provider.DatasetDataProvider(
- dataset,
- num_readers=num_readers,
- num_epochs=None if is_training else 1,
- shuffle=is_training)
- image, label, object_label, image_name, height, width, video_id = _get_data(
- data_provider, dataset_split, video_frames_are_decoded)
-
- sampling_is_valid = tf.constant(True)
- if num_frames_per_video is not None:
- total_num_frames = tf.shape(image)[0]
- if first_frame_finetuning or three_frame_dataset:
- if sample_only_first_frame_for_finetuning:
- assert not sample_adjacent_and_consistent_query_frames, (
- 'this option does not make sense for sampling only first frame.')
- # Sample the first frame num_frames_per_video times.
- sel_indices = tf.tile(tf.constant(0, dtype=tf.int32)[tf.newaxis],
- multiples=[num_frames_per_video])
- else:
- if sample_adjacent_and_consistent_query_frames:
- if add_prev_frame_label:
- num_frames_per_video += 1
- # Since this is first frame fine-tuning, we'll for now assume that
- # each sequence has exactly 3 images: the ref frame and 2 adjacent
- # query frames.
- assert num_frames_per_video == 3
- with tf.control_dependencies([tf.assert_equal(total_num_frames, 3)]):
- sel_indices = tf.constant([1, 2], dtype=tf.int32)
- else:
- # Sample num_frames_per_video - 1 query frames which are not the
- # first frame.
- sel_indices = tf.random_shuffle(
- tf.range(1, total_num_frames))[:(num_frames_per_video - 1)]
- # Concat first frame as reference frame to the front.
- sel_indices = tf.concat([tf.constant(0, dtype=tf.int32)[tf.newaxis],
- sel_indices], axis=0)
- else:
- if sample_adjacent_and_consistent_query_frames:
- if add_prev_frame_label:
- # Sample one more frame which we can use to provide initial softmax
- # feedback.
- num_frames_per_video += 1
- ref_idx = tf.random_shuffle(tf.range(total_num_frames))[0]
- sampling_is_valid = tf.greater_equal(total_num_frames,
- num_frames_per_video)
- def sample_query_start_idx():
- return tf.random_shuffle(
- tf.range(total_num_frames - num_frames_per_video + 1))[0]
- query_start_idx = tf.cond(sampling_is_valid, sample_query_start_idx,
- lambda: tf.constant(0, dtype=tf.int32))
- def sample_sel_indices():
- return tf.concat(
- [ref_idx[tf.newaxis],
- tf.range(
- query_start_idx,
- query_start_idx + (num_frames_per_video - 1))], axis=0)
- sel_indices = tf.cond(
- sampling_is_valid, sample_sel_indices,
- lambda: tf.zeros((num_frames_per_video,), dtype=tf.int32))
- else:
- # Randomly sample some frames from the video.
- sel_indices = tf.random_shuffle(
- tf.range(total_num_frames))[:num_frames_per_video]
- image = tf.gather(image, sel_indices, axis=0)
- if not video_frames_are_decoded:
- image = decode_image_sequence(image)
-
- if label is not None:
- if num_frames_per_video is not None:
- label = tf.gather(label, sel_indices, axis=0)
- if not video_frames_are_decoded:
- label = decode_image_sequence(label, image_format='png', channels=1)
-
- # Sometimes, label is saved as [num_frames_per_video, height, width] or
- # [num_frames_per_video, height, width, 1]. We change it to be
- # [num_frames_per_video, height, width, 1].
- if label.shape.ndims == 3:
- label = tf.expand_dims(label, 3)
- elif label.shape.ndims == 4 and label.shape.dims[3] == 1:
- pass
- else:
- raise ValueError('Input label shape must be '
- '[num_frames_per_video, height, width],'
- ' or [num_frames, height, width, 1]. '
- 'Got {}'.format(label.shape.ndims))
- label.set_shape([None, None, None, 1])
-
- # Add size of first dimension since tf can't figure it out automatically.
- image.set_shape((num_frames_per_video, None, None, None))
- if label is not None:
- label.set_shape((num_frames_per_video, None, None, None))
-
- preceding_frame_label = None
- if preprocess_image_and_label:
- if num_frames_per_video is None:
- raise ValueError('num_frame_per_video must be specified for preproc.')
- original_images = []
- images = []
- labels = []
- if sample_adjacent_and_consistent_query_frames:
- num_frames_individual_preproc = 1
- else:
- num_frames_individual_preproc = num_frames_per_video
- for frame_idx in range(num_frames_individual_preproc):
- original_image_t, image_t, label_t = (
- input_preprocess.preprocess_image_and_label(
- image[frame_idx],
- label[frame_idx],
- crop_height=crop_size[0] if crop_size is not None else None,
- crop_width=crop_size[1] if crop_size is not None else None,
- min_resize_value=min_resize_value,
- max_resize_value=max_resize_value,
- resize_factor=resize_factor,
- min_scale_factor=min_scale_factor,
- max_scale_factor=max_scale_factor,
- scale_factor_step_size=scale_factor_step_size,
- ignore_label=dataset.ignore_label,
- is_training=is_training,
- model_variant=model_variant))
- original_images.append(original_image_t)
- images.append(image_t)
- labels.append(label_t)
- if sample_adjacent_and_consistent_query_frames:
- imgs_for_preproc = [image[frame_idx] for frame_idx in
- range(1, num_frames_per_video)]
- labels_for_preproc = [label[frame_idx] for frame_idx in
- range(1, num_frames_per_video)]
- original_image_rest, image_rest, label_rest = (
- input_preprocess.preprocess_images_and_labels_consistently(
- imgs_for_preproc,
- labels_for_preproc,
- crop_height=crop_size[0] if crop_size is not None else None,
- crop_width=crop_size[1] if crop_size is not None else None,
- min_resize_value=min_resize_value,
- max_resize_value=max_resize_value,
- resize_factor=resize_factor,
- min_scale_factor=min_scale_factor,
- max_scale_factor=max_scale_factor,
- scale_factor_step_size=scale_factor_step_size,
- ignore_label=dataset.ignore_label,
- is_training=is_training,
- model_variant=model_variant))
- original_images.extend(original_image_rest)
- images.extend(image_rest)
- labels.extend(label_rest)
- assert len(original_images) == num_frames_per_video
- assert len(images) == num_frames_per_video
- assert len(labels) == num_frames_per_video
-
- if remap_labels_to_reference_frame:
- # Remap labels to indices into the labels of the (downscaled) reference
- # frame, or 0, i.e. background, for labels which are not present
- # in the reference.
- reference_labels = labels[0][tf.newaxis]
- h, w = train_utils.resolve_shape(reference_labels)[1:3]
- embedding_height = model.scale_dimension(
- h, 1.0 / decoder_output_stride)
- embedding_width = model.scale_dimension(
- w, 1.0 / decoder_output_stride)
- reference_labels_embedding_size = tf.squeeze(
- tf.image.resize_nearest_neighbor(
- reference_labels, tf.stack([embedding_height, embedding_width]),
- align_corners=True),
- axis=0)
- # Get sorted unique labels in the reference frame.
- labels_in_ref_frame, _ = tf.unique(
- tf.reshape(reference_labels_embedding_size, [-1]))
- labels_in_ref_frame = tf.contrib.framework.sort(labels_in_ref_frame)
- for idx in range(1, len(labels)):
- ref_label_mask = tf.equal(
- labels[idx],
- labels_in_ref_frame[tf.newaxis, tf.newaxis, :])
- remapped = tf.argmax(tf.cast(ref_label_mask, tf.uint8), axis=-1,
- output_type=tf.int32)
- # Set to 0 if label is not present
- is_in_ref = tf.reduce_any(ref_label_mask, axis=-1)
- remapped *= tf.cast(is_in_ref, tf.int32)
- labels[idx] = remapped[..., tf.newaxis]
-
- if sample_adjacent_and_consistent_query_frames:
- if first_frame_finetuning and generate_prev_frame_mask_by_mask_damaging:
- preceding_frame_label = mask_damaging.damage_masks(labels[1])
- elif add_prev_frame_label:
- # Discard the image of the additional frame and take the label as
- # initialization for softmax feedback.
- original_images = [original_images[0]] + original_images[2:]
- preceding_frame_label = labels[1]
- images = [images[0]] + images[2:]
- labels = [labels[0]] + labels[2:]
- num_frames_per_video -= 1
-
- original_image = tf.stack(original_images, axis=0)
- image = tf.stack(images, axis=0)
- label = tf.stack(labels, axis=0)
- else:
- if label is not None:
- # Need to set label shape due to batching.
- label.set_shape([num_frames_per_video,
- None if crop_size is None else crop_size[0],
- None if crop_size is None else crop_size[1],
- 1])
- original_image = tf.to_float(tf.zeros_like(label))
- if crop_size is None:
- height = tf.shape(image)[1]
- width = tf.shape(image)[2]
- else:
- height = crop_size[0]
- width = crop_size[1]
-
- sample = {'image': image,
- 'image_name': image_name,
- 'height': height,
- 'width': width,
- 'video_id': video_id}
- if label is not None:
- sample['label'] = label
-
- if object_label is not None:
- sample['object_label'] = object_label
-
- if preceding_frame_label is not None:
- sample['preceding_frame_label'] = preceding_frame_label
-
- if not is_training:
- # Original image is only used during visualization.
- sample['original_image'] = original_image
-
- if is_training:
- if first_frame_finetuning:
- keep_input = tf.constant(True)
- else:
- keep_input = tf.logical_and(sampling_is_valid, tf.logical_and(
- _has_enough_pixels_of_each_object_in_first_frame(
- label, decoder_output_stride),
- _has_foreground_and_background_in_first_frame_2(
- label, decoder_output_stride)))
-
- batched = tf.train.maybe_batch(sample,
- keep_input=keep_input,
- batch_size=batch_size,
- num_threads=num_threads,
- capacity=batch_capacity_factor * batch_size,
- dynamic_pad=True)
- else:
- batched = tf.train.batch(sample,
- batch_size=batch_size,
- num_threads=num_threads,
- capacity=batch_capacity_factor * batch_size,
- dynamic_pad=True)
-
- # Flatten from [batch, num_frames_per_video, ...] to
- # batch * num_frames_per_video, ...].
- cropped_height = train_utils.resolve_shape(batched['image'])[2]
- cropped_width = train_utils.resolve_shape(batched['image'])[3]
- if num_frames_per_video is None:
- first_dim = -1
- else:
- first_dim = batch_size * num_frames_per_video
- batched['image'] = tf.reshape(batched['image'],
- [first_dim, cropped_height, cropped_width, 3])
- if label is not None:
- batched['label'] = tf.reshape(batched['label'],
- [first_dim, cropped_height, cropped_width, 1])
- return batched
diff --git a/research/feelvos/vis_video.py b/research/feelvos/vis_video.py
deleted file mode 100644
index 211bccf52acdef83aca298285fc473748126de02..0000000000000000000000000000000000000000
--- a/research/feelvos/vis_video.py
+++ /dev/null
@@ -1,500 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Segmentation results evaluation and visualization for videos using attention.
-"""
-
-import math
-import os
-import time
-import numpy as np
-
-import tensorflow as tf
-
-from feelvos import common
-from feelvos import model
-from feelvos.datasets import video_dataset
-from feelvos.utils import embedding_utils
-from feelvos.utils import eval_utils
-from feelvos.utils import video_input_generator
-
-
-slim = tf.contrib.slim
-flags = tf.app.flags
-FLAGS = flags.FLAGS
-
-flags.DEFINE_integer('eval_interval_secs', 60 * 5,
- 'How often (in seconds) to run evaluation.')
-
-flags.DEFINE_string('master', '', 'BNS name of the tensorflow server')
-
-flags.DEFINE_integer('vis_batch_size', 1,
- 'The number of images in each batch during evaluation.')
-
-flags.DEFINE_string('vis_logdir', None, 'Where to write the event logs.')
-
-flags.DEFINE_string('checkpoint_dir', None, 'Directory of model checkpoints.')
-
-flags.DEFINE_integer('output_stride', 8,
- 'The ratio of input to output spatial resolution.')
-
-flags.DEFINE_string('dataset', 'davis_2016',
- 'Name of the segmentation dataset.')
-
-flags.DEFINE_string('vis_split', 'val',
- 'Which split of the dataset used for visualizing results')
-
-flags.DEFINE_string(
- 'dataset_dir',
- '/cns/is-d/home/lcchen/data/pascal_voc_seg/example_sstables',
- 'Where the dataset resides.')
-
-flags.DEFINE_integer('num_vis_examples', -1,
- 'Number of examples for visualization. If -1, use all '
- 'samples in the vis data.')
-
-flags.DEFINE_multi_integer('atrous_rates', None,
- 'Atrous rates for atrous spatial pyramid pooling.')
-
-flags.DEFINE_bool('save_segmentations', False, 'Whether to save the '
- 'segmentation masks as '
- 'png images. Might be slow '
- 'on cns.')
-
-flags.DEFINE_bool('save_embeddings', False, 'Whether to save the embeddings as'
- 'pickle. Might be slow on cns.')
-
-flags.DEFINE_bool('eval_once_and_quit', False,
- 'Whether to just run the eval a single time and quit '
- 'afterwards. Otherwise, the eval is run in a loop with '
- 'new checkpoints.')
-
-flags.DEFINE_boolean('first_frame_finetuning', False,
- 'Whether to only sample the first frame for fine-tuning.')
-
-# the folder where segmentations are saved.
-_SEGMENTATION_SAVE_FOLDER = 'segmentation'
-_EMBEDDINGS_SAVE_FOLDER = 'embeddings'
-
-
-def _process_seq_data(segmentation_dir, embeddings_dir, seq_name,
- predicted_labels, gt_labels, embeddings):
- """Calculates the sequence IoU and optionally save the segmentation masks.
-
- Args:
- segmentation_dir: Directory in which the segmentation results are stored.
- embeddings_dir: Directory in which the embeddings are stored.
- seq_name: String, the name of the sequence.
- predicted_labels: Int64 np.array of shape [n_frames, height, width].
- gt_labels: Ground truth labels, Int64 np.array of shape
- [n_frames, height, width].
- embeddings: Float32 np.array of embeddings of shape
- [n_frames, decoder_height, decoder_width, embedding_dim], or None.
-
- Returns:
- The IoU for the sequence (float).
- """
- sequence_dir = os.path.join(segmentation_dir, seq_name)
- tf.gfile.MakeDirs(sequence_dir)
- embeddings_seq_dir = os.path.join(embeddings_dir, seq_name)
- tf.gfile.MakeDirs(embeddings_seq_dir)
- label_set = np.unique(gt_labels[0])
- ious = []
- assert len(predicted_labels) == len(gt_labels)
- if embeddings is not None:
- assert len(predicted_labels) == len(embeddings)
- for t, (predicted_label, gt_label) in enumerate(
- zip(predicted_labels, gt_labels)):
- if FLAGS.save_segmentations:
- seg_filename = os.path.join(segmentation_dir, seq_name, '%05d.png' % t)
- eval_utils.save_segmentation_with_colormap(seg_filename, predicted_label)
- if FLAGS.save_embeddings:
- embedding_filename = os.path.join(embeddings_dir, seq_name,
- '%05d.npy' % t)
- assert embeddings is not None
- eval_utils.save_embeddings(embedding_filename, embeddings[t])
- object_ious_t = eval_utils.calculate_multi_object_ious(
- predicted_label, gt_label, label_set)
- ious.append(object_ious_t)
- # First and last frame are excluded in DAVIS eval.
- seq_ious = np.mean(ious[1:-1], axis=0)
- tf.logging.info('seq ious: %s %s', seq_name, seq_ious)
- return seq_ious
-
-
-def create_predictions(samples, reference_labels, first_frame_img,
- model_options):
- """Predicts segmentation labels for each frame of the video.
-
- Slower version than create_predictions_fast, but does support more options.
-
- Args:
- samples: Dictionary of input samples.
- reference_labels: Int tensor of shape [1, height, width, 1].
- first_frame_img: Float32 tensor of shape [height, width, 3].
- model_options: An InternalModelOptions instance to configure models.
-
- Returns:
- predicted_labels: Int tensor of shape [time, height, width] of
- predicted labels for each frame.
- all_embeddings: Float32 tensor of shape
- [time, height, width, embedding_dim], or None.
- """
-
- def predict(args, imgs):
- """Predicts segmentation labels and softmax probabilities for each image.
-
- Args:
- args: A tuple of (predictions, softmax_probabilities), where predictions
- is an int tensor of shape [1, h, w] and softmax_probabilities is a
- float32 tensor of shape [1, h_decoder, w_decoder, n_objects].
- imgs: Either a one-tuple of the image to predict labels for of shape
- [h, w, 3], or pair of previous frame and current frame image.
-
- Returns:
- predictions: The predicted labels as int tensor of shape [1, h, w].
- softmax_probabilities: The softmax probabilities of shape
- [1, h_decoder, w_decoder, n_objects].
- """
- if FLAGS.save_embeddings:
- last_frame_predictions, last_softmax_probabilities, _ = args
- else:
- last_frame_predictions, last_softmax_probabilities = args
-
- if FLAGS.also_attend_to_previous_frame or FLAGS.use_softmax_feedback:
- ref_labels_to_use = tf.concat(
- [reference_labels, last_frame_predictions[..., tf.newaxis]],
- axis=0)
- else:
- ref_labels_to_use = reference_labels
-
- predictions, softmax_probabilities = model.predict_labels(
- tf.stack((first_frame_img,) + imgs),
- model_options=model_options,
- image_pyramid=FLAGS.image_pyramid,
- embedding_dimension=FLAGS.embedding_dimension,
- reference_labels=ref_labels_to_use,
- k_nearest_neighbors=FLAGS.k_nearest_neighbors,
- use_softmax_feedback=FLAGS.use_softmax_feedback,
- initial_softmax_feedback=last_softmax_probabilities,
- embedding_seg_feature_dimension=
- FLAGS.embedding_seg_feature_dimension,
- embedding_seg_n_layers=FLAGS.embedding_seg_n_layers,
- embedding_seg_kernel_size=FLAGS.embedding_seg_kernel_size,
- embedding_seg_atrous_rates=FLAGS.embedding_seg_atrous_rates,
- also_return_softmax_probabilities=True,
- num_frames_per_video=
- (3 if FLAGS.also_attend_to_previous_frame or
- FLAGS.use_softmax_feedback else 2),
- normalize_nearest_neighbor_distances=
- FLAGS.normalize_nearest_neighbor_distances,
- also_attend_to_previous_frame=FLAGS.also_attend_to_previous_frame,
- use_local_previous_frame_attention=
- FLAGS.use_local_previous_frame_attention,
- previous_frame_attention_window_size=
- FLAGS.previous_frame_attention_window_size,
- use_first_frame_matching=FLAGS.use_first_frame_matching
- )
- predictions = tf.cast(predictions[common.OUTPUT_TYPE], tf.int32)
-
- if FLAGS.save_embeddings:
- names = [n.name for n in tf.get_default_graph().as_graph_def().node]
- embedding_names = [x for x in names if 'embeddings' in x]
- # This will crash when multi-scale inference is used.
- assert len(embedding_names) == 1, len(embedding_names)
- embedding_name = embedding_names[0] + ':0'
- embeddings = tf.get_default_graph().get_tensor_by_name(embedding_name)
- return predictions, softmax_probabilities, embeddings
- else:
- return predictions, softmax_probabilities
-
- init_labels = tf.squeeze(reference_labels, axis=-1)
- init_softmax = embedding_utils.create_initial_softmax_from_labels(
- reference_labels, reference_labels, common.parse_decoder_output_stride(),
- reduce_labels=False)
- if FLAGS.save_embeddings:
- decoder_height = tf.shape(init_softmax)[1]
- decoder_width = tf.shape(init_softmax)[2]
- n_frames = (3 if FLAGS.also_attend_to_previous_frame
- or FLAGS.use_softmax_feedback else 2)
- embeddings_init = tf.zeros((n_frames, decoder_height, decoder_width,
- FLAGS.embedding_dimension))
- init = (init_labels, init_softmax, embeddings_init)
- else:
- init = (init_labels, init_softmax)
- # Do not eval the first frame again but concat the first frame ground
- # truth instead.
- if FLAGS.also_attend_to_previous_frame or FLAGS.use_softmax_feedback:
- elems = (samples[common.IMAGE][:-1], samples[common.IMAGE][1:])
- else:
- elems = (samples[common.IMAGE][1:],)
- res = tf.scan(predict, elems,
- initializer=init,
- parallel_iterations=1,
- swap_memory=True)
- if FLAGS.save_embeddings:
- predicted_labels, _, all_embeddings = res
- first_frame_embeddings = all_embeddings[0, 0, tf.newaxis]
- other_frame_embeddings = all_embeddings[:, -1]
- all_embeddings = tf.concat(
- [first_frame_embeddings, other_frame_embeddings], axis=0)
- else:
- predicted_labels, _ = res
- all_embeddings = None
- predicted_labels = tf.concat([reference_labels[..., 0],
- tf.squeeze(predicted_labels, axis=1)],
- axis=0)
- return predicted_labels, all_embeddings
-
-
-def create_predictions_fast(samples, reference_labels, first_frame_img,
- model_options):
- """Predicts segmentation labels for each frame of the video.
-
- Faster version than create_predictions, but does not support all options.
-
- Args:
- samples: Dictionary of input samples.
- reference_labels: Int tensor of shape [1, height, width, 1].
- first_frame_img: Float32 tensor of shape [height, width, 3].
- model_options: An InternalModelOptions instance to configure models.
-
- Returns:
- predicted_labels: Int tensor of shape [time, height, width] of
- predicted labels for each frame.
- all_embeddings: Float32 tensor of shape
- [time, height, width, embedding_dim], or None.
-
- Raises:
- ValueError: If FLAGS.save_embeddings is True, FLAGS.use_softmax_feedback is
- False, or FLAGS.also_attend_to_previous_frame is False.
- """
- if FLAGS.save_embeddings:
- raise ValueError('save_embeddings does not work with '
- 'create_predictions_fast. Use the slower '
- 'create_predictions instead.')
- if not FLAGS.use_softmax_feedback:
- raise ValueError('use_softmax_feedback must be True for '
- 'create_predictions_fast. Use the slower '
- 'create_predictions instead.')
- if not FLAGS.also_attend_to_previous_frame:
- raise ValueError('also_attend_to_previous_frame must be True for '
- 'create_predictions_fast. Use the slower '
- 'create_predictions instead.')
- # Extract embeddings for first frame and prepare initial predictions.
- first_frame_embeddings = embedding_utils.get_embeddings(
- first_frame_img[tf.newaxis], model_options, FLAGS.embedding_dimension)
- init_labels = tf.squeeze(reference_labels, axis=-1)
- init_softmax = embedding_utils.create_initial_softmax_from_labels(
- reference_labels, reference_labels, common.parse_decoder_output_stride(),
- reduce_labels=False)
- init = (init_labels, init_softmax, first_frame_embeddings)
-
- def predict(args, img):
- """Predicts segmentation labels and softmax probabilities for each image.
-
- Args:
- args: tuple of
- (predictions, softmax_probabilities, last_frame_embeddings), where
- predictions is an int tensor of shape [1, h, w],
- softmax_probabilities is a float32 tensor of shape
- [1, h_decoder, w_decoder, n_objects],
- and last_frame_embeddings is a float32 tensor of shape
- [h_decoder, w_decoder, embedding_dimension].
- img: Image to predict labels for of shape [h, w, 3].
-
- Returns:
- predictions: The predicted labels as int tensor of shape [1, h, w].
- softmax_probabilities: The softmax probabilities of shape
- [1, h_decoder, w_decoder, n_objects].
- """
- (last_frame_predictions, last_softmax_probabilities,
- prev_frame_embeddings) = args
- ref_labels_to_use = tf.concat(
- [reference_labels, last_frame_predictions[..., tf.newaxis]],
- axis=0)
-
- predictions, softmax_probabilities, embeddings = model.predict_labels(
- img[tf.newaxis],
- model_options=model_options,
- image_pyramid=FLAGS.image_pyramid,
- embedding_dimension=FLAGS.embedding_dimension,
- reference_labels=ref_labels_to_use,
- k_nearest_neighbors=FLAGS.k_nearest_neighbors,
- use_softmax_feedback=FLAGS.use_softmax_feedback,
- initial_softmax_feedback=last_softmax_probabilities,
- embedding_seg_feature_dimension=
- FLAGS.embedding_seg_feature_dimension,
- embedding_seg_n_layers=FLAGS.embedding_seg_n_layers,
- embedding_seg_kernel_size=FLAGS.embedding_seg_kernel_size,
- embedding_seg_atrous_rates=FLAGS.embedding_seg_atrous_rates,
- also_return_softmax_probabilities=True,
- num_frames_per_video=1,
- normalize_nearest_neighbor_distances=
- FLAGS.normalize_nearest_neighbor_distances,
- also_attend_to_previous_frame=FLAGS.also_attend_to_previous_frame,
- use_local_previous_frame_attention=
- FLAGS.use_local_previous_frame_attention,
- previous_frame_attention_window_size=
- FLAGS.previous_frame_attention_window_size,
- use_first_frame_matching=FLAGS.use_first_frame_matching,
- also_return_embeddings=True,
- ref_embeddings=(first_frame_embeddings, prev_frame_embeddings)
- )
- predictions = tf.cast(predictions[common.OUTPUT_TYPE], tf.int32)
- return predictions, softmax_probabilities, embeddings
-
- # Do not eval the first frame again but concat the first frame ground
- # truth instead.
- # If you have a lot of GPU memory, you can try to set swap_memory=False,
- # and/or parallel_iterations=2.
- elems = samples[common.IMAGE][1:]
- res = tf.scan(predict, elems,
- initializer=init,
- parallel_iterations=1,
- swap_memory=True)
- predicted_labels, _, _ = res
- predicted_labels = tf.concat([reference_labels[..., 0],
- tf.squeeze(predicted_labels, axis=1)],
- axis=0)
- return predicted_labels
-
-
-def main(unused_argv):
- if FLAGS.vis_batch_size != 1:
- raise ValueError('Only batch size 1 is supported for now')
-
- data_type = 'tf_sequence_example'
- # Get dataset-dependent information.
- dataset = video_dataset.get_dataset(
- FLAGS.dataset,
- FLAGS.vis_split,
- dataset_dir=FLAGS.dataset_dir,
- data_type=data_type)
-
- # Prepare for visualization.
- tf.gfile.MakeDirs(FLAGS.vis_logdir)
- segmentation_dir = os.path.join(FLAGS.vis_logdir, _SEGMENTATION_SAVE_FOLDER)
- tf.gfile.MakeDirs(segmentation_dir)
- embeddings_dir = os.path.join(FLAGS.vis_logdir, _EMBEDDINGS_SAVE_FOLDER)
- tf.gfile.MakeDirs(embeddings_dir)
- num_vis_examples = (dataset.num_videos if (FLAGS.num_vis_examples < 0)
- else FLAGS.num_vis_examples)
- if FLAGS.first_frame_finetuning:
- num_vis_examples = 1
-
- tf.logging.info('Visualizing on %s set', FLAGS.vis_split)
- g = tf.Graph()
- with g.as_default():
- # Without setting device to CPU we run out of memory.
- with tf.device('cpu:0'):
- samples = video_input_generator.get(
- dataset,
- None,
- None,
- FLAGS.vis_batch_size,
- min_resize_value=FLAGS.min_resize_value,
- max_resize_value=FLAGS.max_resize_value,
- resize_factor=FLAGS.resize_factor,
- dataset_split=FLAGS.vis_split,
- is_training=False,
- model_variant=FLAGS.model_variant,
- preprocess_image_and_label=False,
- remap_labels_to_reference_frame=False)
- samples[common.IMAGE] = tf.cast(samples[common.IMAGE], tf.float32)
- samples[common.LABEL] = tf.cast(samples[common.LABEL], tf.int32)
- first_frame_img = samples[common.IMAGE][0]
- reference_labels = samples[common.LABEL][0, tf.newaxis]
- gt_labels = tf.squeeze(samples[common.LABEL], axis=-1)
- seq_name = samples[common.VIDEO_ID][0]
-
- model_options = common.VideoModelOptions(
- outputs_to_num_classes={common.OUTPUT_TYPE: dataset.num_classes},
- crop_size=None,
- atrous_rates=FLAGS.atrous_rates,
- output_stride=FLAGS.output_stride)
-
- all_embeddings = None
- predicted_labels = create_predictions_fast(
- samples, reference_labels, first_frame_img, model_options)
- # If you need more options like saving embeddings, replace the call to
- # create_predictions_fast with create_predictions.
-
- tf.train.get_or_create_global_step()
- saver = tf.train.Saver(slim.get_variables_to_restore())
- sv = tf.train.Supervisor(graph=g,
- logdir=FLAGS.vis_logdir,
- init_op=tf.global_variables_initializer(),
- summary_op=None,
- summary_writer=None,
- global_step=None,
- saver=saver)
- num_batches = int(
- math.ceil(num_vis_examples / float(FLAGS.vis_batch_size)))
- last_checkpoint = None
-
- # Infinite loop to visualize the results when new checkpoint is created.
- while True:
- last_checkpoint = slim.evaluation.wait_for_new_checkpoint(
- FLAGS.checkpoint_dir, last_checkpoint)
- start = time.time()
- tf.logging.info(
- 'Starting visualization at ' + time.strftime('%Y-%m-%d-%H:%M:%S',
- time.gmtime()))
- tf.logging.info('Visualizing with model %s', last_checkpoint)
-
- all_ious = []
- with sv.managed_session(FLAGS.master,
- start_standard_services=False) as sess:
- sv.start_queue_runners(sess)
- sv.saver.restore(sess, last_checkpoint)
-
- for batch in range(num_batches):
- ops = [predicted_labels, gt_labels, seq_name]
- if FLAGS.save_embeddings:
- ops.append(all_embeddings)
- tf.logging.info('Visualizing batch %d / %d', batch + 1, num_batches)
- res = sess.run(ops)
- tf.logging.info('Forwarding done')
- pred_labels_val, gt_labels_val, seq_name_val = res[:3]
- if FLAGS.save_embeddings:
- all_embeddings_val = res[3]
- else:
- all_embeddings_val = None
- seq_ious = _process_seq_data(segmentation_dir, embeddings_dir,
- seq_name_val, pred_labels_val,
- gt_labels_val, all_embeddings_val)
- all_ious.append(seq_ious)
- all_ious = np.concatenate(all_ious, axis=0)
- tf.logging.info('n_seqs %s, mIoU %f', all_ious.shape, all_ious.mean())
- tf.logging.info(
- 'Finished visualization at ' + time.strftime('%Y-%m-%d-%H:%M:%S',
- time.gmtime()))
- result_dir = FLAGS.vis_logdir + '/results/'
- tf.gfile.MakeDirs(result_dir)
- with tf.gfile.GFile(result_dir + seq_name_val + '.txt', 'w') as f:
- f.write(str(all_ious))
- if FLAGS.first_frame_finetuning or FLAGS.eval_once_and_quit:
- break
- time_to_next_eval = start + FLAGS.eval_interval_secs - time.time()
- if time_to_next_eval > 0:
- time.sleep(time_to_next_eval)
-
-
-if __name__ == '__main__':
- flags.mark_flag_as_required('checkpoint_dir')
- flags.mark_flag_as_required('vis_logdir')
- tf.logging.set_verbosity(tf.logging.INFO)
- tf.app.run()
diff --git a/research/fivo/.gitattributes b/research/fivo/.gitattributes
deleted file mode 100644
index f706c0421d718f8af8e62d96d69101fe383d2b4f..0000000000000000000000000000000000000000
--- a/research/fivo/.gitattributes
+++ /dev/null
@@ -1,2 +0,0 @@
-*.pkl binary
-*.tfrecord binary
diff --git a/research/fivo/.gitignore b/research/fivo/.gitignore
deleted file mode 100644
index af2f537516daf33fdaf579436dfa33fdd9044f49..0000000000000000000000000000000000000000
--- a/research/fivo/.gitignore
+++ /dev/null
@@ -1,104 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-# Usually these files are written by a python script from a template
-# before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-.static_storage/
-.media/
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
diff --git a/research/fivo/README.md b/research/fivo/README.md
deleted file mode 100644
index 36d355b1b2961f2c8c8b721b5ce13c0c3eab1e8b..0000000000000000000000000000000000000000
--- a/research/fivo/README.md
+++ /dev/null
@@ -1,215 +0,0 @@
-
-
-
-
-# Filtering Variational Objectives
-
-This folder contains a TensorFlow implementation of the algorithms from
-
-Chris J. Maddison\*, Dieterich Lawson\*, George Tucker\*, Nicolas Heess, Mohammad Norouzi, Andriy Mnih, Arnaud Doucet, and Yee Whye Teh. "Filtering Variational Objectives." NIPS 2017.
-
-[https://arxiv.org/abs/1705.09279](https://arxiv.org/abs/1705.09279)
-
-This code implements 3 different bounds for training sequential latent variable models: the evidence lower bound (ELBO), the importance weighted auto-encoder bound (IWAE), and our bound, the filtering variational objective (FIVO).
-
-Additionally it contains several sequential latent variable model implementations:
-
-* Variational recurrent neural network (VRNN)
-* Stochastic recurrent neural network (SRNN)
-* Gaussian hidden Markov model with linear conditionals (GHMM)
-
-The VRNN and SRNN can be trained for sequence modeling of pianoroll and speech data. The GHMM is trainable on a synthetic dataset, useful as a simple example of an analytically tractable model.
-
-#### Directory Structure
-The important parts of the code are organized as follows.
-
-```
-run_fivo.py # main script, contains flag definitions
-fivo
-├─smc.py # a sequential Monte Carlo implementation
-├─bounds.py # code for computing each bound, uses smc.py
-├─runners.py # code for VRNN and SRNN training and evaluation
-├─ghmm_runners.py # code for GHMM training and evaluation
-├─data
-| ├─datasets.py # readers for pianoroll and speech datasets
-| ├─calculate_pianoroll_mean.py # preprocesses the pianoroll datasets
-| └─create_timit_dataset.py # preprocesses the TIMIT dataset
-└─models
- ├─base.py # base classes used in other models
- ├─vrnn.py # VRNN implementation
- ├─srnn.py # SRNN implementation
- └─ghmm.py # Gaussian hidden Markov model (GHMM) implementation
-bin
-├─run_train.sh # an example script that runs training
-├─run_eval.sh # an example script that runs evaluation
-├─run_sample.sh # an example script that runs sampling
-├─run_tests.sh # a script that runs all tests
-└─download_pianorolls.sh # a script that downloads pianoroll files
-```
-
-### Pianorolls
-
-Requirements before we start:
-
-* TensorFlow (see [tensorflow.org](http://tensorflow.org) for how to install)
-* [scipy](https://www.scipy.org/)
-* [sonnet](https://github.com/deepmind/sonnet)
-
-
-#### Download the Data
-
-The pianoroll datasets are encoded as pickled sparse arrays and are available at [http://www-etud.iro.umontreal.ca/~boulanni/icml2012](http://www-etud.iro.umontreal.ca/~boulanni/icml2012). You can use the script `bin/download_pianorolls.sh` to download the files into a directory of your choosing.
-```
-export PIANOROLL_DIR=~/pianorolls
-mkdir $PIANOROLL_DIR
-sh bin/download_pianorolls.sh $PIANOROLL_DIR
-```
-
-#### Preprocess the Data
-
-The script `calculate_pianoroll_mean.py` loads a pianoroll pickle file, calculates the mean, updates the pickle file to include the mean under the key `train_mean`, and writes the file back to disk in-place. You should do this for all pianoroll datasets you wish to train on.
-
-```
-python data/calculate_pianoroll_mean.py --in_file=$PIANOROLL_DIR/piano-midi.de.pkl
-python data/calculate_pianoroll_mean.py --in_file=$PIANOROLL_DIR/nottingham.de.pkl
-python data/calculate_pianoroll_mean.py --in_file=$PIANOROLL_DIR/musedata.pkl
-python data/calculate_pianoroll_mean.py --in_file=$PIANOROLL_DIR/jsb.pkl
-```
-
-#### Training
-
-Now we can train a model. Here is the command for a standard training run, taken from `bin/run_train.sh`:
-```
-python run_fivo.py \
- --mode=train \
- --logdir=/tmp/fivo \
- --model=vrnn \
- --bound=fivo \
- --summarize_every=100 \
- --batch_size=4 \
- --num_samples=4 \
- --learning_rate=0.0001 \
- --dataset_path="$PIANOROLL_DIR/jsb.pkl" \
- --dataset_type="pianoroll"
-```
-
-You should see output that looks something like this (with extra logging cruft):
-
-```
-Saving checkpoints for 0 into /tmp/fivo/model.ckpt.
-Step 1, fivo bound per timestep: -11.322491
-global_step/sec: 7.49971
-Step 101, fivo bound per timestep: -11.399275
-global_step/sec: 8.04498
-Step 201, fivo bound per timestep: -11.174991
-global_step/sec: 8.03989
-Step 301, fivo bound per timestep: -11.073008
-```
-#### Evaluation
-
-You can also evaluate saved checkpoints. The `eval` mode loads a model checkpoint, tests its performance on all items in a dataset, and reports the log-likelihood averaged over the dataset. For example here is a command, taken from `bin/run_eval.sh`, that will evaluate a JSB model on the test set:
-
-```
-python run_fivo.py \
- --mode=eval \
- --split=test \
- --alsologtostderr \
- --logdir=/tmp/fivo \
- --model=vrnn \
- --batch_size=4 \
- --num_samples=4 \
- --dataset_path="$PIANOROLL_DIR/jsb.pkl" \
- --dataset_type="pianoroll"
-```
-
-You should see output like this:
-```
-Restoring parameters from /tmp/fivo/model.ckpt-0
-Model restored from step 0, evaluating.
-test elbo ll/t: -12.198834, iwae ll/t: -11.981187 fivo ll/t: -11.579776
-test elbo ll/seq: -748.564789, iwae ll/seq: -735.209206 fivo ll/seq: -710.577141
-```
-The evaluation script prints log-likelihood in both nats per timestep (ll/t) and nats per sequence (ll/seq) for all three bounds.
-
-#### Sampling
-
-You can also sample from trained models. The `sample` mode loads a model checkpoint, conditions the model on a prefix of a randomly chosen datapoint, samples a sequence of outputs from the conditioned model, and writes out the samples and prefix to a `.npz` file in `logdir`. For example here is a command that samples from a model trained on JSB, taken from `bin/run_sample.sh`:
-```
-python run_fivo.py \
- --mode=sample \
- --alsologtostderr \
- --logdir="/tmp/fivo" \
- --model=vrnn \
- --bound=fivo \
- --batch_size=4 \
- --num_samples=4 \
- --split=test \
- --dataset_path="$PIANOROLL_DIR/jsb.pkl" \
- --dataset_type="pianoroll" \
- --prefix_length=25 \
- --sample_length=50
-```
-
-Here `num_samples` denotes the number of samples used when conditioning the model as well as the number of trajectories to sample for each prefix.
-
-You should see very little output.
-```
-Restoring parameters from /tmp/fivo/model.ckpt-0
-Running local_init_op.
-Done running local_init_op.
-```
-
-Loading the samples with `np.load` confirms that we conditioned the model on 4
-prefixes of length 25 and sampled 4 sequences of length 50 for each prefix.
-```
->>> import numpy as np
->>> x = np.load("/tmp/fivo/samples.npz")
->>> x[()]['prefixes'].shape
-(25, 4, 88)
->>> x[()]['samples'].shape
-(50, 4, 4, 88)
-```
-
-### Training on TIMIT
-
-The TIMIT speech dataset is available at the [Linguistic Data Consortium website](https://catalog.ldc.upenn.edu/LDC93S1), but is unfortunately not free. These instructions will proceed assuming you have downloaded the TIMIT archive and extracted it into the directory `$RAW_TIMIT_DIR`.
-
-#### Preprocess TIMIT
-
-We preprocess TIMIT (as described in our paper) and write it out to a series of TFRecord files. To prepare the TIMIT dataset use the script `create_timit_dataset.py`
-```
-export $TIMIT_DIR=~/timit_dataset
-mkdir $TIMIT_DIR
-python data/create_timit_dataset.py \
- --raw_timit_dir=$RAW_TIMIT_DIR \
- --out_dir=$TIMIT_DIR
-```
-You should see this exact output:
-```
-4389 train / 231 valid / 1680 test
-train mean: 0.006060 train std: 548.136169
-```
-
-#### Training on TIMIT
-This is very similar to training on pianoroll datasets, with just a few flags switched.
-```
-python run_fivo.py \
- --mode=train \
- --logdir=/tmp/fivo \
- --model=vrnn \
- --bound=fivo \
- --summarize_every=100 \
- --batch_size=4 \
- --num_samples=4 \
- --learning_rate=0.0001 \
- --dataset_path="$TIMIT_DIR/train" \
- --dataset_type="speech"
-```
-Evaluation and sampling are similar.
-
-### Tests
-This codebase comes with a number of tests to verify correctness, runnable via `bin/run_tests.sh`. The tests are also useful to look at for examples of how to use the code.
-
-### Contact
-
-This codebase is maintained by Dieterich Lawson. For questions and issues please open an issue on the tensorflow/models issues tracker and assign it to @dieterichlawson.
diff --git a/research/fivo/bin/download_pianorolls.sh b/research/fivo/bin/download_pianorolls.sh
deleted file mode 100644
index ef7050b4df5fb9815be04d133e659fa31d8d055e..0000000000000000000000000000000000000000
--- a/research/fivo/bin/download_pianorolls.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# A script to download the pianoroll datasets.
-# Accepts one argument, the directory to put the files in
-
-if [ -z "$1" ]
- then
- echo "Error, must provide a directory to download the files to."
- exit
-fi
-
-echo "Downloading datasets into $1"
-curl -s "http://www-etud.iro.umontreal.ca/~boulanni/Piano-midi.de.pickle" > $1/piano-midi.de.pkl
-curl -s "http://www-etud.iro.umontreal.ca/~boulanni/Nottingham.pickle" > $1/nottingham.pkl
-curl -s "http://www-etud.iro.umontreal.ca/~boulanni/MuseData.pickle" > $1/musedata.pkl
-curl -s "http://www-etud.iro.umontreal.ca/~boulanni/JSB%20Chorales.pickle" > $1/jsb.pkl
diff --git a/research/fivo/bin/run_eval.sh b/research/fivo/bin/run_eval.sh
deleted file mode 100644
index b30bcedc2d16e5bdd681386100ecca23612a139a..0000000000000000000000000000000000000000
--- a/research/fivo/bin/run_eval.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# An example of running evaluation.
-
-PIANOROLL_DIR=$HOME/pianorolls
-
-python run_fivo.py \
- --mode=eval \
- --logdir=/tmp/fivo \
- --model=vrnn \
- --batch_size=4 \
- --num_samples=4 \
- --split=test \
- --dataset_path="$PIANOROLL_DIR/jsb.pkl" \
- --dataset_type="pianoroll"
diff --git a/research/fivo/bin/run_sample.sh b/research/fivo/bin/run_sample.sh
deleted file mode 100644
index e0c82a0cb137822e85035a23081ecf6408b7cca1..0000000000000000000000000000000000000000
--- a/research/fivo/bin/run_sample.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# An example of sampling from the model.
-
-PIANOROLL_DIR=$HOME/pianorolls
-
-python run_fivo.py \
- --mode=sample \
- --alsologtostderr \
- --logdir="/tmp/fivo" \
- --model=vrnn \
- --bound=fivo \
- --batch_size=4 \
- --num_samples=4 \
- --split=test \
- --dataset_path="$PIANOROLL_DIR/jsb.pkl" \
- --dataset_type="pianoroll" \
- --prefix_length=25 \
- --sample_length=50
diff --git a/research/fivo/bin/run_tests.sh b/research/fivo/bin/run_tests.sh
deleted file mode 100644
index 2ea58f016620db98e258494919c6d339b5fd996e..0000000000000000000000000000000000000000
--- a/research/fivo/bin/run_tests.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-python -m fivo.smc_test && \
-python -m fivo.bounds_test && \
-python -m fivo.nested_utils_test && \
-python -m fivo.data.datasets_test && \
-python -m fivo.models.ghmm_test && \
-python -m fivo.models.vrnn_test && \
-python -m fivo.models.srnn_test && \
-python -m fivo.ghmm_runners_test && \
-python -m fivo.runners_test
diff --git a/research/fivo/bin/run_train.sh b/research/fivo/bin/run_train.sh
deleted file mode 100644
index a845959770c77cd99528005e1ee69e4593fcae0c..0000000000000000000000000000000000000000
--- a/research/fivo/bin/run_train.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# An example of running training.
-
-PIANOROLL_DIR=$HOME/pianorolls
-
-python run_fivo.py \
- --mode=train \
- --logdir=/tmp/fivo \
- --model=vrnn \
- --bound=fivo \
- --summarize_every=100 \
- --batch_size=4 \
- --num_samples=4 \
- --learning_rate=0.0001 \
- --dataset_path="$PIANOROLL_DIR/jsb.pkl" \
- --dataset_type="pianoroll"
diff --git a/research/fivo/experimental/README.md b/research/fivo/experimental/README.md
deleted file mode 100644
index 649de0ba95cdee2fa1b101a588dc48903b2ca13b..0000000000000000000000000000000000000000
--- a/research/fivo/experimental/README.md
+++ /dev/null
@@ -1 +0,0 @@
-An experimental codebase for running simple examples.
diff --git a/research/fivo/experimental/bounds.py b/research/fivo/experimental/bounds.py
deleted file mode 100644
index afc970c59a1a86dbe8438b4e8bba791d3c95aa63..0000000000000000000000000000000000000000
--- a/research/fivo/experimental/bounds.py
+++ /dev/null
@@ -1,673 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import namedtuple
-
-import tensorflow as tf
-import summary_utils as summ
-
-Loss = namedtuple("Loss", "name loss vars")
-Loss.__new__.__defaults__ = (tf.GraphKeys.TRAINABLE_VARIABLES,)
-
-
-def iwae(model, observation, num_timesteps, num_samples=1,
- summarize=False):
- """Compute the IWAE evidence lower bound.
-
- Args:
- model: A callable that computes one timestep of the model.
- observation: A shape [batch_size*num_samples, state_size] Tensor
- containing z_n, the observation for each sequence in the batch.
- num_timesteps: The number of timesteps in each sequence, an integer.
- num_samples: The number of samples to use to compute the IWAE bound.
- Returns:
- log_p_hat: The IWAE estimator of the lower bound on the log marginal.
- loss: A tensor that you can perform gradient descent on to optimize the
- bound.
- maintain_ema_op: A no-op included for compatibility with FIVO.
- states: The sequence of states sampled.
- """
- # Initialization
- num_instances = tf.shape(observation)[0]
- batch_size = tf.cast(num_instances / num_samples, tf.int32)
- states = [model.zero_state(num_instances)]
- log_weights = []
- log_weight_acc = tf.zeros([num_samples, batch_size], dtype=observation.dtype)
-
- for t in xrange(num_timesteps):
- # run the model for one timestep
- (zt, log_q_zt, log_p_zt, log_p_x_given_z, _) = model(
- states[-1], observation, t)
- # update accumulators
- states.append(zt)
- log_weight = log_p_zt + log_p_x_given_z - log_q_zt
- log_weight_acc += tf.reshape(log_weight, [num_samples, batch_size])
- if summarize:
- weight_dist = tf.contrib.distributions.Categorical(
- logits=tf.transpose(log_weight_acc, perm=[1, 0]),
- allow_nan_stats=False)
- weight_entropy = weight_dist.entropy()
- weight_entropy = tf.reduce_mean(weight_entropy)
- tf.summary.scalar("weight_entropy/%d" % t, weight_entropy)
- log_weights.append(log_weight_acc)
- # Compute the lower bound on the log evidence.
- log_p_hat = (tf.reduce_logsumexp(log_weight_acc, axis=0) -
- tf.log(tf.cast(num_samples, observation.dtype))) / num_timesteps
- loss = -tf.reduce_mean(log_p_hat)
- losses = [Loss("log_p_hat", loss)]
-
- # we clip off the initial state before returning.
- # there are no emas for iwae, so we return a noop for that
- return log_p_hat, losses, tf.no_op(), states[1:], log_weights
-
-
-def multinomial_resampling(log_weights, states, n, b):
- """Resample states with multinomial resampling.
-
- Args:
- log_weights: A (n x b) Tensor representing a batch of b logits for n-ary
- Categorical distribution.
- states: A list of (b*n x d) Tensors that will be resample in from the groups
- of every n-th row.
-
- Returns:
- resampled_states: A list of (b*n x d) Tensors resampled via stratified sampling.
- log_probs: A (n x b) Tensor of the log probabilities of the ancestry decisions.
- resampling_parameters: The Tensor of parameters of the resampling distribution.
- ancestors: An (n x b) Tensor of integral indices representing the ancestry decisions.
- resampling_dist: The distribution object for resampling.
- """
- log_weights = tf.convert_to_tensor(log_weights)
- states = [tf.convert_to_tensor(state) for state in states]
-
- resampling_parameters = tf.transpose(log_weights, perm=[1,0])
- resampling_dist = tf.contrib.distributions.Categorical(logits=resampling_parameters)
- ancestors = tf.stop_gradient(
- resampling_dist.sample(sample_shape=n))
- log_probs = resampling_dist.log_prob(ancestors)
-
- offset = tf.expand_dims(tf.range(b), 0)
- ancestor_inds = tf.reshape(ancestors * b + offset, [-1])
-
- resampled_states = []
- for state in states:
- resampled_states.append(tf.gather(state, ancestor_inds))
- return resampled_states, log_probs, resampling_parameters, ancestors, resampling_dist
-
-def stratified_resampling(log_weights, states, n, b):
- """Resample states with straitified resampling.
-
- Args:
- log_weights: A (n x b) Tensor representing a batch of b logits for n-ary
- Categorical distribution.
- states: A list of (b*n x d) Tensors that will be resample in from the groups
- of every n-th row.
-
- Returns:
- resampled_states: A list of (b*n x d) Tensors resampled via stratified sampling.
- log_probs: A (n x b) Tensor of the log probabilities of the ancestry decisions.
- resampling_parameters: The Tensor of parameters of the resampling distribution.
- ancestors: An (n x b) Tensor of integral indices representing the ancestry decisions.
- resampling_dist: The distribution object for resampling.
- """
- log_weights = tf.convert_to_tensor(log_weights)
- states = [tf.convert_to_tensor(state) for state in states]
-
- log_weights = tf.transpose(log_weights, perm=[1,0])
-
- probs = tf.nn.softmax(
- tf.tile(tf.expand_dims(log_weights, axis=1),
- [1, n, 1])
- )
-
- cdfs = tf.concat([tf.zeros((b,n,1), dtype=probs.dtype), tf.cumsum(probs, axis=2)], 2)
-
- bins = tf.range(n, dtype=probs.dtype) / n
- bins = tf.tile(tf.reshape(bins, [1,-1,1]), [b,1,n+1])
-
- strat_cdfs = tf.minimum(tf.maximum((cdfs - bins) * n, 0.0), 1.0)
- resampling_parameters = strat_cdfs[:,:,1:] - strat_cdfs[:,:,:-1]
-
- resampling_dist = tf.contrib.distributions.Categorical(
- probs = resampling_parameters,
- allow_nan_stats=False)
-
- ancestors = tf.stop_gradient(
- resampling_dist.sample())
- log_probs = resampling_dist.log_prob(ancestors)
-
- ancestors = tf.transpose(ancestors, perm=[1,0])
- log_probs = tf.transpose(log_probs, perm=[1,0])
-
- offset = tf.expand_dims(tf.range(b), 0)
- ancestor_inds = tf.reshape(ancestors * b + offset, [-1])
-
- resampled_states = []
- for state in states:
- resampled_states.append(tf.gather(state, ancestor_inds))
-
- return resampled_states, log_probs, resampling_parameters, ancestors, resampling_dist
-
-def systematic_resampling(log_weights, states, n, b):
- """Resample states with systematic resampling.
-
- Args:
- log_weights: A (n x b) Tensor representing a batch of b logits for n-ary
- Categorical distribution.
- states: A list of (b*n x d) Tensors that will be resample in from the groups
- of every n-th row.
-
- Returns:
- resampled_states: A list of (b*n x d) Tensors resampled via stratified sampling.
- log_probs: A (n x b) Tensor of the log probabilities of the ancestry decisions.
- resampling_parameters: The Tensor of parameters of the resampling distribution.
- ancestors: An (n x b) Tensor of integral indices representing the ancestry decisions.
- resampling_dist: The distribution object for resampling.
- """
-
- log_weights = tf.convert_to_tensor(log_weights)
- states = [tf.convert_to_tensor(state) for state in states]
-
- log_weights = tf.transpose(log_weights, perm=[1,0])
-
- probs = tf.nn.softmax(
- tf.tile(tf.expand_dims(log_weights, axis=1),
- [1, n, 1])
- )
-
- cdfs = tf.concat([tf.zeros((b,n,1), dtype=probs.dtype), tf.cumsum(probs, axis=2)], 2)
-
- bins = tf.range(n, dtype=probs.dtype) / n
- bins = tf.tile(tf.reshape(bins, [1,-1,1]), [b,1,n+1])
-
- strat_cdfs = tf.minimum(tf.maximum((cdfs - bins) * n, 0.0), 1.0)
- resampling_parameters = strat_cdfs[:,:,1:] - strat_cdfs[:,:,:-1]
-
- resampling_dist = tf.contrib.distributions.Categorical(
- probs=resampling_parameters,
- allow_nan_stats=True)
-
- U = tf.random_uniform((b, 1, 1), dtype=probs.dtype)
-
- ancestors = tf.stop_gradient(tf.reduce_sum(tf.to_float(U > strat_cdfs[:,:,1:]), axis=-1))
- log_probs = resampling_dist.log_prob(ancestors)
-
- ancestors = tf.transpose(ancestors, perm=[1,0])
- log_probs = tf.transpose(log_probs, perm=[1,0])
-
- offset = tf.expand_dims(tf.range(b, dtype=probs.dtype), 0)
- ancestor_inds = tf.reshape(ancestors * b + offset, [-1])
-
- resampled_states = []
- for state in states:
- resampled_states.append(tf.gather(state, ancestor_inds))
-
- return resampled_states, log_probs, resampling_parameters, ancestors, resampling_dist
-
-
-def log_blend(inputs, weights):
- """Blends state in the log space.
-
- Args:
- inputs: A set of scalar states, one for each particle in each particle filter.
- Should be [num_samples, batch_size].
- weights: A set of weights used to blend the state. Each set of weights
- should be of dimension [num_samples] (one weight for each previous particle).
- There should be one set of weights for each new particle in each particle filter.
- Thus the shape should be [num_samples, batch_size, num_samples] where
- the first axis indexes new particle and the last axis indexes old particles.
- Returns:
- blended: The blended states, a tensor of shape [num_samples, batch_size].
- """
- raw_max = tf.reduce_max(inputs, axis=0, keepdims=True)
- my_max = tf.stop_gradient(
- tf.where(tf.is_finite(raw_max), raw_max, tf.zeros_like(raw_max))
- )
- # Don't ask.
- blended = tf.log(tf.einsum("ijk,kj->ij", weights, tf.exp(inputs - raw_max))) + my_max
- return blended
-
-
-def relaxed_resampling(log_weights, states, num_samples, batch_size,
- log_r_x=None, blend_type="log", temperature=0.5,
- straight_through=False):
- """Resample states with relaxed resampling.
-
- Args:
- log_weights: A (n x b) Tensor representing a batch of b logits for n-ary
- Categorical distribution.
- states: A list of (b*n x d) Tensors that will be resample in from the groups
- of every n-th row.
-
- Returns:
- resampled_states: A list of (b*n x d) Tensors resampled via stratified sampling.
- log_probs: A (n x b) Tensor of the log probabilities of the ancestry decisions.
- resampling_parameters: The Tensor of parameters of the resampling distribution.
- ancestors: An (n x b x n) Tensor of relaxed one hot representations of the ancestry decisions.
- resampling_dist: The distribution object for resampling.
- """
- assert blend_type in ["log", "linear"], "Blend type must be 'log' or 'linear'."
- log_weights = tf.convert_to_tensor(log_weights)
- states = [tf.convert_to_tensor(state) for state in states]
- state_dim = states[0].get_shape().as_list()[-1]
- # weights are num_samples by batch_size, so we transpose to get a
- # set of batch_size distributions over [0,num_samples).
- resampling_parameters = tf.transpose(log_weights, perm=[1, 0])
- resampling_dist = tf.contrib.distributions.RelaxedOneHotCategorical(
- temperature,
- logits=resampling_parameters)
-
- # sample num_samples samples from the distribution, resulting in a
- # [num_samples, batch_size, num_samples] Tensor that represents a set of
- # [num_samples, batch_size] blending weights. The dimensions represent
- # [sample index, batch index, blending weight index]
- ancestors = resampling_dist.sample(sample_shape=num_samples)
- if straight_through:
- # Forward pass discrete choices, backwards pass soft choices
- hard_ancestor_indices = tf.argmax(ancestors, axis=-1)
- hard_ancestors = tf.one_hot(hard_ancestor_indices, num_samples,
- dtype=ancestors.dtype)
- ancestors = tf.stop_gradient(hard_ancestors - ancestors) + ancestors
- log_probs = resampling_dist.log_prob(ancestors)
- if log_r_x is not None and blend_type == "log":
- log_r_x = tf.reshape(log_r_x, [num_samples, batch_size])
- log_r_x = log_blend(log_r_x, ancestors)
- log_r_x = tf.reshape(log_r_x, [num_samples*batch_size])
- elif log_r_x is not None and blend_type == "linear":
- # If blend type is linear just add log_r to the states that will be blended
- # linearly.
- states.append(log_r_x)
-
- # transpose the 'indices' to be [batch_index, blending weight index, sample index]
- ancestor_inds = tf.transpose(ancestors, perm=[1, 2, 0])
- resampled_states = []
- for state in states:
- # state is currently [num_samples * batch_size, state_dim] so we reshape
- # to [num_samples, batch_size, state_dim] and then transpose to
- # [batch_size, state_size, num_samples]
- state = tf.transpose(tf.reshape(state, [num_samples, batch_size, -1]), perm=[1, 2, 0])
- # state is now (batch_size, state_size, num_samples)
- # and ancestor is (batch index, blending weight index, sample index)
- # multiplying these gives a matrix of size [batch_size, state_size, num_samples]
- next_state = tf.matmul(state, ancestor_inds)
- # transpose the state to be [num_samples, batch_size, state_size]
- # and then reshape it to match the state format.
- next_state = tf.reshape(tf.transpose(next_state, perm=[2,0,1]), [num_samples*batch_size, state_dim])
- resampled_states.append(next_state)
-
- new_dist = tf.contrib.distributions.Categorical(
- logits=resampling_parameters)
-
- if log_r_x is not None and blend_type == "linear":
- # If blend type is linear pop off log_r that we added to the states.
- log_r_x = tf.squeeze(resampled_states[-1])
- resampled_states = resampled_states[:-1]
- return resampled_states, log_probs, log_r_x, resampling_parameters, ancestors, new_dist
-
-
-def fivo(model,
- observation,
- num_timesteps,
- resampling_schedule,
- num_samples=1,
- use_resampling_grads=True,
- resampling_type="multinomial",
- resampling_temperature=0.5,
- aux=True,
- summarize=False):
- """Compute the FIVO evidence lower bound.
-
- Args:
- model: A callable that computes one timestep of the model.
- observation: A shape [batch_size*num_samples, state_size] Tensor
- containing z_n, the observation for each sequence in the batch.
- num_timesteps: The number of timesteps in each sequence, an integer.
- resampling_schedule: A list of booleans of length num_timesteps, contains
- True if a resampling should occur on a specific timestep.
- num_samples: The number of samples to use to compute the IWAE bound.
- use_resampling_grads: Whether or not to include the resampling gradients
- in loss.
- resampling type: The type of resampling, one of "multinomial", "stratified",
- "relaxed-logblend", "relaxed-linearblend", "relaxed-stateblend", or
- "systematic".
- resampling_temperature: A positive temperature only used for relaxed
- resampling.
- aux: If true, compute the FIVO-AUX bound.
- Returns:
- log_p_hat: The IWAE estimator of the lower bound on the log marginal.
- loss: A tensor that you can perform gradient descent on to optimize the
- bound.
- maintain_ema_op: An op to update the baseline ema used for the resampling
- gradients.
- states: The sequence of states sampled.
- """
- # Initialization
- num_instances = tf.cast(tf.shape(observation)[0], tf.int32)
- batch_size = tf.cast(num_instances / num_samples, tf.int32)
- states = [model.zero_state(num_instances)]
- prev_state = states[0]
- log_weight_acc = tf.zeros(shape=[num_samples, batch_size], dtype=observation.dtype)
- prev_log_r_zt = tf.zeros([num_instances], dtype=observation.dtype)
- log_weights = []
- log_weights_all = []
- log_p_hats = []
- resampling_log_probs = []
- for t in xrange(num_timesteps):
- # run the model for one timestep
- (zt, log_q_zt, log_p_zt, log_p_x_given_z, log_r_zt) = model(
- prev_state, observation, t)
- # update accumulators
- states.append(zt)
- log_weight = log_p_zt + log_p_x_given_z - log_q_zt
- if aux:
- if t == num_timesteps - 1:
- log_weight -= prev_log_r_zt
- else:
- log_weight += log_r_zt - prev_log_r_zt
- prev_log_r_zt = log_r_zt
- log_weight_acc += tf.reshape(log_weight, [num_samples, batch_size])
- log_weights_all.append(log_weight_acc)
- if resampling_schedule[t]:
-
- # These objects will be resampled
- to_resample = [states[-1]]
- if aux and "relaxed" not in resampling_type:
- to_resample.append(prev_log_r_zt)
-
- # do the resampling
- if resampling_type == "multinomial":
- (resampled,
- resampling_log_prob,
- _, _, _) = multinomial_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size)
- elif resampling_type == "stratified":
- (resampled,
- resampling_log_prob,
- _, _, _) = stratified_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size)
- elif resampling_type == "systematic":
- (resampled,
- resampling_log_prob,
- _, _, _) = systematic_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size)
- elif "relaxed" in resampling_type:
- if aux:
- if resampling_type == "relaxed-logblend":
- (resampled,
- resampling_log_prob,
- prev_log_r_zt,
- _, _, _) = relaxed_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size,
- temperature=resampling_temperature,
- log_r_x=prev_log_r_zt,
- blend_type="log")
- elif resampling_type == "relaxed-linearblend":
- (resampled,
- resampling_log_prob,
- prev_log_r_zt,
- _, _, _) = relaxed_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size,
- temperature=resampling_temperature,
- log_r_x=prev_log_r_zt,
- blend_type="linear")
- elif resampling_type == "relaxed-stateblend":
- (resampled,
- resampling_log_prob,
- _, _, _, _) = relaxed_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size,
- temperature=resampling_temperature)
- # Calculate prev_log_r_zt from the post-resampling state
- prev_r_zt = model.r.r_xn(resampled[0], t)
- prev_log_r_zt = tf.reduce_sum(
- prev_r_zt.log_prob(observation), axis=[1])
- elif resampling_type == "relaxed-stateblend-st":
- (resampled,
- resampling_log_prob,
- _, _, _, _) = relaxed_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size,
- temperature=resampling_temperature,
- straight_through=True)
- # Calculate prev_log_r_zt from the post-resampling state
- prev_r_zt = model.r.r_xn(resampled[0], t)
- prev_log_r_zt = tf.reduce_sum(
- prev_r_zt.log_prob(observation), axis=[1])
- else:
- (resampled,
- resampling_log_prob,
- _, _, _, _) = relaxed_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size,
- temperature=resampling_temperature)
- #if summarize:
- # resampling_entropy = resampling_dist.entropy()
- # resampling_entropy = tf.reduce_mean(resampling_entropy)
- # tf.summary.scalar("weight_entropy/%d" % t, resampling_entropy)
-
- resampling_log_probs.append(tf.reduce_sum(resampling_log_prob, axis=0))
- prev_state = resampled[0]
- if aux and "relaxed" not in resampling_type:
- # Squeeze out the extra dim potentially added by resampling.
- # prev_log_r_zt should always be [num_instances]
- prev_log_r_zt = tf.squeeze(resampled[1])
- # Update the log p hat estimate, taking a log sum exp over the sample
- # dimension. The appended tensor is [batch_size].
- log_p_hats.append(
- tf.reduce_logsumexp(log_weight_acc, axis=0) - tf.log(
- tf.cast(num_samples, dtype=observation.dtype)))
- # reset the weights
- log_weights.append(log_weight_acc)
- log_weight_acc = tf.zeros_like(log_weight_acc)
- else:
- prev_state = states[-1]
- # Compute the final weight update. If we just resampled this will be zero.
- final_update = (tf.reduce_logsumexp(log_weight_acc, axis=0) -
- tf.log(tf.cast(num_samples, dtype=observation.dtype)))
- # If we ever resampled, then sum up the previous log p hat terms
- if len(log_p_hats) > 0:
- log_p_hat = tf.reduce_sum(log_p_hats, axis=0) + final_update
- else: # otherwise, log_p_hat only comes from the final update
- log_p_hat = final_update
-
- if use_resampling_grads and any(resampling_schedule):
- # compute the rewards
- # cumsum([a, b, c]) => [a, a+b, a+b+c]
- # learning signal at timestep t is
- # [sum from i=t+1 to T of log_p_hat_i for t=1:T]
- # so we will compute (sum from i=1 to T of log_p_hat_i)
- # and at timestep t will subtract off (sum from i=1 to t of log_p_hat_i)
- # rewards is a [num_resampling_events, batch_size] Tensor
- rewards = tf.stop_gradient(
- tf.expand_dims(log_p_hat, 0) - tf.cumsum(log_p_hats, axis=0))
- batch_avg_rewards = tf.reduce_mean(rewards, axis=1)
- # compute ema baseline.
- # centered_rewards is [num_resampling_events, batch_size]
- baseline_ema = tf.train.ExponentialMovingAverage(decay=0.94)
- maintain_baseline_op = baseline_ema.apply([batch_avg_rewards])
- baseline = tf.expand_dims(baseline_ema.average(batch_avg_rewards), 1)
- centered_rewards = rewards - baseline
- if summarize:
- summ.summarize_learning_signal(rewards, "rewards")
- summ.summarize_learning_signal(centered_rewards, "centered_rewards")
- # compute the loss tensor.
- resampling_grads = tf.reduce_sum(
- tf.stop_gradient(centered_rewards) * resampling_log_probs, axis=0)
- losses = [Loss("log_p_hat", -tf.reduce_mean(log_p_hat)/num_timesteps),
- Loss("resampling_grads", -tf.reduce_mean(resampling_grads)/num_timesteps)]
- else:
- losses = [Loss("log_p_hat", -tf.reduce_mean(log_p_hat)/num_timesteps)]
- maintain_baseline_op = tf.no_op()
-
- log_p_hat /= num_timesteps
- # we clip off the initial state before returning.
- return log_p_hat, losses, maintain_baseline_op, states[1:], log_weights_all
-
-
-def fivo_aux_td(
- model,
- observation,
- num_timesteps,
- resampling_schedule,
- num_samples=1,
- summarize=False):
- """Compute the FIVO_AUX evidence lower bound."""
- # Initialization
- num_instances = tf.cast(tf.shape(observation)[0], tf.int32)
- batch_size = tf.cast(num_instances / num_samples, tf.int32)
- states = [model.zero_state(num_instances)]
- prev_state = states[0]
- log_weight_acc = tf.zeros(shape=[num_samples, batch_size], dtype=observation.dtype)
- prev_log_r = tf.zeros([num_instances], dtype=observation.dtype)
- # must be pre-resampling
- log_rs = []
- # must be post-resampling
- r_tilde_params = [model.r_tilde.r_zt(states[0], observation, 0)]
- log_r_tildes = []
- log_p_xs = []
- # contains the weight at each timestep before resampling only on resampling timesteps
- log_weights = []
- # contains weight at each timestep before resampling
- log_weights_all = []
- log_p_hats = []
- for t in xrange(num_timesteps):
- # run the model for one timestep
- # zt is state, [num_instances, state_dim]
- # log_q_zt, log_p_x_given_z is [num_instances]
- # r_tilde_mu, r_tilde_sigma is [num_instances, state_dim]
- # p_ztplus1 is a normal distribution on [num_instances, state_dim]
- (zt, log_q_zt, log_p_zt, log_p_x_given_z,
- r_tilde_mu, r_tilde_sigma_sq, p_ztplus1) = model(prev_state, observation, t)
-
- # Compute the log weight without log r.
- log_weight = log_p_zt + log_p_x_given_z - log_q_zt
-
- # Compute log r.
- if t == num_timesteps - 1:
- log_r = tf.zeros_like(prev_log_r)
- else:
- p_mu = p_ztplus1.mean()
- p_sigma_sq = p_ztplus1.variance()
- log_r = (tf.log(r_tilde_sigma_sq) -
- tf.log(r_tilde_sigma_sq + p_sigma_sq) -
- tf.square(r_tilde_mu - p_mu)/(r_tilde_sigma_sq + p_sigma_sq))
- log_r = 0.5*tf.reduce_sum(log_r, axis=-1)
-
- #log_weight += tf.stop_gradient(log_r - prev_log_r)
- log_weight += log_r - prev_log_r
- log_weight_acc += tf.reshape(log_weight, [num_samples, batch_size])
-
- # Update accumulators
- states.append(zt)
- log_weights_all.append(log_weight_acc)
- log_p_xs.append(log_p_x_given_z)
- log_rs.append(log_r)
-
- # Compute log_r_tilde as [num_instances] Tensor.
- prev_r_tilde_mu, prev_r_tilde_sigma_sq = r_tilde_params[-1]
- prev_log_r_tilde = -0.5*tf.reduce_sum(
- tf.square(zt - prev_r_tilde_mu)/prev_r_tilde_sigma_sq, axis=-1)
- #tf.square(tf.stop_gradient(zt) - r_tilde_mu)/r_tilde_sigma_sq, axis=-1)
- #tf.square(zt - r_tilde_mu)/r_tilde_sigma_sq, axis=-1)
- log_r_tildes.append(prev_log_r_tilde)
-
- # optionally resample
- if resampling_schedule[t]:
- # These objects will be resampled
- if t < num_timesteps - 1:
- to_resample = [zt, log_r, r_tilde_mu, r_tilde_sigma_sq]
- else:
- to_resample = [zt, log_r]
- (resampled,
- _, _, _, _) = multinomial_resampling(log_weight_acc,
- to_resample,
- num_samples,
- batch_size)
- prev_state = resampled[0]
- # Squeeze out the extra dim potentially added by resampling.
- # prev_log_r_zt and log_r_tilde should always be [num_instances]
- prev_log_r = tf.squeeze(resampled[1])
- if t < num_timesteps -1:
- r_tilde_params.append((resampled[2], resampled[3]))
- # Update the log p hat estimate, taking a log sum exp over the sample
- # dimension. The appended tensor is [batch_size].
- log_p_hats.append(
- tf.reduce_logsumexp(log_weight_acc, axis=0) - tf.log(
- tf.cast(num_samples, dtype=observation.dtype)))
- # reset the weights
- log_weights.append(log_weight_acc)
- log_weight_acc = tf.zeros_like(log_weight_acc)
- else:
- prev_state = zt
- prev_log_r = log_r
- if t < num_timesteps - 1:
- r_tilde_params.append((r_tilde_mu, r_tilde_sigma_sq))
-
- # Compute the final weight update. If we just resampled this will be zero.
- final_update = (tf.reduce_logsumexp(log_weight_acc, axis=0) -
- tf.log(tf.cast(num_samples, dtype=observation.dtype)))
- # If we ever resampled, then sum up the previous log p hat terms
- if len(log_p_hats) > 0:
- log_p_hat = tf.reduce_sum(log_p_hats, axis=0) + final_update
- else: # otherwise, log_p_hat only comes from the final update
- log_p_hat = final_update
-
- # Compute the bellman loss.
- # Will remove the first timestep as it is not used.
- # log p(x_t|z_t) is in row t-1.
- log_p_x = tf.reshape(tf.stack(log_p_xs),
- [num_timesteps, num_samples, batch_size])
- # log r_t is contained in row t-1.
- # last column is zeros (because at timestep T (num_timesteps) r is 1.
- log_r = tf.reshape(tf.stack(log_rs),
- [num_timesteps, num_samples, batch_size])
- # [num_timesteps, num_instances]. log r_tilde_t is in row t-1.
- log_r_tilde = tf.reshape(tf.stack(log_r_tildes),
- [num_timesteps, num_samples, batch_size])
- log_lambda = tf.reduce_mean(log_r_tilde - log_p_x - log_r, axis=1,
- keepdims=True)
- bellman_sos = tf.reduce_mean(tf.square(
- log_r_tilde - tf.stop_gradient(log_lambda + log_p_x + log_r)), axis=[0, 1])
- bellman_loss = tf.reduce_mean(bellman_sos)/num_timesteps
- tf.summary.scalar("bellman_loss", bellman_loss)
-
- if len(tf.get_collection("LOG_P_HAT_VARS")) == 0:
- log_p_hat_collection = list(set(tf.trainable_variables()) -
- set(tf.get_collection("R_TILDE_VARS")))
- for v in log_p_hat_collection:
- tf.add_to_collection("LOG_P_HAT_VARS", v)
-
- log_p_hat /= num_timesteps
- losses = [Loss("log_p_hat", -tf.reduce_mean(log_p_hat), "LOG_P_HAT_VARS"),
- Loss("bellman_loss", bellman_loss, "R_TILDE_VARS")]
-
- return log_p_hat, losses, tf.no_op(), states[1:], log_weights_all
diff --git a/research/fivo/experimental/data.py b/research/fivo/experimental/data.py
deleted file mode 100644
index 0842f212991e1651a12cca239c5b8380fea9d0f8..0000000000000000000000000000000000000000
--- a/research/fivo/experimental/data.py
+++ /dev/null
@@ -1,192 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Datasets."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-import models
-
-
-def make_long_chain_dataset(
- state_size=1,
- num_obs=5,
- steps_per_obs=3,
- variance=1.,
- observation_variance=1.,
- batch_size=4,
- num_samples=1,
- observation_type=models.STANDARD_OBSERVATION,
- transition_type=models.STANDARD_TRANSITION,
- fixed_observation=None,
- dtype="float32"):
- """Creates a long chain data generating process.
-
- Creates a tf.data.Dataset that provides batches of data from a long
- chain.
-
- Args:
- state_size: The dimension of the state space of the process.
- num_obs: The number of observations in the chain.
- steps_per_obs: The number of steps between each observation.
- variance: The variance of the normal distributions used at each timestep.
- batch_size: The number of trajectories to include in each batch.
- num_samples: The number of replicas of each trajectory to include in each
- batch.
- dtype: The datatype of the states and observations.
- Returns:
- dataset: A tf.data.Dataset that can be iterated over.
- """
- num_timesteps = num_obs * steps_per_obs
- def data_generator():
- """An infinite generator of latents and observations from the model."""
- while True:
- states = []
- observations = []
- # z0 ~ Normal(0, sqrt(variance)).
- states.append(
- np.random.normal(size=[state_size],
- scale=np.sqrt(variance)).astype(dtype))
- # start at 1 because we've already generated z0
- # go to num_timesteps+1 because we want to include the num_timesteps-th step
- for t in xrange(1, num_timesteps+1):
- if transition_type == models.ROUND_TRANSITION:
- loc = np.round(states[-1])
- elif transition_type == models.STANDARD_TRANSITION:
- loc = states[-1]
- new_state = np.random.normal(size=[state_size],
- loc=loc,
- scale=np.sqrt(variance))
- states.append(new_state.astype(dtype))
- if t % steps_per_obs == 0:
- if fixed_observation is None:
- if observation_type == models.SQUARED_OBSERVATION:
- loc = np.square(states[-1])
- elif observation_type == models.ABS_OBSERVATION:
- loc = np.abs(states[-1])
- elif observation_type == models.STANDARD_OBSERVATION:
- loc = states[-1]
- new_obs = np.random.normal(size=[state_size],
- loc=loc,
- scale=np.sqrt(observation_variance)).astype(dtype)
- else:
- new_obs = np.ones([state_size])* fixed_observation
-
- observations.append(new_obs)
- yield states, observations
-
- dataset = tf.data.Dataset.from_generator(
- data_generator,
- output_types=(tf.as_dtype(dtype), tf.as_dtype(dtype)),
- output_shapes=([num_timesteps+1, state_size], [num_obs, state_size]))
- dataset = dataset.repeat().batch(batch_size)
-
- def tile_batch(state, observation):
- state = tf.tile(state, [num_samples, 1, 1])
- observation = tf.tile(observation, [num_samples, 1, 1])
- return state, observation
-
- dataset = dataset.map(tile_batch, num_parallel_calls=12).prefetch(1024)
- return dataset
-
-
-def make_dataset(bs=None,
- state_size=1,
- num_timesteps=10,
- variance=1.,
- prior_type="unimodal",
- bimodal_prior_weight=0.5,
- bimodal_prior_mean=1,
- transition_type=models.STANDARD_TRANSITION,
- fixed_observation=None,
- batch_size=4,
- num_samples=1,
- dtype='float32'):
- """Creates a data generating process.
-
- Creates a tf.data.Dataset that provides batches of data.
-
- Args:
- bs: The parameters of the data generating process. If None, new bs are
- randomly generated.
- state_size: The dimension of the state space of the process.
- num_timesteps: The length of the state sequences in the process.
- variance: The variance of the normal distributions used at each timestep.
- batch_size: The number of trajectories to include in each batch.
- num_samples: The number of replicas of each trajectory to include in each
- batch.
- Returns:
- bs: The true bs used to generate the data
- dataset: A tf.data.Dataset that can be iterated over.
- """
-
- if bs is None:
- bs = [np.random.uniform(size=[state_size]).astype(dtype) for _ in xrange(num_timesteps)]
- tf.logging.info("data generating processs bs: %s",
- np.array(bs).reshape(num_timesteps))
-
-
- def data_generator():
- """An infinite generator of latents and observations from the model."""
- while True:
- states = []
- if prior_type == "unimodal" or prior_type == "nonlinear":
- # Prior is Normal(0, sqrt(variance)).
- states.append(np.random.normal(size=[state_size], scale=np.sqrt(variance)).astype(dtype))
- elif prior_type == "bimodal":
- if np.random.uniform() > bimodal_prior_weight:
- loc = bimodal_prior_mean
- else:
- loc = - bimodal_prior_mean
- states.append(np.random.normal(size=[state_size],
- loc=loc,
- scale=np.sqrt(variance)
- ).astype(dtype))
-
- for t in xrange(num_timesteps):
- if transition_type == models.ROUND_TRANSITION:
- loc = np.round(states[-1])
- elif transition_type == models.STANDARD_TRANSITION:
- loc = states[-1]
- loc += bs[t]
- new_state = np.random.normal(size=[state_size],
- loc=loc,
- scale=np.sqrt(variance)).astype(dtype)
- states.append(new_state)
-
- if fixed_observation is None:
- observation = states[-1]
- else:
- observation = np.ones_like(states[-1]) * fixed_observation
- yield np.array(states[:-1]), observation
-
- dataset = tf.data.Dataset.from_generator(
- data_generator,
- output_types=(tf.as_dtype(dtype), tf.as_dtype(dtype)),
- output_shapes=([num_timesteps, state_size], [state_size]))
- dataset = dataset.repeat().batch(batch_size)
-
- def tile_batch(state, observation):
- state = tf.tile(state, [num_samples, 1, 1])
- observation = tf.tile(observation, [num_samples, 1])
- return state, observation
-
- dataset = dataset.map(tile_batch, num_parallel_calls=12).prefetch(1024)
- return np.array(bs), dataset
diff --git a/research/fivo/experimental/models.py b/research/fivo/experimental/models.py
deleted file mode 100644
index 62801ca1ee145e64c80b66e0c83dd7d834ac0847..0000000000000000000000000000000000000000
--- a/research/fivo/experimental/models.py
+++ /dev/null
@@ -1,1227 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Model."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import functools
-import sonnet as snt
-import tensorflow as tf
-import numpy as np
-import math
-
-SQUARED_OBSERVATION = "squared"
-ABS_OBSERVATION = "abs"
-STANDARD_OBSERVATION = "standard"
-OBSERVATION_TYPES = [SQUARED_OBSERVATION, ABS_OBSERVATION, STANDARD_OBSERVATION]
-
-ROUND_TRANSITION = "round"
-STANDARD_TRANSITION = "standard"
-TRANSITION_TYPES = [ROUND_TRANSITION, STANDARD_TRANSITION]
-
-
-class Q(object):
-
- def __init__(self,
- state_size,
- num_timesteps,
- sigma_min=1e-5,
- dtype=tf.float32,
- random_seed=None,
- init_mu0_to_zero=False,
- graph_collection_name="Q_VARS"):
- self.sigma_min = sigma_min
- self.dtype = dtype
- self.graph_collection_name = graph_collection_name
- initializers = []
- for t in xrange(num_timesteps):
- if t == 0 and init_mu0_to_zero:
- initializers.append(
- {"w": tf.zeros_initializer, "b": tf.zeros_initializer})
- else:
- initializers.append(
- {"w": tf.random_uniform_initializer(seed=random_seed),
- "b": tf.zeros_initializer})
-
- def custom_getter(getter, *args, **kwargs):
- out = getter(*args, **kwargs)
- ref = tf.get_collection_ref(self.graph_collection_name)
- if out not in ref:
- ref.append(out)
- return out
-
- self.mus = [
- snt.Linear(output_size=state_size,
- initializers=initializers[t],
- name="q_mu_%d" % t,
- custom_getter=custom_getter
- )
- for t in xrange(num_timesteps)
- ]
- self.sigmas = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="q_sigma_%d" % (t + 1),
- collections=[tf.GraphKeys.GLOBAL_VARIABLES, graph_collection_name],
- initializer=tf.random_uniform_initializer(seed=random_seed))
- for t in xrange(num_timesteps)
- ]
-
- def q_zt(self, observation, prev_state, t):
- batch_size = tf.shape(prev_state)[0]
- q_mu = self.mus[t](tf.concat([observation, prev_state], axis=1))
- q_sigma = tf.maximum(tf.nn.softplus(self.sigmas[t]), self.sigma_min)
- q_sigma = tf.tile(q_sigma[tf.newaxis, :], [batch_size, 1])
- q_zt = tf.contrib.distributions.Normal(loc=q_mu, scale=tf.sqrt(q_sigma))
- return q_zt
-
- def summarize_weights(self):
- for t, sigma in enumerate(self.sigmas):
- tf.summary.scalar("q_sigma/%d" % t, sigma[0])
- for t, f in enumerate(self.mus):
- tf.summary.scalar("q_mu/b_%d" % t, f.b[0])
- tf.summary.scalar("q_mu/w_obs_%d" % t, f.w[0,0])
- if t != 0:
- tf.summary.scalar("q_mu/w_prev_state_%d" % t, f.w[1,0])
-
-
-class PreviousStateQ(Q):
-
- def q_zt(self, unused_observation, prev_state, t):
- batch_size = tf.shape(prev_state)[0]
- q_mu = self.mus[t](prev_state)
- q_sigma = tf.maximum(tf.nn.softplus(self.sigmas[t]), self.sigma_min)
- q_sigma = tf.tile(q_sigma[tf.newaxis, :], [batch_size, 1])
- q_zt = tf.contrib.distributions.Normal(loc=q_mu, scale=tf.sqrt(q_sigma))
- return q_zt
-
- def summarize_weights(self):
- for t, sigma in enumerate(self.sigmas):
- tf.summary.scalar("q_sigma/%d" % t, sigma[0])
- for t, f in enumerate(self.mus):
- tf.summary.scalar("q_mu/b_%d" % t, f.b[0])
- tf.summary.scalar("q_mu/w_prev_state_%d" % t, f.w[0,0])
-
-
-class ObservationQ(Q):
-
- def q_zt(self, observation, prev_state, t):
- batch_size = tf.shape(prev_state)[0]
- q_mu = self.mus[t](observation)
- q_sigma = tf.maximum(tf.nn.softplus(self.sigmas[t]), self.sigma_min)
- q_sigma = tf.tile(q_sigma[tf.newaxis, :], [batch_size, 1])
- q_zt = tf.contrib.distributions.Normal(loc=q_mu, scale=tf.sqrt(q_sigma))
- return q_zt
-
- def summarize_weights(self):
- for t, sigma in enumerate(self.sigmas):
- tf.summary.scalar("q_sigma/%d" % t, sigma[0])
- for t, f in enumerate(self.mus):
- tf.summary.scalar("q_mu/b_%d" % t, f.b[0])
- tf.summary.scalar("q_mu/w_obs_%d" % t, f.w[0,0])
-
-
-class SimpleMeanQ(object):
-
- def __init__(self,
- state_size,
- num_timesteps,
- sigma_min=1e-5,
- dtype=tf.float32,
- random_seed=None,
- init_mu0_to_zero=False,
- graph_collection_name="Q_VARS"):
- self.sigma_min = sigma_min
- self.dtype = dtype
- self.graph_collection_name = graph_collection_name
- initializers = []
- for t in xrange(num_timesteps):
- if t == 0 and init_mu0_to_zero:
- initializers.append(tf.zeros_initializer)
- else:
- initializers.append(tf.random_uniform_initializer(seed=random_seed))
-
- self.mus = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="q_mu_%d" % (t + 1),
- collections=[tf.GraphKeys.GLOBAL_VARIABLES, graph_collection_name],
- initializer=initializers[t])
- for t in xrange(num_timesteps)
- ]
- self.sigmas = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="q_sigma_%d" % (t + 1),
- collections=[tf.GraphKeys.GLOBAL_VARIABLES, graph_collection_name],
- initializer=tf.random_uniform_initializer(seed=random_seed))
- for t in xrange(num_timesteps)
- ]
-
- def q_zt(self, unused_observation, prev_state, t):
- batch_size = tf.shape(prev_state)[0]
- q_mu = tf.tile(self.mus[t][tf.newaxis, :], [batch_size, 1])
- q_sigma = tf.maximum(tf.nn.softplus(self.sigmas[t]), self.sigma_min)
- q_sigma = tf.tile(q_sigma[tf.newaxis, :], [batch_size, 1])
- q_zt = tf.contrib.distributions.Normal(loc=q_mu, scale=tf.sqrt(q_sigma))
- return q_zt
-
- def summarize_weights(self):
- for t, sigma in enumerate(self.sigmas):
- tf.summary.scalar("q_sigma/%d" % t, sigma[0])
- for t, f in enumerate(self.mus):
- tf.summary.scalar("q_mu/%d" % t, f[0])
-
-
-class R(object):
-
- def __init__(self,
- state_size,
- num_timesteps,
- sigma_min=1e-5,
- dtype=tf.float32,
- sigma_init=1.,
- random_seed=None,
- graph_collection_name="R_VARS"):
- self.dtype = dtype
- self.sigma_min = sigma_min
- initializers = {"w": tf.truncated_normal_initializer(seed=random_seed),
- "b": tf.zeros_initializer}
- self.graph_collection_name=graph_collection_name
-
- def custom_getter(getter, *args, **kwargs):
- out = getter(*args, **kwargs)
- ref = tf.get_collection_ref(self.graph_collection_name)
- if out not in ref:
- ref.append(out)
- return out
-
- self.mus= [
- snt.Linear(output_size=state_size,
- initializers=initializers,
- name="r_mu_%d" % t,
- custom_getter=custom_getter)
- for t in xrange(num_timesteps)
- ]
-
- self.sigmas = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="r_sigma_%d" % (t + 1),
- collections=[tf.GraphKeys.GLOBAL_VARIABLES, graph_collection_name],
- #initializer=tf.random_uniform_initializer(seed=random_seed, maxval=100))
- initializer=tf.constant_initializer(sigma_init))
- for t in xrange(num_timesteps)
- ]
-
- def r_xn(self, z_t, t):
- batch_size = tf.shape(z_t)[0]
- r_mu = self.mus[t](z_t)
- r_sigma = tf.maximum(tf.nn.softplus(self.sigmas[t]), self.sigma_min)
- r_sigma = tf.tile(r_sigma[tf.newaxis, :], [batch_size, 1])
- return tf.contrib.distributions.Normal(
- loc=r_mu, scale=tf.sqrt(r_sigma))
-
- def summarize_weights(self):
- for t in range(len(self.mus) - 1):
- tf.summary.scalar("r_mu/%d" % t, self.mus[t][0])
- tf.summary.scalar("r_sigma/%d" % t, self.sigmas[t][0])
-
-
-class P(object):
-
- def __init__(self,
- state_size,
- num_timesteps,
- sigma_min=1e-5,
- variance=1.0,
- dtype=tf.float32,
- random_seed=None,
- trainable=True,
- init_bs_to_zero=False,
- graph_collection_name="P_VARS"):
- self.state_size = state_size
- self.num_timesteps = num_timesteps
- self.sigma_min = sigma_min
- self.dtype = dtype
- self.variance = variance
- self.graph_collection_name = graph_collection_name
- if init_bs_to_zero:
- initializers = [tf.zeros_initializer for _ in xrange(num_timesteps)]
- else:
- initializers = [tf.random_uniform_initializer(seed=random_seed) for _ in xrange(num_timesteps)]
-
- self.bs = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="p_b_%d" % (t + 1),
- initializer=initializers[t],
- collections=[tf.GraphKeys.GLOBAL_VARIABLES, graph_collection_name],
- trainable=trainable) for t in xrange(num_timesteps)
- ]
- self.Bs = tf.cumsum(self.bs, reverse=True, axis=0)
-
- def posterior(self, observation, prev_state, t):
- """Computes the true posterior p(z_t|z_{t-1}, z_n)."""
- # bs[0] is really b_1
- # Bs[i] is sum from k=i+1^n b_k
- mu = observation - self.Bs[t]
- if t > 0:
- mu += (prev_state + self.bs[t - 1]) * float(self.num_timesteps - t)
- mu /= float(self.num_timesteps - t + 1)
- sigma = tf.ones_like(mu) * self.variance * (
- float(self.num_timesteps - t) / float(self.num_timesteps - t + 1))
- return tf.contrib.distributions.Normal(loc=mu, scale=tf.sqrt(sigma))
-
- def lookahead(self, state, t):
- """Computes the true lookahead distribution p(z_n|z_t)."""
- mu = state + self.Bs[t]
- sigma = tf.ones_like(state) * self.variance * float(self.num_timesteps - t)
- return tf.contrib.distributions.Normal(loc=mu, scale=tf.sqrt(sigma))
-
- def likelihood(self, observation):
- batch_size = tf.shape(observation)[0]
- mu = tf.tile(tf.reduce_sum(self.bs, axis=0)[tf.newaxis, :], [batch_size, 1])
- sigma = tf.ones_like(mu) * self.variance * (self.num_timesteps + 1)
- dist = tf.contrib.distributions.Normal(loc=mu, scale=tf.sqrt(sigma))
- # Average over the batch and take the sum over the state size
- return tf.reduce_mean(tf.reduce_sum(dist.log_prob(observation), axis=1))
-
- def p_zt(self, prev_state, t):
- """Computes the model p(z_t| z_{t-1})."""
- batch_size = tf.shape(prev_state)[0]
- if t > 0:
- z_mu_p = prev_state + self.bs[t - 1]
- else: # p(z_0) is Normal(0,1)
- z_mu_p = tf.zeros([batch_size, self.state_size], dtype=self.dtype)
- p_zt = tf.contrib.distributions.Normal(
- loc=z_mu_p, scale=tf.sqrt(tf.ones_like(z_mu_p) * self.variance))
- return p_zt
-
- def generative(self, unused_observation, z_nm1):
- """Computes the model's generative distribution p(z_n| z_{n-1})."""
- generative_p_mu = z_nm1 + self.bs[-1]
- return tf.contrib.distributions.Normal(
- loc=generative_p_mu, scale=tf.sqrt(tf.ones_like(generative_p_mu) * self.variance))
-
-
-class ShortChainNonlinearP(object):
-
- def __init__(self,
- state_size,
- num_timesteps,
- sigma_min=1e-5,
- variance=1.0,
- observation_variance=1.0,
- transition_type=STANDARD_TRANSITION,
- transition_dist=tf.contrib.distributions.Normal,
- dtype=tf.float32,
- random_seed=None):
- self.state_size = state_size
- self.num_timesteps = num_timesteps
- self.sigma_min = sigma_min
- self.dtype = dtype
- self.variance = variance
- self.observation_variance = observation_variance
- self.transition_type = transition_type
- self.transition_dist = transition_dist
-
- def p_zt(self, prev_state, t):
- """Computes the model p(z_t| z_{t-1})."""
- batch_size = tf.shape(prev_state)[0]
- if t > 0:
- if self.transition_type == ROUND_TRANSITION:
- loc = tf.round(prev_state)
- tf.logging.info("p(z_%d | z_%d) ~ N(round(z_%d), %0.1f)" % (t, t-1, t-1, self.variance))
- elif self.transition_type == STANDARD_TRANSITION:
- loc = prev_state
- tf.logging.info("p(z_%d | z_%d) ~ N(z_%d, %0.1f)" % (t, t-1, t-1, self.variance))
- else: # p(z_0) is Normal(0,1)
- loc = tf.zeros([batch_size, self.state_size], dtype=self.dtype)
- tf.logging.info("p(z_0) ~ N(0,%0.1f)" % self.variance)
-
- p_zt = self.transition_dist(
- loc=loc,
- scale=tf.sqrt(tf.ones_like(loc) * self.variance))
- return p_zt
-
- def generative(self, unused_obs, z_ni):
- """Computes the model's generative distribution p(x_i| z_{ni})."""
- if self.transition_type == ROUND_TRANSITION:
- loc = tf.round(z_ni)
- elif self.transition_type == STANDARD_TRANSITION:
- loc = z_ni
- generative_sigma_sq = tf.ones_like(loc) * self.observation_variance
- return self.transition_dist(
- loc=loc, scale=tf.sqrt(generative_sigma_sq))
-
-
-class BimodalPriorP(object):
-
- def __init__(self,
- state_size,
- num_timesteps,
- mixing_coeff=0.5,
- prior_mode_mean=1,
- sigma_min=1e-5,
- variance=1.0,
- dtype=tf.float32,
- random_seed=None,
- trainable=True,
- init_bs_to_zero=False,
- graph_collection_name="P_VARS"):
- self.state_size = state_size
- self.num_timesteps = num_timesteps
- self.sigma_min = sigma_min
- self.dtype = dtype
- self.variance = variance
- self.mixing_coeff = mixing_coeff
- self.prior_mode_mean = prior_mode_mean
-
- if init_bs_to_zero:
- initializers = [tf.zeros_initializer for _ in xrange(num_timesteps)]
- else:
- initializers = [tf.random_uniform_initializer(seed=random_seed) for _ in xrange(num_timesteps)]
-
- self.bs = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="b_%d" % (t + 1),
- initializer=initializers[t],
- collections=[tf.GraphKeys.GLOBAL_VARIABLES, graph_collection_name],
- trainable=trainable) for t in xrange(num_timesteps)
- ]
- self.Bs = tf.cumsum(self.bs, reverse=True, axis=0)
-
- def posterior(self, observation, prev_state, t):
- # NOTE: This is currently wrong, but would require a refactoring of
- # summarize_q to fix as kl is not defined for a mixture
- """Computes the true posterior p(z_t|z_{t-1}, z_n)."""
- # bs[0] is really b_1
- # Bs[i] is sum from k=i+1^n b_k
- mu = observation - self.Bs[t]
- if t > 0:
- mu += (prev_state + self.bs[t - 1]) * float(self.num_timesteps - t)
- mu /= float(self.num_timesteps - t + 1)
- sigma = tf.ones_like(mu) * self.variance * (
- float(self.num_timesteps - t) / float(self.num_timesteps - t + 1))
- return tf.contrib.distributions.Normal(loc=mu, scale=tf.sqrt(sigma))
-
- def lookahead(self, state, t):
- """Computes the true lookahead distribution p(z_n|z_t)."""
- mu = state + self.Bs[t]
- sigma = tf.ones_like(state) * self.variance * float(self.num_timesteps - t)
- return tf.contrib.distributions.Normal(loc=mu, scale=tf.sqrt(sigma))
-
- def likelihood(self, observation):
- batch_size = tf.shape(observation)[0]
- sum_of_bs = tf.tile(tf.reduce_sum(self.bs, axis=0)[tf.newaxis, :], [batch_size, 1])
- sigma = tf.ones_like(sum_of_bs) * self.variance * (self.num_timesteps + 1)
- mu_pos = (tf.ones([batch_size, self.state_size], dtype=self.dtype) * self.prior_mode_mean) + sum_of_bs
- mu_neg = (tf.ones([batch_size, self.state_size], dtype=self.dtype) * -self.prior_mode_mean) + sum_of_bs
- zn_pos = tf.contrib.distributions.Normal(
- loc=mu_pos,
- scale=tf.sqrt(sigma))
- zn_neg = tf.contrib.distributions.Normal(
- loc=mu_neg,
- scale=tf.sqrt(sigma))
- mode_probs = tf.convert_to_tensor([self.mixing_coeff, 1-self.mixing_coeff], dtype=tf.float64)
- mode_probs = tf.tile(mode_probs[tf.newaxis, tf.newaxis, :], [batch_size, 1, 1])
- mode_selection_dist = tf.contrib.distributions.Categorical(probs=mode_probs)
- zn_dist = tf.contrib.distributions.Mixture(
- cat=mode_selection_dist,
- components=[zn_pos, zn_neg],
- validate_args=True)
- # Average over the batch and take the sum over the state size
- return tf.reduce_mean(tf.reduce_sum(zn_dist.log_prob(observation), axis=1))
-
- def p_zt(self, prev_state, t):
- """Computes the model p(z_t| z_{t-1})."""
- batch_size = tf.shape(prev_state)[0]
- if t > 0:
- z_mu_p = prev_state + self.bs[t - 1]
- p_zt = tf.contrib.distributions.Normal(
- loc=z_mu_p, scale=tf.sqrt(tf.ones_like(z_mu_p) * self.variance))
- return p_zt
- else: # p(z_0) is mixture of two Normals
- mu_pos = tf.ones([batch_size, self.state_size], dtype=self.dtype) * self.prior_mode_mean
- mu_neg = tf.ones([batch_size, self.state_size], dtype=self.dtype) * -self.prior_mode_mean
- z0_pos = tf.contrib.distributions.Normal(
- loc=mu_pos,
- scale=tf.sqrt(tf.ones_like(mu_pos) * self.variance))
- z0_neg = tf.contrib.distributions.Normal(
- loc=mu_neg,
- scale=tf.sqrt(tf.ones_like(mu_neg) * self.variance))
- mode_probs = tf.convert_to_tensor([self.mixing_coeff, 1-self.mixing_coeff], dtype=tf.float64)
- mode_probs = tf.tile(mode_probs[tf.newaxis, tf.newaxis, :], [batch_size, 1, 1])
- mode_selection_dist = tf.contrib.distributions.Categorical(probs=mode_probs)
- z0_dist = tf.contrib.distributions.Mixture(
- cat=mode_selection_dist,
- components=[z0_pos, z0_neg],
- validate_args=False)
- return z0_dist
-
- def generative(self, unused_observation, z_nm1):
- """Computes the model's generative distribution p(z_n| z_{n-1})."""
- generative_p_mu = z_nm1 + self.bs[-1]
- return tf.contrib.distributions.Normal(
- loc=generative_p_mu, scale=tf.sqrt(tf.ones_like(generative_p_mu) * self.variance))
-
-class Model(object):
-
- def __init__(self,
- p,
- q,
- r,
- state_size,
- num_timesteps,
- dtype=tf.float32):
- self.p = p
- self.q = q
- self.r = r
- self.state_size = state_size
- self.num_timesteps = num_timesteps
- self.dtype = dtype
-
- def zero_state(self, batch_size):
- return tf.zeros([batch_size, self.state_size], dtype=self.dtype)
-
- def __call__(self, prev_state, observation, t):
- # Compute the q distribution over z, q(z_t|z_n, z_{t-1}).
- q_zt = self.q.q_zt(observation, prev_state, t)
- # Compute the p distribution over z, p(z_t|z_{t-1}).
- p_zt = self.p.p_zt(prev_state, t)
- # sample from q
- zt = q_zt.sample()
- r_xn = self.r.r_xn(zt, t)
- # Calculate the logprobs and sum over the state size.
- log_q_zt = tf.reduce_sum(q_zt.log_prob(zt), axis=1)
- log_p_zt = tf.reduce_sum(p_zt.log_prob(zt), axis=1)
- log_r_xn = tf.reduce_sum(r_xn.log_prob(observation), axis=1)
- # If we're at the last timestep, also calc the logprob of the observation.
- if t == self.num_timesteps - 1:
- generative_dist = self.p.generative(observation, zt)
- log_p_x_given_z = tf.reduce_sum(generative_dist.log_prob(observation), axis=1)
- else:
- log_p_x_given_z = tf.zeros_like(log_q_zt)
- return (zt, log_q_zt, log_p_zt, log_p_x_given_z, log_r_xn)
-
- @staticmethod
- def create(state_size,
- num_timesteps,
- sigma_min=1e-5,
- r_sigma_init=1,
- variance=1.0,
- mixing_coeff=0.5,
- prior_mode_mean=1.0,
- dtype=tf.float32,
- random_seed=None,
- train_p=True,
- p_type="unimodal",
- q_type="normal",
- observation_variance=1.0,
- transition_type=STANDARD_TRANSITION,
- use_bs=True):
- if p_type == "unimodal":
- p = P(state_size,
- num_timesteps,
- sigma_min=sigma_min,
- variance=variance,
- dtype=dtype,
- random_seed=random_seed,
- trainable=train_p,
- init_bs_to_zero=not use_bs)
- elif p_type == "bimodal":
- p = BimodalPriorP(
- state_size,
- num_timesteps,
- mixing_coeff=mixing_coeff,
- prior_mode_mean=prior_mode_mean,
- sigma_min=sigma_min,
- variance=variance,
- dtype=dtype,
- random_seed=random_seed,
- trainable=train_p,
- init_bs_to_zero=not use_bs)
- elif "nonlinear" in p_type:
- if "cauchy" in p_type:
- trans_dist = tf.contrib.distributions.Cauchy
- else:
- trans_dist = tf.contrib.distributions.Normal
- p = ShortChainNonlinearP(
- state_size,
- num_timesteps,
- sigma_min=sigma_min,
- variance=variance,
- observation_variance=observation_variance,
- transition_type=transition_type,
- transition_dist=trans_dist,
- dtype=dtype,
- random_seed=random_seed
- )
-
- if q_type == "normal":
- q_class = Q
- elif q_type == "simple_mean":
- q_class = SimpleMeanQ
- elif q_type == "prev_state":
- q_class = PreviousStateQ
- elif q_type == "observation":
- q_class = ObservationQ
-
- q = q_class(state_size,
- num_timesteps,
- sigma_min=sigma_min,
- dtype=dtype,
- random_seed=random_seed,
- init_mu0_to_zero=not use_bs)
- r = R(state_size,
- num_timesteps,
- sigma_min=sigma_min,
- sigma_init=r_sigma_init,
- dtype=dtype,
- random_seed=random_seed)
- model = Model(p, q, r, state_size, num_timesteps, dtype=dtype)
- return model
-
-
-class BackwardsModel(object):
-
- def __init__(self,
- state_size,
- num_timesteps,
- sigma_min=1e-5,
- dtype=tf.float32):
- self.state_size = state_size
- self.num_timesteps = num_timesteps
- self.sigma_min = sigma_min
- self.dtype = dtype
- self.bs = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="b_%d" % (t + 1),
- initializer=tf.zeros_initializer) for t in xrange(num_timesteps)
- ]
- self.Bs = tf.cumsum(self.bs, reverse=True, axis=0)
- self.q_mus = [
- snt.Linear(output_size=state_size) for _ in xrange(num_timesteps)
- ]
- self.q_sigmas = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="q_sigma_%d" % (t + 1),
- initializer=tf.zeros_initializer) for t in xrange(num_timesteps)
- ]
- self.r_mus = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="r_mu_%d" % (t + 1),
- initializer=tf.zeros_initializer) for t in xrange(num_timesteps)
- ]
- self.r_sigmas = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="r_sigma_%d" % (t + 1),
- initializer=tf.zeros_initializer) for t in xrange(num_timesteps)
- ]
-
- def zero_state(self, batch_size):
- return tf.zeros([batch_size, self.state_size], dtype=self.dtype)
-
- def posterior(self, unused_observation, prev_state, unused_t):
- # TODO(dieterichl): Correct this.
- return tf.contrib.distributions.Normal(
- loc=tf.zeros_like(prev_state), scale=tf.zeros_like(prev_state))
-
- def lookahead(self, state, unused_t):
- # TODO(dieterichl): Correct this.
- return tf.contrib.distributions.Normal(
- loc=tf.zeros_like(state), scale=tf.zeros_like(state))
-
- def q_zt(self, observation, next_state, t):
- """Computes the variational posterior q(z_{t}|z_{t+1}, z_n)."""
- t_backwards = self.num_timesteps - t - 1
- batch_size = tf.shape(next_state)[0]
- q_mu = self.q_mus[t_backwards](tf.concat([observation, next_state], axis=1))
- q_sigma = tf.maximum(
- tf.nn.softplus(self.q_sigmas[t_backwards]), self.sigma_min)
- q_sigma = tf.tile(q_sigma[tf.newaxis, :], [batch_size, 1])
- q_zt = tf.contrib.distributions.Normal(loc=q_mu, scale=tf.sqrt(q_sigma))
- return q_zt
-
- def p_zt(self, prev_state, t):
- """Computes the model p(z_{t+1}| z_{t})."""
- t_backwards = self.num_timesteps - t - 1
- z_mu_p = prev_state + self.bs[t_backwards]
- p_zt = tf.contrib.distributions.Normal(
- loc=z_mu_p, scale=tf.ones_like(z_mu_p))
- return p_zt
-
- def generative(self, unused_observation, z_nm1):
- """Computes the model's generative distribution p(z_n| z_{n-1})."""
- generative_p_mu = z_nm1 + self.bs[-1]
- return tf.contrib.distributions.Normal(
- loc=generative_p_mu, scale=tf.ones_like(generative_p_mu))
-
- def r(self, z_t, t):
- t_backwards = self.num_timesteps - t - 1
- batch_size = tf.shape(z_t)[0]
- r_mu = tf.tile(self.r_mus[t_backwards][tf.newaxis, :], [batch_size, 1])
- r_sigma = tf.maximum(
- tf.nn.softplus(self.r_sigmas[t_backwards]), self.sigma_min)
- r_sigma = tf.tile(r_sigma[tf.newaxis, :], [batch_size, 1])
- return tf.contrib.distributions.Normal(loc=r_mu, scale=tf.sqrt(r_sigma))
-
- def likelihood(self, observation):
- batch_size = tf.shape(observation)[0]
- mu = tf.tile(tf.reduce_sum(self.bs, axis=0)[tf.newaxis, :], [batch_size, 1])
- sigma = tf.ones_like(mu) * (self.num_timesteps + 1)
- dist = tf.contrib.distributions.Normal(loc=mu, scale=tf.sqrt(sigma))
- # Average over the batch and take the sum over the state size
- return tf.reduce_mean(tf.reduce_sum(dist.log_prob(observation), axis=1))
-
- def __call__(self, next_state, observation, t):
- # next state = z_{t+1}
- # Compute the q distribution over z, q(z_{t}|z_n, z_{t+1}).
- q_zt = self.q_zt(observation, next_state, t)
- # sample from q
- zt = q_zt.sample()
- # Compute the p distribution over z, p(z_{t+1}|z_{t}).
- p_zt = self.p_zt(zt, t)
- # Compute log p(z_{t+1} | z_t)
- if t == 0:
- log_p_zt = p_zt.log_prob(observation)
- else:
- log_p_zt = p_zt.log_prob(next_state)
-
- # Compute r prior over zt
- r_zt = self.r(zt, t)
- log_r_zt = r_zt.log_prob(zt)
- # Compute proposal density at zt
- log_q_zt = q_zt.log_prob(zt)
- # If we're at the last timestep, also calc the logprob of the observation.
-
- if t == self.num_timesteps - 1:
- p_z0_dist = tf.contrib.distributions.Normal(
- loc=tf.zeros_like(zt), scale=tf.ones_like(zt))
- z0_log_prob = p_z0_dist.log_prob(zt)
- else:
- z0_log_prob = tf.zeros_like(log_q_zt)
- return (zt, log_q_zt, log_p_zt, z0_log_prob, log_r_zt)
-
-
-class LongChainP(object):
-
- def __init__(self,
- state_size,
- num_obs,
- steps_per_obs,
- sigma_min=1e-5,
- variance=1.0,
- observation_variance=1.0,
- observation_type=STANDARD_OBSERVATION,
- transition_type=STANDARD_TRANSITION,
- dtype=tf.float32,
- random_seed=None):
- self.state_size = state_size
- self.steps_per_obs = steps_per_obs
- self.num_obs = num_obs
- self.num_timesteps = steps_per_obs*num_obs + 1
- self.sigma_min = sigma_min
- self.dtype = dtype
- self.variance = variance
- self.observation_variance = observation_variance
- self.observation_type = observation_type
- self.transition_type = transition_type
-
- def likelihood(self, observations):
- """Computes the model's true likelihood of the observations.
-
- Args:
- observations: A [batch_size, m, state_size] Tensor representing each of
- the m observations.
- Returns:
- logprob: The true likelihood of the observations given the model.
- """
- raise ValueError("Likelihood is not defined for long-chain models")
- # batch_size = tf.shape(observations)[0]
- # mu = tf.zeros([batch_size, self.state_size, self.num_obs], dtype=self.dtype)
- # sigma = np.fromfunction(
- # lambda i, j: 1 + self.steps_per_obs*np.minimum(i+1, j+1),
- # [self.num_obs, self.num_obs])
- # sigma += np.eye(self.num_obs)
- # sigma = tf.convert_to_tensor(sigma * self.variance, dtype=self.dtype)
- # sigma = tf.tile(sigma[tf.newaxis, tf.newaxis, ...],
- # [batch_size, self.state_size, 1, 1])
- # dist = tf.contrib.distributions.MultivariateNormalFullCovariance(
- # loc=mu,
- # covariance_matrix=sigma)
- # Average over the batch and take the sum over the state size
- #return tf.reduce_mean(tf.reduce_sum(dist.log_prob(observations), axis=1))
-
- def p_zt(self, prev_state, t):
- """Computes the model p(z_t| z_{t-1})."""
- batch_size = tf.shape(prev_state)[0]
- if t > 0:
- if self.transition_type == ROUND_TRANSITION:
- loc = tf.round(prev_state)
- tf.logging.info("p(z_%d | z_%d) ~ N(round(z_%d), %0.1f)" % (t, t-1, t-1, self.variance))
- elif self.transition_type == STANDARD_TRANSITION:
- loc = prev_state
- tf.logging.info("p(z_%d | z_%d) ~ N(z_%d, %0.1f)" % (t, t-1, t-1, self.variance))
- else: # p(z_0) is Normal(0,1)
- loc = tf.zeros([batch_size, self.state_size], dtype=self.dtype)
- tf.logging.info("p(z_0) ~ N(0,%0.1f)" % self.variance)
-
- p_zt = tf.contrib.distributions.Normal(
- loc=loc,
- scale=tf.sqrt(tf.ones_like(loc) * self.variance))
- return p_zt
-
- def generative(self, z_ni, t):
- """Computes the model's generative distribution p(x_i| z_{ni})."""
- if self.observation_type == SQUARED_OBSERVATION:
- generative_mu = tf.square(z_ni)
- tf.logging.info("p(x_%d | z_%d) ~ N(z_%d^2, %0.1f)" % (t, t, t, self.variance))
- elif self.observation_type == ABS_OBSERVATION:
- generative_mu = tf.abs(z_ni)
- tf.logging.info("p(x_%d | z_%d) ~ N(|z_%d|, %0.1f)" % (t, t, t, self.variance))
- elif self.observation_type == STANDARD_OBSERVATION:
- generative_mu = z_ni
- tf.logging.info("p(x_%d | z_%d) ~ N(z_%d, %0.1f)" % (t, t, t, self.variance))
- generative_sigma_sq = tf.ones_like(generative_mu) * self.observation_variance
- return tf.contrib.distributions.Normal(
- loc=generative_mu, scale=tf.sqrt(generative_sigma_sq))
-
-
-class LongChainQ(object):
-
- def __init__(self,
- state_size,
- num_obs,
- steps_per_obs,
- sigma_min=1e-5,
- dtype=tf.float32,
- random_seed=None):
- self.state_size = state_size
- self.sigma_min = sigma_min
- self.dtype = dtype
- self.steps_per_obs = steps_per_obs
- self.num_obs = num_obs
- self.num_timesteps = num_obs*steps_per_obs +1
-
- initializers = {
- "w": tf.random_uniform_initializer(seed=random_seed),
- "b": tf.zeros_initializer
- }
- self.mus = [
- snt.Linear(output_size=state_size, initializers=initializers)
- for t in xrange(self.num_timesteps)
- ]
- self.sigmas = [
- tf.get_variable(
- shape=[state_size],
- dtype=self.dtype,
- name="q_sigma_%d" % (t + 1),
- initializer=tf.random_uniform_initializer(seed=random_seed))
- for t in xrange(self.num_timesteps)
- ]
-
- def first_relevant_obs_index(self, t):
- return int(max((t-1)/self.steps_per_obs, 0))
-
- def q_zt(self, observations, prev_state, t):
- """Computes a distribution over z_t.
-
- Args:
- observations: a [batch_size, num_observations, state_size] Tensor.
- prev_state: a [batch_size, state_size] Tensor.
- t: The current timestep, an int Tensor.
- """
- # filter out unneeded past obs
- first_relevant_obs_index = int(math.floor(max(t-1, 0) / self.steps_per_obs))
- num_relevant_observations = self.num_obs - first_relevant_obs_index
- observations = observations[:,first_relevant_obs_index:,:]
- batch_size = tf.shape(prev_state)[0]
- # concatenate the prev state and observations along the second axis (that is
- # not the batch or state size axis, and then flatten it to
- # [batch_size, (num_relevant_observations + 1) * state_size] to feed it into
- # the linear layer.
- q_input = tf.concat([observations, prev_state[:,tf.newaxis, :]], axis=1)
- q_input = tf.reshape(q_input,
- [batch_size, (num_relevant_observations + 1) * self.state_size])
- q_mu = self.mus[t](q_input)
- q_sigma = tf.maximum(tf.nn.softplus(self.sigmas[t]), self.sigma_min)
- q_sigma = tf.tile(q_sigma[tf.newaxis, :], [batch_size, 1])
- q_zt = tf.contrib.distributions.Normal(loc=q_mu, scale=tf.sqrt(q_sigma))
- tf.logging.info(
- "q(z_{t} | z_{tm1}, x_{obsf}:{obst}) ~ N(Linear([z_{tm1},x_{obsf}:{obst}]), sigma_{t})".format(
- **{"t": t,
- "tm1": t-1,
- "obsf": (first_relevant_obs_index+1)*self.steps_per_obs,
- "obst":self.steps_per_obs*self.num_obs}))
- return q_zt
-
- def summarize_weights(self):
- pass
-
-class LongChainR(object):
-
- def __init__(self,
- state_size,
- num_obs,
- steps_per_obs,
- sigma_min=1e-5,
- dtype=tf.float32,
- random_seed=None):
- self.state_size = state_size
- self.dtype = dtype
- self.sigma_min = sigma_min
- self.steps_per_obs = steps_per_obs
- self.num_obs = num_obs
- self.num_timesteps = num_obs*steps_per_obs + 1
- self.sigmas = [
- tf.get_variable(
- shape=[self.num_future_obs(t)],
- dtype=self.dtype,
- name="r_sigma_%d" % (t + 1),
- #initializer=tf.random_uniform_initializer(seed=random_seed, maxval=100))
- initializer=tf.constant_initializer(1.0))
- for t in range(self.num_timesteps)
- ]
-
- def first_future_obs_index(self, t):
- return int(math.floor(t / self.steps_per_obs))
-
- def num_future_obs(self, t):
- return int(self.num_obs - self.first_future_obs_index(t))
-
- def r_xn(self, z_t, t):
- """Computes a distribution over the future observations given current latent
- state.
-
- The indexing in these messages is 1 indexed and inclusive. This is
- consistent with the latex documents.
-
- Args:
- z_t: [batch_size, state_size] Tensor
- t: Current timestep
- """
- tf.logging.info(
- "r(x_{start}:{end} | z_{t}) ~ N(z_{t}, sigma_{t})".format(
- **{"t": t,
- "start": (self.first_future_obs_index(t)+1)*self.steps_per_obs,
- "end": self.num_timesteps-1}))
- batch_size = tf.shape(z_t)[0]
- # the mean for all future observations is the same.
- # this tiling results in a [batch_size, num_future_obs, state_size] Tensor
- r_mu = tf.tile(z_t[:,tf.newaxis,:], [1, self.num_future_obs(t), 1])
- # compute the variance
- r_sigma = tf.maximum(tf.nn.softplus(self.sigmas[t]), self.sigma_min)
- # the variance is the same across all state dimensions, so we only have to
- # time sigma to be [batch_size, num_future_obs].
- r_sigma = tf.tile(r_sigma[tf.newaxis,:, tf.newaxis], [batch_size, 1, self.state_size])
- return tf.contrib.distributions.Normal(
- loc=r_mu, scale=tf.sqrt(r_sigma))
-
- def summarize_weights(self):
- pass
-
-
-class LongChainModel(object):
-
- def __init__(self,
- p,
- q,
- r,
- state_size,
- num_obs,
- steps_per_obs,
- dtype=tf.float32,
- disable_r=False):
- self.p = p
- self.q = q
- self.r = r
- self.disable_r = disable_r
- self.state_size = state_size
- self.num_obs = num_obs
- self.steps_per_obs = steps_per_obs
- self.num_timesteps = steps_per_obs*num_obs + 1
- self.dtype = dtype
-
- def zero_state(self, batch_size):
- return tf.zeros([batch_size, self.state_size], dtype=self.dtype)
-
- def next_obs_ind(self, t):
- return int(math.floor(max(t-1,0)/self.steps_per_obs))
-
- def __call__(self, prev_state, observations, t):
- """Computes the importance weight for the model system.
-
- Args:
- prev_state: [batch_size, state_size] Tensor
- observations: [batch_size, num_observations, state_size] Tensor
- """
- # Compute the q distribution over z, q(z_t|z_n, z_{t-1}).
- q_zt = self.q.q_zt(observations, prev_state, t)
- # Compute the p distribution over z, p(z_t|z_{t-1}).
- p_zt = self.p.p_zt(prev_state, t)
- # sample from q and evaluate the logprobs, summing over the state size
- zt = q_zt.sample()
- log_q_zt = tf.reduce_sum(q_zt.log_prob(zt), axis=1)
- log_p_zt = tf.reduce_sum(p_zt.log_prob(zt), axis=1)
- if not self.disable_r and t < self.num_timesteps-1:
- # score the remaining observations using r
- r_xn = self.r.r_xn(zt, t)
- log_r_xn = r_xn.log_prob(observations[:, self.next_obs_ind(t+1):, :])
- # sum over state size and observation, leaving the batch index
- log_r_xn = tf.reduce_sum(log_r_xn, axis=[1,2])
- else:
- log_r_xn = tf.zeros_like(log_p_zt)
- if t != 0 and t % self.steps_per_obs == 0:
- generative_dist = self.p.generative(zt, t)
- log_p_x_given_z = generative_dist.log_prob(observations[:,self.next_obs_ind(t),:])
- log_p_x_given_z = tf.reduce_sum(log_p_x_given_z, axis=1)
- else:
- log_p_x_given_z = tf.zeros_like(log_q_zt)
- return (zt, log_q_zt, log_p_zt, log_p_x_given_z, log_r_xn)
-
- @staticmethod
- def create(state_size,
- num_obs,
- steps_per_obs,
- sigma_min=1e-5,
- variance=1.0,
- observation_variance=1.0,
- observation_type=STANDARD_OBSERVATION,
- transition_type=STANDARD_TRANSITION,
- dtype=tf.float32,
- random_seed=None,
- disable_r=False):
- p = LongChainP(
- state_size,
- num_obs,
- steps_per_obs,
- sigma_min=sigma_min,
- variance=variance,
- observation_variance=observation_variance,
- observation_type=observation_type,
- transition_type=transition_type,
- dtype=dtype,
- random_seed=random_seed)
- q = LongChainQ(
- state_size,
- num_obs,
- steps_per_obs,
- sigma_min=sigma_min,
- dtype=dtype,
- random_seed=random_seed)
- r = LongChainR(
- state_size,
- num_obs,
- steps_per_obs,
- sigma_min=sigma_min,
- dtype=dtype,
- random_seed=random_seed)
- model = LongChainModel(
- p, q, r, state_size, num_obs, steps_per_obs,
- dtype=dtype,
- disable_r=disable_r)
- return model
-
-
-class RTilde(object):
-
- def __init__(self,
- state_size,
- num_timesteps,
- sigma_min=1e-5,
- dtype=tf.float32,
- random_seed=None,
- graph_collection_name="R_TILDE_VARS"):
- self.dtype = dtype
- self.sigma_min = sigma_min
- initializers = {"w": tf.truncated_normal_initializer(seed=random_seed),
- "b": tf.zeros_initializer}
- self.graph_collection_name=graph_collection_name
-
- def custom_getter(getter, *args, **kwargs):
- out = getter(*args, **kwargs)
- ref = tf.get_collection_ref(self.graph_collection_name)
- if out not in ref:
- ref.append(out)
- return out
-
- self.fns = [
- snt.Linear(output_size=2*state_size,
- initializers=initializers,
- name="r_tilde_%d" % t,
- custom_getter=custom_getter)
- for t in xrange(num_timesteps)
- ]
-
- def r_zt(self, z_t, observation, t):
- #out = self.fns[t](tf.stop_gradient(tf.concat([z_t, observation], axis=1)))
- out = self.fns[t](tf.concat([z_t, observation], axis=1))
- mu, raw_sigma_sq = tf.split(out, 2, axis=1)
- sigma_sq = tf.maximum(tf.nn.softplus(raw_sigma_sq), self.sigma_min)
- return mu, sigma_sq
-
-class TDModel(object):
-
- def __init__(self,
- p,
- q,
- r_tilde,
- state_size,
- num_timesteps,
- dtype=tf.float32,
- disable_r=False):
- self.p = p
- self.q = q
- self.r_tilde = r_tilde
- self.disable_r = disable_r
- self.state_size = state_size
- self.num_timesteps = num_timesteps
- self.dtype = dtype
-
- def zero_state(self, batch_size):
- return tf.zeros([batch_size, self.state_size], dtype=self.dtype)
-
- def __call__(self, prev_state, observation, t):
- """Computes the importance weight for the model system.
-
- Args:
- prev_state: [batch_size, state_size] Tensor
- observations: [batch_size, num_observations, state_size] Tensor
- """
- # Compute the q distribution over z, q(z_t|z_n, z_{t-1}).
- q_zt = self.q.q_zt(observation, prev_state, t)
- # Compute the p distribution over z, p(z_t|z_{t-1}).
- p_zt = self.p.p_zt(prev_state, t)
- # sample from q and evaluate the logprobs, summing over the state size
- zt = q_zt.sample()
- # If it isn't the last timestep, compute the distribution over the next z.
- if t < self.num_timesteps - 1:
- p_ztplus1 = self.p.p_zt(zt, t+1)
- else:
- p_ztplus1 = None
- log_q_zt = tf.reduce_sum(q_zt.log_prob(zt), axis=1)
- log_p_zt = tf.reduce_sum(p_zt.log_prob(zt), axis=1)
-
- if not self.disable_r and t < self.num_timesteps-1:
- # score the remaining observations using r
- r_tilde_mu, r_tilde_sigma_sq = self.r_tilde.r_zt(zt, observation, t+1)
- else:
- r_tilde_mu = None
- r_tilde_sigma_sq = None
- if t == self.num_timesteps - 1:
- generative_dist = self.p.generative(observation, zt)
- log_p_x_given_z = tf.reduce_sum(generative_dist.log_prob(observation), axis=1)
- else:
- log_p_x_given_z = tf.zeros_like(log_q_zt)
- return (zt, log_q_zt, log_p_zt, log_p_x_given_z,
- r_tilde_mu, r_tilde_sigma_sq, p_ztplus1)
-
- @staticmethod
- def create(state_size,
- num_timesteps,
- sigma_min=1e-5,
- variance=1.0,
- dtype=tf.float32,
- random_seed=None,
- train_p=True,
- p_type="unimodal",
- q_type="normal",
- mixing_coeff=0.5,
- prior_mode_mean=1.0,
- observation_variance=1.0,
- transition_type=STANDARD_TRANSITION,
- use_bs=True):
- if p_type == "unimodal":
- p = P(state_size,
- num_timesteps,
- sigma_min=sigma_min,
- variance=variance,
- dtype=dtype,
- random_seed=random_seed,
- trainable=train_p,
- init_bs_to_zero=not use_bs)
- elif p_type == "bimodal":
- p = BimodalPriorP(
- state_size,
- num_timesteps,
- mixing_coeff=mixing_coeff,
- prior_mode_mean=prior_mode_mean,
- sigma_min=sigma_min,
- variance=variance,
- dtype=dtype,
- random_seed=random_seed,
- trainable=train_p,
- init_bs_to_zero=not use_bs)
- elif "nonlinear" in p_type:
- if "cauchy" in p_type:
- trans_dist = tf.contrib.distributions.Cauchy
- else:
- trans_dist = tf.contrib.distributions.Normal
-
- p = ShortChainNonlinearP(
- state_size,
- num_timesteps,
- sigma_min=sigma_min,
- variance=variance,
- observation_variance=observation_variance,
- transition_type=transition_type,
- transition_dist=trans_dist,
- dtype=dtype,
- random_seed=random_seed
- )
-
- if q_type == "normal":
- q_class = Q
- elif q_type == "simple_mean":
- q_class = SimpleMeanQ
- elif q_type == "prev_state":
- q_class = PreviousStateQ
- elif q_type == "observation":
- q_class = ObservationQ
-
- q = q_class(state_size,
- num_timesteps,
- sigma_min=sigma_min,
- dtype=dtype,
- random_seed=random_seed,
- init_mu0_to_zero=not use_bs)
- r_tilde = RTilde(
- state_size,
- num_timesteps,
- sigma_min=sigma_min,
- dtype=dtype,
- random_seed=random_seed)
- model = TDModel(p, q, r_tilde, state_size, num_timesteps, dtype=dtype)
- return model
diff --git a/research/fivo/experimental/run.sh b/research/fivo/experimental/run.sh
deleted file mode 100644
index c650f636d5313a196960a92b509202b47e7da518..0000000000000000000000000000000000000000
--- a/research/fivo/experimental/run.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-model="forward"
-T=5
-num_obs=1
-var=0.1
-n=4
-lr=0.0001
-bound="fivo-aux"
-q_type="normal"
-resampling_method="multinomial"
-rgrad="true"
-p_type="unimodal"
-use_bs=false
-
-LOGDIR=/tmp/fivo/model-$model-$bound-$resampling_method-resampling-rgrad-$rgrad-T-$T-var-$var-n-$n-lr-$lr-q-$q_type-p-$p_type
-
-python train.py \
- --logdir=$LOGDIR \
- --model=$model \
- --bound=$bound \
- --q_type=$q_type \
- --p_type=$p_type \
- --variance=$var \
- --use_resampling_grads=$rgrad \
- --resampling=always \
- --resampling_method=$resampling_method \
- --batch_size=4 \
- --num_samples=$n \
- --num_timesteps=$T \
- --num_eval_samples=256 \
- --summarize_every=100 \
- --learning_rate=$lr \
- --decay_steps=1000000 \
- --max_steps=1000000000 \
- --random_seed=1234 \
- --train_p=false \
- --use_bs=$use_bs \
- --alsologtostderr
diff --git a/research/fivo/experimental/summary_utils.py b/research/fivo/experimental/summary_utils.py
deleted file mode 100644
index 04e4aeea257577e60d3651656d0c62355d501ea8..0000000000000000000000000000000000000000
--- a/research/fivo/experimental/summary_utils.py
+++ /dev/null
@@ -1,332 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utils for plotting and summarizing.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import matplotlib.gridspec as gridspec
-import matplotlib.pyplot as plt
-import numpy as np
-import scipy
-
-import tensorflow as tf
-
-import models
-
-
-def summarize_ess(weights, only_last_timestep=False):
- """Plots the effective sample size.
-
- Args:
- weights: List of length num_timesteps Tensors of shape
- [num_samples, batch_size]
- """
- num_timesteps = len(weights)
- batch_size = tf.cast(tf.shape(weights[0])[1], dtype=tf.float64)
- for i in range(num_timesteps):
- if only_last_timestep and i < num_timesteps-1: continue
-
- w = tf.nn.softmax(weights[i], dim=0)
- centered_weights = w - tf.reduce_mean(w, axis=0, keepdims=True)
- variance = tf.reduce_sum(tf.square(centered_weights))/(batch_size-1)
- ess = 1./tf.reduce_mean(tf.reduce_sum(tf.square(w), axis=0))
- tf.summary.scalar("ess/%d" % i, ess)
- tf.summary.scalar("ese/%d" % i, ess / batch_size)
- tf.summary.scalar("weight_variance/%d" % i, variance)
-
-
-def summarize_particles(states, weights, observation, model):
- """Plots particle locations and weights.
-
- Args:
- states: List of length num_timesteps Tensors of shape
- [batch_size*num_particles, state_size].
- weights: List of length num_timesteps Tensors of shape [num_samples,
- batch_size]
- observation: Tensor of shape [batch_size*num_samples, state_size]
- """
- num_timesteps = len(weights)
- num_samples, batch_size = weights[0].get_shape().as_list()
- # get q0 information for plotting
- q0_dist = model.q.q_zt(observation, tf.zeros_like(states[0]), 0)
- q0_loc = q0_dist.loc[0:batch_size, 0]
- q0_scale = q0_dist.scale[0:batch_size, 0]
- # get posterior information for plotting
- post = (model.p.mixing_coeff, model.p.prior_mode_mean, model.p.variance,
- tf.reduce_sum(model.p.bs), model.p.num_timesteps)
-
- # Reshape states and weights to be [time, num_samples, batch_size]
- states = tf.stack(states)
- weights = tf.stack(weights)
- # normalize the weights over the sample dimension
- weights = tf.nn.softmax(weights, dim=1)
- states = tf.reshape(states, tf.shape(weights))
-
- ess = 1./tf.reduce_sum(tf.square(weights), axis=1)
-
- def _plot_states(states_batch, weights_batch, observation_batch, ess_batch, q0, post):
- """
- states: [time, num_samples, batch_size]
- weights [time, num_samples, batch_size]
- observation: [batch_size, 1]
- q0: ([batch_size], [batch_size])
- post: ...
- """
- num_timesteps, _, batch_size = states_batch.shape
- plots = []
- for i in range(batch_size):
- states = states_batch[:,:,i]
- weights = weights_batch[:,:,i]
- observation = observation_batch[i]
- ess = ess_batch[:,i]
- q0_loc = q0[0][i]
- q0_scale = q0[1][i]
-
- fig = plt.figure(figsize=(7, (num_timesteps + 1) * 2))
- # Each timestep gets two plots -- a bar plot and a histogram of state locs.
- # The bar plot will be bar_rows rows tall.
- # The histogram will be 1 row tall.
- # There is also 1 extra plot at the top showing the posterior and q.
- bar_rows = 8
- num_rows = (num_timesteps + 1) * (bar_rows + 1)
- gs = gridspec.GridSpec(num_rows, 1)
-
- # Figure out how wide to make the plot
- prior_lims = (post[1] * -2, post[1] * 2)
- q_lims = (scipy.stats.norm.ppf(0.01, loc=q0_loc, scale=q0_scale),
- scipy.stats.norm.ppf(0.99, loc=q0_loc, scale=q0_scale))
- state_width = states.max() - states.min()
- state_lims = (states.min() - state_width * 0.15,
- states.max() + state_width * 0.15)
-
- lims = (min(prior_lims[0], q_lims[0], state_lims[0]),
- max(prior_lims[1], q_lims[1], state_lims[1]))
- # plot the posterior
- z0 = np.arange(lims[0], lims[1], 0.1)
- alpha, pos_mu, sigma_sq, B, T = post
- neg_mu = -pos_mu
- scale = np.sqrt((T + 1) * sigma_sq)
- p_zn = (
- alpha * scipy.stats.norm.pdf(
- observation, loc=pos_mu + B, scale=scale) + (1 - alpha) *
- scipy.stats.norm.pdf(observation, loc=neg_mu + B, scale=scale))
- p_z0 = (
- alpha * scipy.stats.norm.pdf(z0, loc=pos_mu, scale=np.sqrt(sigma_sq))
- + (1 - alpha) * scipy.stats.norm.pdf(
- z0, loc=neg_mu, scale=np.sqrt(sigma_sq)))
- p_zn_given_z0 = scipy.stats.norm.pdf(
- observation, loc=z0 + B, scale=np.sqrt(T * sigma_sq))
- post_z0 = (p_z0 * p_zn_given_z0) / p_zn
- # plot q
- q_z0 = scipy.stats.norm.pdf(z0, loc=q0_loc, scale=q0_scale)
- ax = plt.subplot(gs[0:bar_rows, :])
- ax.plot(z0, q_z0, color="blue")
- ax.plot(z0, post_z0, color="green")
- ax.plot(z0, p_z0, color="red")
- ax.legend(("q", "posterior", "prior"), loc="best", prop={"size": 10})
-
- ax.set_xticks([])
- ax.set_xlim(*lims)
-
- # plot the states
- for t in range(num_timesteps):
- start = (t + 1) * (bar_rows + 1)
- ax1 = plt.subplot(gs[start:start + bar_rows, :])
- ax2 = plt.subplot(gs[start + bar_rows:start + bar_rows + 1, :])
- # plot the states barplot
- # ax1.hist(
- # states[t, :],
- # weights=weights[t, :],
- # bins=50,
- # edgecolor="none",
- # alpha=0.2)
- ax1.bar(states[t,:], weights[t,:], width=0.02, alpha=0.2, edgecolor = "none")
- ax1.set_ylabel("t=%d" % t)
- ax1.set_xticks([])
- ax1.grid(True, which="both")
- ax1.set_xlim(*lims)
- # plot the observation
- ax1.axvline(x=observation, color="red", linestyle="dashed")
- # add the ESS
- ax1.text(0.1, 0.9, "ESS: %0.2f" % ess[t],
- ha='center', va='center', transform=ax1.transAxes)
-
- # plot the state location histogram
- ax2.hist2d(
- states[t, :], np.zeros_like(states[t, :]), bins=[50, 1], cmap="Greys")
- ax2.grid(False)
- ax2.set_yticks([])
- ax2.set_xlim(*lims)
- if t != num_timesteps - 1:
- ax2.set_xticks([])
-
- fig.canvas.draw()
- p = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
- plots.append(p.reshape(fig.canvas.get_width_height()[::-1] + (3,)))
- plt.close(fig)
- return np.stack(plots)
-
- plots = tf.py_func(_plot_states,
- [states, weights, observation, ess, (q0_loc, q0_scale), post],
- [tf.uint8])[0]
- tf.summary.image("states", plots, 5, collections=["infrequent_summaries"])
-
-
-def plot_weights(weights, resampled=None):
- """Plots the weights and effective sample size from an SMC rollout.
-
- Args:
- weights: [num_timesteps, num_samples, batch_size] importance weights
- resampled: [num_timesteps] 0/1 indicating if resampling ocurred
- """
- weights = tf.convert_to_tensor(weights)
-
- def _make_plots(weights, resampled):
- num_timesteps, num_samples, batch_size = weights.shape
- plots = []
- for i in range(batch_size):
- fig, axes = plt.subplots(nrows=1, sharex=True, figsize=(8, 4))
- axes.stackplot(np.arange(num_timesteps), np.transpose(weights[:, :, i]))
- axes.set_title("Weights")
- axes.set_xlabel("Steps")
- axes.set_ylim([0, 1])
- axes.set_xlim([0, num_timesteps - 1])
- for j in np.where(resampled > 0)[0]:
- axes.axvline(x=j, color="red", linestyle="dashed", ymin=0.0, ymax=1.0)
- fig.canvas.draw()
- data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
- data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
- plots.append(data)
- plt.close(fig)
- return np.stack(plots, axis=0)
-
- if resampled is None:
- num_timesteps, _, batch_size = weights.get_shape().as_list()
- resampled = tf.zeros([num_timesteps], dtype=tf.float32)
- plots = tf.py_func(_make_plots,
- [tf.nn.softmax(weights, dim=1),
- tf.to_float(resampled)], [tf.uint8])[0]
- batch_size = weights.get_shape().as_list()[-1]
- tf.summary.image(
- "weights", plots, batch_size, collections=["infrequent_summaries"])
-
-
-def summarize_weights(weights, num_timesteps, num_samples):
- # weights is [num_timesteps, num_samples, batch_size]
- weights = tf.convert_to_tensor(weights)
- mean = tf.reduce_mean(weights, axis=1, keepdims=True)
- squared_diff = tf.square(weights - mean)
- variances = tf.reduce_sum(squared_diff, axis=1) / (num_samples - 1)
- # average the variance over the batch
- variances = tf.reduce_mean(variances, axis=1)
- avg_magnitude = tf.reduce_mean(tf.abs(weights), axis=[1, 2])
- for t in xrange(num_timesteps):
- tf.summary.scalar("weights/variance_%d" % t, variances[t])
- tf.summary.scalar("weights/magnitude_%d" % t, avg_magnitude[t])
- tf.summary.histogram("weights/step_%d" % t, weights[t])
-
-
-def summarize_learning_signal(rewards, tag):
- num_resampling_events, _ = rewards.get_shape().as_list()
- mean = tf.reduce_mean(rewards, axis=1)
- avg_magnitude = tf.reduce_mean(tf.abs(rewards), axis=1)
- reward_square = tf.reduce_mean(tf.square(rewards), axis=1)
- for t in xrange(num_resampling_events):
- tf.summary.scalar("%s/mean_%d" % (tag, t), mean[t])
- tf.summary.scalar("%s/magnitude_%d" % (tag, t), avg_magnitude[t])
- tf.summary.scalar("%s/squared_%d" % (tag, t), reward_square[t])
- tf.summary.histogram("%s/step_%d" % (tag, t), rewards[t])
-
-
-def summarize_qs(model, observation, states):
- model.q.summarize_weights()
- if hasattr(model.p, "posterior") and callable(getattr(model.p, "posterior")):
- states = [tf.zeros_like(states[0])] + states[:-1]
- for t, prev_state in enumerate(states):
- p = model.p.posterior(observation, prev_state, t)
- q = model.q.q_zt(observation, prev_state, t)
- kl = tf.reduce_mean(tf.contrib.distributions.kl_divergence(p, q))
- tf.summary.scalar("kl_q/%d" % t, tf.reduce_mean(kl))
- mean_diff = q.loc - p.loc
- mean_abs_err = tf.abs(mean_diff)
- mean_rel_err = tf.abs(mean_diff / p.loc)
- tf.summary.scalar("q_mean_convergence/absolute_error_%d" % t,
- tf.reduce_mean(mean_abs_err))
- tf.summary.scalar("q_mean_convergence/relative_error_%d" % t,
- tf.reduce_mean(mean_rel_err))
- sigma_diff = tf.square(q.scale) - tf.square(p.scale)
- sigma_abs_err = tf.abs(sigma_diff)
- sigma_rel_err = tf.abs(sigma_diff / tf.square(p.scale))
- tf.summary.scalar("q_variance_convergence/absolute_error_%d" % t,
- tf.reduce_mean(sigma_abs_err))
- tf.summary.scalar("q_variance_convergence/relative_error_%d" % t,
- tf.reduce_mean(sigma_rel_err))
-
-
-def summarize_rs(model, states):
- model.r.summarize_weights()
- for t, state in enumerate(states):
- true_r = model.p.lookahead(state, t)
- r = model.r.r_xn(state, t)
- kl = tf.reduce_mean(tf.contrib.distributions.kl_divergence(true_r, r))
- tf.summary.scalar("kl_r/%d" % t, tf.reduce_mean(kl))
- mean_diff = true_r.loc - r.loc
- mean_abs_err = tf.abs(mean_diff)
- mean_rel_err = tf.abs(mean_diff / true_r.loc)
- tf.summary.scalar("r_mean_convergence/absolute_error_%d" % t,
- tf.reduce_mean(mean_abs_err))
- tf.summary.scalar("r_mean_convergence/relative_error_%d" % t,
- tf.reduce_mean(mean_rel_err))
- sigma_diff = tf.square(r.scale) - tf.square(true_r.scale)
- sigma_abs_err = tf.abs(sigma_diff)
- sigma_rel_err = tf.abs(sigma_diff / tf.square(true_r.scale))
- tf.summary.scalar("r_variance_convergence/absolute_error_%d" % t,
- tf.reduce_mean(sigma_abs_err))
- tf.summary.scalar("r_variance_convergence/relative_error_%d" % t,
- tf.reduce_mean(sigma_rel_err))
-
-
-def summarize_model(model, true_bs, observation, states, bound, summarize_r=True):
- if hasattr(model.p, "bs"):
- model_b = tf.reduce_sum(model.p.bs, axis=0)
- true_b = tf.reduce_sum(true_bs, axis=0)
- abs_err = tf.abs(model_b - true_b)
- rel_err = abs_err / true_b
- tf.summary.scalar("sum_of_bs/data_generating_process", tf.reduce_mean(true_b))
- tf.summary.scalar("sum_of_bs/model", tf.reduce_mean(model_b))
- tf.summary.scalar("sum_of_bs/absolute_error", tf.reduce_mean(abs_err))
- tf.summary.scalar("sum_of_bs/relative_error", tf.reduce_mean(rel_err))
- #summarize_qs(model, observation, states)
- #if bound == "fivo-aux" and summarize_r:
- # summarize_rs(model, states)
-
-
-def summarize_grads(grads, loss_name):
- grad_ema = tf.train.ExponentialMovingAverage(decay=0.99)
- vectorized_grads = tf.concat(
- [tf.reshape(g, [-1]) for g, _ in grads if g is not None], axis=0)
- new_second_moments = tf.square(vectorized_grads)
- new_first_moments = vectorized_grads
- maintain_grad_ema_op = grad_ema.apply([new_first_moments, new_second_moments])
- first_moments = grad_ema.average(new_first_moments)
- second_moments = grad_ema.average(new_second_moments)
- variances = second_moments - tf.square(first_moments)
- tf.summary.scalar("grad_variance/%s" % loss_name, tf.reduce_mean(variances))
- tf.summary.histogram("grad_variance/%s" % loss_name, variances)
- tf.summary.histogram("grad_mean/%s" % loss_name, first_moments)
- return maintain_grad_ema_op
diff --git a/research/fivo/experimental/train.py b/research/fivo/experimental/train.py
deleted file mode 100644
index 8abc9909b115298a30151a332d340f7b25e3cf90..0000000000000000000000000000000000000000
--- a/research/fivo/experimental/train.py
+++ /dev/null
@@ -1,637 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Main script for running fivo"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import defaultdict
-
-import numpy as np
-import tensorflow as tf
-
-import bounds
-import data
-import models
-import summary_utils as summ
-
-tf.logging.set_verbosity(tf.logging.INFO)
-
-tf.app.flags.DEFINE_integer("random_seed", None,
- "A random seed for the data generating process. Same seed "
- "-> same data generating process and initialization.")
-tf.app.flags.DEFINE_enum("bound", "fivo", ["iwae", "fivo", "fivo-aux", "fivo-aux-td"],
- "The bound to optimize.")
-tf.app.flags.DEFINE_enum("model", "forward", ["forward", "long_chain"],
- "The model to use.")
-tf.app.flags.DEFINE_enum("q_type", "normal",
- ["normal", "simple_mean", "prev_state", "observation"],
- "The parameterization to use for q")
-tf.app.flags.DEFINE_enum("p_type", "unimodal", ["unimodal", "bimodal", "nonlinear"],
- "The type of prior.")
-tf.app.flags.DEFINE_boolean("train_p", True,
- "If false, do not train the model p.")
-
-tf.app.flags.DEFINE_integer("state_size", 1,
- "The dimensionality of the state space.")
-tf.app.flags.DEFINE_float("variance", 1.0,
- "The variance of the data generating process.")
-
-tf.app.flags.DEFINE_boolean("use_bs", True,
- "If False, initialize all bs to 0.")
-tf.app.flags.DEFINE_float("bimodal_prior_weight", 0.5,
- "The weight assigned to the positive mode of the prior in "
- "both the data generating process and p.")
-tf.app.flags.DEFINE_float("bimodal_prior_mean", None,
- "If supplied, sets the mean of the 2 modes of the prior to "
- "be 1 and -1 times the supplied value. This is for both the "
- "data generating process and p.")
-tf.app.flags.DEFINE_float("fixed_observation", None,
- "If supplied, fix the observation to a constant value in the"
- " data generating process only.")
-tf.app.flags.DEFINE_float("r_sigma_init", 1.,
- "Value to initialize variance of r to.")
-tf.app.flags.DEFINE_enum("observation_type",
- models.STANDARD_OBSERVATION, models.OBSERVATION_TYPES,
- "The type of observation for the long chain model.")
-tf.app.flags.DEFINE_enum("transition_type",
- models.STANDARD_TRANSITION, models.TRANSITION_TYPES,
- "The type of transition for the long chain model.")
-tf.app.flags.DEFINE_float("observation_variance", None,
- "The variance of the observation. Defaults to 'variance'")
-
-tf.app.flags.DEFINE_integer("num_timesteps", 5,
- "Number of timesteps in the sequence.")
-tf.app.flags.DEFINE_integer("num_observations", 1,
- "The number of observations.")
-tf.app.flags.DEFINE_integer("steps_per_observation", 5,
- "The number of timesteps between each observation.")
-
-tf.app.flags.DEFINE_integer("batch_size", 4,
- "The number of examples per batch.")
-tf.app.flags.DEFINE_integer("num_samples", 4,
- "The number particles to use.")
-tf.app.flags.DEFINE_integer("num_eval_samples", 512,
- "The batch size and # of particles to use for eval.")
-
-tf.app.flags.DEFINE_string("resampling", "always",
- "How to resample. Accepts 'always','never', or a "
- "comma-separated list of booleans like 'true,true,false'.")
-tf.app.flags.DEFINE_enum("resampling_method", "multinomial", ["multinomial",
- "stratified",
- "systematic",
- "relaxed-logblend",
- "relaxed-stateblend",
- "relaxed-linearblend",
- "relaxed-stateblend-st",],
- "Type of resampling method to use.")
-tf.app.flags.DEFINE_boolean("use_resampling_grads", True,
- "Whether or not to use resampling grads to optimize FIVO."
- "Disabled automatically if resampling_method=relaxed.")
-tf.app.flags.DEFINE_boolean("disable_r", False,
- "If false, r is not used for fivo-aux and is set to zeros.")
-
-tf.app.flags.DEFINE_float("learning_rate", 1e-4,
- "The learning rate to use for ADAM or SGD.")
-tf.app.flags.DEFINE_integer("decay_steps", 25000,
- "The number of steps before the learning rate is halved.")
-tf.app.flags.DEFINE_integer("max_steps", int(1e6),
- "The number of steps to run training for.")
-
-tf.app.flags.DEFINE_string("logdir", "/tmp/fivo-aux",
- "Directory for summaries and checkpoints.")
-
-tf.app.flags.DEFINE_integer("summarize_every", int(1e3),
- "The number of steps between each evaluation.")
-FLAGS = tf.app.flags.FLAGS
-
-
-def combine_grad_lists(grad_lists):
- # grads is num_losses by num_variables.
- # each list could have different variables.
- # for each variable, sum the grads across all losses.
- grads_dict = defaultdict(list)
- var_dict = {}
- for grad_list in grad_lists:
- for grad, var in grad_list:
- if grad is not None:
- grads_dict[var.name].append(grad)
- var_dict[var.name] = var
-
- final_grads = []
- for var_name, var in var_dict.iteritems():
- grads = grads_dict[var_name]
- if len(grads) > 0:
- tf.logging.info("Var %s has combined grads from %s." %
- (var_name, [g.name for g in grads]))
- grad = tf.reduce_sum(grads, axis=0)
- else:
- tf.logging.info("Var %s has no grads" % var_name)
- grad = None
- final_grads.append((grad, var))
- return final_grads
-
-
-def make_apply_grads_op(losses, global_step, learning_rate, lr_decay_steps):
- for l in losses:
- assert isinstance(l, bounds.Loss)
-
- lr = tf.train.exponential_decay(
- learning_rate, global_step, lr_decay_steps, 0.5, staircase=False)
- tf.summary.scalar("learning_rate", lr)
- opt = tf.train.AdamOptimizer(lr)
-
- ema_ops = []
- grads = []
- for loss_name, loss, loss_var_collection in losses:
- tf.logging.info("Computing grads of %s w.r.t. vars in collection %s" %
- (loss_name, loss_var_collection))
- g = opt.compute_gradients(loss,
- var_list=tf.get_collection(loss_var_collection))
- ema_ops.append(summ.summarize_grads(g, loss_name))
- grads.append(g)
-
- all_grads = combine_grad_lists(grads)
- apply_grads_op = opt.apply_gradients(all_grads, global_step=global_step)
-
- # Update the emas after applying the grads.
- with tf.control_dependencies([apply_grads_op]):
- train_op = tf.group(*ema_ops)
- return train_op
-
-
-def add_check_numerics_ops():
- check_op = []
- for op in tf.get_default_graph().get_operations():
- bad = ["logits/Log", "sample/Reshape", "log_prob/mul",
- "log_prob/SparseSoftmaxCrossEntropyWithLogits/Reshape",
- "entropy/Reshape", "entropy/LogSoftmax", "Categorical", "Mean"]
- if all([x not in op.name for x in bad]):
- for output in op.outputs:
- if output.dtype in [tf.float16, tf.float32, tf.float64]:
- if op._get_control_flow_context() is not None: # pylint: disable=protected-access
- raise ValueError("`tf.add_check_numerics_ops() is not compatible "
- "with TensorFlow control flow operations such as "
- "`tf.cond()` or `tf.while_loop()`.")
-
- message = op.name + ":" + str(output.value_index)
- with tf.control_dependencies(check_op):
- check_op = [tf.check_numerics(output, message=message)]
- return tf.group(*check_op)
-
-
-def create_long_chain_graph(bound, state_size, num_obs, steps_per_obs,
- batch_size, num_samples, num_eval_samples,
- resampling_schedule, use_resampling_grads,
- learning_rate, lr_decay_steps, dtype="float64"):
- num_timesteps = num_obs * steps_per_obs + 1
- # Make the dataset.
- dataset = data.make_long_chain_dataset(
- state_size=state_size,
- num_obs=num_obs,
- steps_per_obs=steps_per_obs,
- batch_size=batch_size,
- num_samples=num_samples,
- variance=FLAGS.variance,
- observation_variance=FLAGS.observation_variance,
- dtype=dtype,
- observation_type=FLAGS.observation_type,
- transition_type=FLAGS.transition_type,
- fixed_observation=FLAGS.fixed_observation)
- itr = dataset.make_one_shot_iterator()
- _, observations = itr.get_next()
- # Make the dataset for eval
- eval_dataset = data.make_long_chain_dataset(
- state_size=state_size,
- num_obs=num_obs,
- steps_per_obs=steps_per_obs,
- batch_size=batch_size,
- num_samples=num_eval_samples,
- variance=FLAGS.variance,
- observation_variance=FLAGS.observation_variance,
- dtype=dtype,
- observation_type=FLAGS.observation_type,
- transition_type=FLAGS.transition_type,
- fixed_observation=FLAGS.fixed_observation)
- eval_itr = eval_dataset.make_one_shot_iterator()
- _, eval_observations = eval_itr.get_next()
-
- # Make the model.
- model = models.LongChainModel.create(
- state_size,
- num_obs,
- steps_per_obs,
- observation_type=FLAGS.observation_type,
- transition_type=FLAGS.transition_type,
- variance=FLAGS.variance,
- observation_variance=FLAGS.observation_variance,
- dtype=tf.as_dtype(dtype),
- disable_r=FLAGS.disable_r)
-
- # Compute the bound and loss
- if bound == "iwae":
- (_, losses, ema_op, _, _) = bounds.iwae(
- model,
- observations,
- num_timesteps,
- num_samples=num_samples)
- (eval_log_p_hat, _, _, _, eval_log_weights) = bounds.iwae(
- model,
- eval_observations,
- num_timesteps,
- num_samples=num_eval_samples,
- summarize=False)
- eval_log_p_hat = tf.reduce_mean(eval_log_p_hat)
- elif bound == "fivo" or "fivo-aux":
- (_, losses, ema_op, _, _) = bounds.fivo(
- model,
- observations,
- num_timesteps,
- resampling_schedule=resampling_schedule,
- use_resampling_grads=use_resampling_grads,
- resampling_type=FLAGS.resampling_method,
- aux=("aux" in bound),
- num_samples=num_samples)
- (eval_log_p_hat, _, _, _, eval_log_weights) = bounds.fivo(
- model,
- eval_observations,
- num_timesteps,
- resampling_schedule=resampling_schedule,
- use_resampling_grads=False,
- resampling_type="multinomial",
- aux=("aux" in bound),
- num_samples=num_eval_samples,
- summarize=False)
- eval_log_p_hat = tf.reduce_mean(eval_log_p_hat)
-
- summ.summarize_ess(eval_log_weights, only_last_timestep=True)
-
- tf.summary.scalar("log_p_hat", eval_log_p_hat)
-
- # Compute and apply grads.
- global_step = tf.train.get_or_create_global_step()
-
- apply_grads = make_apply_grads_op(losses,
- global_step,
- learning_rate,
- lr_decay_steps)
-
- # Update the emas after applying the grads.
- with tf.control_dependencies([apply_grads]):
- train_op = tf.group(ema_op)
-
- # We can't calculate the likelihood for most of these models
- # so we just return zeros.
- eval_likelihood = tf.zeros([], dtype=dtype)
- return global_step, train_op, eval_log_p_hat, eval_likelihood
-
-
-def create_graph(bound, state_size, num_timesteps, batch_size,
- num_samples, num_eval_samples, resampling_schedule,
- use_resampling_grads, learning_rate, lr_decay_steps,
- train_p, dtype='float64'):
- if FLAGS.use_bs:
- true_bs = None
- else:
- true_bs = [np.zeros([state_size]).astype(dtype) for _ in xrange(num_timesteps)]
-
- # Make the dataset.
- true_bs, dataset = data.make_dataset(
- bs=true_bs,
- state_size=state_size,
- num_timesteps=num_timesteps,
- batch_size=batch_size,
- num_samples=num_samples,
- variance=FLAGS.variance,
- prior_type=FLAGS.p_type,
- bimodal_prior_weight=FLAGS.bimodal_prior_weight,
- bimodal_prior_mean=FLAGS.bimodal_prior_mean,
- transition_type=FLAGS.transition_type,
- fixed_observation=FLAGS.fixed_observation,
- dtype=dtype)
- itr = dataset.make_one_shot_iterator()
- _, observations = itr.get_next()
- # Make the dataset for eval
- _, eval_dataset = data.make_dataset(
- bs=true_bs,
- state_size=state_size,
- num_timesteps=num_timesteps,
- batch_size=num_eval_samples,
- num_samples=num_eval_samples,
- variance=FLAGS.variance,
- prior_type=FLAGS.p_type,
- bimodal_prior_weight=FLAGS.bimodal_prior_weight,
- bimodal_prior_mean=FLAGS.bimodal_prior_mean,
- transition_type=FLAGS.transition_type,
- fixed_observation=FLAGS.fixed_observation,
- dtype=dtype)
- eval_itr = eval_dataset.make_one_shot_iterator()
- _, eval_observations = eval_itr.get_next()
-
- # Make the model.
- if bound == "fivo-aux-td":
- model = models.TDModel.create(
- state_size,
- num_timesteps,
- variance=FLAGS.variance,
- train_p=train_p,
- p_type=FLAGS.p_type,
- q_type=FLAGS.q_type,
- mixing_coeff=FLAGS.bimodal_prior_weight,
- prior_mode_mean=FLAGS.bimodal_prior_mean,
- observation_variance=FLAGS.observation_variance,
- transition_type=FLAGS.transition_type,
- use_bs=FLAGS.use_bs,
- dtype=tf.as_dtype(dtype),
- random_seed=FLAGS.random_seed)
- else:
- model = models.Model.create(
- state_size,
- num_timesteps,
- variance=FLAGS.variance,
- train_p=train_p,
- p_type=FLAGS.p_type,
- q_type=FLAGS.q_type,
- mixing_coeff=FLAGS.bimodal_prior_weight,
- prior_mode_mean=FLAGS.bimodal_prior_mean,
- observation_variance=FLAGS.observation_variance,
- transition_type=FLAGS.transition_type,
- use_bs=FLAGS.use_bs,
- r_sigma_init=FLAGS.r_sigma_init,
- dtype=tf.as_dtype(dtype),
- random_seed=FLAGS.random_seed)
-
- # Compute the bound and loss
- if bound == "iwae":
- (_, losses, ema_op, _, _) = bounds.iwae(
- model,
- observations,
- num_timesteps,
- num_samples=num_samples)
- (eval_log_p_hat, _, _, eval_states, eval_log_weights) = bounds.iwae(
- model,
- eval_observations,
- num_timesteps,
- num_samples=num_eval_samples,
- summarize=True)
-
- eval_log_p_hat = tf.reduce_mean(eval_log_p_hat)
-
- elif "fivo" in bound:
- if bound == "fivo-aux-td":
- (_, losses, ema_op, _, _) = bounds.fivo_aux_td(
- model,
- observations,
- num_timesteps,
- resampling_schedule=resampling_schedule,
- num_samples=num_samples)
- (eval_log_p_hat, _, _, eval_states, eval_log_weights) = bounds.fivo_aux_td(
- model,
- eval_observations,
- num_timesteps,
- resampling_schedule=resampling_schedule,
- num_samples=num_eval_samples,
- summarize=True)
- else:
- (_, losses, ema_op, _, _) = bounds.fivo(
- model,
- observations,
- num_timesteps,
- resampling_schedule=resampling_schedule,
- use_resampling_grads=use_resampling_grads,
- resampling_type=FLAGS.resampling_method,
- aux=("aux" in bound),
- num_samples=num_samples)
- (eval_log_p_hat, _, _, eval_states, eval_log_weights) = bounds.fivo(
- model,
- eval_observations,
- num_timesteps,
- resampling_schedule=resampling_schedule,
- use_resampling_grads=False,
- resampling_type="multinomial",
- aux=("aux" in bound),
- num_samples=num_eval_samples,
- summarize=True)
- eval_log_p_hat = tf.reduce_mean(eval_log_p_hat)
-
- summ.summarize_ess(eval_log_weights, only_last_timestep=True)
-
- # if FLAGS.p_type == "bimodal":
- # # create the observations that showcase the model.
- # mode_odds_ratio = tf.convert_to_tensor([1., 3., 1./3., 512., 1./512.],
- # dtype=tf.float64)
- # mode_odds_ratio = tf.expand_dims(mode_odds_ratio, 1)
- # k = ((num_timesteps+1) * FLAGS.variance) / (2*FLAGS.bimodal_prior_mean)
- # explain_obs = tf.reduce_sum(model.p.bs) + tf.log(mode_odds_ratio) * k
- # explain_obs = tf.tile(explain_obs, [num_eval_samples, 1])
- # # run the model on the explainable observations
- # if bound == "iwae":
- # (_, _, _, explain_states, explain_log_weights) = bounds.iwae(
- # model,
- # explain_obs,
- # num_timesteps,
- # num_samples=num_eval_samples)
- # elif bound == "fivo" or "fivo-aux":
- # (_, _, _, explain_states, explain_log_weights) = bounds.fivo(
- # model,
- # explain_obs,
- # num_timesteps,
- # resampling_schedule=resampling_schedule,
- # use_resampling_grads=False,
- # resampling_type="multinomial",
- # aux=("aux" in bound),
- # num_samples=num_eval_samples)
- # summ.summarize_particles(explain_states,
- # explain_log_weights,
- # explain_obs,
- # model)
-
- # Calculate the true likelihood.
- if hasattr(model.p, 'likelihood') and callable(getattr(model.p, 'likelihood')):
- eval_likelihood = model.p.likelihood(eval_observations)/ FLAGS.num_timesteps
- else:
- eval_likelihood = tf.zeros_like(eval_log_p_hat)
-
- tf.summary.scalar("log_p_hat", eval_log_p_hat)
- tf.summary.scalar("likelihood", eval_likelihood)
- tf.summary.scalar("bound_gap", eval_likelihood - eval_log_p_hat)
- summ.summarize_model(model, true_bs, eval_observations, eval_states, bound,
- summarize_r=not bound == "fivo-aux-td")
-
- # Compute and apply grads.
- global_step = tf.train.get_or_create_global_step()
-
- apply_grads = make_apply_grads_op(losses,
- global_step,
- learning_rate,
- lr_decay_steps)
-
- # Update the emas after applying the grads.
- with tf.control_dependencies([apply_grads]):
- train_op = tf.group(ema_op)
- #train_op = tf.group(ema_op, add_check_numerics_ops())
-
- return global_step, train_op, eval_log_p_hat, eval_likelihood
-
-
-def parse_resampling_schedule(schedule, num_timesteps):
- schedule = schedule.strip().lower()
- if schedule == "always":
- return [True] * (num_timesteps - 1) + [False]
- elif schedule == "never":
- return [False] * num_timesteps
- elif "every" in schedule:
- n = int(schedule.split("_")[1])
- return [(i+1) % n == 0 for i in xrange(num_timesteps)]
- else:
- sched = [x.strip() == "true" for x in schedule.split(",")]
- assert len(
- sched
- ) == num_timesteps, "Wrong number of timesteps in resampling schedule."
- return sched
-
-
-def create_log_hook(step, eval_log_p_hat, eval_likelihood):
- def summ_formatter(d):
- return ("Step {step}, log p_hat: {log_p_hat:.5f} likelihood: {likelihood:.5f}".format(**d))
- hook = tf.train.LoggingTensorHook(
- {
- "step": step,
- "log_p_hat": eval_log_p_hat,
- "likelihood": eval_likelihood,
- },
- every_n_iter=FLAGS.summarize_every,
- formatter=summ_formatter)
- return hook
-
-
-def create_infrequent_summary_hook():
- infrequent_summary_hook = tf.train.SummarySaverHook(
- save_steps=10000,
- output_dir=FLAGS.logdir,
- summary_op=tf.summary.merge_all(key="infrequent_summaries")
- )
- return infrequent_summary_hook
-
-
-def main(unused_argv):
- if FLAGS.model == "long_chain":
- resampling_schedule = parse_resampling_schedule(FLAGS.resampling,
- FLAGS.num_timesteps + 1)
- else:
- resampling_schedule = parse_resampling_schedule(FLAGS.resampling,
- FLAGS.num_timesteps)
- if FLAGS.random_seed is None:
- seed = np.random.randint(0, high=10000)
- else:
- seed = FLAGS.random_seed
- tf.logging.info("Using random seed %d", seed)
-
- if FLAGS.model == "long_chain":
- assert FLAGS.q_type == "normal", "Q type %s not supported for long chain models" % FLAGS.q_type
- assert FLAGS.p_type == "unimodal", "Bimodal priors are not supported for long chain models"
- assert not FLAGS.use_bs, "Bs are not supported with long chain models"
- assert FLAGS.num_timesteps == FLAGS.num_observations * FLAGS.steps_per_observation, "Num timesteps does not match."
- assert FLAGS.bound != "fivo-aux-td", "TD Training is not compatible with long chain models."
-
- if FLAGS.model == "forward":
- if "nonlinear" not in FLAGS.p_type:
- assert FLAGS.transition_type == models.STANDARD_TRANSITION, "Non-standard transitions not supported by the forward model."
- assert FLAGS.observation_type == models.STANDARD_OBSERVATION, "Non-standard observations not supported by the forward model."
- assert FLAGS.observation_variance is None, "Forward model does not support observation variance."
- assert FLAGS.num_observations == 1, "Forward model only supports 1 observation."
-
- if "relaxed" in FLAGS.resampling_method:
- FLAGS.use_resampling_grads = False
- assert FLAGS.bound != "fivo-aux-td", "TD Training is not compatible with relaxed resampling."
-
- if FLAGS.observation_variance is None:
- FLAGS.observation_variance = FLAGS.variance
-
- if FLAGS.p_type == "bimodal":
- assert FLAGS.bimodal_prior_mean is not None, "Must specify prior mean if using bimodal p."
-
- if FLAGS.p_type == "nonlinear" or FLAGS.p_type == "nonlinear-cauchy":
- assert not FLAGS.use_bs, "Using bs is not compatible with the nonlinear model."
-
- g = tf.Graph()
- with g.as_default():
- # Set the seeds.
- tf.set_random_seed(seed)
- np.random.seed(seed)
- if FLAGS.model == "long_chain":
- (global_step, train_op, eval_log_p_hat,
- eval_likelihood) = create_long_chain_graph(
- FLAGS.bound,
- FLAGS.state_size,
- FLAGS.num_observations,
- FLAGS.steps_per_observation,
- FLAGS.batch_size,
- FLAGS.num_samples,
- FLAGS.num_eval_samples,
- resampling_schedule,
- FLAGS.use_resampling_grads,
- FLAGS.learning_rate,
- FLAGS.decay_steps)
- else:
- (global_step, train_op,
- eval_log_p_hat, eval_likelihood) = create_graph(
- FLAGS.bound,
- FLAGS.state_size,
- FLAGS.num_timesteps,
- FLAGS.batch_size,
- FLAGS.num_samples,
- FLAGS.num_eval_samples,
- resampling_schedule,
- FLAGS.use_resampling_grads,
- FLAGS.learning_rate,
- FLAGS.decay_steps,
- FLAGS.train_p)
-
- log_hooks = [create_log_hook(global_step, eval_log_p_hat, eval_likelihood)]
- if len(tf.get_collection("infrequent_summaries")) > 0:
- log_hooks.append(create_infrequent_summary_hook())
-
- tf.logging.info("trainable variables:")
- tf.logging.info([v.name for v in tf.trainable_variables()])
- tf.logging.info("p vars:")
- tf.logging.info([v.name for v in tf.get_collection("P_VARS")])
- tf.logging.info("q vars:")
- tf.logging.info([v.name for v in tf.get_collection("Q_VARS")])
- tf.logging.info("r vars:")
- tf.logging.info([v.name for v in tf.get_collection("R_VARS")])
- tf.logging.info("r tilde vars:")
- tf.logging.info([v.name for v in tf.get_collection("R_TILDE_VARS")])
-
- with tf.train.MonitoredTrainingSession(
- master="",
- is_chief=True,
- hooks=log_hooks,
- checkpoint_dir=FLAGS.logdir,
- save_checkpoint_secs=120,
- save_summaries_steps=FLAGS.summarize_every,
- log_step_count_steps=FLAGS.summarize_every) as sess:
- cur_step = -1
- while True:
- if sess.should_stop() or cur_step > FLAGS.max_steps:
- break
- # run a step
- _, cur_step = sess.run([train_op, global_step])
-
-
-if __name__ == "__main__":
- tf.app.run(main)
diff --git a/research/fivo/fivo/__init__.py b/research/fivo/fivo/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/fivo/fivo/bounds.py b/research/fivo/fivo/bounds.py
deleted file mode 100644
index 088519033dd80669e99015b8e465888bd94a4cb1..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/bounds.py
+++ /dev/null
@@ -1,317 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Implementation of objectives for training stochastic latent variable models.
-
-Contains implementations of the Importance Weighted Autoencoder objective (IWAE)
-and the Filtering Variational objective (FIVO).
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import functools
-import tensorflow as tf
-
-from fivo import nested_utils as nested
-from fivo import smc
-
-
-def iwae(model,
- observations,
- seq_lengths,
- num_samples=1,
- parallel_iterations=30,
- swap_memory=True):
- """Computes the IWAE lower bound on the log marginal probability.
-
- This method accepts a stochastic latent variable model and some observations
- and computes a stochastic lower bound on the log marginal probability of the
- observations. The IWAE estimator is defined by averaging multiple importance
- weights. For more details see "Importance Weighted Autoencoders" by Burda
- et al. https://arxiv.org/abs/1509.00519.
-
- When num_samples = 1, this bound becomes the evidence lower bound (ELBO).
-
- Args:
- model: A subclass of ELBOTrainableSequenceModel that implements one
- timestep of the model. See models/vrnn.py for an example.
- observations: The inputs to the model. A potentially nested list or tuple of
- Tensors each of shape [max_seq_len, batch_size, ...]. The Tensors must
- have a rank at least two and have matching shapes in the first two
- dimensions, which represent time and the batch respectively. The model
- will be provided with the observations before computing the bound.
- seq_lengths: A [batch_size] Tensor of ints encoding the length of each
- sequence in the batch (sequences can be padded to a common length).
- num_samples: The number of samples to use.
- parallel_iterations: The number of parallel iterations to use for the
- internal while loop.
- swap_memory: Whether GPU-CPU memory swapping should be enabled for the
- internal while loop.
-
- Returns:
- log_p_hat: A Tensor of shape [batch_size] containing IWAE's estimate of the
- log marginal probability of the observations.
- log_weights: A Tensor of shape [max_seq_len, batch_size, num_samples]
- containing the log weights at each timestep. Will not be valid for
- timesteps past the end of a sequence.
- """
- log_p_hat, log_weights, _, final_state = fivo(
- model,
- observations,
- seq_lengths,
- num_samples=num_samples,
- resampling_criterion=smc.never_resample_criterion,
- parallel_iterations=parallel_iterations,
- swap_memory=swap_memory)
- return log_p_hat, log_weights, final_state
-
-
-def fivo(model,
- observations,
- seq_lengths,
- num_samples=1,
- resampling_criterion=smc.ess_criterion,
- resampling_type='multinomial',
- relaxed_resampling_temperature=0.5,
- parallel_iterations=30,
- swap_memory=True,
- random_seed=None):
- """Computes the FIVO lower bound on the log marginal probability.
-
- This method accepts a stochastic latent variable model and some observations
- and computes a stochastic lower bound on the log marginal probability of the
- observations. The lower bound is defined by a particle filter's unbiased
- estimate of the marginal probability of the observations. For more details see
- "Filtering Variational Objectives" by Maddison et al.
- https://arxiv.org/abs/1705.09279.
-
- When the resampling criterion is "never resample", this bound becomes IWAE.
-
- Args:
- model: A subclass of ELBOTrainableSequenceModel that implements one
- timestep of the model. See models/vrnn.py for an example.
- observations: The inputs to the model. A potentially nested list or tuple of
- Tensors each of shape [max_seq_len, batch_size, ...]. The Tensors must
- have a rank at least two and have matching shapes in the first two
- dimensions, which represent time and the batch respectively. The model
- will be provided with the observations before computing the bound.
- seq_lengths: A [batch_size] Tensor of ints encoding the length of each
- sequence in the batch (sequences can be padded to a common length).
- num_samples: The number of particles to use in each particle filter.
- resampling_criterion: The resampling criterion to use for this particle
- filter. Must accept the number of samples, the current log weights,
- and the current timestep and return a boolean Tensor of shape [batch_size]
- indicating whether each particle filter should resample. See
- ess_criterion and related functions for examples. When
- resampling_criterion is never_resample_criterion, resampling_fn is ignored
- and never called.
- resampling_type: The type of resampling, one of "multinomial" or "relaxed".
- relaxed_resampling_temperature: A positive temperature only used for relaxed
- resampling.
- parallel_iterations: The number of parallel iterations to use for the
- internal while loop. Note that values greater than 1 can introduce
- non-determinism even when random_seed is provided.
- swap_memory: Whether GPU-CPU memory swapping should be enabled for the
- internal while loop.
- random_seed: The random seed to pass to the resampling operations in
- the particle filter. Mainly useful for testing.
-
- Returns:
- log_p_hat: A Tensor of shape [batch_size] containing FIVO's estimate of the
- log marginal probability of the observations.
- log_weights: A Tensor of shape [max_seq_len, batch_size, num_samples]
- containing the log weights at each timestep of the particle filter. Note
- that on timesteps when a resampling operation is performed the log weights
- are reset to 0. Will not be valid for timesteps past the end of a
- sequence.
- resampled: A Tensor of shape [max_seq_len, batch_size] indicating when the
- particle filters resampled. Will be 1.0 on timesteps when resampling
- occurred and 0.0 on timesteps when it did not.
- """
- # batch_size is the number of particle filters running in parallel.
- batch_size = tf.shape(seq_lengths)[0]
-
- # Each sequence in the batch will be the input data for a different
- # particle filter. The batch will be laid out as:
- # particle 1 of particle filter 1
- # particle 1 of particle filter 2
- # ...
- # particle 1 of particle filter batch_size
- # particle 2 of particle filter 1
- # ...
- # particle num_samples of particle filter batch_size
- observations = nested.tile_tensors(observations, [1, num_samples])
- tiled_seq_lengths = tf.tile(seq_lengths, [num_samples])
- model.set_observations(observations, tiled_seq_lengths)
-
- if resampling_type == 'multinomial':
- resampling_fn = smc.multinomial_resampling
- elif resampling_type == 'relaxed':
- resampling_fn = functools.partial(
- smc.relaxed_resampling, temperature=relaxed_resampling_temperature)
- resampling_fn = functools.partial(resampling_fn, random_seed=random_seed)
-
- def transition_fn(prev_state, t):
- if prev_state is None:
- return model.zero_state(batch_size * num_samples, tf.float32)
- return model.propose_and_weight(prev_state, t)
-
- log_p_hat, log_weights, resampled, final_state, _ = smc.smc(
- transition_fn,
- seq_lengths,
- num_particles=num_samples,
- resampling_criterion=resampling_criterion,
- resampling_fn=resampling_fn,
- parallel_iterations=parallel_iterations,
- swap_memory=swap_memory)
-
- return log_p_hat, log_weights, resampled, final_state
-
-def fivo_aux_td(
- model,
- observations,
- seq_lengths,
- num_samples=1,
- resampling_criterion=smc.ess_criterion,
- resampling_type='multinomial',
- relaxed_resampling_temperature=0.5,
- parallel_iterations=30,
- swap_memory=True,
- random_seed=None):
- """Experimental."""
- # batch_size is the number of particle filters running in parallel.
- batch_size = tf.shape(seq_lengths)[0]
- max_seq_len = tf.reduce_max(seq_lengths)
-
- # Each sequence in the batch will be the input data for a different
- # particle filter. The batch will be laid out as:
- # particle 1 of particle filter 1
- # particle 1 of particle filter 2
- # ...
- # particle 1 of particle filter batch_size
- # particle 2 of particle filter 1
- # ...
- # particle num_samples of particle filter batch_size
- observations = nested.tile_tensors(observations, [1, num_samples])
- tiled_seq_lengths = tf.tile(seq_lengths, [num_samples])
- model.set_observations(observations, tiled_seq_lengths)
-
- if resampling_type == 'multinomial':
- resampling_fn = smc.multinomial_resampling
- elif resampling_type == 'relaxed':
- resampling_fn = functools.partial(
- smc.relaxed_resampling, temperature=relaxed_resampling_temperature)
- resampling_fn = functools.partial(resampling_fn, random_seed=random_seed)
-
- def transition_fn(prev_state, t):
- if prev_state is None:
- model_init_state = model.zero_state(batch_size * num_samples, tf.float32)
- return (tf.zeros([num_samples*batch_size], dtype=tf.float32),
- (tf.zeros([num_samples*batch_size, model.latent_size], dtype=tf.float32),
- tf.zeros([num_samples*batch_size, model.latent_size], dtype=tf.float32)),
- model_init_state)
-
- prev_log_r, prev_log_r_tilde, prev_model_state = prev_state
- (new_model_state, zt, log_q_zt, log_p_zt,
- log_p_x_given_z, log_r_tilde, p_ztplus1) = model(prev_model_state, t)
- r_tilde_mu, r_tilde_sigma_sq = log_r_tilde
- # Compute the weight without r.
- log_weight = log_p_zt + log_p_x_given_z - log_q_zt
- # Compute log_r and log_r_tilde.
- p_mu = tf.stop_gradient(p_ztplus1.mean())
- p_sigma_sq = tf.stop_gradient(p_ztplus1.variance())
- log_r = (tf.log(r_tilde_sigma_sq) -
- tf.log(r_tilde_sigma_sq + p_sigma_sq) -
- tf.square(r_tilde_mu - p_mu)/(r_tilde_sigma_sq + p_sigma_sq))
- # log_r is [num_samples*batch_size, latent_size]. We sum it along the last
- # dimension to compute log r.
- log_r = 0.5*tf.reduce_sum(log_r, axis=-1)
- # Compute prev log r tilde
- prev_r_tilde_mu, prev_r_tilde_sigma_sq = prev_log_r_tilde
- prev_log_r_tilde = -0.5*tf.reduce_sum(
- tf.square(tf.stop_gradient(zt) - prev_r_tilde_mu)/prev_r_tilde_sigma_sq, axis=-1)
- # If the sequence is on the last timestep, log_r and log_r_tilde are just zeros.
- last_timestep = t >= (tiled_seq_lengths - 1)
- log_r = tf.where(last_timestep,
- tf.zeros_like(log_r),
- log_r)
- prev_log_r_tilde = tf.where(last_timestep,
- tf.zeros_like(prev_log_r_tilde),
- prev_log_r_tilde)
- log_weight += tf.stop_gradient(log_r - prev_log_r)
- new_state = (log_r, log_r_tilde, new_model_state)
- loop_fn_args = (log_r, prev_log_r_tilde, log_p_x_given_z, log_r - prev_log_r)
- return log_weight, new_state, loop_fn_args
-
- def loop_fn(loop_state, loop_args, unused_model_state, log_weights, resampled, mask, t):
- if loop_state is None:
- return (tf.zeros([batch_size], dtype=tf.float32),
- tf.zeros([batch_size], dtype=tf.float32),
- tf.zeros([num_samples, batch_size], dtype=tf.float32))
- log_p_hat_acc, bellman_loss_acc, log_r_diff_acc = loop_state
- log_r, prev_log_r_tilde, log_p_x_given_z, log_r_diff = loop_args
- # Compute the log_p_hat update
- log_p_hat_update = tf.reduce_logsumexp(
- log_weights, axis=0) - tf.log(tf.to_float(num_samples))
- # If it is the last timestep, we always add the update.
- log_p_hat_acc += tf.cond(t >= max_seq_len-1,
- lambda: log_p_hat_update,
- lambda: log_p_hat_update * resampled)
- # Compute the Bellman update.
- log_r = tf.reshape(log_r, [num_samples, batch_size])
- prev_log_r_tilde = tf.reshape(prev_log_r_tilde, [num_samples, batch_size])
- log_p_x_given_z = tf.reshape(log_p_x_given_z, [num_samples, batch_size])
- mask = tf.reshape(mask, [num_samples, batch_size])
- # On the first timestep there is no bellman error because there is no
- # prev_log_r_tilde.
- mask = tf.cond(tf.equal(t, 0),
- lambda: tf.zeros_like(mask),
- lambda: mask)
- # On the first timestep also fix up prev_log_r_tilde, which will be -inf.
- prev_log_r_tilde = tf.where(
- tf.is_inf(prev_log_r_tilde),
- tf.zeros_like(prev_log_r_tilde),
- prev_log_r_tilde)
- # log_lambda is [num_samples, batch_size]
- log_lambda = tf.reduce_mean(prev_log_r_tilde - log_p_x_given_z - log_r,
- axis=0, keepdims=True)
- bellman_error = mask * tf.square(
- prev_log_r_tilde -
- tf.stop_gradient(log_lambda + log_p_x_given_z + log_r)
- )
- bellman_loss_acc += tf.reduce_mean(bellman_error, axis=0)
- # Compute the log_r_diff update
- log_r_diff_acc += mask * tf.reshape(log_r_diff, [num_samples, batch_size])
- return (log_p_hat_acc, bellman_loss_acc, log_r_diff_acc)
-
- log_weights, resampled, accs = smc.smc(
- transition_fn,
- seq_lengths,
- num_particles=num_samples,
- resampling_criterion=resampling_criterion,
- resampling_fn=resampling_fn,
- loop_fn=loop_fn,
- parallel_iterations=parallel_iterations,
- swap_memory=swap_memory)
-
- log_p_hat, bellman_loss, log_r_diff = accs
- loss_per_seq = [- log_p_hat, bellman_loss]
- tf.summary.scalar("bellman_loss",
- tf.reduce_mean(bellman_loss / tf.to_float(seq_lengths)))
- tf.summary.scalar("log_r_diff",
- tf.reduce_mean(tf.reduce_mean(log_r_diff, axis=0) / tf.to_float(seq_lengths)))
- return loss_per_seq, log_p_hat, log_weights, resampled
diff --git a/research/fivo/fivo/bounds_test.py b/research/fivo/fivo/bounds_test.py
deleted file mode 100644
index c970f74f4cec36a855c54bbe6cdf8d76c3f86599..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/bounds_test.py
+++ /dev/null
@@ -1,183 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.bounds"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-from fivo.test_utils import create_vrnn
-from fivo import bounds
-
-
-class BoundsTest(tf.test.TestCase):
-
- def test_elbo(self):
- """A golden-value test for the ELBO (the IWAE bound with num_samples=1)."""
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- model, inputs, targets, lengths = create_vrnn(random_seed=1234)
- outs = bounds.iwae(model, (inputs, targets), lengths, num_samples=1,
- parallel_iterations=1)
- sess.run(tf.global_variables_initializer())
- log_p_hat, _, _ = sess.run(outs)
- self.assertAllClose([-21.615765, -13.614225], log_p_hat)
-
- def test_iwae(self):
- """A golden-value test for the IWAE bound."""
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- model, inputs, targets, lengths = create_vrnn(random_seed=1234)
- outs = bounds.iwae(model, (inputs, targets), lengths, num_samples=4,
- parallel_iterations=1)
- sess.run(tf.global_variables_initializer())
- log_p_hat, weights, _ = sess.run(outs)
- self.assertAllClose([-23.301426, -13.64028], log_p_hat)
- weights_gt = np.array(
- [[[-3.66708851, -2.07074022, -4.91751671, -5.03293562],
- [-2.99690723, -3.17782736, -4.50084877, -3.48536515]],
- [[-6.2539978, -4.37615728, -7.43738699, -7.85044909],
- [-8.27518654, -6.71545124, -8.96198845, -7.05567837]],
- [[-9.19093227, -8.01637268, -11.64603615, -10.51128292],
- [-12.34527206, -11.54284477, -11.8667469, -9.69417381]],
- [[-12.20609856, -10.47217369, -13.66270638, -13.46115875],
- [-17.17656708, -16.25190353, -15.28658581, -12.33067703]],
- [[-16.14766312, -15.57472229, -17.47755432, -17.98189926],
- [-17.17656708, -16.25190353, -15.28658581, -12.33067703]],
- [[-20.07182884, -18.43191147, -20.1606636, -21.45263863],
- [-17.17656708, -16.25190353, -15.28658581, -12.33067703]],
- [[-24.10270691, -22.20865822, -24.14675522, -25.27248383],
- [-17.17656708, -16.25190353, -15.28658581, -12.33067703]]])
- self.assertAllClose(weights_gt, weights)
-
- def test_fivo(self):
- """A golden-value test for the FIVO bound."""
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- model, inputs, targets, lengths = create_vrnn(random_seed=1234)
- outs = bounds.fivo(model, (inputs, targets), lengths, num_samples=4,
- random_seed=1234, parallel_iterations=1)
- sess.run(tf.global_variables_initializer())
- log_p_hat, weights, resampled, _ = sess.run(outs)
- self.assertAllClose([-22.98902512, -14.21689224], log_p_hat)
- weights_gt = np.array(
- [[[-3.66708851, -2.07074022, -4.91751671, -5.03293562],
- [-2.99690723, -3.17782736, -4.50084877, -3.48536515]],
- [[-2.67100811, -2.30541706, -2.34178066, -2.81751347],
- [-8.27518654, -6.71545124, -8.96198845, -7.05567837]],
- [[-5.65190411, -5.94563246, -6.55041981, -5.4783473],
- [-12.34527206, -11.54284477, -11.8667469, -9.69417381]],
- [[-8.71947861, -8.40143299, -8.54593086, -8.42822266],
- [-4.28782988, -4.50591278, -3.40847206, -2.63650274]],
- [[-12.7003831, -13.5039815, -12.3569726, -12.9489622],
- [-4.28782988, -4.50591278, -3.40847206, -2.63650274]],
- [[-16.4520301, -16.3611698, -15.0314846, -16.4197006],
- [-4.28782988, -4.50591278, -3.40847206, -2.63650274]],
- [[-20.7010765, -20.1379165, -19.0020351, -20.2395458],
- [-4.28782988, -4.50591278, -3.40847206, -2.63650274]]])
- self.assertAllClose(weights_gt, weights)
- resampled_gt = np.array(
- [[1., 0.],
- [0., 0.],
- [0., 1.],
- [0., 0.],
- [0., 0.],
- [0., 0.],
- [0., 0.]])
- self.assertAllClose(resampled_gt, resampled)
-
- def test_fivo_relaxed(self):
- """A golden-value test for the FIVO bound with relaxed sampling."""
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- model, inputs, targets, lengths = create_vrnn(random_seed=1234)
- outs = bounds.fivo(model, (inputs, targets), lengths, num_samples=4,
- random_seed=1234, parallel_iterations=1,
- resampling_type="relaxed")
- sess.run(tf.global_variables_initializer())
- log_p_hat, weights, resampled, _ = sess.run(outs)
- self.assertAllClose([-22.942394, -14.273882], log_p_hat)
- weights_gt = np.array(
- [[[-3.66708851, -2.07074118, -4.91751575, -5.03293514],
- [-2.99690628, -3.17782831, -4.50084877, -3.48536515]],
- [[-2.84939098, -2.30087185, -2.35649204, -2.48417377],
- [-8.27518654, -6.71545172, -8.96199131, -7.05567837]],
- [[-5.92327023, -5.9433074, -6.5826683, -5.04259014],
- [-12.34527206, -11.54284668, -11.86675072, -9.69417477]],
- [[-8.95323944, -8.40061855, -8.52760506, -7.99130583],
- [-4.58102798, -4.56017351, -3.46283388, -2.65550804]],
- [[-12.87836456, -13.49628639, -12.31680107, -12.74228859],
- [-4.58102798, -4.56017351, -3.46283388, -2.65550804]],
- [[-16.78347397, -16.35150909, -14.98797417, -16.35162735],
- [-4.58102798, -4.56017351, -3.46283388, -2.65550804]],
- [[-20.81165886, -20.1307621, -18.92229652, -20.17458153],
- [-4.58102798, -4.56017351, -3.46283388, -2.65550804]]])
- self.assertAllClose(weights_gt, weights)
- resampled_gt = np.array(
- [[1., 0.],
- [0., 0.],
- [0., 1.],
- [0., 0.],
- [0., 0.],
- [0., 0.],
- [0., 0.]])
- self.assertAllClose(resampled_gt, resampled)
-
- def test_fivo_aux_relaxed(self):
- """A golden-value test for the FIVO-AUX bound with relaxed sampling."""
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- model, inputs, targets, lengths = create_vrnn(random_seed=1234,
- use_tilt=True)
- outs = bounds.fivo(model, (inputs, targets), lengths, num_samples=4,
- random_seed=1234, parallel_iterations=1,
- resampling_type="relaxed")
- sess.run(tf.global_variables_initializer())
- log_p_hat, weights, resampled, _ = sess.run(outs)
- self.assertAllClose([-23.1395, -14.271059], log_p_hat)
- weights_gt = np.array(
- [[[-5.19826221, -3.55476403, -5.98663855, -6.08058834],
- [-6.31685925, -5.70243931, -7.07638931, -6.18138981]],
- [[-3.97986865, -3.58831525, -3.85753584, -3.5010016],
- [-11.38203049, -8.66213989, -11.23646641, -10.02024746]],
- [[-6.62269831, -6.36680222, -6.78096485, -5.80072498],
- [-3.55419445, -8.11326408, -3.48766923, -3.08593249]],
- [[-10.56472301, -10.16084099, -9.96741676, -8.5270071],
- [-6.04880285, -7.80853653, -4.72652149, -3.49711013]],
- [[-13.36585426, -16.08720398, -13.33416367, -13.1017189],
- [-0., -0., -0., -0.]],
- [[-17.54233551, -17.35167503, -16.79163361, -16.51471138],
- [0., -0., -0., -0.]],
- [[-19.74024963, -18.69452858, -17.76246452, -18.76182365],
- [0., -0., -0., -0.]]])
- self.assertAllClose(weights_gt, weights)
- resampled_gt = np.array([[1., 0.],
- [0., 1.],
- [0., 0.],
- [0., 1.],
- [0., 0.],
- [0., 0.],
- [0., 0.]])
- self.assertAllClose(resampled_gt, resampled)
-
-
-if __name__ == "__main__":
- np.set_printoptions(threshold=np.nan) # Used to easily see the gold values.
- # Use print(repr(numpy_array)) to print the values.
- tf.test.main()
diff --git a/research/fivo/fivo/data/__init__.py b/research/fivo/fivo/data/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/fivo/fivo/data/calculate_pianoroll_mean.py b/research/fivo/fivo/data/calculate_pianoroll_mean.py
deleted file mode 100644
index 93f712bd328f61a83faffc55ad2cf6ca33b47fb7..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/data/calculate_pianoroll_mean.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Script to calculate the mean of a pianoroll dataset.
-
-Given a pianoroll pickle file, this script loads the dataset and
-calculates the mean of the training set. Then it updates the pickle file
-so that the key "train_mean" points to the mean vector.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import pickle
-import numpy as np
-
-import tensorflow as tf
-
-
-from datasets import sparse_pianoroll_to_dense
-
-tf.app.flags.DEFINE_string('in_file', None,
- 'Filename of the pickled pianoroll dataset to load.')
-tf.app.flags.DEFINE_string('out_file', None,
- 'Name of the output pickle file. Defaults to in_file, '
- 'updating the input pickle file.')
-tf.app.flags.mark_flag_as_required('in_file')
-
-FLAGS = tf.app.flags.FLAGS
-
-MIN_NOTE = 21
-MAX_NOTE = 108
-NUM_NOTES = MAX_NOTE - MIN_NOTE + 1
-
-
-def main(unused_argv):
- if FLAGS.out_file is None:
- FLAGS.out_file = FLAGS.in_file
- with tf.gfile.Open(FLAGS.in_file, 'r') as f:
- pianorolls = pickle.load(f)
- dense_pianorolls = [sparse_pianoroll_to_dense(p, MIN_NOTE, NUM_NOTES)[0]
- for p in pianorolls['train']]
- # Concatenate all elements along the time axis.
- concatenated = np.concatenate(dense_pianorolls, axis=0)
- mean = np.mean(concatenated, axis=0)
- pianorolls['train_mean'] = mean
- # Write out the whole pickle file, including the train mean.
- pickle.dump(pianorolls, open(FLAGS.out_file, 'wb'))
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/fivo/fivo/data/create_timit_dataset.py b/research/fivo/fivo/data/create_timit_dataset.py
deleted file mode 100644
index ea1cd3b10cb0812c2d6aad51491924ecfe8eec37..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/data/create_timit_dataset.py
+++ /dev/null
@@ -1,180 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Preprocesses TIMIT from raw wavfiles to create a set of TFRecords.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import glob
-import os
-import random
-import re
-
-import numpy as np
-import tensorflow as tf
-
-tf.app.flags.DEFINE_string("raw_timit_dir", None,
- "Directory containing TIMIT files.")
-tf.app.flags.DEFINE_string("out_dir", None,
- "Output directory for TFRecord files.")
-tf.app.flags.DEFINE_float("valid_frac", 0.05,
- "Fraction of train set to use as valid set. "
- "Must be between 0.0 and 1.0.")
-
-tf.app.flags.mark_flag_as_required("raw_timit_dir")
-tf.app.flags.mark_flag_as_required("out_dir")
-
-FLAGS = tf.app.flags.FLAGS
-
-NUM_TRAIN_FILES = 4620
-NUM_TEST_FILES = 1680
-SAMPLES_PER_TIMESTEP = 200
-
-# Regexes for reading SPHERE header files.
-SAMPLE_COUNT_REGEX = re.compile(r"sample_count -i (\d+)")
-SAMPLE_MIN_REGEX = re.compile(r"sample_min -i (-?\d+)")
-SAMPLE_MAX_REGEX = re.compile(r"sample_max -i (-?\d+)")
-
-
-def get_filenames(split):
- """Get all wav filenames from the TIMIT archive."""
- path = os.path.join(FLAGS.raw_timit_dir, "TIMIT", split, "*", "*", "*.WAV")
- # Sort the output by name so the order is deterministic.
- files = sorted(glob.glob(path))
- return files
-
-
-def load_timit_wav(filename):
- """Loads a TIMIT wavfile into a numpy array.
-
- TIMIT wavfiles include a SPHERE header, detailed in the TIMIT docs. The first
- line is the header type and the second is the length of the header in bytes.
- After the header, the remaining bytes are actual WAV data.
-
- The header includes information about the WAV data such as the number of
- samples and minimum and maximum amplitude. This function asserts that the
- loaded wav data matches the header.
-
- Args:
- filename: The name of the TIMIT wavfile to load.
- Returns:
- wav: A numpy array containing the loaded wav data.
- """
- wav_file = open(filename, "rb")
- header_type = wav_file.readline()
- header_length_str = wav_file.readline()
- # The header length includes the length of the first two lines.
- header_remaining_bytes = (int(header_length_str) - len(header_type) -
- len(header_length_str))
- header = wav_file.read(header_remaining_bytes)
- # Read the relevant header fields.
- sample_count = int(SAMPLE_COUNT_REGEX.search(header).group(1))
- sample_min = int(SAMPLE_MIN_REGEX.search(header).group(1))
- sample_max = int(SAMPLE_MAX_REGEX.search(header).group(1))
- wav = np.fromstring(wav_file.read(), dtype="int16").astype("float32")
- # Check that the loaded data conforms to the header description.
- assert len(wav) == sample_count
- assert wav.min() == sample_min
- assert wav.max() == sample_max
- return wav
-
-
-def preprocess(wavs, block_size, mean, std):
- """Normalize the wav data and reshape it into chunks."""
- processed_wavs = []
- for wav in wavs:
- wav = (wav - mean) / std
- wav_length = wav.shape[0]
- if wav_length % block_size != 0:
- pad_width = block_size - (wav_length % block_size)
- wav = np.pad(wav, (0, pad_width), "constant")
- assert wav.shape[0] % block_size == 0
- wav = wav.reshape((-1, block_size))
- processed_wavs.append(wav)
- return processed_wavs
-
-
-def create_tfrecord_from_wavs(wavs, output_file):
- """Writes processed wav files to disk as sharded TFRecord files."""
- with tf.python_io.TFRecordWriter(output_file) as builder:
- for wav in wavs:
- builder.write(wav.astype(np.float32).tobytes())
-
-
-def main(unused_argv):
- train_filenames = get_filenames("TRAIN")
- test_filenames = get_filenames("TEST")
-
- num_train_files = len(train_filenames)
- num_test_files = len(test_filenames)
- num_valid_files = int(num_train_files * FLAGS.valid_frac)
- num_train_files -= num_valid_files
-
- print("%d train / %d valid / %d test" % (
- num_train_files, num_valid_files, num_test_files))
-
- random.seed(1234)
- random.shuffle(train_filenames)
-
- valid_filenames = train_filenames[:num_valid_files]
- train_filenames = train_filenames[num_valid_files:]
-
- # Make sure there is no overlap in the train, test, and valid sets.
- train_s = set(train_filenames)
- test_s = set(test_filenames)
- valid_s = set(valid_filenames)
- # Disable explicit length testing to make the assertions more readable.
- # pylint: disable=g-explicit-length-test
- assert len(train_s & test_s) == 0
- assert len(train_s & valid_s) == 0
- assert len(valid_s & test_s) == 0
- # pylint: enable=g-explicit-length-test
-
- train_wavs = [load_timit_wav(f) for f in train_filenames]
- valid_wavs = [load_timit_wav(f) for f in valid_filenames]
- test_wavs = [load_timit_wav(f) for f in test_filenames]
- assert len(train_wavs) + len(valid_wavs) == NUM_TRAIN_FILES
- assert len(test_wavs) == NUM_TEST_FILES
-
- # Calculate the mean and standard deviation of the train set.
- train_stacked = np.hstack(train_wavs)
- train_mean = np.mean(train_stacked)
- train_std = np.std(train_stacked)
- print("train mean: %f train std: %f" % (train_mean, train_std))
-
- # Process all data, normalizing with the train set statistics.
- processed_train_wavs = preprocess(train_wavs, SAMPLES_PER_TIMESTEP,
- train_mean, train_std)
- processed_valid_wavs = preprocess(valid_wavs, SAMPLES_PER_TIMESTEP,
- train_mean, train_std)
- processed_test_wavs = preprocess(test_wavs, SAMPLES_PER_TIMESTEP, train_mean,
- train_std)
-
- # Write the datasets to disk.
- create_tfrecord_from_wavs(
- processed_train_wavs,
- os.path.join(FLAGS.out_dir, "train"))
- create_tfrecord_from_wavs(
- processed_valid_wavs,
- os.path.join(FLAGS.out_dir, "valid"))
- create_tfrecord_from_wavs(
- processed_test_wavs,
- os.path.join(FLAGS.out_dir, "test"))
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/fivo/fivo/data/datasets.py b/research/fivo/fivo/data/datasets.py
deleted file mode 100644
index 6d5324623250e31d65b23c97e7e684de59da1ba6..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/data/datasets.py
+++ /dev/null
@@ -1,453 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Code for creating sequence datasets.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import pickle
-
-import numpy as np
-from scipy.sparse import coo_matrix
-import tensorflow as tf
-
-# The default number of threads used to process data in parallel.
-DEFAULT_PARALLELISM = 12
-
-
-def sparse_pianoroll_to_dense(pianoroll, min_note, num_notes):
- """Converts a sparse pianoroll to a dense numpy array.
-
- Given a sparse pianoroll, converts it to a dense numpy array of shape
- [num_timesteps, num_notes] where entry i,j is 1.0 if note j is active on
- timestep i and 0.0 otherwise.
-
- Args:
- pianoroll: A sparse pianoroll object, a list of tuples where the i'th tuple
- contains the indices of the notes active at timestep i.
- min_note: The minimum note in the pianoroll, subtracted from all notes so
- that the minimum note becomes 0.
- num_notes: The number of possible different note indices, determines the
- second dimension of the resulting dense array.
- Returns:
- dense_pianoroll: A [num_timesteps, num_notes] numpy array of floats.
- num_timesteps: A python int, the number of timesteps in the pianoroll.
- """
- num_timesteps = len(pianoroll)
- inds = []
- for time, chord in enumerate(pianoroll):
- # Re-index the notes to start from min_note.
- inds.extend((time, note-min_note) for note in chord)
- shape = [num_timesteps, num_notes]
- values = [1.] * len(inds)
- sparse_pianoroll = coo_matrix(
- (values, ([x[0] for x in inds], [x[1] for x in inds])),
- shape=shape)
- return sparse_pianoroll.toarray(), num_timesteps
-
-
-def create_pianoroll_dataset(path,
- split,
- batch_size,
- num_parallel_calls=DEFAULT_PARALLELISM,
- shuffle=False,
- repeat=False,
- min_note=21,
- max_note=108):
- """Creates a pianoroll dataset.
-
- Args:
- path: The path of a pickle file containing the dataset to load.
- split: The split to use, can be train, test, or valid.
- batch_size: The batch size. If repeat is False then it is not guaranteed
- that the true batch size will match for all batches since batch_size
- may not necessarily evenly divide the number of elements.
- num_parallel_calls: The number of threads to use for parallel processing of
- the data.
- shuffle: If true, shuffles the order of the dataset.
- repeat: If true, repeats the dataset endlessly.
- min_note: The minimum note number of the dataset. For all pianoroll datasets
- the minimum note is number 21, and changing this affects the dimension of
- the data. This is useful mostly for testing.
- max_note: The maximum note number of the dataset. For all pianoroll datasets
- the maximum note is number 108, and changing this affects the dimension of
- the data. This is useful mostly for testing.
- Returns:
- inputs: A batch of input sequences represented as a dense Tensor of shape
- [time, batch_size, data_dimension]. The sequences in inputs are the
- sequences in targets shifted one timestep into the future, padded with
- zeros. This tensor is mean-centered, with the mean taken from the pickle
- file key 'train_mean'.
- targets: A batch of target sequences represented as a dense Tensor of
- shape [time, batch_size, data_dimension].
- lens: An int Tensor of shape [batch_size] representing the lengths of each
- sequence in the batch.
- mean: A float Tensor of shape [data_dimension] containing the mean loaded
- from the pickle file.
- """
- # Load the data from disk.
- num_notes = max_note - min_note + 1
- with tf.gfile.Open(path, "r") as f:
- raw_data = pickle.load(f)
- pianorolls = raw_data[split]
- mean = raw_data["train_mean"]
- num_examples = len(pianorolls)
-
- def pianoroll_generator():
- for sparse_pianoroll in pianorolls:
- yield sparse_pianoroll_to_dense(sparse_pianoroll, min_note, num_notes)
-
- dataset = tf.data.Dataset.from_generator(
- pianoroll_generator,
- output_types=(tf.float64, tf.int64),
- output_shapes=([None, num_notes], []))
-
- if repeat: dataset = dataset.repeat()
- if shuffle: dataset = dataset.shuffle(num_examples)
-
- # Batch sequences togther, padding them to a common length in time.
- dataset = dataset.padded_batch(batch_size,
- padded_shapes=([None, num_notes], []))
-
- def process_pianoroll_batch(data, lengths):
- """Create mean-centered and time-major next-step prediction Tensors."""
- data = tf.to_float(tf.transpose(data, perm=[1, 0, 2]))
- lengths = tf.to_int32(lengths)
- targets = data
- # Mean center the inputs.
- inputs = data - tf.constant(mean, dtype=tf.float32,
- shape=[1, 1, mean.shape[0]])
- # Shift the inputs one step forward in time. Also remove the last timestep
- # so that targets and inputs are the same length.
- inputs = tf.pad(inputs, [[1, 0], [0, 0], [0, 0]], mode="CONSTANT")[:-1]
- # Mask out unused timesteps.
- inputs *= tf.expand_dims(tf.transpose(
- tf.sequence_mask(lengths, dtype=inputs.dtype)), 2)
- return inputs, targets, lengths
-
- dataset = dataset.map(process_pianoroll_batch,
- num_parallel_calls=num_parallel_calls)
- dataset = dataset.prefetch(num_examples)
-
- itr = dataset.make_one_shot_iterator()
- inputs, targets, lengths = itr.get_next()
- return inputs, targets, lengths, tf.constant(mean, dtype=tf.float32)
-
-
-def create_human_pose_dataset(
- path,
- split,
- batch_size,
- num_parallel_calls=DEFAULT_PARALLELISM,
- shuffle=False,
- repeat=False,):
- """Creates a human pose dataset.
-
- Args:
- path: The path of a pickle file containing the dataset to load.
- split: The split to use, can be train, test, or valid.
- batch_size: The batch size. If repeat is False then it is not guaranteed
- that the true batch size will match for all batches since batch_size
- may not necessarily evenly divide the number of elements.
- num_parallel_calls: The number of threads to use for parallel processing of
- the data.
- shuffle: If true, shuffles the order of the dataset.
- repeat: If true, repeats the dataset endlessly.
- Returns:
- inputs: A batch of input sequences represented as a dense Tensor of shape
- [time, batch_size, data_dimension]. The sequences in inputs are the
- sequences in targets shifted one timestep into the future, padded with
- zeros. This tensor is mean-centered, with the mean taken from the pickle
- file key 'train_mean'.
- targets: A batch of target sequences represented as a dense Tensor of
- shape [time, batch_size, data_dimension].
- lens: An int Tensor of shape [batch_size] representing the lengths of each
- sequence in the batch.
- mean: A float Tensor of shape [data_dimension] containing the mean loaded
- from the pickle file.
- """
- # Load the data from disk.
- with tf.gfile.Open(path, "r") as f:
- raw_data = pickle.load(f)
-
- mean = raw_data["train_mean"]
- pose_sequences = raw_data[split]
- num_examples = len(pose_sequences)
- num_features = pose_sequences[0].shape[1]
-
- def pose_generator():
- """A generator that yields pose data sequences."""
- # Each timestep has 32 x values followed by 32 y values so is 64
- # dimensional.
- for pose_sequence in pose_sequences:
- yield pose_sequence, pose_sequence.shape[0]
-
- dataset = tf.data.Dataset.from_generator(
- pose_generator,
- output_types=(tf.float64, tf.int64),
- output_shapes=([None, num_features], []))
-
- if repeat:
- dataset = dataset.repeat()
- if shuffle:
- dataset = dataset.shuffle(num_examples)
-
- # Batch sequences togther, padding them to a common length in time.
- dataset = dataset.padded_batch(
- batch_size, padded_shapes=([None, num_features], []))
-
- # Post-process each batch, ensuring that it is mean-centered and time-major.
- def process_pose_data(data, lengths):
- """Creates Tensors for next step prediction and mean-centers the input."""
- data = tf.to_float(tf.transpose(data, perm=[1, 0, 2]))
- lengths = tf.to_int32(lengths)
- targets = data
- # Mean center the inputs.
- inputs = data - tf.constant(
- mean, dtype=tf.float32, shape=[1, 1, mean.shape[0]])
- # Shift the inputs one step forward in time. Also remove the last timestep
- # so that targets and inputs are the same length.
- inputs = tf.pad(inputs, [[1, 0], [0, 0], [0, 0]], mode="CONSTANT")[:-1]
- # Mask out unused timesteps.
- inputs *= tf.expand_dims(
- tf.transpose(tf.sequence_mask(lengths, dtype=inputs.dtype)), 2)
- return inputs, targets, lengths
-
- dataset = dataset.map(
- process_pose_data,
- num_parallel_calls=num_parallel_calls)
- dataset = dataset.prefetch(num_examples)
-
- itr = dataset.make_one_shot_iterator()
- inputs, targets, lengths = itr.get_next()
- return inputs, targets, lengths, tf.constant(mean, dtype=tf.float32)
-
-
-def create_speech_dataset(path,
- batch_size,
- samples_per_timestep=200,
- num_parallel_calls=DEFAULT_PARALLELISM,
- prefetch_buffer_size=2048,
- shuffle=False,
- repeat=False):
- """Creates a speech dataset.
-
- Args:
- path: The path of a possibly sharded TFRecord file containing the data.
- batch_size: The batch size. If repeat is False then it is not guaranteed
- that the true batch size will match for all batches since batch_size
- may not necessarily evenly divide the number of elements.
- samples_per_timestep: The number of audio samples per timestep. Used to
- reshape the data into sequences of shape [time, samples_per_timestep].
- Should not change except for testing -- in all speech datasets 200 is the
- number of samples per timestep.
- num_parallel_calls: The number of threads to use for parallel processing of
- the data.
- prefetch_buffer_size: The size of the prefetch queues to use after reading
- and processing the raw data.
- shuffle: If true, shuffles the order of the dataset.
- repeat: If true, repeats the dataset endlessly.
- Returns:
- inputs: A batch of input sequences represented as a dense Tensor of shape
- [time, batch_size, samples_per_timestep]. The sequences in inputs are the
- sequences in targets shifted one timestep into the future, padded with
- zeros.
- targets: A batch of target sequences represented as a dense Tensor of
- shape [time, batch_size, samples_per_timestep].
- lens: An int Tensor of shape [batch_size] representing the lengths of each
- sequence in the batch.
- """
- filenames = [path]
-
- def read_speech_example(value):
- """Parses a single tf.Example from the TFRecord file."""
- decoded = tf.decode_raw(value, out_type=tf.float32)
- example = tf.reshape(decoded, [-1, samples_per_timestep])
- length = tf.shape(example)[0]
- return example, length
-
- # Create the dataset from the TFRecord files
- dataset = tf.data.TFRecordDataset(filenames).map(
- read_speech_example, num_parallel_calls=num_parallel_calls)
- dataset = dataset.prefetch(prefetch_buffer_size)
-
- if repeat: dataset = dataset.repeat()
- if shuffle: dataset = dataset.shuffle(prefetch_buffer_size)
-
- dataset = dataset.padded_batch(
- batch_size, padded_shapes=([None, samples_per_timestep], []))
-
- def process_speech_batch(data, lengths):
- """Creates Tensors for next step prediction."""
- data = tf.transpose(data, perm=[1, 0, 2])
- lengths = tf.to_int32(lengths)
- targets = data
- # Shift the inputs one step forward in time. Also remove the last timestep
- # so that targets and inputs are the same length.
- inputs = tf.pad(data, [[1, 0], [0, 0], [0, 0]], mode="CONSTANT")[:-1]
- # Mask out unused timesteps.
- inputs *= tf.expand_dims(
- tf.transpose(tf.sequence_mask(lengths, dtype=inputs.dtype)), 2)
- return inputs, targets, lengths
-
- dataset = dataset.map(process_speech_batch,
- num_parallel_calls=num_parallel_calls)
- dataset = dataset.prefetch(prefetch_buffer_size)
-
- itr = dataset.make_one_shot_iterator()
- inputs, targets, lengths = itr.get_next()
- return inputs, targets, lengths
-
-
-SQUARED_OBSERVATION = "squared"
-ABS_OBSERVATION = "abs"
-STANDARD_OBSERVATION = "standard"
-OBSERVATION_TYPES = [SQUARED_OBSERVATION, ABS_OBSERVATION, STANDARD_OBSERVATION]
-
-ROUND_TRANSITION = "round"
-STANDARD_TRANSITION = "standard"
-TRANSITION_TYPES = [ROUND_TRANSITION, STANDARD_TRANSITION]
-
-
-def create_chain_graph_dataset(
- batch_size,
- num_timesteps,
- steps_per_observation=None,
- state_size=1,
- transition_variance=1.,
- observation_variance=1.,
- transition_type=STANDARD_TRANSITION,
- observation_type=STANDARD_OBSERVATION,
- fixed_observation=None,
- prefetch_buffer_size=2048,
- dtype="float32"):
- """Creates a toy chain graph dataset.
-
- Creates a dataset where the data are sampled from a diffusion process. The
- 'latent' states of the process are sampled as a chain of Normals:
-
- z0 ~ N(0, transition_variance)
- z1 ~ N(transition_fn(z0), transition_variance)
- ...
-
- where transition_fn could be round z0 or pass it through unchanged.
-
- The observations are produced every steps_per_observation timesteps as a
- function of the latent zs. For example if steps_per_observation is 3 then the
- first observation will be produced as a function of z3:
-
- x1 ~ N(observation_fn(z3), observation_variance)
-
- where observation_fn could square z3, take the absolute value, or pass
- it through unchanged.
-
- Only the observations are returned.
-
- Args:
- batch_size: The batch size. The number of trajectories to run in parallel.
- num_timesteps: The length of the chain of latent states (i.e. the
- number of z's excluding z0.
- steps_per_observation: The number of latent states between each observation,
- must evenly divide num_timesteps.
- state_size: The size of the latent state and observation, must be a
- python int.
- transition_variance: The variance of the transition density.
- observation_variance: The variance of the observation density.
- transition_type: Must be one of "round" or "standard". "round" means that
- the transition density is centered at the rounded previous latent state.
- "standard" centers the transition density at the previous latent state,
- unchanged.
- observation_type: Must be one of "squared", "abs" or "standard". "squared"
- centers the observation density at the squared latent state. "abs"
- centers the observaiton density at the absolute value of the current
- latent state. "standard" centers the observation density at the current
- latent state.
- fixed_observation: If not None, fixes all observations to be a constant.
- Must be a scalar.
- prefetch_buffer_size: The size of the prefetch queues to use after reading
- and processing the raw data.
- dtype: A string convertible to a tensorflow datatype. The datatype used
- to represent the states and observations.
- Returns:
- observations: A batch of observations represented as a dense Tensor of
- shape [num_observations, batch_size, state_size]. num_observations is
- num_timesteps/steps_per_observation.
- lens: An int Tensor of shape [batch_size] representing the lengths of each
- sequence in the batch. Will contain num_observations as each entry.
- Raises:
- ValueError: Raised if steps_per_observation does not evenly divide
- num_timesteps.
- """
- if steps_per_observation is None:
- steps_per_observation = num_timesteps
- if num_timesteps % steps_per_observation != 0:
- raise ValueError("steps_per_observation must evenly divide num_timesteps.")
- num_observations = int(num_timesteps / steps_per_observation)
- def data_generator():
- """An infinite generator of latents and observations from the model."""
- transition_std = np.sqrt(transition_variance)
- observation_std = np.sqrt(observation_variance)
- while True:
- states = []
- observations = []
- # Sample z0 ~ Normal(0, sqrt(variance)).
- states.append(
- np.random.normal(size=[state_size],
- scale=observation_std).astype(dtype))
- # Start the range at 1 because we've already generated z0.
- # The range ends at num_timesteps+1 because we want to include the
- # num_timesteps-th step.
- for t in xrange(1, num_timesteps+1):
- if transition_type == ROUND_TRANSITION:
- loc = np.round(states[-1])
- elif transition_type == STANDARD_TRANSITION:
- loc = states[-1]
- z_t = np.random.normal(size=[state_size], loc=loc, scale=transition_std)
- states.append(z_t.astype(dtype))
- if t % steps_per_observation == 0:
- if fixed_observation is None:
- if observation_type == SQUARED_OBSERVATION:
- loc = np.square(states[-1])
- elif observation_type == ABS_OBSERVATION:
- loc = np.abs(states[-1])
- elif observation_type == STANDARD_OBSERVATION:
- loc = states[-1]
- x_t = np.random.normal(size=[state_size],
- loc=loc,
- scale=observation_std).astype(dtype)
- else:
- x_t = np.ones([state_size]) * fixed_observation
-
- observations.append(x_t)
- yield states, observations
-
- dataset = tf.data.Dataset.from_generator(
- data_generator,
- output_types=(tf.as_dtype(dtype), tf.as_dtype(dtype)),
- output_shapes=([num_timesteps+1, state_size],
- [num_observations, state_size])
- )
- dataset = dataset.repeat().batch(batch_size)
- dataset = dataset.prefetch(prefetch_buffer_size)
- itr = dataset.make_one_shot_iterator()
- _, observations = itr.get_next()
- # Transpose observations from [batch, time, state_size] to
- # [time, batch, state_size].
- observations = tf.transpose(observations, perm=[1, 0, 2])
- lengths = tf.ones([batch_size], dtype=tf.int32) * num_observations
- return observations, lengths
diff --git a/research/fivo/fivo/data/datasets_test.py b/research/fivo/fivo/data/datasets_test.py
deleted file mode 100644
index e6bbfda67aa44efc0bc4b1a34eb0cb9f09d53de5..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/data/datasets_test.py
+++ /dev/null
@@ -1,303 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.data.datasets."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import pickle
-import os
-
-import numpy as np
-import tensorflow as tf
-
-from fivo.data import datasets
-
-FLAGS = tf.app.flags.FLAGS
-
-
-class DatasetsTest(tf.test.TestCase):
-
- def test_sparse_pianoroll_to_dense_empty_at_end(self):
- sparse_pianoroll = [(0, 1), (1, 0), (), (1,), (), ()]
- dense_pianoroll, num_timesteps = datasets.sparse_pianoroll_to_dense(
- sparse_pianoroll, min_note=0, num_notes=2)
- self.assertEqual(num_timesteps, 6)
- self.assertAllEqual([[1, 1],
- [1, 1],
- [0, 0],
- [0, 1],
- [0, 0],
- [0, 0]], dense_pianoroll)
-
- def test_sparse_pianoroll_to_dense_with_chord(self):
- sparse_pianoroll = [(0, 1), (1, 0), (), (1,)]
- dense_pianoroll, num_timesteps = datasets.sparse_pianoroll_to_dense(
- sparse_pianoroll, min_note=0, num_notes=2)
- self.assertEqual(num_timesteps, 4)
- self.assertAllEqual([[1, 1],
- [1, 1],
- [0, 0],
- [0, 1]], dense_pianoroll)
-
- def test_sparse_pianoroll_to_dense_simple(self):
- sparse_pianoroll = [(0,), (), (1,)]
- dense_pianoroll, num_timesteps = datasets.sparse_pianoroll_to_dense(
- sparse_pianoroll, min_note=0, num_notes=2)
- self.assertEqual(num_timesteps, 3)
- self.assertAllEqual([[1, 0],
- [0, 0],
- [0, 1]], dense_pianoroll)
-
- def test_sparse_pianoroll_to_dense_subtracts_min_note(self):
- sparse_pianoroll = [(4, 5), (5, 4), (), (5,), (), ()]
- dense_pianoroll, num_timesteps = datasets.sparse_pianoroll_to_dense(
- sparse_pianoroll, min_note=4, num_notes=2)
- self.assertEqual(num_timesteps, 6)
- self.assertAllEqual([[1, 1],
- [1, 1],
- [0, 0],
- [0, 1],
- [0, 0],
- [0, 0]], dense_pianoroll)
-
- def test_sparse_pianoroll_to_dense_uses_num_notes(self):
- sparse_pianoroll = [(4, 5), (5, 4), (), (5,), (), ()]
- dense_pianoroll, num_timesteps = datasets.sparse_pianoroll_to_dense(
- sparse_pianoroll, min_note=4, num_notes=3)
- self.assertEqual(num_timesteps, 6)
- self.assertAllEqual([[1, 1, 0],
- [1, 1, 0],
- [0, 0, 0],
- [0, 1, 0],
- [0, 0, 0],
- [0, 0, 0]], dense_pianoroll)
-
- def test_pianoroll_dataset(self):
- pianoroll_data = [[(0,), (), (1,)],
- [(0, 1), (1,)],
- [(1,), (0,), (), (0, 1), (), ()]]
- pianoroll_mean = np.zeros([3])
- pianoroll_mean[-1] = 1
- data = {"train": pianoroll_data, "train_mean": pianoroll_mean}
- path = os.path.join(tf.test.get_temp_dir(), "test.pkl")
- pickle.dump(data, open(path, "wb"))
- with self.test_session() as sess:
- inputs, targets, lens, mean = datasets.create_pianoroll_dataset(
- path, "train", 2, num_parallel_calls=1,
- shuffle=False, repeat=False,
- min_note=0, max_note=2)
- i1, t1, l1 = sess.run([inputs, targets, lens])
- i2, t2, l2 = sess.run([inputs, targets, lens])
- m = sess.run(mean)
- # Check the lengths.
- self.assertAllEqual([3, 2], l1)
- self.assertAllEqual([6], l2)
- # Check the mean.
- self.assertAllEqual(pianoroll_mean, m)
- # Check the targets. The targets should not be mean-centered and should
- # be padded with zeros to a common length within a batch.
- self.assertAllEqual([[1, 0, 0],
- [0, 0, 0],
- [0, 1, 0]], t1[:, 0, :])
- self.assertAllEqual([[1, 1, 0],
- [0, 1, 0],
- [0, 0, 0]], t1[:, 1, :])
- self.assertAllEqual([[0, 1, 0],
- [1, 0, 0],
- [0, 0, 0],
- [1, 1, 0],
- [0, 0, 0],
- [0, 0, 0]], t2[:, 0, :])
- # Check the inputs. Each sequence should start with zeros on the first
- # timestep. Each sequence should be padded with zeros to a common length
- # within a batch. The mean should be subtracted from all timesteps except
- # the first and the padding.
- self.assertAllEqual([[0, 0, 0],
- [1, 0, -1],
- [0, 0, -1]], i1[:, 0, :])
- self.assertAllEqual([[0, 0, 0],
- [1, 1, -1],
- [0, 0, 0]], i1[:, 1, :])
- self.assertAllEqual([[0, 0, 0],
- [0, 1, -1],
- [1, 0, -1],
- [0, 0, -1],
- [1, 1, -1],
- [0, 0, -1]], i2[:, 0, :])
-
- def test_human_pose_dataset(self):
- pose_data = [
- [[0, 0], [2, 2]],
- [[2, 2]],
- [[0, 0], [0, 0], [2, 2], [2, 2], [0, 0]],
- ]
- pose_data = [np.array(x, dtype=np.float64) for x in pose_data]
- pose_data_mean = np.array([1, 1], dtype=np.float64)
- data = {
- "train": pose_data,
- "train_mean": pose_data_mean,
- }
- path = os.path.join(tf.test.get_temp_dir(), "test_human_pose_dataset.pkl")
- with open(path, "wb") as out:
- pickle.dump(data, out)
- with self.test_session() as sess:
- inputs, targets, lens, mean = datasets.create_human_pose_dataset(
- path, "train", 2, num_parallel_calls=1, shuffle=False, repeat=False)
- i1, t1, l1 = sess.run([inputs, targets, lens])
- i2, t2, l2 = sess.run([inputs, targets, lens])
- m = sess.run(mean)
- # Check the lengths.
- self.assertAllEqual([2, 1], l1)
- self.assertAllEqual([5], l2)
- # Check the mean.
- self.assertAllEqual(pose_data_mean, m)
- # Check the targets. The targets should not be mean-centered and should
- # be padded with zeros to a common length within a batch.
- self.assertAllEqual([[0, 0], [2, 2]], t1[:, 0, :])
- self.assertAllEqual([[2, 2], [0, 0]], t1[:, 1, :])
- self.assertAllEqual([[0, 0], [0, 0], [2, 2], [2, 2], [0, 0]], t2[:, 0, :])
- # Check the inputs. Each sequence should start with zeros on the first
- # timestep. Each sequence should be padded with zeros to a common length
- # within a batch. The mean should be subtracted from all timesteps except
- # the first and the padding.
- self.assertAllEqual([[0, 0], [-1, -1]], i1[:, 0, :])
- self.assertAllEqual([[0, 0], [0, 0]], i1[:, 1, :])
- self.assertAllEqual([[0, 0], [-1, -1], [-1, -1], [1, 1], [1, 1]],
- i2[:, 0, :])
-
- def test_speech_dataset(self):
- with self.test_session() as sess:
- path = os.path.join(
- os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
- "test_data",
- "tiny_speech_dataset.tfrecord")
- inputs, targets, lens = datasets.create_speech_dataset(
- path, 3, samples_per_timestep=2, num_parallel_calls=1,
- prefetch_buffer_size=3, shuffle=False, repeat=False)
- inputs1, targets1, lengths1 = sess.run([inputs, targets, lens])
- inputs2, targets2, lengths2 = sess.run([inputs, targets, lens])
- # Check the lengths.
- self.assertAllEqual([1, 2, 3], lengths1)
- self.assertAllEqual([4], lengths2)
- # Check the targets. The targets should be padded with zeros to a common
- # length within a batch.
- self.assertAllEqual([[[0., 1.], [0., 1.], [0., 1.]],
- [[0., 0.], [2., 3.], [2., 3.]],
- [[0., 0.], [0., 0.], [4., 5.]]],
- targets1)
- self.assertAllEqual([[[0., 1.]],
- [[2., 3.]],
- [[4., 5.]],
- [[6., 7.]]],
- targets2)
- # Check the inputs. Each sequence should start with zeros on the first
- # timestep. Each sequence should be padded with zeros to a common length
- # within a batch.
- self.assertAllEqual([[[0., 0.], [0., 0.], [0., 0.]],
- [[0., 0.], [0., 1.], [0., 1.]],
- [[0., 0.], [0., 0.], [2., 3.]]],
- inputs1)
- self.assertAllEqual([[[0., 0.]],
- [[0., 1.]],
- [[2., 3.]],
- [[4., 5.]]],
- inputs2)
-
- def test_chain_graph_raises_error_on_wrong_steps_per_observation(self):
- with self.assertRaises(ValueError):
- datasets.create_chain_graph_dataset(
- batch_size=4,
- num_timesteps=10,
- steps_per_observation=9)
-
- def test_chain_graph_single_obs(self):
- with self.test_session() as sess:
- np.random.seed(1234)
- num_observations = 1
- num_timesteps = 5
- batch_size = 2
- state_size = 1
- observations, lengths = datasets.create_chain_graph_dataset(
- batch_size=batch_size,
- num_timesteps=num_timesteps,
- state_size=state_size)
- out_observations, out_lengths = sess.run([observations, lengths])
- self.assertAllEqual([num_observations, num_observations], out_lengths)
- self.assertAllClose(
- [[[1.426677], [-1.789461]]],
- out_observations)
-
- def test_chain_graph_multiple_obs(self):
- with self.test_session() as sess:
- np.random.seed(1234)
- num_observations = 3
- num_timesteps = 6
- batch_size = 2
- state_size = 1
- observations, lengths = datasets.create_chain_graph_dataset(
- batch_size=batch_size,
- num_timesteps=num_timesteps,
- steps_per_observation=num_timesteps/num_observations,
- state_size=state_size)
- out_observations, out_lengths = sess.run([observations, lengths])
- self.assertAllEqual([num_observations, num_observations], out_lengths)
- self.assertAllClose(
- [[[0.40051451], [1.07405114]],
- [[1.73932898], [3.16880035]],
- [[-1.98377144], [2.82669163]]],
- out_observations)
-
- def test_chain_graph_state_dims(self):
- with self.test_session() as sess:
- np.random.seed(1234)
- num_observations = 1
- num_timesteps = 5
- batch_size = 2
- state_size = 3
- observations, lengths = datasets.create_chain_graph_dataset(
- batch_size=batch_size,
- num_timesteps=num_timesteps,
- state_size=state_size)
- out_observations, out_lengths = sess.run([observations, lengths])
- self.assertAllEqual([num_observations, num_observations], out_lengths)
- self.assertAllClose(
- [[[1.052287, -4.560759, 3.07988],
- [2.008926, 0.495567, 3.488678]]],
- out_observations)
-
- def test_chain_graph_fixed_obs(self):
- with self.test_session() as sess:
- np.random.seed(1234)
- num_observations = 3
- num_timesteps = 6
- batch_size = 2
- state_size = 1
- observations, lengths = datasets.create_chain_graph_dataset(
- batch_size=batch_size,
- num_timesteps=num_timesteps,
- steps_per_observation=num_timesteps/num_observations,
- state_size=state_size,
- fixed_observation=4.)
- out_observations, out_lengths = sess.run([observations, lengths])
- self.assertAllEqual([num_observations, num_observations], out_lengths)
- self.assertAllClose(
- np.ones([num_observations, batch_size, state_size]) * 4.,
- out_observations)
-
-if __name__ == "__main__":
- tf.test.main()
diff --git a/research/fivo/fivo/ghmm_runners.py b/research/fivo/fivo/ghmm_runners.py
deleted file mode 100644
index 1f1ba6d4f9ea9ed9dee7d95449ba73285c77f24d..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/ghmm_runners.py
+++ /dev/null
@@ -1,235 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Creates and runs Gaussian HMM-related graphs."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import numpy as np
-import tensorflow as tf
-
-from fivo import smc
-from fivo import bounds
-from fivo.data import datasets
-from fivo.models import ghmm
-
-
-def run_train(config):
- """Runs training for a Gaussian HMM setup."""
-
- def create_logging_hook(step, bound_value, likelihood, bound_gap):
- """Creates a logging hook that prints the bound value periodically."""
- bound_label = config.bound + "/t"
- def summary_formatter(log_dict):
- string = ("Step {step}, %s: {value:.3f}, "
- "likelihood: {ll:.3f}, gap: {gap:.3e}") % bound_label
- return string.format(**log_dict)
- logging_hook = tf.train.LoggingTensorHook(
- {"step": step, "value": bound_value,
- "ll": likelihood, "gap": bound_gap},
- every_n_iter=config.summarize_every,
- formatter=summary_formatter)
- return logging_hook
-
- def create_losses(model, observations, lengths):
- """Creates the loss to be optimized.
-
- Args:
- model: A Trainable GHMM model.
- observations: A set of observations.
- lengths: The lengths of each sequence in the observations.
- Returns:
- loss: A float Tensor that when differentiated yields the gradients
- to apply to the model. Should be optimized via gradient descent.
- bound: A float Tensor containing the value of the bound that is
- being optimized.
- true_ll: The true log-likelihood of the data under the model.
- bound_gap: The gap between the bound and the true log-likelihood.
- """
- # Compute lower bounds on the log likelihood.
- if config.bound == "elbo":
- ll_per_seq, _, _ = bounds.iwae(
- model, observations, lengths, num_samples=1,
- parallel_iterations=config.parallel_iterations
- )
- elif config.bound == "iwae":
- ll_per_seq, _, _ = bounds.iwae(
- model, observations, lengths, num_samples=config.num_samples,
- parallel_iterations=config.parallel_iterations
- )
- elif config.bound == "fivo":
- if config.resampling_type == "relaxed":
- ll_per_seq, _, _, _ = bounds.fivo(
- model,
- observations,
- lengths,
- num_samples=config.num_samples,
- resampling_criterion=smc.ess_criterion,
- resampling_type=config.resampling_type,
- relaxed_resampling_temperature=config.
- relaxed_resampling_temperature,
- random_seed=config.random_seed,
- parallel_iterations=config.parallel_iterations)
- else:
- ll_per_seq, _, _, _ = bounds.fivo(
- model, observations, lengths,
- num_samples=config.num_samples,
- resampling_criterion=smc.ess_criterion,
- resampling_type=config.resampling_type,
- random_seed=config.random_seed,
- parallel_iterations=config.parallel_iterations
- )
- ll_per_t = tf.reduce_mean(ll_per_seq / tf.to_float(lengths))
- # Compute the data's true likelihood under the model and the bound gap.
- true_ll_per_seq = model.likelihood(tf.squeeze(observations))
- true_ll_per_t = tf.reduce_mean(true_ll_per_seq / tf.to_float(lengths))
- bound_gap = true_ll_per_seq - ll_per_seq
- bound_gap = tf.reduce_mean(bound_gap/ tf.to_float(lengths))
- tf.summary.scalar("train_ll_bound", ll_per_t)
- tf.summary.scalar("train_true_ll", true_ll_per_t)
- tf.summary.scalar("bound_gap", bound_gap)
- return -ll_per_t, ll_per_t, true_ll_per_t, bound_gap
-
- def create_graph():
- """Creates the training graph."""
- global_step = tf.train.get_or_create_global_step()
- xs, lengths = datasets.create_chain_graph_dataset(
- config.batch_size,
- config.num_timesteps,
- steps_per_observation=1,
- state_size=1,
- transition_variance=config.variance,
- observation_variance=config.variance)
- model = ghmm.TrainableGaussianHMM(
- config.num_timesteps,
- config.proposal_type,
- transition_variances=config.variance,
- emission_variances=config.variance,
- random_seed=config.random_seed)
- loss, bound, true_ll, gap = create_losses(model, xs, lengths)
- opt = tf.train.AdamOptimizer(config.learning_rate)
- grads = opt.compute_gradients(loss, var_list=tf.trainable_variables())
- train_op = opt.apply_gradients(grads, global_step=global_step)
- return bound, true_ll, gap, train_op, global_step
-
- with tf.Graph().as_default():
- if config.random_seed:
- tf.set_random_seed(config.random_seed)
- np.random.seed(config.random_seed)
- bound, true_ll, gap, train_op, global_step = create_graph()
- log_hook = create_logging_hook(global_step, bound, true_ll, gap)
- with tf.train.MonitoredTrainingSession(
- master="",
- hooks=[log_hook],
- checkpoint_dir=config.logdir,
- save_checkpoint_secs=120,
- save_summaries_steps=config.summarize_every,
- log_step_count_steps=config.summarize_every*20) as sess:
- cur_step = -1
- while cur_step <= config.max_steps and not sess.should_stop():
- cur_step = sess.run(global_step)
- _, cur_step = sess.run([train_op, global_step])
-
-
-def run_eval(config):
- """Evaluates a Gaussian HMM using the given config."""
-
- def create_bound(model, xs, lengths):
- """Creates the bound to be evaluated."""
- if config.bound == "elbo":
- ll_per_seq, log_weights, _ = bounds.iwae(
- model, xs, lengths, num_samples=1,
- parallel_iterations=config.parallel_iterations
- )
- elif config.bound == "iwae":
- ll_per_seq, log_weights, _ = bounds.iwae(
- model, xs, lengths, num_samples=config.num_samples,
- parallel_iterations=config.parallel_iterations
- )
- elif config.bound == "fivo":
- ll_per_seq, log_weights, resampled, _ = bounds.fivo(
- model, xs, lengths,
- num_samples=config.num_samples,
- resampling_criterion=smc.ess_criterion,
- resampling_type=config.resampling_type,
- random_seed=config.random_seed,
- parallel_iterations=config.parallel_iterations
- )
- # Compute bound scaled by number of timesteps.
- bound_per_t = ll_per_seq / tf.to_float(lengths)
- if config.bound == "fivo":
- return bound_per_t, log_weights, resampled
- else:
- return bound_per_t, log_weights
-
- def create_graph():
- """Creates the dataset, model, and bound."""
- xs, lengths = datasets.create_chain_graph_dataset(
- config.batch_size,
- config.num_timesteps,
- steps_per_observation=1,
- state_size=1,
- transition_variance=config.variance,
- observation_variance=config.variance)
- model = ghmm.TrainableGaussianHMM(
- config.num_timesteps,
- config.proposal_type,
- transition_variances=config.variance,
- emission_variances=config.variance,
- random_seed=config.random_seed)
- true_likelihood = tf.reduce_mean(
- model.likelihood(tf.squeeze(xs)) / tf.to_float(lengths))
- outs = [true_likelihood]
- outs.extend(list(create_bound(model, xs, lengths)))
- return outs
-
- with tf.Graph().as_default():
- if config.random_seed:
- tf.set_random_seed(config.random_seed)
- np.random.seed(config.random_seed)
- graph_outs = create_graph()
- with tf.train.SingularMonitoredSession(
- checkpoint_dir=config.logdir) as sess:
- outs = sess.run(graph_outs)
- likelihood = outs[0]
- avg_bound = np.mean(outs[1])
- std = np.std(outs[1])
- log_weights = outs[2]
- log_weight_variances = np.var(log_weights, axis=2)
- avg_log_weight_variance = np.var(log_weight_variances, axis=1)
- avg_log_weight = np.mean(log_weights, axis=(1, 2))
- data = {"mean": avg_bound, "std": std, "log_weights": log_weights,
- "log_weight_means": avg_log_weight,
- "log_weight_variances": avg_log_weight_variance}
- if len(outs) == 4:
- data["resampled"] = outs[3]
- data["avg_resampled"] = np.mean(outs[3], axis=1)
- # Log some useful statistics.
- tf.logging.info("Evaled bound %s with batch_size: %d, num_samples: %d."
- % (config.bound, config.batch_size, config.num_samples))
- tf.logging.info("mean: %f, std: %f" % (avg_bound, std))
- tf.logging.info("true likelihood: %s" % likelihood)
- tf.logging.info("avg log weight: %s" % avg_log_weight)
- tf.logging.info("log weight variance: %s" % avg_log_weight_variance)
- if len(outs) == 4:
- tf.logging.info("avg resamples per t: %s" % data["avg_resampled"])
- if not tf.gfile.Exists(config.logdir):
- tf.gfile.MakeDirs(config.logdir)
- with tf.gfile.Open(os.path.join(config.logdir, "out.npz"), "w") as fout:
- np.save(fout, data)
diff --git a/research/fivo/fivo/ghmm_runners_test.py b/research/fivo/fivo/ghmm_runners_test.py
deleted file mode 100644
index 50044ad475b3458858b580a6ff7664267485757b..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/ghmm_runners_test.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.ghmm_runners."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import numpy as np
-import tensorflow as tf
-
-from fivo import ghmm_runners
-
-
-class GHMMRunnersTest(tf.test.TestCase):
-
- def default_config(self):
- class Config(object):
- pass
- config = Config()
- config.model = "ghmm"
- config.bound = "fivo"
- config.proposal_type = "prior"
- config.batch_size = 4
- config.num_samples = 4
- config.num_timesteps = 10
- config.variance = 0.1
- config.resampling_type = "multinomial"
- config.random_seed = 1234
- config.parallel_iterations = 1
- config.learning_rate = 1e-4
- config.summarize_every = 1
- config.max_steps = 1
- return config
-
- def test_eval_ghmm_notraining_fivo_prior(self):
- self.eval_ghmm_notraining("fivo", "prior", -3.063864)
-
- def test_eval_ghmm_notraining_fivo_true_filtering(self):
- self.eval_ghmm_notraining("fivo", "true-filtering", -1.1409812)
-
- def test_eval_ghmm_notraining_fivo_true_smoothing(self):
- self.eval_ghmm_notraining("fivo", "true-smoothing", -0.85592091)
-
- def test_eval_ghmm_notraining_iwae_prior(self):
- self.eval_ghmm_notraining("iwae", "prior", -5.9730167)
-
- def test_eval_ghmm_notraining_iwae_true_filtering(self):
- self.eval_ghmm_notraining("iwae", "true-filtering", -1.1485999)
-
- def test_eval_ghmm_notraining_iwae_true_smoothing(self):
- self.eval_ghmm_notraining("iwae", "true-smoothing", -0.85592091)
-
- def eval_ghmm_notraining(self, bound, proposal_type, expected_bound_avg):
- config = self.default_config()
- config.proposal_type = proposal_type
- config.bound = bound
- config.logdir = os.path.join(
- tf.test.get_temp_dir(), "test-ghmm-%s-%s" % (proposal_type, bound))
-
- ghmm_runners.run_eval(config)
-
- data = np.load(os.path.join(config.logdir, "out.npz")).item()
- self.assertAlmostEqual(expected_bound_avg, data["mean"], places=3)
-
- def test_train_ghmm_for_one_step_and_eval_fivo_filtering(self):
- self.train_ghmm_for_one_step_and_eval("fivo", "filtering", -16.727108)
-
- def test_train_ghmm_for_one_step_and_eval_fivo_smoothing(self):
- self.train_ghmm_for_one_step_and_eval("fivo", "smoothing", -19.381277)
-
- def test_train_ghmm_for_one_step_and_eval_iwae_filtering(self):
- self.train_ghmm_for_one_step_and_eval("iwae", "filtering", -33.31966)
-
- def test_train_ghmm_for_one_step_and_eval_iwae_smoothing(self):
- self.train_ghmm_for_one_step_and_eval("iwae", "smoothing", -46.388447)
-
- def train_ghmm_for_one_step_and_eval(self, bound, proposal_type, expected_bound_avg):
- config = self.default_config()
- config.proposal_type = proposal_type
- config.bound = bound
- config.max_steps = 1
- config.logdir = os.path.join(
- tf.test.get_temp_dir(), "test-ghmm-training-%s-%s" % (proposal_type, bound))
- ghmm_runners.run_train(config)
- ghmm_runners.run_eval(config)
- data = np.load(os.path.join(config.logdir, "out.npz")).item()
- self.assertAlmostEqual(expected_bound_avg, data["mean"], places=2)
-
-
-if __name__ == "__main__":
- tf.test.main()
diff --git a/research/fivo/fivo/models/__init__.py b/research/fivo/fivo/models/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/fivo/fivo/models/base.py b/research/fivo/fivo/models/base.py
deleted file mode 100644
index 5ffcb7af216f5659e71d7425eeb4e2c3158b3d47..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/models/base.py
+++ /dev/null
@@ -1,342 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Reusable model classes for FIVO."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import sonnet as snt
-import tensorflow as tf
-
-from fivo import nested_utils as nested
-
-tfd = tf.contrib.distributions
-
-
-class ELBOTrainableSequenceModel(object):
- """An abstract class for ELBO-trainable sequence models to extend.
-
- Because the ELBO, IWAE, and FIVO bounds all accept the same arguments,
- any model that is ELBO-trainable is also IWAE- and FIVO-trainable.
- """
-
- def zero_state(self, batch_size, dtype):
- """Returns the initial state of the model as a Tensor or tuple of Tensors.
-
- Args:
- batch_size: The batch size.
- dtype: The datatype to use for the state.
- """
- raise NotImplementedError("zero_state not yet implemented.")
-
- def set_observations(self, observations, seq_lengths):
- """Sets the observations for the model.
-
- This method provides the model with all observed variables including both
- inputs and targets. It will be called before running any computations with
- the model that require the observations, e.g. training the model or
- computing bounds, and should be used to run any necessary preprocessing
- steps.
-
- Args:
- observations: A potentially nested set of Tensors containing
- all observations for the model, both inputs and targets. Typically
- a set of Tensors with shape [max_seq_len, batch_size, data_size].
- seq_lengths: A [batch_size] Tensor of ints encoding the length of each
- sequence in the batch (sequences can be padded to a common length).
- """
- self.observations = observations
- self.max_seq_len = tf.reduce_max(seq_lengths)
- self.observations_ta = nested.tas_for_tensors(
- observations, self.max_seq_len, clear_after_read=False)
- self.seq_lengths = seq_lengths
-
- def propose_and_weight(self, state, t):
- """Propogates model state one timestep and computes log weights.
-
- This method accepts the current state of the model and computes the state
- for the next timestep as well as the incremental log weight of each
- element in the batch.
-
- Args:
- state: The current state of the model.
- t: A scalar integer Tensor representing the current timestep.
- Returns:
- next_state: The state of the model after one timestep.
- log_weights: A [batch_size] Tensor containing the incremental log weights.
- """
- raise NotImplementedError("propose_and_weight not yet implemented.")
-
-DEFAULT_INITIALIZERS = {"w": tf.contrib.layers.xavier_initializer(),
- "b": tf.zeros_initializer()}
-
-
-class ConditionalNormalDistribution(object):
- """A Normal distribution conditioned on Tensor inputs via a fc network."""
-
- def __init__(self, size, hidden_layer_sizes, sigma_min=0.0,
- raw_sigma_bias=0.25, hidden_activation_fn=tf.nn.relu,
- initializers=None, name="conditional_normal_distribution"):
- """Creates a conditional Normal distribution.
-
- Args:
- size: The dimension of the random variable.
- hidden_layer_sizes: The sizes of the hidden layers of the fully connected
- network used to condition the distribution on the inputs.
- sigma_min: The minimum standard deviation allowed, a scalar.
- raw_sigma_bias: A scalar that is added to the raw standard deviation
- output from the fully connected network. Set to 0.25 by default to
- prevent standard deviations close to 0.
- hidden_activation_fn: The activation function to use on the hidden layers
- of the fully connected network.
- initializers: The variable intitializers to use for the fully connected
- network. The network is implemented using snt.nets.MLP so it must
- be a dictionary mapping the keys 'w' and 'b' to the initializers for
- the weights and biases. Defaults to xavier for the weights and zeros
- for the biases when initializers is None.
- name: The name of this distribution, used for sonnet scoping.
- """
- self.sigma_min = sigma_min
- self.raw_sigma_bias = raw_sigma_bias
- self.name = name
- self.size = size
- if initializers is None:
- initializers = DEFAULT_INITIALIZERS
- self.fcnet = snt.nets.MLP(
- output_sizes=hidden_layer_sizes + [2*size],
- activation=hidden_activation_fn,
- initializers=initializers,
- activate_final=False,
- use_bias=True,
- name=name + "_fcnet")
-
- def condition(self, tensor_list, **unused_kwargs):
- """Computes the parameters of a normal distribution based on the inputs."""
- inputs = tf.concat(tensor_list, axis=1)
- outs = self.fcnet(inputs)
- mu, sigma = tf.split(outs, 2, axis=1)
- sigma = tf.maximum(tf.nn.softplus(sigma + self.raw_sigma_bias),
- self.sigma_min)
- return mu, sigma
-
- def __call__(self, *args, **kwargs):
- """Creates a normal distribution conditioned on the inputs."""
- mu, sigma = self.condition(args, **kwargs)
- return tf.contrib.distributions.Normal(loc=mu, scale=sigma)
-
-
-class ConditionalBernoulliDistribution(object):
- """A Bernoulli distribution conditioned on Tensor inputs via a fc net."""
-
- def __init__(self, size, hidden_layer_sizes, hidden_activation_fn=tf.nn.relu,
- initializers=None, bias_init=0.0,
- name="conditional_bernoulli_distribution"):
- """Creates a conditional Bernoulli distribution.
-
- Args:
- size: The dimension of the random variable.
- hidden_layer_sizes: The sizes of the hidden layers of the fully connected
- network used to condition the distribution on the inputs.
- hidden_activation_fn: The activation function to use on the hidden layers
- of the fully connected network.
- initializers: The variable intiializers to use for the fully connected
- network. The network is implemented using snt.nets.MLP so it must
- be a dictionary mapping the keys 'w' and 'b' to the initializers for
- the weights and biases. Defaults to xavier for the weights and zeros
- for the biases when initializers is None.
- bias_init: A scalar or vector Tensor that is added to the output of the
- fully-connected network that parameterizes the mean of this
- distribution.
- name: The name of this distribution, used for sonnet scoping.
- """
- self.bias_init = bias_init
- self.size = size
- if initializers is None:
- initializers = DEFAULT_INITIALIZERS
- self.fcnet = snt.nets.MLP(
- output_sizes=hidden_layer_sizes + [size],
- activation=hidden_activation_fn,
- initializers=initializers,
- activate_final=False,
- use_bias=True,
- name=name + "_fcnet")
-
- def condition(self, tensor_list):
- """Computes the p parameter of the Bernoulli distribution."""
- inputs = tf.concat(tensor_list, axis=1)
- return self.fcnet(inputs) + self.bias_init
-
- def __call__(self, *args):
- p = self.condition(args)
- return tf.contrib.distributions.Bernoulli(logits=p)
-
-
-class NormalApproximatePosterior(ConditionalNormalDistribution):
- """A Normally-distributed approx. posterior with res_q parameterization."""
-
- def __init__(self, size, hidden_layer_sizes, sigma_min=0.0,
- raw_sigma_bias=0.25, hidden_activation_fn=tf.nn.relu,
- initializers=None, smoothing=False,
- name="conditional_normal_distribution"):
- super(NormalApproximatePosterior, self).__init__(
- size, hidden_layer_sizes, sigma_min=sigma_min,
- raw_sigma_bias=raw_sigma_bias,
- hidden_activation_fn=hidden_activation_fn, initializers=initializers,
- name=name)
- self.smoothing = smoothing
-
- def condition(self, tensor_list, prior_mu, smoothing_tensors=None):
- """Generates the mean and variance of the normal distribution.
-
- Args:
- tensor_list: The list of Tensors to condition on. Will be concatenated and
- fed through a fully connected network.
- prior_mu: The mean of the prior distribution associated with this
- approximate posterior. Will be added to the mean produced by
- this approximate posterior, in res_q fashion.
- smoothing_tensors: A list of Tensors. If smoothing is True, these Tensors
- will be concatenated with the tensors in tensor_list.
- Returns:
- mu: The mean of the approximate posterior.
- sigma: The standard deviation of the approximate posterior.
- """
- if self.smoothing:
- tensor_list.extend(smoothing_tensors)
- mu, sigma = super(NormalApproximatePosterior, self).condition(tensor_list)
- return mu + prior_mu, sigma
-
-
-class NonstationaryLinearDistribution(object):
- """A set of loc-scale distributions that are linear functions of inputs.
-
- This class defines a series of location-scale distributions such that
- the means are learnable linear functions of the inputs and the log variances
- are learnable constants. The functions and log variances are different across
- timesteps, allowing the distributions to be nonstationary.
- """
-
- def __init__(self,
- num_timesteps,
- inputs_per_timestep=None,
- outputs_per_timestep=None,
- initializers=None,
- variance_min=0.0,
- output_distribution=tfd.Normal,
- dtype=tf.float32):
- """Creates a NonstationaryLinearDistribution.
-
- Args:
- num_timesteps: The number of timesteps, i.e. the number of distributions.
- inputs_per_timestep: A list of python ints, the dimension of inputs to the
- linear function at each timestep. If not provided, the dimension at each
- timestep is assumed to be 1.
- outputs_per_timestep: A list of python ints, the dimension of the output
- distribution at each timestep. If not provided, the dimension at each
- timestep is assumed to be 1.
- initializers: A dictionary containing intializers for the variables. The
- initializer under the key 'w' is used for the weights in the linear
- function and the initializer under the key 'b' is used for the biases.
- Defaults to xavier initialization for the weights and zeros for the
- biases.
- variance_min: Python float, the minimum variance of each distribution.
- output_distribution: A locatin-scale subclass of tfd.Distribution that
- defines the output distribution, e.g. Normal.
- dtype: The dtype of the weights and biases.
- """
- if not initializers:
- initializers = DEFAULT_INITIALIZERS
- if not inputs_per_timestep:
- inputs_per_timestep = [1] * num_timesteps
- if not outputs_per_timestep:
- outputs_per_timestep = [1] * num_timesteps
- self.num_timesteps = num_timesteps
- self.variance_min = variance_min
- self.initializers = initializers
- self.dtype = dtype
- self.output_distribution = output_distribution
-
- def _get_variables_ta(shapes, name, initializer, trainable=True):
- """Creates a sequence of variables and stores them in a TensorArray."""
- # Infer shape if all shapes are equal.
- first_shape = shapes[0]
- infer_shape = all(shape == first_shape for shape in shapes)
- ta = tf.TensorArray(
- dtype=dtype, size=len(shapes), dynamic_size=False,
- clear_after_read=False, infer_shape=infer_shape)
- for t, shape in enumerate(shapes):
- var = tf.get_variable(
- name % t, shape=shape, initializer=initializer, trainable=trainable)
- ta = ta.write(t, var)
- return ta
-
- bias_shapes = [[num_outputs] for num_outputs in outputs_per_timestep]
- self.log_variances = _get_variables_ta(
- bias_shapes, "proposal_log_variance_%d", initializers["b"])
- self.mean_biases = _get_variables_ta(
- bias_shapes, "proposal_b_%d", initializers["b"])
- weight_shapes = zip(inputs_per_timestep, outputs_per_timestep)
- self.mean_weights = _get_variables_ta(
- weight_shapes, "proposal_w_%d", initializers["w"])
- self.shapes = tf.TensorArray(
- dtype=tf.int32, size=num_timesteps,
- dynamic_size=False, clear_after_read=False).unstack(weight_shapes)
-
- def __call__(self, t, inputs):
- """Computes the distribution at timestep t.
-
- Args:
- t: Scalar integer Tensor, the current timestep. Must be in
- [0, num_timesteps).
- inputs: The inputs to the linear function parameterizing the mean of
- the current distribution. A Tensor of shape [batch_size, num_inputs_t].
- Returns:
- A tfd.Distribution subclass representing the distribution at timestep t.
- """
- b = self.mean_biases.read(t)
- w = self.mean_weights.read(t)
- shape = self.shapes.read(t)
- w = tf.reshape(w, shape)
- b = tf.reshape(b, [shape[1], 1])
- log_variance = self.log_variances.read(t)
- scale = tf.sqrt(tf.maximum(tf.exp(log_variance), self.variance_min))
- loc = tf.matmul(w, inputs, transpose_a=True) + b
- return self.output_distribution(loc=loc, scale=scale)
-
-
-def encode_all(inputs, encoder):
- """Encodes a timeseries of inputs with a time independent encoder.
-
- Args:
- inputs: A [time, batch, feature_dimensions] tensor.
- encoder: A network that takes a [batch, features_dimensions] input and
- encodes the input.
- Returns:
- A [time, batch, encoded_feature_dimensions] output tensor.
- """
- input_shape = tf.shape(inputs)
- num_timesteps, batch_size = input_shape[0], input_shape[1]
- reshaped_inputs = tf.reshape(inputs, [-1, inputs.shape[-1]])
- inputs_encoded = encoder(reshaped_inputs)
- inputs_encoded = tf.reshape(inputs_encoded,
- [num_timesteps, batch_size, encoder.output_size])
- return inputs_encoded
-
-
-def ta_for_tensor(x, **kwargs):
- """Creates a TensorArray for the input tensor."""
- return tf.TensorArray(
- x.dtype, tf.shape(x)[0], dynamic_size=False, **kwargs).unstack(x)
diff --git a/research/fivo/fivo/models/ghmm.py b/research/fivo/fivo/models/ghmm.py
deleted file mode 100644
index 07cf6c50e803383ef5690e8d24010e4706286eb7..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/models/ghmm.py
+++ /dev/null
@@ -1,483 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A Gaussian hidden markov model.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from fivo.models import base
-
-tfd = tf.contrib.distributions
-
-
-class GaussianHMM(object):
- """A hidden markov model with 1-D Gaussian latent space and observations.
-
- This is a hidden markov model where the state and observations are
- one-dimensional Gaussians. The mean of each latent state is a linear
- function of the previous latent state, and the mean of each observation
- is a linear function of the current latent state.
-
- The description that follows is 0-indexed instead of 1-indexed to make
- it easier to reason about the parameters passed to the model.
-
- The parameters of the model are:
- T: The number timesteps, latent states, and observations.
- vz_t, t=0 to T-1: The variance of the latent state at timestep t.
- vx_t, t=0 to T-1: The variance of the observation at timestep t.
- wz_t, t=1 to T-1: The weight that defines the latent transition at t.
- wx_t, t=0 to T-1: The weight that defines the observation function at t.
-
- There are T vz_t, vx_t, and wx_t but only T-1 wz_t because there are only
- T-1 transitions in the model.
-
- Given these parameters, sampling from the model is defined as
-
- z_0 ~ N(0, vz_0)
- x_0 | z_0 ~ N(wx_0 * z_0, vx_0)
- z_1 | z_0 ~ N(wz_1 * z_0, vz_1)
- x_1 | z_1 ~ N(wx_1 * z_1, vx_1)
- ...
- z_{T-1} | z_{T-2} ~ N(wz_{T-1} * z_{T-2}, vz_{T-1})
- x_{T-1} | z_{T-1} ~ N(wx_{T-1} * z_{T-1}, vx_{T-1}).
- """
-
- def __init__(self,
- num_timesteps,
- transition_variances=1.,
- emission_variances=1.,
- transition_weights=1.,
- emission_weights=1.,
- dtype=tf.float32):
- """Creates a gaussian hidden markov model.
-
- Args:
- num_timesteps: A python int, the number of timesteps in the model.
- transition_variances: The variance of p(z_t | z_t-1). Can be a scalar,
- setting all variances to be the same, or a Tensor of shape
- [num_timesteps].
- emission_variances: The variance of p(x_t | z_t). Can be a scalar,
- setting all variances to be the same, or a Tensor of shape
- [num_timesteps].
- transition_weights: The weight that defines the linear function that
- produces the mean of z_t given z_{t-1}. Can be a scalar, setting
- all weights to be the same, or a Tensor of shape [num_timesteps-1].
- emission_weights: The weight that defines the linear function that
- produces the mean of x_t given z_t. Can be a scalar, setting
- all weights to be the same, or a Tensor of shape [num_timesteps].
- dtype: The datatype of the state.
- """
- self.num_timesteps = num_timesteps
- self.dtype = dtype
-
- def _expand_param(param, size):
- param = tf.convert_to_tensor(param, dtype=self.dtype)
- if not param.get_shape().as_list():
- param = tf.tile(param[tf.newaxis], [size])
-
- return param
-
- def _ta_for_param(param):
- size = tf.shape(param)[0]
- ta = tf.TensorArray(dtype=param.dtype,
- size=size,
- dynamic_size=False,
- clear_after_read=False).unstack(param)
- return ta
-
- self.transition_variances = _ta_for_param(
- _expand_param(transition_variances, num_timesteps))
- self.transition_weights = _ta_for_param(
- _expand_param(transition_weights, num_timesteps-1))
- em_var = _expand_param(emission_variances, num_timesteps)
- self.emission_variances = _ta_for_param(em_var)
- em_w = _expand_param(emission_weights, num_timesteps)
- self.emission_weights = _ta_for_param(em_w)
- self._compute_covariances(em_w, em_var)
-
- def _compute_covariances(self, emission_weights, emission_variances):
- """Compute all covariance matrices.
-
- Computes the covaraince matrix for the latent variables, the observations,
- and the covariance between the latents and observations.
-
- Args:
- emission_weights: A Tensor of shape [num_timesteps] containing
- the emission distribution weights at each timestep.
- emission_variances: A Tensor of shape [num_timesteps] containing
- the emiision distribution variances at each timestep.
- """
- # Compute the marginal variance of each latent.
- z_variances = [self.transition_variances.read(0)]
- for i in range(1, self.num_timesteps):
- z_variances.append(
- z_variances[i-1] * tf.square(self.transition_weights.read(i-1)) +
- self.transition_variances.read(i))
- # Compute the latent covariance matrix.
- sigma_z = []
- for i in range(self.num_timesteps):
- sigma_z_row = []
- for j in range(self.num_timesteps):
- if i == j:
- sigma_z_row.append(z_variances[i])
- continue
- min_ind = min(i, j)
- max_ind = max(i, j)
- weight = tf.reduce_prod(
- self.transition_weights.gather(tf.range(min_ind, max_ind)))
- sigma_z_row.append(z_variances[min_ind] * weight)
- sigma_z.append(tf.stack(sigma_z_row))
- self.sigma_z = tf.stack(sigma_z)
- # Compute the observation covariance matrix.
- x_weights_outer = tf.einsum("i,j->ij", emission_weights, emission_weights)
- self.sigma_x = x_weights_outer * self.sigma_z + tf.diag(emission_variances)
- # Compute the latent - observation covariance matrix.
- # The first axis will index latents, the second axis will index observtions.
- self.sigma_zx = emission_weights[tf.newaxis, :] * self.sigma_z
- self.obs_dist = tfd.MultivariateNormalFullCovariance(
- loc=tf.zeros([self.num_timesteps], dtype=tf.float32),
- covariance_matrix=self.sigma_x)
-
- def transition(self, t, z_prev):
- """Compute the transition distribution p(z_t | z_t-1).
-
- Args:
- t: The current timestep, a scalar integer Tensor. When t=0 z_prev is
- mostly ignored and the distribution p(z_0) is returned. z_prev is
- 'mostly' ignored because it is still used to derive batch_size.
- z_prev: A [batch_size] set of states.
- Returns:
- p(z_t | z_t-1) as a univariate normal distribution.
- """
- batch_size = tf.shape(z_prev)[0]
- scale = tf.sqrt(self.transition_variances.read(t))
- scale = tf.tile(scale[tf.newaxis], [batch_size])
- loc = tf.cond(tf.greater(t, 0),
- lambda: self.transition_weights.read(t-1)*z_prev,
- lambda: tf.zeros_like(scale))
- return tfd.Normal(loc=loc, scale=scale)
-
- def emission(self, t, z):
- """Compute the emission distribution p(x_t | z_t).
-
- Args:
- t: The current timestep, a scalar integer Tensor.
- z: A [batch_size] set of the current states.
- Returns:
- p(x_t | z_t) as a univariate normal distribution.
- """
- batch_size = tf.shape(z)[0]
- scale = tf.sqrt(self.emission_variances.read(t))
- scale = tf.tile(scale[tf.newaxis], [batch_size])
- loc = self.emission_weights.read(t)*z
- return tfd.Normal(loc=loc, scale=scale)
-
- def filtering(self, t, z_prev, x_cur):
- """Computes the filtering distribution p(z_t | z_{t-1}, x_t).
-
- Args:
- t: A python int, the index for z_t. When t is 0, z_prev is ignored,
- giving p(z_0 | x_0).
- z_prev: z_{t-1}, the previous z to condition on. A Tensor of shape
- [batch_size].
- x_cur: x_t, the current x to condition on. A Tensor of shape [batch_size].
- Returns:
- p(z_t | z_{t-1}, x_t) as a univariate normal distribution.
- """
- z_prev = tf.convert_to_tensor(z_prev)
- x_cur = tf.convert_to_tensor(x_cur)
- batch_size = tf.shape(z_prev)[0]
- z_var = self.transition_variances.read(t)
- x_var = self.emission_variances.read(t)
- x_weight = self.emission_weights.read(t)
- prev_state_weight = x_var/(tf.square(x_weight)*z_var + x_var)
- prev_state_weight *= tf.cond(tf.greater(t, 0),
- lambda: self.transition_weights.read(t-1),
- lambda: tf.zeros_like(prev_state_weight))
- cur_obs_weight = (x_weight*z_var)/(tf.square(x_weight)*z_var + x_var)
- loc = prev_state_weight*z_prev + cur_obs_weight*x_cur
- scale = tf.sqrt((z_var*x_var)/(tf.square(x_weight)*z_var + x_var))
- scale = tf.tile(scale[tf.newaxis], [batch_size])
- return tfd.Normal(loc=loc, scale=scale)
-
- def smoothing(self, t, z_prev, xs):
- """Computes the smoothing distribution p(z_t | z_{t-1}, x_{t:num_timesteps).
-
- Args:
- t: A python int, the index for z_t. When t is 0, z_prev is ignored,
- giving p(z_0 | x_{0:num_timesteps-1}).
- z_prev: z_{t-1}, the previous z to condition on. A Tensor of shape
- [batch_size].
- xs: x_{t:num_timesteps}, the future xs to condition on. A Tensor of shape
- [num_timesteps - t, batch_size].
- Returns:
- p(z_t | z_{t-1}, x_{t:num_timesteps}) as a univariate normal distribution.
- """
- xs = tf.convert_to_tensor(xs)
- z_prev = tf.convert_to_tensor(z_prev)
- batch_size = tf.shape(xs)[1]
- mess_mean, mess_prec = tf.cond(
- tf.less(t, self.num_timesteps-1),
- lambda: tf.unstack(self._compute_backwards_messages(xs[1:]).read(0)),
- lambda: [tf.zeros([batch_size]), tf.zeros([batch_size])])
- return self._smoothing_from_message(t, z_prev, xs[0], mess_mean, mess_prec)
-
- def _smoothing_from_message(self, t, z_prev, x_t, mess_mean, mess_prec):
- """Computes the smoothing distribution given message incoming to z_t.
-
- Computes p(z_t | z_{t-1}, x_{t:num_timesteps}) given the message incoming
- to the node for z_t.
-
- Args:
- t: A python int, the index for z_t. When t is 0, z_prev is ignored.
- z_prev: z_{t-1}, the previous z to condition on. A Tensor of shape
- [batch_size].
- x_t: The observation x at timestep t.
- mess_mean: The mean of the message incoming to z_t, in information form.
- mess_prec: The precision of the message incoming to z_t.
- Returns:
- p(z_t | z_{t-1}, x_{t:num_timesteps}) as a univariate normal distribution.
- """
-
- batch_size = tf.shape(x_t)[0]
- z_var = self.transition_variances.read(t)
- x_var = self.emission_variances.read(t)
- w_x = self.emission_weights.read(t)
-
- def transition_term():
- return (tf.square(self.transition_weights.read(t))/
- self.transition_variances.read(t+1))
-
- prec = 1./z_var + tf.square(w_x)/x_var + mess_prec
- prec += tf.cond(tf.less(t, self.num_timesteps-1),
- transition_term, lambda: 0.)
- mean = x_t*(w_x/x_var) + mess_mean
- mean += tf.cond(tf.greater(t, 0),
- lambda: z_prev*(self.transition_weights.read(t-1)/z_var),
- lambda: 0.)
- mean = tf.reshape(mean / prec, [batch_size])
- scale = tf.reshape(tf.sqrt(1./prec), [batch_size])
- return tfd.Normal(loc=mean, scale=scale)
-
- def _compute_backwards_messages(self, xs):
- """Computes the backwards messages used in smoothing."""
- batch_size = tf.shape(xs)[1]
- num_xs = tf.shape(xs)[0]
- until_t = self.num_timesteps - num_xs
- xs = tf.TensorArray(dtype=xs.dtype,
- size=num_xs,
- dynamic_size=False,
- clear_after_read=True).unstack(xs)
- messages_ta = tf.TensorArray(dtype=xs.dtype,
- size=num_xs,
- dynamic_size=False,
- clear_after_read=False)
-
- def compute_message(t, prev_mean, prev_prec, messages_ta):
- """Computes one step of the backwards messages."""
- z_var = self.transition_variances.read(t)
- w_z = self.transition_weights.read(t-1)
- x_var = self.emission_variances.read(t)
- w_x = self.emission_weights.read(t)
- cur_x = xs.read(t - until_t)
-
- # If it isn't the first message, add the terms from the transition.
- def transition_term():
- return (tf.square(self.transition_weights.read(t))/
- self.transition_variances.read(t+1))
-
- unary_prec = 1/z_var + tf.square(w_x)/x_var
- unary_prec += tf.cond(tf.less(t, self.num_timesteps-1),
- transition_term, lambda: 0.)
-
- unary_mean = (w_x / x_var) * cur_x
- pairwise_prec = w_z / z_var
-
- next_prec = -tf.square(pairwise_prec)/(unary_prec + prev_prec)
- next_mean = (pairwise_prec * (unary_mean + prev_mean) /
- (unary_prec + prev_prec))
- next_prec = tf.reshape(next_prec, [batch_size])
- next_mean = tf.reshape(next_mean, [batch_size])
- messages_ta = messages_ta.write(t - until_t,
- tf.stack([next_mean, next_prec]))
- return t-1, next_mean, next_prec, messages_ta
-
- def pred(t, *unused_args):
- return tf.greater_equal(t, until_t)
-
- init_prec = tf.zeros([batch_size], dtype=xs.dtype)
- init_mean = tf.zeros([batch_size], dtype=xs.dtype)
- t0 = tf.constant(self.num_timesteps - 1, dtype=tf.int32)
-
- outs = tf.while_loop(pred, compute_message,
- (t0, init_mean, init_prec, messages_ta))
- messages = outs[-1]
- return messages
-
- def lookahead(self, t, z_prev):
- """Compute the 'lookahead' distribution, p(x_{t:T} | z_{t-1}).
-
- Args:
- t: A scalar Tensor int, the current timestep. Must be at least 1.
- z_prev: The latent state at time t-1. A Tensor of shape [batch_size].
- Returns:
- p(x_{t:T} | z_{t-1}) as a multivariate normal distribution.
- """
- z_prev = tf.convert_to_tensor(z_prev)
- sigma_zx = self.sigma_zx[t-1, t:]
- z_var = self.sigma_z[t-1, t-1]
- mean = tf.einsum("i,j->ij", z_prev, sigma_zx) / z_var
- variance = (self.sigma_x[t:, t:] -
- tf.einsum("i,j->ij", sigma_zx, sigma_zx) / z_var)
- return tfd.MultivariateNormalFullCovariance(
- loc=mean, covariance_matrix=variance)
-
- def likelihood(self, xs):
- """Compute the true marginal likelihood of the data.
-
- Args:
- xs: The observations, a [num_timesteps, batch_size] float Tensor.
- Returns:
- likelihoods: A [batch_size] float Tensor representing the likelihood of
- each sequence of observations in the batch.
- """
- return self.obs_dist.log_prob(tf.transpose(xs))
-
-
-class TrainableGaussianHMM(GaussianHMM, base.ELBOTrainableSequenceModel):
- """An interface between importance-sampling training methods and the GHMM."""
-
- def __init__(self,
- num_timesteps,
- proposal_type,
- transition_variances=1.,
- emission_variances=1.,
- transition_weights=1.,
- emission_weights=1.,
- random_seed=None,
- dtype=tf.float32):
- """Constructs a trainable Gaussian HMM.
-
- Args:
- num_timesteps: A python int, the number of timesteps in the model.
- proposal_type: The type of proposal to use in the importance sampling
- setup. Could be "filtering", "smoothing", "prior", "true-filtering",
- or "true-smoothing". If "true-filtering" or "true-smoothing" are
- selected, then the true filtering or smoothing distributions are used to
- propose new states. If "learned-filtering" is selected then a
- distribution with learnable parameters is used. Specifically at each
- timestep the proposal is Gaussian with mean that is a learnable linear
- function of the previous state and current observation. The log variance
- is a per-timestep learnable constant. "learned-smoothing" is similar,
- but the mean is a learnable linear function of the previous state and
- all future observations. Note that this proposal class includes the true
- posterior. If "prior" is selected then states are proposed from the
- model's prior.
- transition_variances: The variance of p(z_t | z_t-1). Can be a scalar,
- setting all variances to be the same, or a Tensor of shape
- [num_timesteps].
- emission_variances: The variance of p(x_t | z_t). Can be a scalar,
- setting all variances to be the same, or a Tensor of shape
- [num_timesteps].
- transition_weights: The weight that defines the linear function that
- produces the mean of z_t given z_{t-1}. Can be a scalar, setting
- all weights to be the same, or a Tensor of shape [num_timesteps-1].
- emission_weights: The weight that defines the linear function that
- produces the mean of x_t given z_t. Can be a scalar, setting
- all weights to be the same, or a Tensor of shape [num_timesteps].
- random_seed: A seed for the proposal sampling, mainly useful for testing.
- dtype: The datatype of the state.
- """
- super(TrainableGaussianHMM, self).__init__(
- num_timesteps, transition_variances, emission_variances,
- transition_weights, emission_weights, dtype=dtype)
- self.random_seed = random_seed
- assert proposal_type in ["filtering", "smoothing", "prior",
- "true-filtering", "true-smoothing"]
- if proposal_type == "true-filtering":
- self.proposal = self._filtering_proposal
- elif proposal_type == "true-smoothing":
- self.proposal = self._smoothing_proposal
- elif proposal_type == "prior":
- self.proposal = self.transition
- elif proposal_type == "filtering":
- self._learned_proposal_fn = base.NonstationaryLinearDistribution(
- num_timesteps, inputs_per_timestep=[1] + [2] * (num_timesteps-1))
- self.proposal = self._learned_filtering_proposal
- elif proposal_type == "smoothing":
- inputs_per_timestep = [num_timesteps] + [num_timesteps - t
- for t in range(num_timesteps-1)]
- self._learned_proposal_fn = base.NonstationaryLinearDistribution(
- num_timesteps, inputs_per_timestep=inputs_per_timestep)
- self.proposal = self._learned_smoothing_proposal
-
- def set_observations(self, xs, seq_lengths):
- """Sets the observations and stores the backwards messages."""
- # Squeeze out data dimension since everything is 1-d.
- xs = tf.squeeze(xs)
- self.batch_size = tf.shape(xs)[1]
- super(TrainableGaussianHMM, self).set_observations(xs, seq_lengths)
- self.messages = self._compute_backwards_messages(xs[1:])
-
- def zero_state(self, batch_size, dtype):
- return tf.zeros([batch_size], dtype=dtype)
-
- def propose_and_weight(self, state, t):
- """Computes the next state and log weights for the GHMM."""
- state_shape = tf.shape(state)
- xt = self.observations[t]
- p_zt = self.transition(t, state)
- q_zt = self.proposal(t, state)
- zt = q_zt.sample(seed=self.random_seed)
- zt = tf.reshape(zt, state_shape)
- p_xt_given_zt = self.emission(t, zt)
- log_p_zt = p_zt.log_prob(zt)
- log_q_zt = q_zt.log_prob(zt)
- log_p_xt_given_zt = p_xt_given_zt.log_prob(xt)
- weight = log_p_zt + log_p_xt_given_zt - log_q_zt
- return weight, zt
-
- def _filtering_proposal(self, t, state):
- """Uses the stored observations to compute the filtering distribution."""
- cur_x = self.observations[t]
- return self.filtering(t, state, cur_x)
-
- def _smoothing_proposal(self, t, state):
- """Uses the stored messages to compute the smoothing distribution."""
- mess_mean, mess_prec = tf.cond(
- tf.less(t, self.num_timesteps-1),
- lambda: tf.unstack(self.messages.read(t)),
- lambda: [tf.zeros([self.batch_size]), tf.zeros([self.batch_size])])
- return self._smoothing_from_message(t, state, self.observations[t],
- mess_mean, mess_prec)
-
- def _learned_filtering_proposal(self, t, state):
- cur_x = self.observations[t]
- inputs = tf.cond(tf.greater(t, 0),
- lambda: tf.stack([state, cur_x], axis=0),
- lambda: cur_x[tf.newaxis, :])
- return self._learned_proposal_fn(t, inputs)
-
- def _learned_smoothing_proposal(self, t, state):
- xs = self.observations_ta.gather(tf.range(t, self.num_timesteps))
- inputs = tf.cond(tf.greater(t, 0),
- lambda: tf.concat([state[tf.newaxis, :], xs], axis=0),
- lambda: xs)
- return self._learned_proposal_fn(t, inputs)
diff --git a/research/fivo/fivo/models/ghmm_test.py b/research/fivo/fivo/models/ghmm_test.py
deleted file mode 100644
index 15a03c0c7abeae09bd1cfc87f917ef53ecac205f..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/models/ghmm_test.py
+++ /dev/null
@@ -1,313 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.models.ghmm"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-from fivo.models.ghmm import GaussianHMM
-from fivo.models.ghmm import TrainableGaussianHMM
-
-
-class GHMMTest(tf.test.TestCase):
-
- def test_transition_no_weights(self):
- with self.test_session() as sess:
- ghmm = GaussianHMM(3,
- transition_variances=[1., 2., 3.])
- prev_z = tf.constant([1., 2.], dtype=tf.float32)
- z0 = ghmm.transition(0, prev_z)
- z1 = ghmm.transition(1, prev_z)
- z2 = ghmm.transition(2, prev_z)
- outs = sess.run([z0.mean(), z0.variance(),
- z1.mean(), z1.variance(),
- z2.mean(), z2.variance()])
- self.assertAllClose(outs, [[0., 0.], [1., 1.],
- [1., 2.], [2., 2.],
- [1., 2.], [3., 3.]])
-
- def test_transition_with_weights(self):
- with self.test_session() as sess:
- ghmm = GaussianHMM(3,
- transition_variances=[1., 2., 3.],
- transition_weights=[2., 3.])
- prev_z = tf.constant([1., 2.], dtype=tf.float32)
- z0 = ghmm.transition(0, prev_z)
- z1 = ghmm.transition(1, prev_z)
- z2 = ghmm.transition(2, prev_z)
- outs = sess.run([z0.mean(), z0.variance(),
- z1.mean(), z1.variance(),
- z2.mean(), z2.variance()])
- self.assertAllClose(outs, [[0., 0.], [1., 1.],
- [2., 4.], [2., 2.],
- [3., 6.], [3., 3.]])
-
- def test_emission_no_weights(self):
- with self.test_session() as sess:
- ghmm = GaussianHMM(3, emission_variances=[1., 2., 3.])
- z = tf.constant([1., 2.], dtype=tf.float32)
- x0 = ghmm.emission(0, z)
- x1 = ghmm.emission(1, z)
- x2 = ghmm.emission(2, z)
- outs = sess.run([x0.mean(), x0.variance(),
- x1.mean(), x1.variance(),
- x2.mean(), x2.variance()])
- self.assertAllClose(outs, [[1., 2.], [1., 1.],
- [1., 2.], [2., 2.],
- [1., 2.], [3., 3.]])
-
- def test_emission_with_weights(self):
- with self.test_session() as sess:
- ghmm = GaussianHMM(3,
- emission_variances=[1., 2., 3.],
- emission_weights=[1., 2., 3.])
- z = tf.constant([1., 2.], dtype=tf.float32)
- x0 = ghmm.emission(0, z)
- x1 = ghmm.emission(1, z)
- x2 = ghmm.emission(2, z)
- outs = sess.run([x0.mean(), x0.variance(),
- x1.mean(), x1.variance(),
- x2.mean(), x2.variance()])
- self.assertAllClose(outs, [[1., 2.], [1., 1.],
- [2., 4.], [2., 2.],
- [3., 6.], [3., 3.]])
-
- def test_filtering_no_weights(self):
- with self.test_session() as sess:
- ghmm = GaussianHMM(3,
- transition_variances=[1., 2., 3.],
- emission_variances=[4., 5., 6.])
- z_prev = tf.constant([1., 2.], dtype=tf.float32)
- x_cur = tf.constant([3., 4.], dtype=tf.float32)
- expected_outs = [[[3./5., 4./5.], [4./5., 4./5.]],
- [[11./7., 18./7.], [10./7., 10./7.]],
- [[5./3., 8./3.], [2., 2.]]]
- f_post_0 = ghmm.filtering(0, z_prev, x_cur)
- f_post_1 = ghmm.filtering(1, z_prev, x_cur)
- f_post_2 = ghmm.filtering(2, z_prev, x_cur)
- outs = sess.run([[f_post_0.mean(), f_post_0.variance()],
- [f_post_1.mean(), f_post_1.variance()],
- [f_post_2.mean(), f_post_2.variance()]])
- self.assertAllClose(expected_outs, outs)
-
- def test_filtering_with_weights(self):
- with self.test_session() as sess:
- ghmm = GaussianHMM(3,
- transition_variances=[1., 2., 3.],
- emission_variances=[4., 5., 6.],
- transition_weights=[7., 8.],
- emission_weights=[9., 10., 11])
- z_prev = tf.constant([1., 2.], dtype=tf.float32)
- x_cur = tf.constant([3., 4.], dtype=tf.float32)
- expected_outs = [[[27./85., 36./85.], [4./85., 4./85.]],
- [[95./205., 150./205.], [10./205., 10./205.]],
- [[147./369., 228./369.], [18./369., 18./369.]]]
- f_post_0 = ghmm.filtering(0, z_prev, x_cur)
- f_post_1 = ghmm.filtering(1, z_prev, x_cur)
- f_post_2 = ghmm.filtering(2, z_prev, x_cur)
- outs = sess.run([[f_post_0.mean(), f_post_0.variance()],
- [f_post_1.mean(), f_post_1.variance()],
- [f_post_2.mean(), f_post_2.variance()]])
- self.assertAllClose(expected_outs, outs)
-
- def test_smoothing(self):
- with self.test_session() as sess:
- ghmm = GaussianHMM(3,
- transition_variances=[1., 2., 3.],
- emission_variances=[4., 5., 6.])
- z_prev = tf.constant([1., 2.], dtype=tf.float32)
- xs = tf.constant([[1., 2.],
- [3., 4.],
- [5., 6.]], dtype=tf.float32)
- s_post1 = ghmm.smoothing(0, z_prev, xs)
- outs = sess.run([s_post1.mean(), s_post1.variance()])
- expected_outs = [[281./421., 410./421.], [292./421., 292./421.]]
- self.assertAllClose(expected_outs, outs)
-
- expected_outs = [[149./73., 222./73.], [90./73., 90./73.]]
- s_post2 = ghmm.smoothing(1, z_prev, xs[1:])
- outs = sess.run([s_post2.mean(), s_post2.variance()])
- self.assertAllClose(expected_outs, outs)
-
- s_post3 = ghmm.smoothing(2, z_prev, xs[2:])
- outs = sess.run([s_post3.mean(), s_post3.variance()])
- expected_outs = [[7./3., 10./3.], [2., 2.]]
- self.assertAllClose(expected_outs, outs)
-
- def test_smoothing_with_weights(self):
- with self.test_session() as sess:
- x_weight = np.array([4, 5, 6, 7], dtype=np.float32)
- sigma_x = np.array([5, 6, 7, 8], dtype=np.float32)
- z_weight = np.array([1, 2, 3], dtype=np.float32)
- sigma_z = np.array([1, 2, 3, 4], dtype=np.float32)
- z_prev = np.array([1, 2], dtype=np.float32)
- batch_size = 2
- xs = np.array([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=np.float32)
-
- z_cov, x_cov, z_x_cov = self._compute_covariance_matrices(
- x_weight, z_weight, sigma_x, sigma_z)
-
- expected_outs = []
- # Compute mean and variance for z_0 when we don't condition
- # on previous zs.
- sigma_12 = z_x_cov[0, :]
- sigma_12_22 = np.dot(sigma_12, np.linalg.inv(x_cov))
- mean = np.dot(sigma_12_22, xs)
- variance = np.squeeze(z_cov[0, 0] - np.dot(sigma_12_22, sigma_12))
- expected_outs.append([mean, np.tile(variance, [batch_size])])
-
- # Compute mean and variance for remaining z_ts.
- for t in xrange(1, 4):
- sigma_12 = np.concatenate([[z_cov[t, t - 1]], z_x_cov[t, t:]])
- sigma_22 = np.vstack((
- np.hstack((z_cov[t-1, t-1], z_x_cov[t-1, t:])),
- np.hstack((np.transpose([z_x_cov[t-1, t:]]), x_cov[t:, t:]))
- ))
- sigma_12_22 = np.dot(sigma_12, np.linalg.inv(sigma_22))
- mean = np.dot(sigma_12_22, np.vstack((z_prev, xs[t:])))
- variance = np.squeeze(z_cov[t, t] - np.dot(sigma_12_22, sigma_12))
- expected_outs.append([mean, np.tile(variance, [batch_size])])
-
- ghmm = GaussianHMM(4,
- transition_variances=sigma_z,
- emission_variances=sigma_x,
- transition_weights=z_weight,
- emission_weights=x_weight)
- out_dists = [ghmm.smoothing(t, z_prev, xs[t:]) for t in range(0, 4)]
- outs = [[d.mean(), d.variance()] for d in out_dists]
- run_outs = sess.run(outs)
- self.assertAllClose(expected_outs, run_outs)
-
- def test_covariance_matrices(self):
- with self.test_session() as sess:
- x_weight = np.array([4, 5, 6, 7], dtype=np.float32)
- sigma_x = np.array([5, 6, 7, 8], dtype=np.float32)
- z_weight = np.array([1, 2, 3], dtype=np.float32)
- sigma_z = np.array([1, 2, 3, 4], dtype=np.float32)
-
- z_cov, x_cov, z_x_cov = self._compute_covariance_matrices(
- x_weight, z_weight, sigma_x, sigma_z)
-
- ghmm = GaussianHMM(4,
- transition_variances=sigma_z,
- emission_variances=sigma_x,
- transition_weights=z_weight,
- emission_weights=x_weight)
- self.assertAllClose(z_cov, sess.run(ghmm.sigma_z))
- self.assertAllClose(x_cov, sess.run(ghmm.sigma_x))
- self.assertAllClose(z_x_cov, sess.run(ghmm.sigma_zx))
-
- def _compute_covariance_matrices(self, x_weight, z_weight, sigma_x, sigma_z):
- # Create z covariance matrix from the definitions.
- z_cov = np.zeros([4, 4])
- z_cov[0, 0] = sigma_z[0]
- for i in range(1, 4):
- z_cov[i, i] = (z_cov[i - 1, i - 1] * np.square(z_weight[i - 1]) +
- sigma_z[i])
- for i in range(4):
- for j in range(4):
- if i == j: continue
- min_ind = min(i, j)
- max_ind = max(i, j)
- weights = np.prod(z_weight[min_ind:max_ind])
- z_cov[i, j] = z_cov[min_ind, min_ind] * weights
- # Compute the x covariance matrix and the z-x covariance matrix.
- x_weights_outer = np.outer(x_weight, x_weight)
- x_cov = x_weights_outer * z_cov + np.diag(sigma_x)
- z_x_cov = x_weight * z_cov
- return z_cov, x_cov, z_x_cov
-
- def test_lookahead(self):
- x_weight = np.array([4, 5, 6, 7], dtype=np.float32)
- sigma_x = np.array([5, 6, 7, 8], dtype=np.float32)
- z_weight = np.array([1, 2, 3], dtype=np.float32)
- sigma_z = np.array([1, 2, 3, 4], dtype=np.float32)
- z_prev = np.array([1, 2], dtype=np.float32)
-
- with self.test_session() as sess:
- z_cov, x_cov, z_x_cov = self._compute_covariance_matrices(
- x_weight, z_weight, sigma_x, sigma_z)
-
- expected_outs = []
- for t in range(1, 4):
- sigma_12 = z_x_cov[t-1, t:]
- z_var = z_cov[t-1, t-1]
- mean = np.outer(z_prev, sigma_12/z_var)
- variance = x_cov[t:, t:] - np.outer(sigma_12, sigma_12)/ z_var
- expected_outs.append([mean, variance])
-
- ghmm = GaussianHMM(4,
- transition_variances=sigma_z,
- emission_variances=sigma_x,
- transition_weights=z_weight,
- emission_weights=x_weight)
- out_dists = [ghmm.lookahead(t, z_prev) for t in range(1, 4)]
- outs = [[d.mean(), d.covariance()] for d in out_dists]
- run_outs = sess.run(outs)
- self.assertAllClose(expected_outs, run_outs)
-
-
-class TrainableGHMMTest(tf.test.TestCase):
-
- def test_filtering_proposal(self):
- """Check that stashing the xs doesn't change the filtering distributions."""
- with self.test_session() as sess:
- ghmm = TrainableGaussianHMM(
- 3, "filtering",
- transition_variances=[1., 2., 3.],
- emission_variances=[4., 5., 6.],
- transition_weights=[7., 8.],
- emission_weights=[9., 10., 11])
- observations = tf.constant([[3., 4.],
- [3., 4.],
- [3., 4.]], dtype=tf.float32)
- ghmm.set_observations(observations, [3, 3])
- z_prev = tf.constant([1., 2.], dtype=tf.float32)
-
- proposals = [ghmm._filtering_proposal(t, z_prev) for t in range(3)]
- dist_params = [[p.mean(), p.variance()] for p in proposals]
-
- expected_outs = [[[27./85., 36./85.], [4./85., 4./85.]],
- [[95./205., 150./205.], [10./205., 10./205.]],
- [[147./369., 228./369.], [18./369., 18./369.]]]
- self.assertAllClose(expected_outs, sess.run(dist_params))
-
- def test_smoothing_proposal(self):
- with self.test_session() as sess:
- ghmm = TrainableGaussianHMM(
- 3, "smoothing",
- transition_variances=[1., 2., 3.],
- emission_variances=[4., 5., 6.])
- xs = tf.constant([[1., 2.],
- [3., 4.],
- [5., 6.]], dtype=tf.float32)
- ghmm.set_observations(xs, [3, 3])
- z_prev = tf.constant([1., 2.], dtype=tf.float32)
-
- proposals = [ghmm._smoothing_proposal(t, z_prev) for t in range(3)]
- dist_params = [[p.mean(), p.variance()] for p in proposals]
-
- expected_outs = [[[281./421., 410./421.], [292./421., 292./421.]],
- [[149./73., 222./73.], [90./73., 90./73.]],
- [[7./3., 10./3.], [2., 2.]]]
- self.assertAllClose(expected_outs, sess.run(dist_params))
-
-if __name__ == "__main__":
- tf.test.main()
diff --git a/research/fivo/fivo/models/srnn.py b/research/fivo/fivo/models/srnn.py
deleted file mode 100644
index cdfb560eedffccf8edf41dbab4e85bbd8bbfab46..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/models/srnn.py
+++ /dev/null
@@ -1,587 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""SRNN classes."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import namedtuple
-import functools
-
-import sonnet as snt
-import tensorflow as tf
-
-from fivo.models import base
-
-
-SRNNState = namedtuple("SRNNState", "rnn_state latent_encoded")
-
-
-class SRNN(object):
- """Implementation of a Stochastic Recurrent Neural Network (SRNN).
-
- Introduced in "Sequential Neural Models with Stochastic Layers"
- by Fraccaro et al. https://arxiv.org/pdf/1605.07571.pdf.
-
- The SRNN is a sequence model similar to an RNN that uses stochastic latent
- variables to improve its representational power. It can be thought of as a
- sequential analogue to the variational auto-encoder (VAE).
-
- The SRNN has a deterministic RNN as its backbone, represented by the
- sequence of RNN hidden states h_t. The latent state is conditioned on
- the deterministic RNN states and previous latent state. Unlike the VRNN, the
- the RNN state is not conditioned on the previous latent state. The latent
- states have a Markov structure and it is assumed that
- p(z_t | z_{1:t-1}) = p(z_t | z_{t-1}).
-
- In this implementation of the SRNN the latent state z_t is Gaussian. The
- model's prior over z_t (also called the transition distribution) is
- distributed as Normal(mu_t, diag(sigma_t^2)) where mu_t and sigma_t are the
- mean and standard deviation output from a fully connected network that accepts
- the rnn hidden state h_t and previous latent state z_{t-1} as input.
-
- The emission distribution p(x_t|z_t, h_t) is conditioned on the latent state
- z_t as well as the current RNN hidden state h_t via a fully connected network.
-
- To increase the modeling power of the SRNN, two additional networks are
- used to extract features from the data and the latent state. Those networks
- are called data_encoder and latent_encoder respectively.
-
- For an example of how to call the SRNN's methods see sample_step.
-
- There are a few differences between this exposition and the paper. The main
- goal was to be consistent with the VRNN code. A few components are renamed.
- The backward RNN for approximating the posterior, g_phi_a in the paper, is the
- rev_rnn_cell. The forward RNN that conditions the latent distribution, d in
- the paper, is the rnn_cell. The paper doesn't name the NN's that serve as
- feature extractors, and we name them here as the data_encoder and
- latent_encoder.
- """
-
- def __init__(self,
- rnn_cell,
- data_encoder,
- latent_encoder,
- transition,
- emission,
- random_seed=None):
- """Create a SRNN.
-
- Args:
- rnn_cell: A subclass of tf.nn.rnn_cell.RNNCell that will form the
- deterministic backbone of the SRNN. The inputs to the RNN will be the
- the encoded input of the current timestep, a Tensor of shape
- [batch_size, encoded_data_size].
- data_encoder: A callable that accepts a batch of data x_t and
- 'encodes' it, e.g. runs it through a fully connected network. Must
- accept as argument the inputs x_t, a Tensor of the shape
- [batch_size, data_size] and return a Tensor of shape
- [batch_size, encoded_data_size]. This callable will be called multiple
- times in the SRNN cell so if scoping is not handled correctly then
- multiple copies of the variables in this network could be made. It is
- recommended to use a snt.nets.MLP module, which takes care of this for
- you.
- latent_encoder: A callable that accepts a latent state z_t and
- 'encodes' it, e.g. runs it through a fully connected network. Must
- accept as argument a Tensor of shape [batch_size, latent_size] and
- return a Tensor of shape [batch_size, encoded_latent_size].
- This callable must also have the property 'output_size' defined,
- returning encoded_latent_size.
- transition: A callable that implements the transition distribution
- p(z_t|h_t, z_t-1). Must accept as argument the previous RNN hidden state
- and previous encoded latent state then return a tf.distributions.Normal
- distribution conditioned on the input.
- emission: A callable that implements the emission distribution
- p(x_t|z_t, h_t). Must accept as arguments the encoded latent state
- and the RNN hidden state and return a subclass of
- tf.distributions.Distribution that can be used to evaluate the logprob
- of the targets.
- random_seed: The seed for the random ops. Sets the seed for sample_step.
- """
- self.random_seed = random_seed
- self.rnn_cell = rnn_cell
- self.data_encoder = data_encoder
- self.latent_encoder = latent_encoder
- self.encoded_z_size = latent_encoder.output_size
- self.state_size = (self.rnn_cell.state_size)
- self._transition = transition
- self._emission = emission
-
- def zero_state(self, batch_size, dtype):
- """The initial state of the SRNN.
-
- Contains the initial state of the RNN and the inital encoded latent.
-
- Args:
- batch_size: The batch size.
- dtype: The data type of the SRNN.
- Returns:
- zero_state: The initial state of the SRNN.
- """
- return SRNNState(
- rnn_state=self.rnn_cell.zero_state(batch_size, dtype),
- latent_encoded=tf.zeros(
- [batch_size, self.latent_encoder.output_size], dtype=dtype))
-
- def run_rnn(self, prev_rnn_state, inputs):
- """Runs the deterministic RNN for one step.
-
- Args:
- prev_rnn_state: The state of the RNN from the previous timestep.
- inputs: A Tensor of shape [batch_size, data_size], the current inputs to
- the model. Most often this is x_{t-1}, the previous token in the
- observation sequence.
- Returns:
- rnn_out: The output of the RNN.
- rnn_state: The new state of the RNN.
- """
- rnn_inputs = self.data_encoder(tf.to_float(inputs))
- rnn_out, rnn_state = self.rnn_cell(rnn_inputs, prev_rnn_state)
- return rnn_out, rnn_state
-
- def transition(self, rnn_out, prev_latent_encoded):
- """Computes the transition distribution p(z_t|h_t, z_{t-1}).
-
- Note that p(z_t | h_t, z_{t-1}) = p(z_t| z_{t-1}, x_{1:t-1})
-
- Args:
- rnn_out: The output of the rnn for the current timestep.
- prev_latent_encoded: Float Tensor of shape
- [batch_size, encoded_latent_size], the previous latent state z_{t-1}
- run through latent_encoder.
- Returns:
- p(z_t | h_t): A normal distribution with event shape
- [batch_size, latent_size].
- """
- return self._transition(rnn_out, prev_latent_encoded)
-
- def emission(self, latent, rnn_out):
- """Computes the emission distribution p(x_t | z_t, h_t).
-
- Note that p(x_t | z_t, h_t) = p(x_t | z_t, x_{1:t-1})
-
- Args:
- latent: The stochastic latent state z_t.
- rnn_out: The output of the rnn for the current timestep.
- Returns:
- p(x_t | z_t, h_t): A distribution with event shape
- [batch_size, data_size].
- latent_encoded: The latent state encoded with latent_encoder. Should be
- passed to transition() on the next timestep.
- """
- latent_encoded = self.latent_encoder(latent)
- return self._emission(latent_encoded, rnn_out), latent_encoded
-
- def sample_step(self, prev_state, inputs, unused_t):
- """Samples one output from the model.
-
- Args:
- prev_state: The previous state of the model, a SRNNState containing the
- previous rnn state and the previous encoded latent.
- inputs: A Tensor of shape [batch_size, data_size], the current inputs to
- the model. Most often this is x_{t-1}, the previous token in the
- observation sequence.
- unused_t: The current timestep. Not used currently.
- Returns:
- new_state: The next state of the model, a SRNNState.
- xt: A float Tensor of shape [batch_size, data_size], an output sampled
- from the emission distribution.
- """
- rnn_out, rnn_state = self.run_rnn(prev_state.rnn_state,
- inputs)
- p_zt = self.transition(rnn_out, prev_state.latent_encoded)
- zt = p_zt.sample(seed=self.random_seed)
- p_xt_given_zt, latent_encoded = self.emission(zt, rnn_out)
- xt = p_xt_given_zt.sample(seed=self.random_seed)
- new_state = SRNNState(rnn_state=rnn_state, latent_encoded=latent_encoded)
- return new_state, tf.to_float(xt)
-
-# pylint: disable=invalid-name
-# pylint thinks this is a top-level constant.
-TrainableSRNNState = namedtuple("TrainableSRNNState",
- SRNNState._fields + ("rnn_out",))
-# pylint: enable=g-invalid-name
-
-
-class TrainableSRNN(SRNN, base.ELBOTrainableSequenceModel):
- """A SRNN subclass with proposals and methods for training and evaluation.
-
- This class adds proposals used for training with importance-sampling based
- methods such as the ELBO. The model can be configured to propose from one
- of three proposals: a learned filtering proposal, a learned smoothing
- proposal, or the prior (i.e. the transition distribution).
-
- As described in the SRNN paper, the learned filtering proposal is
- parameterized by a fully connected neural network that accepts as input the
- current target x_t and the current rnn output h_t. The learned smoothing
- proposal is also given the hidden state of an RNN run in reverse over the
- inputs, so as to incorporate information about future observations.
-
- All learned proposals use the 'res_q' parameterization, meaning that instead
- of directly producing the mean of z_t, the proposal network predicts the
- 'residual' from the prior's mean. This is explored more in section 3.3 of
- https://arxiv.org/pdf/1605.07571.pdf.
-
- During training, the latent state z_t is sampled from the proposal and the
- reparameterization trick is used to provide low-variance gradients.
-
- Note that the SRNN paper refers to the proposals as the approximate posterior,
- but we match the VRNN convention of referring to it as the encoder.
- """
-
- def __init__(self,
- rnn_cell,
- data_encoder,
- latent_encoder,
- transition,
- emission,
- proposal_type,
- proposal=None,
- rev_rnn_cell=None,
- tilt=None,
- random_seed=None):
- """Create a trainable RNN.
-
- Args:
- rnn_cell: A subclass of tf.nn.rnn_cell.RNNCell that will form the
- deterministic backbone of the SRNN. The inputs to the RNN will be the
- the encoded input of the current timestep, a Tensor of shape
- [batch_size, encoded_data_size].
- data_encoder: A callable that accepts a batch of data x_t and
- 'encodes' it, e.g. runs it through a fully connected network. Must
- accept as argument the inputs x_t, a Tensor of the shape
- [batch_size, data_size] and return a Tensor of shape
- [batch_size, encoded_data_size]. This callable will be called multiple
- times in the SRNN cell so if scoping is not handled correctly then
- multiple copies of the variables in this network could be made. It is
- recommended to use a snt.nets.MLP module, which takes care of this for
- you.
- latent_encoder: A callable that accepts a latent state z_t and
- 'encodes' it, e.g. runs it through a fully connected network. Must
- accept as argument a Tensor of shape [batch_size, latent_size] and
- return a Tensor of shape [batch_size, encoded_latent_size].
- This callable must also have the property 'output_size' defined,
- returning encoded_latent_size.
- transition: A callable that implements the transition distribution
- p(z_t|h_t, z_t-1). Must accept as argument the previous RNN hidden state
- and previous encoded latent state then return a tf.distributions.Normal
- distribution conditioned on the input.
- emission: A callable that implements the emission distribution
- p(x_t|z_t, h_t). Must accept as arguments the encoded latent state
- and the RNN hidden state and return a subclass of
- tf.distributions.Distribution that can be used to evaluate the logprob
- of the targets.
- proposal_type: A string indicating the type of proposal to use. Can
- be either "filtering", "smoothing", or "prior". When proposal_type is
- "filtering" or "smoothing", proposal must be provided. When
- proposal_type is "smoothing", rev_rnn_cell must also be provided.
- proposal: A callable that implements the proposal q(z_t| h_t, x_{1:T}).
- If proposal_type is "filtering" then proposal must accept as arguments
- the current rnn output, the encoded target of the current timestep,
- and the mean of the prior. If proposal_type is "smoothing" then
- in addition to the current rnn output and the mean of the prior
- proposal must accept as arguments the output of the reverse rnn.
- proposal should return a tf.distributions.Normal distribution
- conditioned on its inputs. If proposal_type is "prior" this argument is
- ignored.
- rev_rnn_cell: A subclass of tf.nn.rnn_cell.RNNCell that will aggregate
- forward rnn outputs in the reverse direction. The inputs to the RNN
- will be the encoded reverse input of the current timestep, a Tensor of
- shape [batch_size, encoded_data_size].
- tilt: A callable that implements the log of a positive tilting function
- (ideally approximating log p(x_{t+1}|z_t, h_t). Must accept as arguments
- the encoded latent state and the RNN hidden state and return a subclass
- of tf.distributions.Distribution that can be used to evaluate the
- logprob of x_{t+1}. Optionally, None and then no tilt is used.
- random_seed: The seed for the random ops. Sets the seed for sample_step
- and __call__.
- """
- super(TrainableSRNN, self).__init__(
- rnn_cell, data_encoder, latent_encoder,
- transition, emission, random_seed=random_seed)
- self.rev_rnn_cell = rev_rnn_cell
- self._tilt = tilt
- assert proposal_type in ["filtering", "smoothing", "prior"]
- self._proposal = proposal
- self.proposal_type = proposal_type
- if proposal_type != "prior":
- assert proposal, "If not proposing from the prior, must provide proposal."
- if proposal_type == "smoothing":
- assert rev_rnn_cell, "Must provide rev_rnn_cell for smoothing proposal."
-
- def zero_state(self, batch_size, dtype):
- super_state = super(TrainableSRNN, self).zero_state(batch_size, dtype)
- return TrainableSRNNState(
- rnn_out=tf.zeros([batch_size, self.rnn_cell.output_size], dtype=dtype),
- **super_state._asdict())
-
- def set_observations(self, observations, seq_lengths):
- """Stores the model's observations.
-
- Stores the observations (inputs and targets) in TensorArrays and precomputes
- things for later like the reverse RNN output and encoded targets.
-
- Args:
- observations: The observations of the model, a tuple containing two
- Tensors of shape [max_seq_len, batch_size, data_size]. The Tensors
- should be the inputs and targets, respectively.
- seq_lengths: An int Tensor of shape [batch_size] containing the length
- of each sequence in observations.
- """
- inputs, targets = observations
- self.seq_lengths = seq_lengths
- self.max_seq_len = tf.reduce_max(seq_lengths)
- self.targets_ta = base.ta_for_tensor(targets, clear_after_read=False)
- targets_encoded = base.encode_all(targets, self.data_encoder)
- self.targets_encoded_ta = base.ta_for_tensor(targets_encoded,
- clear_after_read=False)
- inputs_encoded = base.encode_all(inputs, self.data_encoder)
- rnn_out, _ = tf.nn.dynamic_rnn(self.rnn_cell,
- inputs_encoded,
- time_major=True,
- dtype=tf.float32,
- scope="forward_rnn")
- self.rnn_ta = base.ta_for_tensor(rnn_out,
- clear_after_read=False)
- if self.rev_rnn_cell:
- targets_and_rnn_out = tf.concat([rnn_out, targets_encoded], 2)
- reversed_targets_and_rnn_out = tf.reverse_sequence(
- targets_and_rnn_out, seq_lengths, seq_axis=0, batch_axis=1)
- # Compute the reverse rnn over the targets.
- reverse_rnn_out, _ = tf.nn.dynamic_rnn(self.rev_rnn_cell,
- reversed_targets_and_rnn_out,
- time_major=True,
- dtype=tf.float32,
- scope="reverse_rnn")
- reverse_rnn_out = tf.reverse_sequence(reverse_rnn_out, seq_lengths,
- seq_axis=0, batch_axis=1)
- self.reverse_rnn_ta = base.ta_for_tensor(reverse_rnn_out,
- clear_after_read=False)
-
- def _filtering_proposal(self, rnn_out, prev_latent_encoded, prior, t):
- """Computes the filtering proposal distribution."""
- return self._proposal(rnn_out,
- prev_latent_encoded,
- self.targets_encoded_ta.read(t),
- prior_mu=prior.mean())
-
- def _smoothing_proposal(self, rnn_out, prev_latent_encoded, prior, t):
- """Computes the smoothing proposal distribution."""
- return self._proposal(rnn_out,
- prev_latent_encoded,
- smoothing_tensors=[self.reverse_rnn_ta.read(t)],
- prior_mu=prior.mean())
-
- def proposal(self, rnn_out, prev_latent_encoded, prior, t):
- """Computes the proposal distribution specified by proposal_type.
-
- Args:
- rnn_out: The output of the rnn for the current timestep.
- prev_latent_encoded: Float Tensor of shape
- [batch_size, encoded_latent_size], the previous latent state z_{t-1}
- run through latent_encoder.
- prior: A tf.distributions.Normal distribution representing the prior
- over z_t, p(z_t | z_{1:t-1}, x_{1:t-1}). Used for 'res_q'.
- t: A scalar int Tensor, the current timestep.
- """
- if self.proposal_type == "filtering":
- return self._filtering_proposal(rnn_out, prev_latent_encoded, prior, t)
- elif self.proposal_type == "smoothing":
- return self._smoothing_proposal(rnn_out, prev_latent_encoded, prior, t)
- elif self.proposal_type == "prior":
- return self.transition(rnn_out, prev_latent_encoded)
-
- def tilt(self, rnn_out, latent_encoded, targets):
- r_func = self._tilt(rnn_out, latent_encoded)
- return tf.reduce_sum(r_func.log_prob(targets), axis=-1)
-
- def propose_and_weight(self, state, t):
- """Runs the model and computes importance weights for one timestep.
-
- Runs the model and computes importance weights, sampling from the proposal
- instead of the transition/prior.
-
- Args:
- state: The previous state of the model, a TrainableSRNNState containing
- the previous rnn state, the previous rnn outs, and the previous encoded
- latent.
- t: A scalar integer Tensor, the current timestep.
- Returns:
- weights: A float Tensor of shape [batch_size].
- new_state: The new state of the model.
- """
- targets = self.targets_ta.read(t)
- rnn_out = self.rnn_ta.read(t)
- p_zt = self.transition(rnn_out, state.latent_encoded)
- q_zt = self.proposal(rnn_out, state.latent_encoded, p_zt, t)
- zt = q_zt.sample(seed=self.random_seed)
- p_xt_given_zt, latent_encoded = self.emission(zt, rnn_out)
- log_p_xt_given_zt = tf.reduce_sum(p_xt_given_zt.log_prob(targets), axis=-1)
- log_p_zt = tf.reduce_sum(p_zt.log_prob(zt), axis=-1)
- log_q_zt = tf.reduce_sum(q_zt.log_prob(zt), axis=-1)
- weights = log_p_zt + log_p_xt_given_zt - log_q_zt
- if self._tilt:
- prev_log_r = tf.cond(
- tf.greater(t, 0),
- lambda: self.tilt(state.rnn_out, state.latent_encoded, targets),
- lambda: 0.) # On the first step, prev_log_r = 0.
- log_r = tf.cond(
- tf.less(t + 1, self.max_seq_len),
- lambda: self.tilt(rnn_out, latent_encoded, self.targets_ta.read(t+1)),
- lambda: 0.)
- # On the last step, log_r = 0.
- log_r *= tf.to_float(t < self.seq_lengths - 1)
- weights += log_r - prev_log_r
-
- # This reshape is required because the TensorArray reports different shapes
- # than the initial state provides (where the first dimension is unknown).
- # The difference breaks the while_loop. Reshape prevents the error.
- rnn_out = tf.reshape(rnn_out, tf.shape(state.rnn_out))
-
- new_state = TrainableSRNNState(rnn_out=rnn_out,
- rnn_state=state.rnn_state, # unmodified
- latent_encoded=latent_encoded)
- return weights, new_state
-
-
-_DEFAULT_INITIALIZERS = {"w": tf.contrib.layers.xavier_initializer(),
- "b": tf.zeros_initializer()}
-
-
-def create_srnn(
- data_size,
- latent_size,
- emission_class,
- rnn_hidden_size=None,
- fcnet_hidden_sizes=None,
- encoded_data_size=None,
- encoded_latent_size=None,
- sigma_min=0.0,
- raw_sigma_bias=0.25,
- emission_bias_init=0.0,
- use_tilt=False,
- proposal_type="filtering",
- initializers=None,
- random_seed=None):
- """A factory method for creating SRNN cells.
-
- Args:
- data_size: The dimension of the vectors that make up the data sequences.
- latent_size: The size of the stochastic latent state of the SRNN.
- emission_class: The class of the emission distribution. Can be either
- ConditionalNormalDistribution or ConditionalBernoulliDistribution.
- rnn_hidden_size: The hidden state dimension of the RNN that forms the
- deterministic part of this SRNN. If None, then it defaults
- to latent_size.
- fcnet_hidden_sizes: A list of python integers, the size of the hidden
- layers of the fully connected networks that parameterize the conditional
- distributions of the SRNN. If None, then it defaults to one hidden
- layer of size latent_size.
- encoded_data_size: The size of the output of the data encoding network. If
- None, defaults to latent_size.
- encoded_latent_size: The size of the output of the latent state encoding
- network. If None, defaults to latent_size.
- sigma_min: The minimum value that the standard deviation of the
- distribution over the latent state can take.
- raw_sigma_bias: A scalar that is added to the raw standard deviation
- output from the neural networks that parameterize the prior and
- approximate posterior. Useful for preventing standard deviations close
- to zero.
- emission_bias_init: A bias to added to the raw output of the fully
- connected network that parameterizes the emission distribution. Useful
- for initalizing the mean of the distribution to a sensible starting point
- such as the mean of the training data. Only used with Bernoulli generative
- distributions.
- use_tilt: If true, create a SRNN with a tilting function.
- proposal_type: The type of proposal to use. Can be "filtering", "smoothing",
- or "prior".
- initializers: The variable intitializers to use for the fully connected
- networks and RNN cell. Must be a dictionary mapping the keys 'w' and 'b'
- to the initializers for the weights and biases. Defaults to xavier for
- the weights and zeros for the biases when initializers is None.
- random_seed: A random seed for the SRNN resampling operations.
- Returns:
- model: A TrainableSRNN object.
- """
- if rnn_hidden_size is None:
- rnn_hidden_size = latent_size
- if fcnet_hidden_sizes is None:
- fcnet_hidden_sizes = [latent_size]
- if encoded_data_size is None:
- encoded_data_size = latent_size
- if encoded_latent_size is None:
- encoded_latent_size = latent_size
- if initializers is None:
- initializers = _DEFAULT_INITIALIZERS
- data_encoder = snt.nets.MLP(
- output_sizes=fcnet_hidden_sizes + [encoded_data_size],
- initializers=initializers,
- name="data_encoder")
- latent_encoder = snt.nets.MLP(
- output_sizes=fcnet_hidden_sizes + [encoded_latent_size],
- initializers=initializers,
- name="latent_encoder")
- transition = base.ConditionalNormalDistribution(
- size=latent_size,
- hidden_layer_sizes=fcnet_hidden_sizes,
- sigma_min=sigma_min,
- raw_sigma_bias=raw_sigma_bias,
- initializers=initializers,
- name="prior")
- # Construct the emission distribution.
- if emission_class == base.ConditionalBernoulliDistribution:
- # For Bernoulli distributed outputs, we initialize the bias so that the
- # network generates on average the mean from the training set.
- emission_dist = functools.partial(base.ConditionalBernoulliDistribution,
- bias_init=emission_bias_init)
- else:
- emission_dist = base.ConditionalNormalDistribution
- emission = emission_dist(
- size=data_size,
- hidden_layer_sizes=fcnet_hidden_sizes,
- initializers=initializers,
- name="generative")
- # Construct the proposal distribution.
- if proposal_type in ["filtering", "smoothing"]:
- proposal = base.NormalApproximatePosterior(
- size=latent_size,
- hidden_layer_sizes=fcnet_hidden_sizes,
- sigma_min=sigma_min,
- raw_sigma_bias=raw_sigma_bias,
- initializers=initializers,
- smoothing=(proposal_type == "smoothing"),
- name="approximate_posterior")
- else:
- proposal = None
-
- if use_tilt:
- tilt = emission_dist(
- size=data_size,
- hidden_layer_sizes=fcnet_hidden_sizes,
- initializers=initializers,
- name="tilt")
- else:
- tilt = None
-
- rnn_cell = tf.nn.rnn_cell.LSTMCell(rnn_hidden_size,
- initializer=initializers["w"])
- rev_rnn_cell = tf.nn.rnn_cell.LSTMCell(rnn_hidden_size,
- initializer=initializers["w"])
- return TrainableSRNN(
- rnn_cell, data_encoder, latent_encoder, transition,
- emission, proposal_type, proposal=proposal, rev_rnn_cell=rev_rnn_cell,
- tilt=tilt, random_seed=random_seed)
diff --git a/research/fivo/fivo/models/srnn_test.py b/research/fivo/fivo/models/srnn_test.py
deleted file mode 100644
index 39e10da134d3834babcf2eef1bb3e97fce12a07a..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/models/srnn_test.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.models.srnn."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from fivo.models import base
-from fivo.test_utils import create_srnn
-
-
-class SrnnTest(tf.test.TestCase):
-
- def test_srnn_normal_emission(self):
- self.run_srnn(base.ConditionalNormalDistribution, [-5.947752, -1.182961])
-
- def test_srnn_bernoulli_emission(self):
- self.run_srnn(base.ConditionalBernoulliDistribution, [-2.566631, -2.479234])
-
- def run_srnn(self, generative_class, gt_log_alpha):
- """Tests the SRNN.
-
- All test values are 'golden values' derived by running the code and copying
- the output.
-
- Args:
- generative_class: The class of the generative distribution to use.
- gt_log_alpha: The ground-truth value of log alpha.
- """
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- batch_size = 2
- model, inputs, targets, _ = create_srnn(generative_class=generative_class,
- batch_size=batch_size,
- data_lengths=(1, 1),
- random_seed=1234)
- zero_state = model.zero_state(batch_size=batch_size, dtype=tf.float32)
- model.set_observations([inputs, targets], tf.convert_to_tensor([1, 1]))
- model_out = model.propose_and_weight(zero_state, 0)
- sess.run(tf.global_variables_initializer())
- log_alpha, state = sess.run(model_out)
- self.assertAllClose(
- state.latent_encoded,
- [[0.591787, 1.310583], [-1.523136, 0.953918]])
- self.assertAllClose(state.rnn_out,
- [[0.041675, -0.056038, -0.001823, 0.005224],
- [0.042925, -0.044619, 0.021401, 0.016998]])
- self.assertAllClose(log_alpha, gt_log_alpha)
-
- def test_srnn_with_tilt_normal_emission(self):
- self.run_srnn_with_tilt(base.ConditionalNormalDistribution, [-9.13577, -4.56725])
-
-
- def test_srnn_with_tilt_bernoulli_emission(self):
- self.run_srnn_with_tilt(base.ConditionalBernoulliDistribution, [-4.617461, -5.079248])
-
- def run_srnn_with_tilt(self, generative_class, gt_log_alpha):
- """Tests the SRNN with a tilting function.
-
- All test values are 'golden values' derived by running the code and copying
- the output.
-
- Args:
- generative_class: The class of the generative distribution to use.
- gt_log_alpha: The ground-truth value of log alpha.
- """
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- batch_size = 2
- model, inputs, targets, _ = create_srnn(generative_class=generative_class,
- batch_size=batch_size,
- data_lengths=(3, 2),
- random_seed=1234,
- use_tilt=True)
- zero_state = model.zero_state(batch_size=batch_size, dtype=tf.float32)
- model.set_observations([inputs, targets], tf.convert_to_tensor([3, 2]))
- model_out = model.propose_and_weight(zero_state, 0)
- sess.run(tf.global_variables_initializer())
- log_alpha, state = sess.run(model_out)
- self.assertAllClose(
- state.latent_encoded,
- [[0.591787, 1.310583], [-1.523136, 0.953918]])
- self.assertAllClose(state.rnn_out,
- [[0.041675, -0.056038, -0.001823, 0.005224],
- [0.042925, -0.044619, 0.021401, 0.016998]])
- self.assertAllClose(log_alpha, gt_log_alpha)
-
-if __name__ == "__main__":
- tf.test.main()
diff --git a/research/fivo/fivo/models/vrnn.py b/research/fivo/fivo/models/vrnn.py
deleted file mode 100644
index 4e2552088c19f141a75d791d2be0d0a5238ed87c..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/models/vrnn.py
+++ /dev/null
@@ -1,572 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""VRNN classes."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import namedtuple
-import functools
-
-import sonnet as snt
-import tensorflow as tf
-
-from fivo.models import base
-
-
-VRNNState = namedtuple("VRNNState", "rnn_state latent_encoded")
-
-
-class VRNN(object):
- """Implementation of a Variational Recurrent Neural Network (VRNN).
-
- Introduced in "A Recurrent Latent Variable Model for Sequential data"
- by Chung et al. https://arxiv.org/pdf/1506.02216.pdf.
-
- The VRNN is a sequence model similar to an RNN that uses stochastic latent
- variables to improve its representational power. It can be thought of as a
- sequential analogue to the variational auto-encoder (VAE).
-
- The VRNN has a deterministic RNN as its backbone, represented by the
- sequence of RNN hidden states h_t. At each timestep, the RNN hidden state h_t
- is conditioned on the previous sequence element, x_{t-1}, as well as the
- latent state from the previous timestep, z_{t-1}.
-
- In this implementation of the VRNN the latent state z_t is Gaussian. The
- model's prior over z_t (also called the transition distribution) is
- distributed as Normal(mu_t, diag(sigma_t^2)) where mu_t and sigma_t are the
- mean and standard deviation output from a fully connected network that accepts
- the rnn hidden state h_t as input.
-
- The emission distribution p(x_t|z_t, h_t) is conditioned on the latent state
- z_t as well as the current RNN hidden state h_t via a fully connected network.
-
- To increase the modeling power of the VRNN, two additional networks are
- used to extract features from the data and the latent state. Those networks
- are called data_encoder and latent_encoder respectively.
-
- For an example of how to call the VRNN's methods see sample_step.
-
- There are a few differences between this exposition and the paper.
- First, the indexing scheme for h_t is different than the paper's -- what the
- paper calls h_t we call h_{t+1}. This is the same notation used by Fraccaro
- et al. to describe the VRNN in the paper linked above. Also, the VRNN paper
- uses VAE terminology to refer to the different internal networks, so it
- refers to the emission distribution as the decoder. This implementation also
- renames the functions phi_x and phi_z in the paper to data_encoder and
- latent_encoder.
- """
-
- def __init__(self,
- rnn_cell,
- data_encoder,
- latent_encoder,
- transition,
- emission,
- random_seed=None):
- """Create a VRNN.
-
- Args:
- rnn_cell: A subclass of tf.nn.rnn_cell.RNNCell that will form the
- deterministic backbone of the VRNN. The inputs to the RNN will be the
- encoded latent state of the previous timestep with shape
- [batch_size, encoded_latent_size] as well as the encoded input of the
- current timestep, a Tensor of shape [batch_size, encoded_data_size].
- data_encoder: A callable that accepts a batch of data x_t and
- 'encodes' it, e.g. runs it through a fully connected network. Must
- accept as argument the inputs x_t, a Tensor of the shape
- [batch_size, data_size] and return a Tensor of shape
- [batch_size, encoded_data_size]. This callable will be called multiple
- times in the VRNN cell so if scoping is not handled correctly then
- multiple copies of the variables in this network could be made. It is
- recommended to use a snt.nets.MLP module, which takes care of this for
- you.
- latent_encoder: A callable that accepts a latent state z_t and
- 'encodes' it, e.g. runs it through a fully connected network. Must
- accept as argument a Tensor of shape [batch_size, latent_size] and
- return a Tensor of shape [batch_size, encoded_latent_size].
- This callable must also have the property 'output_size' defined,
- returning encoded_latent_size.
- transition: A callable that implements the transition distribution
- p(z_t|h_t). Must accept as argument the previous RNN hidden state and
- return a tf.distributions.Normal distribution conditioned on the input.
- emission: A callable that implements the emission distribution
- p(x_t|z_t, h_t). Must accept as arguments the encoded latent state
- and the RNN hidden state and return a subclass of
- tf.distributions.Distribution that can be used to evaluate the logprob
- of the targets.
- random_seed: The seed for the random ops. Sets the seed for sample_step.
- """
- self.random_seed = random_seed
- self.rnn_cell = rnn_cell
- self.data_encoder = data_encoder
- self.latent_encoder = latent_encoder
- self.encoded_z_size = latent_encoder.output_size
- self.state_size = (self.rnn_cell.state_size)
- self._transition = transition
- self._emission = emission
-
- def zero_state(self, batch_size, dtype):
- """The initial state of the VRNN.
-
- Contains the initial state of the RNN and the inital encoded latent.
-
- Args:
- batch_size: The batch size.
- dtype: The data type of the VRNN.
- Returns:
- zero_state: The initial state of the VRNN.
- """
- return VRNNState(
- rnn_state=self.rnn_cell.zero_state(batch_size, dtype),
- latent_encoded=tf.zeros(
- [batch_size, self.latent_encoder.output_size], dtype=dtype))
-
- def run_rnn(self, prev_rnn_state, prev_latent_encoded, inputs):
- """Runs the deterministic RNN for one step.
-
- Args:
- prev_rnn_state: The state of the RNN from the previous timestep.
- prev_latent_encoded: Float Tensor of shape
- [batch_size, encoded_latent_size], the previous latent state z_{t-1}
- run through latent_encoder.
- inputs: A Tensor of shape [batch_size, data_size], the current inputs to
- the model. Most often this is x_{t-1}, the previous token in the
- observation sequence.
- Returns:
- rnn_out: The output of the RNN.
- rnn_state: The new state of the RNN.
- """
- inputs_encoded = self.data_encoder(tf.to_float(inputs))
- rnn_inputs = tf.concat([inputs_encoded, prev_latent_encoded], axis=1)
- rnn_out, rnn_state = self.rnn_cell(rnn_inputs, prev_rnn_state)
- return rnn_out, rnn_state
-
- def transition(self, rnn_out):
- """Computes the transition distribution p(z_t|h_t).
-
- Note that p(z_t | h_t) = p(z_t| z_{1:t-1}, x_{1:t-1})
-
- Args:
- rnn_out: The output of the rnn for the current timestep.
- Returns:
- p(z_t | h_t): A normal distribution with event shape
- [batch_size, latent_size].
- """
- return self._transition(rnn_out)
-
- def emission(self, latent, rnn_out):
- """Computes the emission distribution p(x_t | z_t, h_t).
-
- Note that p(x_t | z_t, h_t) = p(x_t | z_{1:t}, x_{1:t-1}).
-
- Args:
- latent: The stochastic latent state z_t.
- rnn_out: The output of the rnn for the current timestep.
- Returns:
- p(x_t | z_t, h_t): A distribution with event shape
- [batch_size, data_size].
- latent_encoded: The latent state encoded with latent_encoder. Should be
- passed to run_rnn on the next timestep.
- """
- latent_encoded = self.latent_encoder(latent)
- return self._emission(latent_encoded, rnn_out), latent_encoded
-
- def sample_step(self, prev_state, inputs, unused_t):
- """Samples one output from the model.
-
- Args:
- prev_state: The previous state of the model, a VRNNState containing the
- previous rnn state and the previous encoded latent.
- inputs: A Tensor of shape [batch_size, data_size], the current inputs to
- the model. Most often this is x_{t-1}, the previous token in the
- observation sequence.
- unused_t: The current timestep. Not used currently.
- Returns:
- new_state: The next state of the model, a VRNNState.
- xt: A float Tensor of shape [batch_size, data_size], an output sampled
- from the emission distribution.
- """
- rnn_out, rnn_state = self.run_rnn(prev_state.rnn_state,
- prev_state.latent_encoded,
- inputs)
- p_zt = self.transition(rnn_out)
- zt = p_zt.sample(seed=self.random_seed)
- p_xt_given_zt, latent_encoded = self.emission(zt, rnn_out)
- xt = p_xt_given_zt.sample(seed=self.random_seed)
- new_state = VRNNState(rnn_state=rnn_state, latent_encoded=latent_encoded)
- return new_state, tf.to_float(xt)
-
-# pylint: disable=invalid-name
-# pylint thinks this is a top-level constant.
-TrainableVRNNState = namedtuple("TrainableVRNNState",
- VRNNState._fields + ("rnn_out",))
-# pylint: enable=g-invalid-name
-
-
-class TrainableVRNN(VRNN, base.ELBOTrainableSequenceModel):
- """A VRNN subclass with proposals and methods for training and evaluation.
-
- This class adds proposals used for training with importance-sampling based
- methods such as the ELBO. The model can be configured to propose from one
- of three proposals: a learned filtering proposal, a learned smoothing
- proposal, or the prior (i.e. the transition distribution).
-
- As described in the VRNN paper, the learned filtering proposal is
- parameterized by a fully connected neural network that accepts as input the
- current target x_t and the current rnn output h_t. The learned smoothing
- proposal is also given the hidden state of an RNN run in reverse over the
- inputs, so as to incorporate information about future observations. This
- smoothing proposal is not described in the VRNN paper.
-
- All learned proposals use the 'res_q' parameterization, meaning that instead
- of directly producing the mean of z_t, the proposal network predicts the
- 'residual' from the prior's mean. This is explored more in section 3.3 of
- https://arxiv.org/pdf/1605.07571.pdf.
-
- During training, the latent state z_t is sampled from the proposal and the
- reparameterization trick is used to provide low-variance gradients.
-
- Note that the VRNN paper uses VAE terminology to refer to the different
- internal networks, so the proposal is referred to as the encoder.
- """
-
- def __init__(self,
- rnn_cell,
- data_encoder,
- latent_encoder,
- transition,
- emission,
- proposal_type,
- proposal=None,
- rev_rnn_cell=None,
- tilt=None,
- random_seed=None):
- """Create a trainable RNN.
-
- Args:
- rnn_cell: A subclass of tf.nn.rnn_cell.RNNCell that will form the
- deterministic backbone of the VRNN. The inputs to the RNN will be the
- encoded latent state of the previous timestep with shape
- [batch_size, encoded_latent_size] as well as the encoded input of the
- current timestep, a Tensor of shape [batch_size, encoded_data_size].
- data_encoder: A callable that accepts a batch of data x_t and
- 'encodes' it, e.g. runs it through a fully connected network. Must
- accept as argument the inputs x_t, a Tensor of the shape
- [batch_size, data_size] and return a Tensor of shape
- [batch_size, encoded_data_size]. This callable will be called multiple
- times in the VRNN cell so if scoping is not handled correctly then
- multiple copies of the variables in this network could be made. It is
- recommended to use a snt.nets.MLP module, which takes care of this for
- you.
- latent_encoder: A callable that accepts a latent state z_t and
- 'encodes' it, e.g. runs it through a fully connected network. Must
- accept as argument a Tensor of shape [batch_size, latent_size] and
- return a Tensor of shape [batch_size, encoded_latent_size].
- This callable must also have the property 'output_size' defined,
- returning encoded_latent_size.
- transition: A callable that implements the transition distribution
- p(z_t|h_t). Must accept as argument the previous RNN hidden state and
- return a tf.distributions.Normal distribution conditioned on the input.
- emission: A callable that implements the emission distribution
- p(x_t|z_t, h_t). Must accept as arguments the encoded latent state
- and the RNN hidden state and return a subclass of
- tf.distributions.Distribution that can be used to evaluate the logprob
- of the targets.
- proposal_type: A string indicating the type of proposal to use. Can
- be either "filtering", "smoothing", or "prior". When proposal_type is
- "filtering" or "smoothing", proposal must be provided. When
- proposal_type is "smoothing", rev_rnn_cell must also be provided.
- proposal: A callable that implements the proposal q(z_t| h_t, x_{1:T}).
- If proposal_type is "filtering" then proposal must accept as arguments
- the current rnn output, the encoded target of the current timestep,
- and the mean of the prior. If proposal_type is "smoothing" then
- in addition to the current rnn output and the mean of the prior
- proposal must accept as arguments the output of the reverse rnn.
- proposal should return a tf.distributions.Normal distribution
- conditioned on its inputs. If proposal_type is "prior" this argument is
- ignored.
- rev_rnn_cell: A subclass of tf.nn.rnn_cell.RNNCell that will aggregate
- observation statistics in the reverse direction. The inputs to the RNN
- will be the encoded reverse input of the current timestep, a Tensor of
- shape [batch_size, encoded_data_size].
- tilt: A callable that implements the log of a positive tilting function
- (ideally approximating log p(x_{t+1}|z_t, h_t). Must accept as arguments
- the encoded latent state and the RNN hidden state and return a subclass
- of tf.distributions.Distribution that can be used to evaluate the
- logprob of x_{t+1}. Optionally, None and then no tilt is used.
- random_seed: The seed for the random ops. Sets the seed for sample_step
- and __call__.
- """
- super(TrainableVRNN, self).__init__(
- rnn_cell, data_encoder, latent_encoder,
- transition, emission, random_seed=random_seed)
- self.rev_rnn_cell = rev_rnn_cell
- self._tilt = tilt
- assert proposal_type in ["filtering", "smoothing", "prior"]
- self._proposal = proposal
- self.proposal_type = proposal_type
- if proposal_type != "prior":
- assert proposal, "If not proposing from the prior, must provide proposal."
- if proposal_type == "smoothing":
- assert rev_rnn_cell, "Must provide rev_rnn_cell for smoothing proposal."
-
- def zero_state(self, batch_size, dtype):
- super_state = super(TrainableVRNN, self).zero_state(batch_size, dtype)
- return TrainableVRNNState(
- rnn_out=tf.zeros([batch_size, self.rnn_cell.output_size], dtype=dtype),
- **super_state._asdict())
-
- def set_observations(self, observations, seq_lengths):
- """Stores the model's observations.
-
- Stores the observations (inputs and targets) in TensorArrays and precomputes
- things for later like the reverse RNN output and encoded targets.
-
- Args:
- observations: The observations of the model, a tuple containing two
- Tensors of shape [max_seq_len, batch_size, data_size]. The Tensors
- should be the inputs and targets, respectively.
- seq_lengths: An int Tensor of shape [batch_size] containing the length
- of each sequence in observations.
- """
- inputs, targets = observations
- self.seq_lengths = seq_lengths
- self.max_seq_len = tf.reduce_max(seq_lengths)
- self.inputs_ta = base.ta_for_tensor(inputs, clear_after_read=False)
- self.targets_ta = base.ta_for_tensor(targets, clear_after_read=False)
- targets_encoded = base.encode_all(targets, self.data_encoder)
- self.targets_encoded_ta = base.ta_for_tensor(targets_encoded,
- clear_after_read=False)
- if self.rev_rnn_cell:
- reverse_targets_encoded = tf.reverse_sequence(
- targets_encoded, seq_lengths, seq_axis=0, batch_axis=1)
- # Compute the reverse rnn over the targets.
- reverse_rnn_out, _ = tf.nn.dynamic_rnn(self.rev_rnn_cell,
- reverse_targets_encoded,
- time_major=True,
- dtype=tf.float32)
- reverse_rnn_out = tf.reverse_sequence(reverse_rnn_out, seq_lengths,
- seq_axis=0, batch_axis=1)
- self.reverse_rnn_ta = base.ta_for_tensor(reverse_rnn_out,
- clear_after_read=False)
-
- def _filtering_proposal(self, rnn_out, prior, t):
- """Computes the filtering proposal distribution."""
- return self._proposal(rnn_out,
- self.targets_encoded_ta.read(t),
- prior_mu=prior.mean())
-
- def _smoothing_proposal(self, rnn_out, prior, t):
- """Computes the smoothing proposal distribution."""
- return self._proposal(rnn_out,
- smoothing_tensors=[self.reverse_rnn_ta.read(t)],
- prior_mu=prior.mean())
-
- def proposal(self, rnn_out, prior, t):
- """Computes the proposal distribution specified by proposal_type.
-
- Args:
- rnn_out: The output of the rnn for the current timestep.
- prior: A tf.distributions.Normal distribution representing the prior
- over z_t, p(z_t | z_{1:t-1}, x_{1:t-1}). Used for 'res_q'.
- t: A scalar int Tensor, the current timestep.
- """
- if self.proposal_type == "filtering":
- return self._filtering_proposal(rnn_out, prior, t)
- elif self.proposal_type == "smoothing":
- return self._smoothing_proposal(rnn_out, prior, t)
- elif self.proposal_type == "prior":
- return self.transition(rnn_out)
-
- def tilt(self, rnn_out, latent_encoded, targets):
- r_func = self._tilt(rnn_out, latent_encoded)
- return tf.reduce_sum(r_func.log_prob(targets), axis=-1)
-
- def propose_and_weight(self, state, t):
- """Runs the model and computes importance weights for one timestep.
-
- Runs the model and computes importance weights, sampling from the proposal
- instead of the transition/prior.
-
- Args:
- state: The previous state of the model, a TrainableVRNNState containing
- the previous rnn state, the previous rnn outs, and the previous encoded
- latent.
- t: A scalar integer Tensor, the current timestep.
- Returns:
- weights: A float Tensor of shape [batch_size].
- new_state: The new state of the model.
- """
- inputs = self.inputs_ta.read(t)
- targets = self.targets_ta.read(t)
- rnn_out, next_rnn_state = self.run_rnn(state.rnn_state,
- state.latent_encoded,
- inputs)
- p_zt = self.transition(rnn_out)
- q_zt = self.proposal(rnn_out, p_zt, t)
- zt = q_zt.sample(seed=self.random_seed)
- p_xt_given_zt, latent_encoded = self.emission(zt, rnn_out)
- log_p_xt_given_zt = tf.reduce_sum(p_xt_given_zt.log_prob(targets), axis=-1)
- log_p_zt = tf.reduce_sum(p_zt.log_prob(zt), axis=-1)
- log_q_zt = tf.reduce_sum(q_zt.log_prob(zt), axis=-1)
- weights = log_p_zt + log_p_xt_given_zt - log_q_zt
- if self._tilt:
- prev_log_r = tf.cond(
- tf.greater(t, 0),
- lambda: self.tilt(state.rnn_out, state.latent_encoded, targets),
- lambda: 0.) # On the first step, prev_log_r = 0.
- log_r = tf.cond(
- tf.less(t + 1, self.max_seq_len),
- lambda: self.tilt(rnn_out, latent_encoded, self.targets_ta.read(t+1)),
- lambda: 0.)
- # On the last step, log_r = 0.
- log_r *= tf.to_float(t < self.seq_lengths - 1)
- weights += log_r - prev_log_r
- new_state = TrainableVRNNState(rnn_state=next_rnn_state,
- rnn_out=rnn_out,
- latent_encoded=latent_encoded)
- return weights, new_state
-
-
-_DEFAULT_INITIALIZERS = {"w": tf.contrib.layers.xavier_initializer(),
- "b": tf.zeros_initializer()}
-
-
-def create_vrnn(
- data_size,
- latent_size,
- emission_class,
- rnn_hidden_size=None,
- fcnet_hidden_sizes=None,
- encoded_data_size=None,
- encoded_latent_size=None,
- sigma_min=0.0,
- raw_sigma_bias=0.25,
- emission_bias_init=0.0,
- use_tilt=False,
- proposal_type="filtering",
- initializers=None,
- random_seed=None):
- """A factory method for creating VRNN cells.
-
- Args:
- data_size: The dimension of the vectors that make up the data sequences.
- latent_size: The size of the stochastic latent state of the VRNN.
- emission_class: The class of the emission distribution. Can be either
- ConditionalNormalDistribution or ConditionalBernoulliDistribution.
- rnn_hidden_size: The hidden state dimension of the RNN that forms the
- deterministic part of this VRNN. If None, then it defaults
- to latent_size.
- fcnet_hidden_sizes: A list of python integers, the size of the hidden
- layers of the fully connected networks that parameterize the conditional
- distributions of the VRNN. If None, then it defaults to one hidden
- layer of size latent_size.
- encoded_data_size: The size of the output of the data encoding network. If
- None, defaults to latent_size.
- encoded_latent_size: The size of the output of the latent state encoding
- network. If None, defaults to latent_size.
- sigma_min: The minimum value that the standard deviation of the
- distribution over the latent state can take.
- raw_sigma_bias: A scalar that is added to the raw standard deviation
- output from the neural networks that parameterize the prior and
- approximate posterior. Useful for preventing standard deviations close
- to zero.
- emission_bias_init: A bias to added to the raw output of the fully
- connected network that parameterizes the emission distribution. Useful
- for initalizing the mean of the distribution to a sensible starting point
- such as the mean of the training data. Only used with Bernoulli generative
- distributions.
- use_tilt: If true, create a VRNN with a tilting function.
- proposal_type: The type of proposal to use. Can be "filtering", "smoothing",
- or "prior".
- initializers: The variable intitializers to use for the fully connected
- networks and RNN cell. Must be a dictionary mapping the keys 'w' and 'b'
- to the initializers for the weights and biases. Defaults to xavier for
- the weights and zeros for the biases when initializers is None.
- random_seed: A random seed for the VRNN resampling operations.
- Returns:
- model: A TrainableVRNN object.
- """
- if rnn_hidden_size is None:
- rnn_hidden_size = latent_size
- if fcnet_hidden_sizes is None:
- fcnet_hidden_sizes = [latent_size]
- if encoded_data_size is None:
- encoded_data_size = latent_size
- if encoded_latent_size is None:
- encoded_latent_size = latent_size
- if initializers is None:
- initializers = _DEFAULT_INITIALIZERS
- data_encoder = snt.nets.MLP(
- output_sizes=fcnet_hidden_sizes + [encoded_data_size],
- initializers=initializers,
- name="data_encoder")
- latent_encoder = snt.nets.MLP(
- output_sizes=fcnet_hidden_sizes + [encoded_latent_size],
- initializers=initializers,
- name="latent_encoder")
- transition = base.ConditionalNormalDistribution(
- size=latent_size,
- hidden_layer_sizes=fcnet_hidden_sizes,
- sigma_min=sigma_min,
- raw_sigma_bias=raw_sigma_bias,
- initializers=initializers,
- name="prior")
- # Construct the emission distribution.
- if emission_class == base.ConditionalBernoulliDistribution:
- # For Bernoulli distributed outputs, we initialize the bias so that the
- # network generates on average the mean from the training set.
- emission_dist = functools.partial(base.ConditionalBernoulliDistribution,
- bias_init=emission_bias_init)
- else:
- emission_dist = base.ConditionalNormalDistribution
- emission = emission_dist(
- size=data_size,
- hidden_layer_sizes=fcnet_hidden_sizes,
- initializers=initializers,
- name="generative")
- # Construct the proposal distribution.
- if proposal_type in ["filtering", "smoothing"]:
- proposal = base.NormalApproximatePosterior(
- size=latent_size,
- hidden_layer_sizes=fcnet_hidden_sizes,
- sigma_min=sigma_min,
- raw_sigma_bias=raw_sigma_bias,
- initializers=initializers,
- smoothing=(proposal_type == "smoothing"),
- name="approximate_posterior")
- else:
- proposal = None
-
- if use_tilt:
- tilt = emission_dist(
- size=data_size,
- hidden_layer_sizes=fcnet_hidden_sizes,
- initializers=initializers,
- name="tilt")
- else:
- tilt = None
-
- rnn_cell = tf.nn.rnn_cell.LSTMCell(rnn_hidden_size,
- initializer=initializers["w"])
- rev_rnn_cell = tf.nn.rnn_cell.LSTMCell(rnn_hidden_size,
- initializer=initializers["w"])
- return TrainableVRNN(
- rnn_cell, data_encoder, latent_encoder, transition,
- emission, proposal_type, proposal=proposal, rev_rnn_cell=rev_rnn_cell,
- tilt=tilt, random_seed=random_seed)
diff --git a/research/fivo/fivo/models/vrnn_test.py b/research/fivo/fivo/models/vrnn_test.py
deleted file mode 100644
index 2d9bde3d5b6c6f66a82bd331cf50a87737864239..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/models/vrnn_test.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.models.vrnn."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-import tensorflow as tf
-
-from fivo.models import base
-from fivo.test_utils import create_vrnn
-
-
-class VrnnTest(tf.test.TestCase):
-
- def test_vrnn_normal_emission(self):
- self.run_vrnn(base.ConditionalNormalDistribution, [-4.509767, -3.242221])
-
- def test_vrnn_bernoulli_emission(self):
- self.run_vrnn(base.ConditionalBernoulliDistribution, [-2.63812733, -2.02216434]),
-
- def run_vrnn(self, generative_class, gt_log_p_x_given_z):
- """Tests the VRNN.
-
- All test values are 'golden values' derived by running the code and copying
- the output.
-
- Args:
- generative_class: The class of the generative distribution to use.
- gt_log_p_x_given_z: The ground-truth value of log p(x|z).
- """
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- batch_size = 2
- model, inputs, targets, _ = create_vrnn(generative_class=generative_class,
- batch_size=batch_size,
- data_lengths=(1, 1),
- random_seed=1234)
- zero_state = model.zero_state(batch_size=batch_size, dtype=tf.float32)
- model.set_observations([inputs, targets], tf.convert_to_tensor([1, 1]))
- model_out = model.propose_and_weight(zero_state, 0)
- sess.run(tf.global_variables_initializer())
- log_alpha, state = sess.run(model_out)
- rnn_state, latent_state, rnn_out = state
- self.assertAllClose(
- rnn_state.c,
- [[-0.15014534, 0.0143046, 0.00160489, -0.12899463],
- [-0.25015137, 0.09377634, -0.05000039, -0.17123522]])
- self.assertAllClose(
- rnn_state.h,
- [[-0.06842659, 0.00760155, 0.00096106, -0.05434214],
- [-0.1109542, 0.0441804, -0.03121299, -0.07882939]]
- )
- self.assertAllClose(
- latent_state,
- [[0.025241, 0.122011, 1.066661, 0.316209, -0.25369, 0.108215,
- -1.501128, -0.440111, -0.40447, -0.156649, 1.206028],
- [0.066824, 0.519937, 0.610973, 0.977739, -0.121889, -0.223429,
- -0.32687, -0.578763, -0.56965, 0.751886, 0.681606]]
- )
- self.assertAllClose(rnn_out, [[-0.068427, 0.007602, 0.000961, -0.054342],
- [-0.110954, 0.04418, -0.031213, -0.078829]])
- gt_log_q_z = [-8.0895052, -6.75819111]
- gt_log_p_z = [-7.246827, -6.512877]
- gt_log_alpha = (np.array(gt_log_p_z) +
- np.array(gt_log_p_x_given_z) -
- np.array(gt_log_q_z))
- self.assertAllClose(log_alpha, gt_log_alpha)
-
- def test_vrnn_with_tilt_normal_emission(self):
- self.run_vrnn_with_tilt(base.ConditionalNormalDistribution, [-5.198263, -6.31686])
-
- def test_vrnn_with_tilt_bernoulli_emission(self):
- self.run_vrnn_with_tilt(base.ConditionalBernoulliDistribution, [-4.66985, -3.802245])
-
- def run_vrnn_with_tilt(self, generative_class, gt_log_alpha):
- """Tests the VRNN with a tilting function.
-
- All test values are 'golden values' derived by running the code and copying
- the output.
-
- Args:
- generative_class: The class of the generative distribution to use.
- gt_log_alpha: The ground-truth value of log alpha.
- """
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- batch_size = 2
- model, inputs, targets, _ = create_vrnn(generative_class=generative_class,
- batch_size=batch_size,
- data_lengths=(3, 2),
- random_seed=1234,
- use_tilt=True)
- zero_state = model.zero_state(batch_size=batch_size, dtype=tf.float32)
- model.set_observations([inputs, targets], tf.convert_to_tensor([3, 2]))
- model_out = model.propose_and_weight(zero_state, 0)
- sess.run(tf.global_variables_initializer())
- log_alpha, state = sess.run(model_out)
- rnn_state, latent_state, rnn_out = state
- self.assertAllClose(
- rnn_state.c,
- [[-0.15014534, 0.0143046, 0.00160489, -0.12899463],
- [-0.25015137, 0.09377634, -0.05000039, -0.17123522]])
- self.assertAllClose(
- rnn_state.h,
- [[-0.06842659, 0.00760155, 0.00096106, -0.05434214],
- [-0.1109542, 0.0441804, -0.03121299, -0.07882939]]
- )
- self.assertAllClose(
- latent_state,
- [[0.025241, 0.122011, 1.066661, 0.316209, -0.25369, 0.108215,
- -1.501128, -0.440111, -0.40447, -0.156649, 1.206028],
- [0.066824, 0.519937, 0.610973, 0.977739, -0.121889, -0.223429,
- -0.32687, -0.578763, -0.56965, 0.751886, 0.681606]]
- )
- self.assertAllClose(rnn_out, [[-0.068427, 0.007602, 0.000961, -0.054342],
- [-0.110954, 0.04418, -0.031213, -0.078829]])
- self.assertAllClose(log_alpha, gt_log_alpha)
-
-if __name__ == "__main__":
- tf.test.main()
diff --git a/research/fivo/fivo/nested_utils.py b/research/fivo/fivo/nested_utils.py
deleted file mode 100644
index ef956a80c40d55331a3acbfe78111e099559ddea..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/nested_utils.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A set of utils for dealing with nested lists and tuples of Tensors."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import itertools
-import tensorflow as tf
-
-from tensorflow.python.util import nest
-
-
-def map_nested(map_fn, nested):
- """Executes map_fn on every element in a (potentially) nested structure.
-
- Args:
- map_fn: A callable to execute on each element in 'nested'.
- nested: A potentially nested combination of sequence objects. Sequence
- objects include tuples, lists, namedtuples, and all subclasses of
- collections.Sequence except strings. See nest.is_sequence for details.
- For example [1, ('hello', 4.3)] is a nested structure containing elements
- 1, 'hello', and 4.3.
- Returns:
- out_structure: A potentially nested combination of sequence objects with the
- same structure as the 'nested' input argument. out_structure
- contains the result of applying map_fn to each element in 'nested'. For
- example map_nested(lambda x: x+1, [1, (3, 4.3)]) returns [2, (4, 5.3)].
- """
- out = map(map_fn, nest.flatten(nested))
- return nest.pack_sequence_as(nested, out)
-
-
-def tile_tensors(tensors, multiples):
- """Tiles a set of Tensors.
-
- Args:
- tensors: A potentially nested tuple or list of Tensors with rank
- greater than or equal to the length of 'multiples'. The Tensors do not
- need to have the same rank, but their rank must not be dynamic.
- multiples: A python list of ints indicating how to tile each Tensor
- in 'tensors'. Similar to the 'multiples' argument to tf.tile.
- Returns:
- tiled_tensors: A potentially nested tuple or list of Tensors with the same
- structure as the 'tensors' input argument. Contains the result of
- applying tf.tile to each Tensor in 'tensors'. When the rank of a Tensor
- in 'tensors' is greater than the length of multiples, multiples is padded
- at the end with 1s. For example when tiling a 4-dimensional Tensor with
- multiples [3, 4], multiples would be padded to [3, 4, 1, 1] before tiling.
- """
- def tile_fn(x):
- return tf.tile(x, multiples + [1] * (x.shape.ndims - len(multiples)))
-
- return map_nested(tile_fn, tensors)
-
-
-def where_tensors(condition, x_tensors, y_tensors):
- """Performs a tf.where operation on a two sets of Tensors.
-
- Args:
- condition: The condition tensor to use for the where operation.
- x_tensors: A potentially nested tuple or list of Tensors.
- y_tensors: A potentially nested tuple or list of Tensors. Must have the
- same structure as x_tensors.
- Returns:
- whered_tensors: A potentially nested tuple or list of Tensors with the
- same structure as the 'tensors' input argument. Contains the result of
- applying tf.where(condition, x, y) on each pair of elements in x_tensors
- and y_tensors.
- """
- flat_x = nest.flatten(x_tensors)
- flat_y = nest.flatten(y_tensors)
- result = [tf.where(condition, x, y) for x, y in
- itertools.izip(flat_x, flat_y)]
-
- return nest.pack_sequence_as(x_tensors, result)
-
-
-def gather_tensors(tensors, indices):
- """Performs a tf.gather operation on a set of Tensors.
-
- Args:
- tensors: A potentially nested tuple or list of Tensors.
- indices: The indices to use for the gather operation.
- Returns:
- gathered_tensors: A potentially nested tuple or list of Tensors with the
- same structure as the 'tensors' input argument. Contains the result of
- applying tf.gather(x, indices) on each element x in 'tensors'.
- """
- return map_nested(lambda x: tf.gather(x, indices), tensors)
-
-
-def tas_for_tensors(tensors, length, **kwargs):
- """Unstacks a set of Tensors into TensorArrays.
-
- Args:
- tensors: A potentially nested tuple or list of Tensors with length in the
- first dimension greater than or equal to the 'length' input argument.
- length: The desired length of the TensorArrays.
- **kwargs: Keyword args for TensorArray constructor.
- Returns:
- tensorarrays: A potentially nested tuple or list of TensorArrays with the
- same structure as 'tensors'. Contains the result of unstacking each Tensor
- in 'tensors'.
- """
- def map_fn(x):
- ta = tf.TensorArray(x.dtype, length,
- name=x.name.split(':')[0] + '_ta', **kwargs)
- return ta.unstack(x[:length, :])
- return map_nested(map_fn, tensors)
-
-
-def read_tas(tas, index):
- """Performs a read operation on a set of TensorArrays.
-
- Args:
- tas: A potentially nested tuple or list of TensorArrays with length greater
- than 'index'.
- index: The location to read from.
- Returns:
- read_tensors: A potentially nested tuple or list of Tensors with the same
- structure as the 'tas' input argument. Contains the result of
- performing a read operation at 'index' on each TensorArray in 'tas'.
- """
- return map_nested(lambda ta: ta.read(index), tas)
diff --git a/research/fivo/fivo/nested_utils_test.py b/research/fivo/fivo/nested_utils_test.py
deleted file mode 100644
index 87991dd79cdb29d12944f9afa3fd0c5178dc4eb5..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/nested_utils_test.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.nested_utils."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import tensorflow as tf
-nest = tf.contrib.framework.nest
-
-from fivo import nested_utils
-
-# An example namedtuple for use in the following tests.
-ExampleTuple = collections.namedtuple('ExampleTuple', ['a', 'b'])
-
-
-class NestedUtilsTest(tf.test.TestCase):
-
- def test_map_nested_works_on_nested_structures(self):
- """Check that map_nested works with nested structures."""
- original = [1, (2, 3.2, (4., ExampleTuple(5, 6)))]
- expected = [2, (3, 4.2, (5., ExampleTuple(6, 7)))]
- out = nested_utils.map_nested(lambda x: x+1, original)
- self.assertEqual(expected, out)
-
- def test_map_nested_works_on_single_objects(self):
- """Check that map_nested works with raw objects."""
- original = 1
- expected = 2
- out = nested_utils.map_nested(lambda x: x+1, original)
- self.assertEqual(expected, out)
-
- def test_map_nested_works_on_flat_lists(self):
- """Check that map_nested works with a flat list."""
- original = [1, 2, 3]
- expected = [2, 3, 4]
- out = nested_utils.map_nested(lambda x: x+1, original)
- self.assertEqual(expected, out)
-
- def test_tile_tensors(self):
- """Checks that tile_tensors correctly tiles tensors of different ranks."""
- a = tf.range(20)
- b = tf.reshape(a, [2, 10])
- c = tf.reshape(a, [2, 2, 5])
- a_tiled = tf.tile(a, [3])
- b_tiled = tf.tile(b, [3, 1])
- c_tiled = tf.tile(c, [3, 1, 1])
- tensors = [a, (b, ExampleTuple(c, c))]
- expected_tensors = [a_tiled, (b_tiled, ExampleTuple(c_tiled, c_tiled))]
- tiled = nested_utils.tile_tensors(tensors, [3])
- nest.assert_same_structure(expected_tensors, tiled)
- with self.test_session() as sess:
- expected, out = sess.run([expected_tensors, tiled])
- expected = nest.flatten(expected)
- out = nest.flatten(out)
- # Check that the tiling is correct.
- for x, y in zip(expected, out):
- self.assertAllClose(x, y)
-
- def test_gather_tensors(self):
- a = tf.reshape(tf.range(20), [5, 4])
- inds = [0, 0, 1, 4]
- a_gathered = tf.gather(a, inds)
- tensors = [a, (a, ExampleTuple(a, a))]
- gt_gathered = [a_gathered, (a_gathered,
- ExampleTuple(a_gathered, a_gathered))]
- gathered = nested_utils.gather_tensors(tensors, inds)
- nest.assert_same_structure(gt_gathered, gathered)
- with self.test_session() as sess:
- gt, out = sess.run([gt_gathered, gathered])
- gt = nest.flatten(gt)
- out = nest.flatten(out)
- # Check that the gathering is correct.
- for x, y in zip(gt, out):
- self.assertAllClose(x, y)
-
- def test_tas_for_tensors(self):
- a = tf.reshape(tf.range(20), [5, 4])
- tensors = [a, (a, ExampleTuple(a, a))]
- tas = nested_utils.tas_for_tensors(tensors, 5)
- nest.assert_same_structure(tensors, tas)
- # We can't pass TensorArrays to sess.run so instead we turn then back into
- # tensors to check that they were created correctly.
- stacked = nested_utils.map_nested(lambda x: x.stack(), tas)
- with self.test_session() as sess:
- gt, out = sess.run([tensors, stacked])
- gt = nest.flatten(gt)
- out = nest.flatten(out)
- # Check that the tas were created correctly.
- for x, y in zip(gt, out):
- self.assertAllClose(x, y)
-
- def test_read_tas(self):
- a = tf.reshape(tf.range(20), [5, 4])
- a_read = a[3, :]
- tensors = [a, (a, ExampleTuple(a, a))]
- gt_read = [a_read, (a_read, ExampleTuple(a_read, a_read))]
- tas = nested_utils.tas_for_tensors(tensors, 5)
- tas_read = nested_utils.read_tas(tas, 3)
- nest.assert_same_structure(tas, tas_read)
- with self.test_session() as sess:
- gt, out = sess.run([gt_read, tas_read])
- gt = nest.flatten(gt)
- out = nest.flatten(out)
- # Check that the tas were read correctly.
- for x, y in zip(gt, out):
- self.assertAllClose(x, y)
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/fivo/fivo/runners.py b/research/fivo/fivo/runners.py
deleted file mode 100644
index ec6fb91bf51fa2c7c44d7402e635d257f80c3f7a..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/runners.py
+++ /dev/null
@@ -1,489 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""High-level code for creating and running FIVO-related Tensorflow graphs.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import os
-import time
-
-import numpy as np
-import tensorflow as tf
-
-from fivo import bounds
-from fivo import smc
-
-from fivo.data import datasets
-from fivo.models import base
-from fivo.models import srnn
-from fivo.models import vrnn
-
-
-def create_dataset_and_model(config, split, shuffle, repeat):
- """Creates the dataset and model for a given config.
-
- Args:
- config: A configuration object with config values accessible as properties.
- Most likely a FLAGS object. This function expects the properties
- batch_size, dataset_path, dataset_type, and latent_size to be defined.
- split: The dataset split to load.
- shuffle: If true, shuffle the dataset randomly.
- repeat: If true, repeat the dataset endlessly.
- Returns:
- inputs: A batch of input sequences represented as a dense Tensor of shape
- [time, batch_size, data_dimension].
- targets: A batch of target sequences represented as a dense Tensor of
- shape [time, batch_size, data_dimension].
- lens: An int Tensor of shape [batch_size] representing the lengths of each
- sequence in the batch.
- model: A vrnn.VRNNCell model object.
- Raises:
- ValueError: if the config is invalid.
- """
- sigma_min = 0.0
- if config.dataset_type == "pianoroll":
- inputs, targets, lengths, mean = datasets.create_pianoroll_dataset(
- config.dataset_path, split, config.batch_size, shuffle=shuffle,
- repeat=repeat)
- # Convert the mean of the training set to logit space so it can be used to
- # initialize the bias of the generative distribution.
- emission_bias_init = -tf.log(
- 1. / tf.clip_by_value(mean, 0.0001, 0.9999) - 1)
- emission_distribution_class = base.ConditionalBernoulliDistribution
- elif config.dataset_type == "speech":
- inputs, targets, lengths = datasets.create_speech_dataset(
- config.dataset_path, config.batch_size,
- samples_per_timestep=config.data_dimension, prefetch_buffer_size=1,
- shuffle=False, repeat=False)
- # There is no bias for the generative distribution because the test set
- # is assumed to be already standardized with the training set statistics.
- mean = None
- emission_bias_init = None
- emission_distribution_class = base.ConditionalNormalDistribution
- if config.model == "vrnn":
- model = vrnn.create_vrnn(inputs.get_shape().as_list()[2],
- config.latent_size,
- emission_distribution_class,
- emission_bias_init=emission_bias_init,
- proposal_type=config.proposal_type,
- sigma_min=sigma_min,
- raw_sigma_bias=0.5,
- use_tilt=(config.bound == "fivo-aux"))
- elif config.model == "srnn":
- model = srnn.create_srnn(inputs.get_shape().as_list()[2],
- config.latent_size,
- emission_distribution_class,
- emission_bias_init=emission_bias_init,
- proposal_type=config.proposal_type,
- sigma_min=sigma_min,
- raw_sigma_bias=0.5,
- use_tilt=(config.bound == "fivo-aux"))
- else:
- raise ValueError("model flag: %s is unrecognized" % config.model)
- return inputs, targets, lengths, model, mean
-
-
-def restore_checkpoint_if_exists(saver, sess, logdir):
- """Looks for a checkpoint and restores the session from it if found.
-
- Args:
- saver: A tf.train.Saver for restoring the session.
- sess: A TensorFlow session.
- logdir: The directory to look for checkpoints in.
- Returns:
- True if a checkpoint was found and restored, False otherwise.
- """
- checkpoint = tf.train.get_checkpoint_state(logdir)
- if checkpoint:
- checkpoint_name = os.path.basename(checkpoint.model_checkpoint_path)
- full_checkpoint_path = os.path.join(logdir, checkpoint_name)
- saver.restore(sess, full_checkpoint_path)
- return True
- return False
-
-
-def wait_for_checkpoint(saver, sess, logdir):
- """Loops until the session is restored from a checkpoint in logdir.
-
- Args:
- saver: A tf.train.Saver for restoring the session.
- sess: A TensorFlow session.
- logdir: The directory to look for checkpoints in.
- """
- while not restore_checkpoint_if_exists(saver, sess, logdir):
- tf.logging.info("Checkpoint not found in %s, sleeping for 60 seconds."
- % logdir)
- time.sleep(60)
-
-
-def run_train(config, create_dataset_and_model_fn=create_dataset_and_model):
- """Runs training for a sequential latent variable model.
-
- Args:
- config: A configuration object with config values accessible as properties.
- Most likely a FLAGS object. For a list of expected properties and their
- meaning see the flags defined in fivo.py.
- create_dataset_and_model_fn: If present, calls this function to create a
- dataset and model instead of create_dataset_and_model() above. The
- signature must be the same.
- """
-
- def create_logging_hook(step, bound_value):
- """Creates a logging hook that prints the bound value periodically."""
- bound_label = config.bound + " bound"
- if config.normalize_by_seq_len:
- bound_label += " per timestep"
- else:
- bound_label += " per sequence"
- def summary_formatter(log_dict):
- return "Step %d, %s: %f" % (
- log_dict["step"], bound_label, log_dict["bound_value"])
- logging_hook = tf.train.LoggingTensorHook(
- {"step": step, "bound_value": bound_value},
- every_n_iter=config.summarize_every,
- formatter=summary_formatter)
- return logging_hook
-
- def create_loss():
- """Creates the loss to be optimized.
-
- Returns:
- bound: A float Tensor containing the value of the bound that is
- being optimized.
- loss: A float Tensor that when differentiated yields the gradients
- to apply to the model. Should be optimized via gradient descent.
- """
- inputs, targets, lengths, model, _ = create_dataset_and_model_fn(
- config, split="train", shuffle=True, repeat=True)
- # Compute lower bounds on the log likelihood.
- if config.bound == "elbo":
- ll_per_seq, _, _ = bounds.iwae(
- model, (inputs, targets), lengths, num_samples=1,
- parallel_iterations=config.parallel_iterations
- )
- elif config.bound == "iwae":
- ll_per_seq, _, _ = bounds.iwae(
- model, (inputs, targets), lengths, num_samples=config.num_samples,
- parallel_iterations=config.parallel_iterations
- )
- elif config.bound in ("fivo", "fivo-aux"):
- if config.resampling_type == "relaxed":
- ll_per_seq, _, _, _ = bounds.fivo(
- model, (inputs, targets),
- lengths,
- num_samples=config.num_samples,
- resampling_criterion=smc.ess_criterion,
- resampling_type=config.resampling_type,
- random_seed=config.random_seed,
- relaxed_resampling_temperature=config.
- relaxed_resampling_temperature,
- parallel_iterations=config.parallel_iterations
- )
- else:
- ll_per_seq, _, _, _ = bounds.fivo(
- model, (inputs, targets), lengths, num_samples=config.num_samples,
- resampling_criterion=smc.ess_criterion,
- resampling_type=config.resampling_type,
- random_seed=config.random_seed,
- parallel_iterations=config.parallel_iterations
- )
- # Compute loss scaled by number of timesteps.
- ll_per_t = tf.reduce_mean(ll_per_seq / tf.to_float(lengths))
- ll_per_seq = tf.reduce_mean(ll_per_seq)
-
- tf.summary.scalar("train_ll_per_seq", ll_per_seq)
- tf.summary.scalar("train_ll_per_t", ll_per_t)
-
- if config.normalize_by_seq_len:
- return ll_per_t, -ll_per_t
- else:
- return ll_per_seq, -ll_per_seq
-
- def create_graph():
- """Creates the training graph."""
- global_step = tf.train.get_or_create_global_step()
- bound, loss = create_loss()
- opt = tf.train.AdamOptimizer(config.learning_rate)
- grads = opt.compute_gradients(loss, var_list=tf.trainable_variables())
- train_op = opt.apply_gradients(grads, global_step=global_step)
- return bound, train_op, global_step
-
- device = tf.train.replica_device_setter(ps_tasks=config.ps_tasks)
- with tf.Graph().as_default():
- if config.random_seed: tf.set_random_seed(config.random_seed)
- with tf.device(device):
- bound, train_op, global_step = create_graph()
- log_hook = create_logging_hook(global_step, bound)
- start_training = not config.stagger_workers
- with tf.train.MonitoredTrainingSession(
- master=config.master,
- is_chief=config.task == 0,
- hooks=[log_hook],
- checkpoint_dir=config.logdir,
- save_checkpoint_secs=120,
- save_summaries_steps=config.summarize_every,
- log_step_count_steps=config.summarize_every) as sess:
- cur_step = -1
- while not sess.should_stop() and cur_step <= config.max_steps:
- if config.task > 0 and not start_training:
- cur_step = sess.run(global_step)
- tf.logging.info("task %d not active yet, sleeping at step %d" %
- (config.task, cur_step))
- time.sleep(30)
- if cur_step >= config.task * 1000:
- start_training = True
- else:
- _, cur_step = sess.run([train_op, global_step])
-
-
-def run_eval(config, create_dataset_and_model_fn=create_dataset_and_model):
- """Runs evaluation for a sequential latent variable model.
-
- This method runs only one evaluation over the dataset, writes summaries to
- disk, and then terminates. It does not loop indefinitely.
-
- Args:
- config: A configuration object with config values accessible as properties.
- Most likely a FLAGS object. For a list of expected properties and their
- meaning see the flags defined in fivo.py.
- create_dataset_and_model_fn: If present, calls this function to create a
- dataset and model instead of create_dataset_and_model() above. The
- signature must be the same.
- """
-
- def create_graph():
- """Creates the evaluation graph.
-
- Returns:
- lower_bounds: A tuple of float Tensors containing the values of the 3
- evidence lower bounds, summed across the batch.
- total_batch_length: The total number of timesteps in the batch, summed
- across batch examples.
- batch_size: The batch size.
- global_step: The global step the checkpoint was loaded from.
- """
- global_step = tf.train.get_or_create_global_step()
- inputs, targets, lengths, model, _ = create_dataset_and_model_fn(
- config, split=config.split, shuffle=False, repeat=False)
- # Compute lower bounds on the log likelihood.
- elbo_ll_per_seq, _, _ = bounds.iwae(
- model, (inputs, targets), lengths, num_samples=1,
- parallel_iterations=config.parallel_iterations
- )
- iwae_ll_per_seq, _, _ = bounds.iwae(
- model, (inputs, targets), lengths, num_samples=config.num_samples,
- parallel_iterations=config.parallel_iterations
- )
- # The resampling type should only be used for training, so we ignore it.
- fivo_ll_per_seq, _, _, _ = bounds.fivo(
- model, (inputs, targets), lengths, num_samples=config.num_samples,
- resampling_criterion=smc.ess_criterion, random_seed=config.random_seed,
- parallel_iterations=config.parallel_iterations
- )
- elbo_ll = tf.reduce_sum(elbo_ll_per_seq)
- iwae_ll = tf.reduce_sum(iwae_ll_per_seq)
- fivo_ll = tf.reduce_sum(fivo_ll_per_seq)
- batch_size = tf.shape(lengths)[0]
- total_batch_length = tf.reduce_sum(lengths)
- return ((elbo_ll, iwae_ll, fivo_ll), total_batch_length, batch_size,
- global_step)
-
- def average_bounds_over_dataset(lower_bounds, total_batch_length, batch_size,
- sess):
- """Computes the values of the bounds, averaged over the datset.
-
- Args:
- lower_bounds: Tuple of float Tensors containing the values of the bounds
- evaluated on a single batch.
- total_batch_length: Integer Tensor that represents the total number of
- timesteps in the current batch.
- batch_size: Integer Tensor containing the batch size. This can vary if the
- requested batch_size does not evenly divide the size of the dataset.
- sess: A TensorFlow Session object.
- Returns:
- ll_per_t: A length 3 numpy array of floats containing each bound's average
- value, normalized by the total number of timesteps in the datset. Can
- be interpreted as a lower bound on the average log likelihood per
- timestep in the dataset.
- ll_per_seq: A length 3 numpy array of floats containing each bound's
- average value, normalized by the number of sequences in the dataset.
- Can be interpreted as a lower bound on the average log likelihood per
- sequence in the datset.
- """
- total_ll = np.zeros(3, dtype=np.float64)
- total_n_elems = 0.0
- total_length = 0.0
- while True:
- try:
- outs = sess.run([lower_bounds, batch_size, total_batch_length])
- except tf.errors.OutOfRangeError:
- break
- total_ll += outs[0]
- total_n_elems += outs[1]
- total_length += outs[2]
- ll_per_t = total_ll / total_length
- ll_per_seq = total_ll / total_n_elems
- return ll_per_t, ll_per_seq
-
- def summarize_lls(lls_per_t, lls_per_seq, summary_writer, step):
- """Creates log-likelihood lower bound summaries and writes them to disk.
-
- Args:
- lls_per_t: An array of 3 python floats, contains the values of the
- evaluated bounds normalized by the number of timesteps.
- lls_per_seq: An array of 3 python floats, contains the values of the
- evaluated bounds normalized by the number of sequences.
- summary_writer: A tf.SummaryWriter.
- step: The current global step.
- """
- def scalar_summary(name, value):
- value = tf.Summary.Value(tag=name, simple_value=value)
- return tf.Summary(value=[value])
-
- for i, bound in enumerate(["elbo", "iwae", "fivo"]):
- per_t_summary = scalar_summary("%s/%s_ll_per_t" % (config.split, bound),
- lls_per_t[i])
- per_seq_summary = scalar_summary("%s/%s_ll_per_seq" %
- (config.split, bound),
- lls_per_seq[i])
- summary_writer.add_summary(per_t_summary, global_step=step)
- summary_writer.add_summary(per_seq_summary, global_step=step)
- summary_writer.flush()
-
- with tf.Graph().as_default():
- if config.random_seed: tf.set_random_seed(config.random_seed)
- lower_bounds, total_batch_length, batch_size, global_step = create_graph()
- summary_dir = config.logdir + "/" + config.split
- summary_writer = tf.summary.FileWriter(
- summary_dir, flush_secs=15, max_queue=100)
- saver = tf.train.Saver()
- with tf.train.SingularMonitoredSession() as sess:
- wait_for_checkpoint(saver, sess, config.logdir)
- step = sess.run(global_step)
- tf.logging.info("Model restored from step %d, evaluating." % step)
- ll_per_t, ll_per_seq = average_bounds_over_dataset(
- lower_bounds, total_batch_length, batch_size, sess)
- summarize_lls(ll_per_t, ll_per_seq, summary_writer, step)
- tf.logging.info("%s elbo ll/t: %f, iwae ll/t: %f fivo ll/t: %f",
- config.split, ll_per_t[0], ll_per_t[1], ll_per_t[2])
- tf.logging.info("%s elbo ll/seq: %f, iwae ll/seq: %f fivo ll/seq: %f",
- config.split, ll_per_seq[0], ll_per_seq[1], ll_per_seq[2])
-
-
-def run_sample(config, create_dataset_and_model_fn=create_dataset_and_model):
- """Sample from the model. Only pianorolls and pose datasets are supported."""
-
- def sample_from_model(model, initial_state, initial_inputs, mean):
- """Samples a sequence of outputs from the model.
-
- The mean must be supplied -- if it isn't the results will be incorrect.
-
- Args:
- model: A model with sample_step implemented. See models/vrnn.py for an
- example.
- initial_state: The initial state of the model.
- initial_inputs: The initial inputs to feed into the model.
- mean: The mean of the training set, a Tensor of shape [data_dimension].
- Returns:
- samples: A Tensor of shape [sample_length, batch_size, num_timesteps,
- data_dimension] containing the samples from the model.
- """
- initial_state, initial_output = model.sample_step(initial_state,
- initial_inputs, 0)
- output_ta = tf.TensorArray(size=config.sample_length,
- dtype=tf.float32,
- dynamic_size=False,
- clear_after_read=True)
- output_ta = output_ta.write(0, initial_output)
- t0 = tf.constant(1, dtype=tf.int32)
-
- def sample_step(t, state, prev_outputs, output_ta):
- state, output = model.sample_step(state, prev_outputs, t)
- output_ta = output_ta.write(t, output)
- centered_output = output - mean[tf.newaxis, :]
- return t+1, state, centered_output, output_ta
-
- def sample_predicate(t, *unused_args):
- return t < config.sample_length
-
- _, _, _, output_ta = tf.while_loop(
- sample_predicate,
- sample_step,
- loop_vars=(t0, initial_state, initial_output, output_ta),
- parallel_iterations=config.parallel_iterations
- )
- samples = output_ta.stack()
- samples = tf.reshape(samples, [config.sample_length, config.batch_size,
- config.num_samples, config.data_dimension])
- return samples
-
- def create_graph():
- """Creates the graph to sample from the model.
-
- First, the model is conditioned on a prefix by sampling a batch of data
- and trimming it to prefix_length. The configured bound is used to do the
- conditioning. Then the final state from the conditioning is used to sample
- from the model.
-
- Returns:
- samples: A Tensor of shape [sample_length, batch_size,
- num_samples, data_dimension] representing samples from the model.
- prefixes: A Tensor of shape [prefix_length, batch_size, data_dimension]
- representing the prefixes the model was conditioned on.
- """
- inputs, targets, lengths, model, mean = create_dataset_and_model_fn(
- config, split=config.split, shuffle=True, repeat=True)
- input_prefixes = inputs[:config.prefix_length]
- target_prefixes = targets[:config.prefix_length]
- prefix_lengths = tf.ones_like(lengths) * config.prefix_length
- if config.bound == "elbo":
- _, _, state = bounds.iwae(
- model, (input_prefixes, target_prefixes),
- prefix_lengths, num_samples=1)
- elif config.bound == "iwae":
- _, _, state = bounds.iwae(
- model, (input_prefixes, target_prefixes),
- prefix_lengths, num_samples=config.num_samples)
- elif config.bound == "fivo":
- _, _, _, state = bounds.fivo(
- model, (input_prefixes, target_prefixes), prefix_lengths,
- num_samples=config.num_samples,
- resampling_criterion=smc.ess_criterion,
- random_seed=config.random_seed)
- sample_inputs = tf.tile(inputs[config.prefix_length],
- [config.num_samples, 1])
- samples = sample_from_model(model, state, sample_inputs, mean)
- return samples, target_prefixes
-
- with tf.Graph().as_default():
- if config.random_seed:
- tf.set_random_seed(config.random_seed)
- samples, prefixes = create_graph()
- if config.sample_out_dir:
- out_dir = config.sample_our_dir
- else:
- out_dir = config.logdir
- if not tf.gfile.Exists(out_dir):
- tf.gfile.MakeDirs(out_dir)
- with tf.train.SingularMonitoredSession(
- checkpoint_dir=config.logdir) as sess:
- samples_out, prefixes_out = sess.run([samples, prefixes])
- with tf.gfile.Open(os.path.join(out_dir, "samples.npz"), "w") as fout:
- np.save(fout, {"prefixes": prefixes_out, "samples": samples_out})
diff --git a/research/fivo/fivo/runners_test.py b/research/fivo/fivo/runners_test.py
deleted file mode 100644
index eb050c0a0b38b2511f3d2fb9ec846e63ead3b5ac..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/runners_test.py
+++ /dev/null
@@ -1,242 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.runners"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import numpy as np
-import tensorflow as tf
-
-from fivo import runners
-from fivo.models import base
-from fivo.models import vrnn
-
-FLAGS = tf.app.flags.FLAGS
-
-
-class RunnersTest(tf.test.TestCase):
-
- def default_config(self):
- class Config(object):
- pass
- config = Config()
- config.model = "vrnn"
- config.latent_size = 64
- config.batch_size = 4
- config.num_samples = 4
- config.resampling_type = "multinomial"
- config.normalize_by_seq_len = True
- config.learning_rate = 0.0001
- config.max_steps = int(1e6)
- config.summarize_every = 50
- # Master must be "" to prevent state from persisting between sessions.
- config.master = ""
- config.task = 0
- config.ps_tasks = 0
- config.stagger_workers = True
- config.random_seed = 1234
- config.parallel_iterations = 1
- config.dataset_type = "pianoroll"
- config.data_dimension = None
- config.dataset_path = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "test_data", "tiny_pianoroll.pkl")
- config.proposal_type = "filtering"
- return config
-
- def run_training_one_step(self, bound, dataset_type, data_dimension,
- dataset_filename, dir_prefix, resampling_type,
- model, batch_size=2, num_samples=3,
- create_dataset_and_model_fn=(runners.create_dataset_and_model)):
- config = self.default_config()
- config.model = model
- config.resampling_type = resampling_type
- config.relaxed_resampling_temperature = 0.5
- config.bound = bound
- config.split = "train"
- config.dataset_type = dataset_type
- config.dataset_path = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "test_data",
- dataset_filename)
- config.max_steps = 1
- config.batch_size = batch_size
- config.num_samples = num_samples
- config.latent_size = 4
- config.data_dimension = data_dimension
- config.logdir = os.path.join(tf.test.get_temp_dir(), "%s-%s-%s-%s" %
- (dir_prefix, bound, dataset_type, model))
- runners.run_train(config,
- create_dataset_and_model_fn=create_dataset_and_model_fn)
- return config
-
- def dummmy_dataset_and_model_fn(self, *unused_args, **unused_kwargs):
- # We ignore the arguments in the dummy but need to preserve prototype.
- batch_elements = 5
- sequence_length = 4
- data_dimensions = 3
- dataset = tf.data.Dataset.from_tensors(
- tf.zeros((sequence_length, batch_elements, data_dimensions),
- dtype=tf.float32))
- inputs = dataset.make_one_shot_iterator().get_next()
- targets = tf.zeros_like(inputs)
- lengths = tf.constant([sequence_length] * batch_elements)
- mean = tf.constant((0.0, 0.0, 0.0))
- model = vrnn.create_vrnn(data_dimensions, 1,
- base.ConditionalNormalDistribution)
- return inputs, targets, lengths, model, mean
-
- def test_training_one_step_fivo_pianoroll_vrnn(self):
- self.run_training_one_step("fivo", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "multinomial", "vrnn")
-
- def test_training_one_step_iwae_pianoroll_vrnn(self):
- self.run_training_one_step("iwae", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "multinomial", "vrnn")
-
- def test_training_one_step_elbo_pianoroll_vrnn(self):
- self.run_training_one_step("elbo", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "multinomial", "vrnn")
-
- def test_training_one_step_fivo_speech_vrnn(self):
- self.run_training_one_step("fivo", "speech", 2, "tiny_speech_dataset.tfrecord",
- "test-training", "multinomial", "vrnn")
-
- def test_training_one_step_iwae_speech_vrnn(self):
- self.run_training_one_step("iwae", "speech", 2, "tiny_speech_dataset.tfrecord",
- "test-training", "multinomial", "vrnn")
-
- def test_training_one_step_elbo_speech_vrnn(self):
- self.run_training_one_step("elbo", "speech", 2, "tiny_speech_dataset.tfrecord",
- "test-training", "multinomial", "vrnn")
-
- def test_training_one_step_fivo_pianoroll_srnn(self):
- self.run_training_one_step("fivo", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "multinomial", "srnn")
-
- def test_training_one_step_iwae_pianoroll_srnn(self):
- self.run_training_one_step("iwae", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "multinomial", "srnn")
-
- def test_training_one_step_elbo_pianoroll_srnn(self):
- self.run_training_one_step("elbo", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "multinomial", "srnn")
-
- def test_training_one_step_fivo_speech_srnn(self):
- self.run_training_one_step("fivo", "speech", 2, "tiny_speech_dataset.tfrecord",
- "test-training", "multinomial", "srnn")
-
- def test_training_one_step_iwae_speech_srnn(self):
- self.run_training_one_step("iwae", "speech", 2, "tiny_speech_dataset.tfrecord",
- "test-training", "multinomial", "srnn")
-
- def test_training_one_step_elbo_speech_srnn(self):
- self.run_training_one_step("elbo", "speech", 2, "tiny_speech_dataset.tfrecord",
- "test-training", "multinomial", "srnn")
-
- def test_training_one_step_fivo_pianoroll_vrnn_relaxed(self):
- self.run_training_one_step("fivo", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "relaxed", "vrnn")
-
- def test_training_one_step_iwae_pianoroll_vrnn_relaxed(self):
- self.run_training_one_step("iwae", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "relaxed", "vrnn")
-
- def test_training_one_step_elbo_pianoroll_vrnn_relaxed(self):
- self.run_training_one_step("elbo", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "relaxed", "vrnn")
-
- def test_training_one_step_fivo_pianoroll_srnn_relaxed(self):
- self.run_training_one_step("fivo", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "relaxed", "srnn")
-
- def test_training_one_step_iwae_pianoroll_srnn_relaxed(self):
- self.run_training_one_step("iwae", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "relaxed", "srnn")
-
- def test_training_one_step_elbo_pianoroll_srnn_relaxed(self):
- self.run_training_one_step("elbo", "pianoroll", 88, "tiny_pianoroll.pkl",
- "test-training", "relaxed", "srnn")
-
- def test_eval_vrnn(self):
- self.run_eval("vrnn")
-
- def test_eval_srnn(self):
- self.run_eval("srnn")
-
- def run_eval(self, model):
- config = self.run_training_one_step(
- "fivo", "pianoroll", 88, "tiny_pianoroll.pkl", "test-eval-" + model,
- "multinomial", model)
- config.split = "train"
- runners.run_eval(config)
-
- def test_sampling_vrnn(self):
- self.run_sampling("vrnn")
-
- def test_sampling_srnn(self):
- self.run_sampling("srnn")
-
- def run_sampling(self, model):
- """Test sampling from the model."""
- config = self.run_training_one_step(
- "fivo", "pianoroll", 88, "tiny_pianoroll.pkl", "test-sampling", "multinomial",
- model)
- config.prefix_length = 3
- config.sample_length = 6
- config.split = "train"
- config.sample_out_dir = None
-
- runners.run_sample(config)
- unused_samples = np.load(os.path.join(config.logdir, "samples.npz"))
-
- def test_training_with_custom_fn(self):
- self.run_training_one_step(
- "fivo", "pianoroll", 3, "tiny_pianoroll.pkl",
- "test-training-custom-fn", "multinomial", "vrnn", batch_size=5,
- create_dataset_and_model_fn=self.dummmy_dataset_and_model_fn)
-
- def test_eval_with_custom_fn(self):
- config = self.run_training_one_step(
- "fivo", "pianoroll", 1, "tiny_pianoroll.pkl",
- "test-eval-custom-fn", "multinomial", "vrnn", batch_size=1,
- create_dataset_and_model_fn=self.dummmy_dataset_and_model_fn)
- config.split = "train"
- runners.run_eval(
- config,
- create_dataset_and_model_fn=self.dummmy_dataset_and_model_fn)
-
- def test_sampling_with_custom_fn(self):
- config = self.run_training_one_step(
- "fivo", "pianoroll", 3, "tiny_pianoroll.pkl",
- "test-sample-custom-fn", "multinomial", "vrnn", batch_size=5,
- create_dataset_and_model_fn=self.dummmy_dataset_and_model_fn)
- config.prefix_length = 2
- config.sample_length = 3
- config.split = "train"
- config.sample_out_dir = None
-
- runners.run_sample(
- config,
- create_dataset_and_model_fn=self.dummmy_dataset_and_model_fn)
- unused_samples = np.load(os.path.join(config.logdir, "samples.npz"))
-
-
-if __name__ == "__main__":
- tf.test.main()
diff --git a/research/fivo/fivo/smc.py b/research/fivo/fivo/smc.py
deleted file mode 100644
index 25d4969043e2cb8bc2c2c7a3770d3d2dfcca0bef..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/smc.py
+++ /dev/null
@@ -1,338 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Implementation of sequential Monte Carlo algorithms.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-import fivo.nested_utils as nested
-
-
-def ess_criterion(log_weights, unused_t):
- """A criterion that resamples based on effective sample size."""
- num_particles = tf.shape(log_weights)[0]
- # Calculate the effective sample size.
- ess_num = 2 * tf.reduce_logsumexp(log_weights, axis=0)
- ess_denom = tf.reduce_logsumexp(2 * log_weights, axis=0)
- log_ess = ess_num - ess_denom
- return log_ess <= tf.log(tf.to_float(num_particles) / 2.0)
-
-
-def never_resample_criterion(log_weights, unused_t):
- """A criterion that never resamples."""
- batch_size = tf.shape(log_weights)[1]
- return tf.cast(tf.zeros([batch_size]), tf.bool)
-
-
-def always_resample_criterion(log_weights, unused_t):
- """A criterion resamples at every timestep."""
- batch_size = tf.shape(log_weights)[1]
- return tf.cast(tf.ones([batch_size]), tf.bool)
-
-
-def multinomial_resampling(log_weights, states, num_particles, batch_size,
- random_seed=None):
- """Resample states with multinomial resampling.
-
- Args:
- log_weights: A [num_particles, batch_size] Tensor representing a batch
- of batch_size logits for num_particles-ary Categorical distribution.
- states: A nested list of [batch_size*num_particles, data_size] Tensors that
- will be resampled from the groups of every num_particles-th row.
- num_particles: The number of particles/samples.
- batch_size: The batch size.
- random_seed: The random seed to pass to the resampling operations in
- the particle filter. Mainly useful for testing.
-
- Returns:
- resampled_states: A nested list of [batch_size*num_particles, data_size]
- Tensors resampled via multinomial sampling.
- """
- # Calculate the ancestor indices via resampling. Because we maintain the
- # log unnormalized weights, we pass the weights in as logits, allowing
- # the distribution object to apply a softmax and normalize them.
- resampling_parameters = tf.transpose(log_weights, perm=[1, 0])
- resampling_dist = tf.contrib.distributions.Categorical(
- logits=resampling_parameters)
- ancestors = tf.stop_gradient(
- resampling_dist.sample(sample_shape=num_particles, seed=random_seed))
-
- # Because the batch is flattened, we must modify ancestor_inds to index the
- # proper samples. The particles in the ith filter are distributed every
- # batch_size rows in the batch, and offset i rows from the top. So, to
- # correct the indices we multiply by the batch_size and add the proper offset.
- # Crucially, when ancestor_inds is flattened the layout of the batch is
- # maintained.
- offset = tf.expand_dims(tf.range(batch_size), 0)
- ancestor_inds = tf.reshape(ancestors * batch_size + offset, [-1])
-
- resampled_states = nested.gather_tensors(states, ancestor_inds)
- return resampled_states
-
-
-def _blend_tensor(blending_weights, tensor, num_particles, batch_size):
- """Blend tensor according to the weights.
-
- The first dimension of tensor is actually a 2d index compacted to a 1d
- index and similarly for blended_tensor. So if we index these Tensors
- by [(i, j), k], then
-
- blended_tensor[(i, j), k] =
- sum_l tensor[(l, j), :] * blending_weights[i, j, l].
-
- Args:
- blending_weights: [num_particles, batch_size, num_particles] weights where
- the indices represent [sample index, batch index, blending weight index].
- tensor: [num_particles * batch_size, state_dim] Tensor to be blended.
- num_particles: The number of particles/samples.
- batch_size: The batch size.
-
- Returns:
- blended_tensor: [num_particles*batch_size, state_dim] blended Tensor.
- """
- # tensor is currently [num_particles * batch_size, state_dim], so we reshape
- # it to [num_particles, batch_size, state_dim]. Then, transpose it to
- # [batch_size, state_size, num_particles].
- tensor = tf.transpose(
- tf.reshape(tensor, [num_particles, batch_size, -1]), perm=[1, 2, 0])
- blending_weights = tf.transpose(blending_weights, perm=[1, 2, 0])
- # blendeding_weights is [batch index, blending weight index, sample index].
- # Multiplying these gives a matrix of size [batch_size, state_size,
- # num_particles].
- tensor = tf.matmul(tensor, blending_weights)
- # transpose the tensor to be [num_particles, batch_size, state_size]
- # and then reshape it to match the original format.
- tensor = tf.reshape(tf.transpose(tensor, perm=[2, 0, 1]),
- [num_particles*batch_size, -1])
- return tensor
-
-
-def relaxed_resampling(log_weights, states, num_particles, batch_size,
- temperature=0.5, random_seed=None):
- """Resample states with relaxed resampling.
-
- Draw soft "ancestors" using the Gumbel-Softmax distribution.
-
- Args:
- log_weights: A [num_particles, batch_size] Tensor representing a batch
- of batch_size logits for num_particles-ary Categorical distribution.
- states: A nested list of [batch_size * num_particles, d] Tensors that will
- be resampled from the groups of every num_particles-th row.
- num_particles: The number of particles/samples.
- batch_size: The batch size.
- temperature: The temperature used for the relaxed one hot distribution.
- random_seed: The random seed to pass to the resampling operations in
- the particle filter. Mainly useful for testing.
-
- Returns:
- resampled_states: A nested list of [batch_size * num_particles, d]
- Tensors resampled via multinomial sampling.
- """
- # log_weights are [num_particles, batch_size], so we transpose to get a
- # set of batch_size distributions over [0, num_particles).
- resampling_parameters = tf.transpose(log_weights, perm=[1, 0])
- resampling_dist = tf.contrib.distributions.RelaxedOneHotCategorical(
- temperature,
- logits=resampling_parameters)
-
- # Sample num_particles samples from the distribution, resulting in a
- # [num_particles, batch_size, num_particles] Tensor that represents a set of
- # [num_particles, batch_size] blending weights. The dimensions represent
- # [particle index, batch index, blending weight index].
- ancestors = resampling_dist.sample(sample_shape=num_particles,
- seed=random_seed)
- def map_fn(tensor):
- return _blend_tensor(ancestors, tensor, num_particles, batch_size)
-
- resampled_states = nested.map_nested(map_fn, states)
- return resampled_states
-
-
-def smc(
- transition_fn,
- num_steps,
- num_particles=1,
- resampling_criterion=ess_criterion,
- resampling_fn=multinomial_resampling,
- loop_fn=None,
- parallel_iterations=30,
- swap_memory=True):
- """Run a sequential Monte Carlo (SMC) algorithm.
-
- This method runs an SMC algorithm that evolves systems of particles
- using the supplied transition function for the specified number of steps. The
- particles are optionally resampled using resampling_fn when indicated by
- resampling_criterion.
-
- Args:
- transition_fn: A callable that propogates a batch of particles one step.
- Must accept as arguments a batch of particle states and the current
- timestep. Must return the particle states one timestep in the future, the
- incremental weights of each particle as a [num_samples*batch_size] float
- Tensor, and optionally a set of arguments to pass to the loop_fn. If
- the loop args are not provided, they will be set to None. Before the
- first timestep transition_fn will be called with the arguments None, -1
- and should return the initial particle states.
- num_steps: A [batch_size] Tensor of ints representing the number of steps
- to run each filter for.
- num_particles: A scalar int, the number of particles to use in each filter.
- resampling_criterion: The resampling criterion to use for this particle
- filter. Must accept the current log weights and timestep and
- return a boolean Tensor of shape [batch_size] indicating whether each
- particle filter should resample. See ess_criterion and related functions
- for examples. When resampling_criterion is never_resample_criterion,
- resampling_fn is ignored and never called.
- resampling_fn: A callable that performs the resampling operation. Must
- accept as arguments the log weights, particle states, num_particles,
- and batch_size and return the resampled particle states. See
- multinomial_resampling and relaxed_resampling for examples.
- loop_fn: A callable that performs operations on the weights and
- particle states, useful for accumulating and processing state that
- shouldn't be resampled. At each timestep after (possibly) resampling
- loop_fn will be called with the previous loop_state, a set of arguments
- produced by transition_fn called loop_args, the resampled particle states,
- the current log weights as [num_particles, batch_size] float Tensor, a
- [batch_size] float Tensor representing whether or not each filter
- resampled, the current mask indicating which filters are active, and the
- current timestep. It must return the next loop state. Before the first
- timestep loop_fn will be called with the arguments None, None, None, None,
- -1 and must return the initial loop state. The loop state can be a
- possibly nested structure of Tensors and TensorArrays.
- parallel_iterations: The number of parallel iterations to use for the
- internal while loop. Note that values greater than 1 can introduce
- non-determinism even when resampling is deterministic.
- swap_memory: Whether GPU-CPU memory swapping should be enabled for the
- internal while loop.
-
- Returns:
- log_z_hat: A Tensor of shape [batch_size] containing an estimate of the log
- normalizing constant that converts between the unormalized target
- distribution (as defined by the weights) and the true target distribution.
- log_weights: A Tensor of shape [max_num_steps, batch_size, num_particles]
- containing the log weights at each timestep of the particle filter.
- Will not be valid for timesteps past the supplied num_steps.
- resampled: A float Tensor of shape [max_num_steps, batch_size] indicating
- when the particle filters resampled. Will be 1.0 on timesteps when
- resampling occurred and 0.0 on timesteps when it did not.
- final_loop_state: The final state returned by loop_fn. If loop_fn is None
- then 0 will be returned.
- """
- # batch_size represents the number of particle filters running in parallel.
- batch_size = tf.shape(num_steps)[0]
- # Create a TensorArray where element t is the [num_particles*batch_size]
- # sequence mask for timestep t.
- max_num_steps = tf.reduce_max(num_steps)
- seq_mask = tf.transpose(
- tf.sequence_mask(num_steps, maxlen=max_num_steps, dtype=tf.float32),
- perm=[1, 0])
- seq_mask = tf.tile(seq_mask, [1, num_particles])
- mask_ta = tf.TensorArray(seq_mask.dtype,
- max_num_steps,
- name='mask_ta')
- mask_ta = mask_ta.unstack(seq_mask)
- # Initialize the state.
- t0 = tf.constant(0, tf.int32)
- init_particle_state = transition_fn(None, -1)
-
- def transition(*args):
- transition_outs = transition_fn(*args)
- if len(transition_outs) == 2:
- return transition_outs + (None,)
- else:
- return transition_outs
-
- if loop_fn is None:
- loop_fn = lambda *args: 0
-
- init_loop_state = loop_fn(None, None, None, None, None, None, -1)
- init_states = (init_particle_state, init_loop_state)
- ta_names = ['log_weights', 'resampled']
- tas = [tf.TensorArray(tf.float32, max_num_steps, name='%s_ta' % n)
- for n in ta_names]
- log_weights_acc = tf.zeros([num_particles, batch_size], dtype=tf.float32)
- log_z_hat_acc = tf.zeros([batch_size], dtype=tf.float32)
-
- def while_predicate(t, *unused_args):
- return t < max_num_steps
-
- def while_step(t, state, tas, log_weights_acc, log_z_hat_acc):
- """Implements one timestep of the particle filter."""
- particle_state, loop_state = state
- cur_mask = nested.read_tas(mask_ta, t)
- # Propagate the particles one step.
- log_alpha, new_particle_state, loop_args = transition(particle_state, t)
- # Update the current weights with the incremental weights.
- log_alpha *= cur_mask
- log_alpha = tf.reshape(log_alpha, [num_particles, batch_size])
- log_weights_acc += log_alpha
-
- should_resample = resampling_criterion(log_weights_acc, t)
-
- if resampling_criterion == never_resample_criterion:
- resampled = tf.to_float(should_resample)
- else:
- # Compute the states as if we did resample.
- resampled_states = resampling_fn(
- log_weights_acc,
- new_particle_state,
- num_particles,
- batch_size)
- # Decide whether or not we should resample; don't resample if we are past
- # the end of a sequence.
- should_resample = tf.logical_and(should_resample,
- cur_mask[:batch_size] > 0.)
- float_should_resample = tf.to_float(should_resample)
- new_particle_state = nested.where_tensors(
- tf.tile(should_resample, [num_particles]),
- resampled_states,
- new_particle_state)
- resampled = float_should_resample
-
- new_loop_state = loop_fn(loop_state, loop_args, new_particle_state,
- log_weights_acc, resampled, cur_mask, t)
- # Update log Z hat.
- log_z_hat_update = tf.reduce_logsumexp(
- log_weights_acc, axis=0) - tf.log(tf.to_float(num_particles))
- # If it is the last timestep, always add the update.
- log_z_hat_acc += tf.cond(t < max_num_steps - 1,
- lambda: log_z_hat_update * resampled,
- lambda: log_z_hat_update)
- # Update the TensorArrays before we reset the weights so that we capture
- # the incremental weights and not zeros.
- ta_updates = [log_weights_acc, resampled]
- new_tas = [ta.write(t, x) for ta, x in zip(tas, ta_updates)]
- # For the particle filters that resampled, reset weights to zero.
- log_weights_acc *= (1. - tf.tile(resampled[tf.newaxis, :],
- [num_particles, 1]))
- new_state = (new_particle_state, new_loop_state)
- return t + 1, new_state, new_tas, log_weights_acc, log_z_hat_acc
-
- _, final_state, tas, _, log_z_hat = tf.while_loop(
- while_predicate,
- while_step,
- loop_vars=(t0, init_states, tas, log_weights_acc, log_z_hat_acc),
- parallel_iterations=parallel_iterations,
- swap_memory=swap_memory)
-
- log_weights, resampled = [x.stack() for x in tas]
- log_weights = tf.transpose(log_weights, perm=[0, 2, 1])
- final_particle_state, final_loop_state = final_state
- return (log_z_hat, log_weights, resampled,
- final_particle_state, final_loop_state)
diff --git a/research/fivo/fivo/smc_test.py b/research/fivo/fivo/smc_test.py
deleted file mode 100644
index ae32a62f21e037252bda44e3e1f47e007c9b7b9b..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/smc_test.py
+++ /dev/null
@@ -1,241 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for fivo.smc."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import scipy
-import tensorflow as tf
-
-from fivo import smc
-
-lse = scipy.special.logsumexp
-
-
-def _simple_transition_fn(state, unused_t):
- if state is None:
- return tf.zeros([4], dtype=tf.float32)
- return tf.constant([5., 4., 1., 0.5]), tf.zeros([4], dtype=tf.float32)
-
-
-def _resample_at_step_criterion(step):
- """A criterion that resamples once at a specific timestep."""
- def criterion(log_weights, t):
- batch_size = tf.shape(log_weights)[1]
- return tf.fill([batch_size], tf.equal(t, step))
- return criterion
-
-
-class SMCTest(tf.test.TestCase):
-
- def test_never_resampling(self):
- """Test that never_resample_criterion makes smc not resample.
-
- Also test that the weights and log_z_hat are computed correctly when never
- resampling.
- """
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- outs = smc.smc(
- _simple_transition_fn,
- num_steps=tf.convert_to_tensor([5, 3]),
- num_particles=2,
- resampling_criterion=smc.never_resample_criterion)
- log_z_hat, weights, resampled = sess.run(outs[0:3])
- gt_weights = np.array(
- [[[5, 1], [4, .5]],
- [[10, 2], [8, 1]],
- [[15, 3], [12, 1.5]],
- [[20, 4], [12, 1.5]],
- [[25, 5], [12, 1.5]]],
- dtype=np.float32)
- gt_log_z_hat = np.array(
- [lse([25, 5]) - np.log(2),
- lse([12, 1.5]) - np.log(2)],
- dtype=np.float32)
- self.assertAllClose(gt_log_z_hat, log_z_hat)
- self.assertAllClose(gt_weights, weights)
- self.assertAllEqual(np.zeros_like(resampled), resampled)
-
- def test_always_resampling(self):
- """Test always_resample_criterion makes smc always resample.
-
- Past a sequence end the filter should not resample, however.
- Also check that weights and log_z_hat estimate are correct.
- """
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- outs = smc.smc(
- _simple_transition_fn,
- num_steps=tf.convert_to_tensor([5, 3]),
- num_particles=2,
- resampling_criterion=smc.always_resample_criterion)
- log_z_hat, weights, resampled = sess.run(outs[0:3])
- gt_weights = np.array(
- [[[5, 1], [4, .5]],
- [[5, 1], [4, .5]],
- [[5, 1], [4, .5]],
- [[5, 1], [0., 0.]],
- [[5, 1], [0., 0.]]],
- dtype=np.float32)
- gt_log_z_hat = np.array(
- [5*lse([5, 1]) - 5*np.log(2),
- 3*lse([4, .5]) - 3*np.log(2)],
- dtype=np.float32)
- gt_resampled = np.array(
- [[1, 1], [1, 1], [1, 1], [1, 0], [1, 0]],
- dtype=np.float32)
- self.assertAllClose(gt_log_z_hat, log_z_hat)
- self.assertAllClose(gt_weights, weights)
- self.assertAllEqual(gt_resampled, resampled)
-
- def test_weights_reset_when_resampling_at_sequence_end(self):
- """Test that the weights are reset when resampling at the sequence end.
-
- When resampling happens on the last timestep of a sequence the weights
- should be set to zero on the next timestep and remain zero afterwards.
- """
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- outs = smc.smc(
- _simple_transition_fn,
- num_steps=tf.convert_to_tensor([5, 3]),
- num_particles=2,
- resampling_criterion=_resample_at_step_criterion(2))
- log_z_hat, weights, resampled = sess.run(outs[0:3])
- gt_log_z = np.array(
- [lse([15, 3]) + lse([10, 2]) - 2*np.log(2),
- lse([12, 1.5]) - np.log(2)],
- dtype=np.float32)
- gt_resampled = np.array(
- [[0, 0], [0, 0], [1, 1], [0, 0], [0, 0]],
- dtype=np.float32)
- gt_weights = np.array(
- [[[5, 1], [4, .5]],
- [[10, 2], [8, 1]],
- [[15, 3], [12, 1.5]],
- [[5, 1], [0, 0]],
- [[10, 2], [0, 0]]],
- dtype=np.float32)
- self.assertAllClose(gt_log_z, log_z_hat)
- self.assertAllEqual(gt_resampled, resampled)
- self.assertAllEqual(gt_weights, weights)
-
- def test_weights_not_updated_past_sequence_end(self):
- """Test that non-zero weights are not updated past the end of a sequence."""
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- outs = smc.smc(
- _simple_transition_fn,
- num_steps=tf.convert_to_tensor([6, 4]),
- num_particles=2,
- resampling_criterion=_resample_at_step_criterion(1))
- log_z_hat, weights, resampled = sess.run(outs[0:3])
- gt_log_z_hat = np.array(
- [lse([10, 2]) + lse([20, 4]) - 2*np.log(2),
- lse([8, 1]) + lse([8, 1]) - 2*np.log(2)],
- dtype=np.float32)
- # Ensure that we only resample on the 2nd timestep.
- gt_resampled = np.array(
- [[0, 0], [1, 1], [0, 0], [0, 0], [0, 0], [0, 0]],
- dtype=np.float32)
- # Ensure that the weights after the end of the sequence don't change.
- # Ensure that the weights after resampling before the end of the sequence
- # do change.
- gt_weights = np.array(
- [[[5, 1], [4, .5]],
- [[10, 2], [8, 1]],
- [[5, 1], [4, .5]],
- [[10, 2], [8, 1]],
- [[15, 3], [8, 1]],
- [[20, 4], [8, 1]]],
- dtype=np.float32)
- self.assertAllClose(gt_log_z_hat, log_z_hat)
- self.assertAllEqual(gt_resampled, resampled)
- self.assertAllEqual(gt_weights, weights)
-
- def test_resampling_on_max_num_steps(self):
- """Test that everything is correct when resampling on step max_num_steps.
-
- When resampling on step max_num_steps (i.e. the last step of the longest
- sequence), ensure that there are no off-by-one errors preventing resampling
- and also that the weights are not updated.
- """
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- outs = smc.smc(
- _simple_transition_fn,
- num_steps=tf.convert_to_tensor([4, 2]),
- num_particles=2,
- resampling_criterion=_resample_at_step_criterion(3))
- log_z_hat, weights, resampled = sess.run(outs[0:3])
- gt_log_z_hat = np.array(
- [lse([20, 4]) - np.log(2),
- lse([8, 1]) - np.log(2)],
- dtype=np.float32)
- # Ensure that we only resample on the 3rd timestep and that the second
- # filter doesn't resample at all because it is only run for 2 steps.
- gt_resampled = np.array(
- [[0, 0], [0, 0], [0, 0], [1, 0]],
- dtype=np.float32)
- gt_weights = np.array(
- [[[5, 1], [4, .5]],
- [[10, 2], [8, 1]],
- [[15, 3], [8, 1]],
- [[20, 4], [8, 1]]],
- dtype=np.float32)
- self.assertAllClose(gt_log_z_hat, log_z_hat)
- self.assertAllEqual(gt_resampled, resampled)
- self.assertAllEqual(gt_weights, weights)
-
- def test_multinomial_resampling(self):
- """Test that mulitnomial resampling selects the correct states."""
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- # Setup input.
- inf = 1000.0 # Very large value in log space.
- num_samples = 2
- batch_size = 2
- log_weights = tf.convert_to_tensor([[inf, 0], [0, inf]])
- states = tf.convert_to_tensor([1, 2, 3, 4])
- # Run test.
- resampled_states = smc.multinomial_resampling(
- log_weights, states, num_samples, batch_size, random_seed=0)
- resampled_states_values = sess.run(resampled_states)
- self.assertAllEqual(resampled_states_values, [1, 4, 1, 4])
-
- def test_blend_tensor(self):
- """Test that relaxed resampling blends the correct states."""
- tf.set_random_seed(1234)
- with self.test_session() as sess:
- # Setup input.
- num_samples = 2
- batch_size = 2
- blending_weights = tf.convert_to_tensor(
- [[[0.5, 0.5], [0.25, 0.75]], [[0.75, 0.25], [0.5, 0.5]]])
- states = tf.convert_to_tensor([4., 8., 12., 16.])
- # Run test.
- blended_states = smc._blend_tensor(blending_weights, states,
- num_samples, batch_size)
- blended_states_values = sess.run(blended_states)
- self.assertAllClose(blended_states_values[:, 0], [8., 14., 6., 12.])
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/fivo/fivo/test_data/tiny_pianoroll.pkl b/research/fivo/fivo/test_data/tiny_pianoroll.pkl
deleted file mode 100644
index c5501c6ceac1a6601b5f1be4c422be4f2c1baa86..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/test_data/tiny_pianoroll.pkl
+++ /dev/null
@@ -1,10979 +0,0 @@
-(dp1
-S'train_mean'
-p2
-cnumpy.core.multiarray
-_reconstruct
-p3
-(cnumpy
-ndarray
-p4
-(I0
-tS'b'
-tRp5
-(I1
-(I88
-tcnumpy
-dtype
-p6
-(S'f8'
-I0
-I1
-tRp7
-(I3
-S'<'
-NNNI-1
-I-1
-I0
-tbI00
-S'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9e0^X\xbez,?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x9e0^X\xbez\x00\x00\x00\x00\x00\x00\x00'
-tRp101
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp102
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp103
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp104
-tp105
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp106
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp107
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp108
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp109
-tp110
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp111
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp112
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp113
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp114
-tp115
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp116
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp117
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp118
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp119
-tp120
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp121
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp122
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp123
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp124
-tp125
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp126
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp127
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp128
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp129
-tp130
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp131
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp132
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp133
-tp134
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp135
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp136
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp137
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp138
-tp139
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp140
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp141
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp142
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp143
-tp144
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp145
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp146
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp147
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp148
-tp149
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp150
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp151
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp152
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp153
-tp154
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp155
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp156
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp157
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp158
-tp159
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp160
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp161
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp162
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp163
-tp164
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp165
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp166
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp167
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp168
-tp169
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp170
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp171
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp172
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp173
-tp174
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp175
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp176
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp177
-tp178
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp179
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp180
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp181
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp182
-tp183
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp184
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp185
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp186
-tp187
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp188
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp189
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp190
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp191
-tp192
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp193
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp194
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp195
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp196
-tp197
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp198
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp199
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp200
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp201
-tp202
-a(g11
-(g12
-S':\x00\x00\x00\x00\x00\x00\x00'
-tRp203
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp204
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp205
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp206
-tp207
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp208
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp209
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp210
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp211
-tp212
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp213
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp214
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp215
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp216
-tp217
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp218
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp219
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp220
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp221
-tp222
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp223
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp224
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp225
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp226
-tp227
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp228
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp229
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp230
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp231
-tp232
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp233
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp234
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp235
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp236
-tp237
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp238
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp239
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp240
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp241
-tp242
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp243
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp244
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp245
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp246
-tp247
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp248
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp249
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp250
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp251
-tp252
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp253
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp254
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp255
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp256
-tp257
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp258
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp259
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp260
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp261
-tp262
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp263
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp264
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp265
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp266
-tp267
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp268
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp269
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp270
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp271
-tp272
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp273
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp274
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp275
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp276
-tp277
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp278
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp279
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp280
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp281
-tp282
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp283
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp284
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp285
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp286
-tp287
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp288
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp289
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp290
-tp291
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp292
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp293
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp294
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp295
-tp296
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp297
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp298
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp299
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp300
-tp301
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp302
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp303
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp304
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp305
-tp306
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp307
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp308
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp309
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp310
-tp311
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp312
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp313
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp314
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp315
-tp316
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp317
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp318
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp319
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp320
-tp321
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp322
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp323
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp324
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp325
-tp326
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp327
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp328
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp329
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp330
-tp331
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp332
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp333
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp334
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp335
-tp336
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp337
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp338
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp339
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp340
-tp341
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp342
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp343
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp344
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp345
-tp346
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp347
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp348
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp349
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp350
-tp351
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp352
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp353
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp354
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp355
-tp356
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp357
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp358
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp359
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp360
-tp361
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp362
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp363
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp364
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp365
-tp366
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp367
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp368
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp369
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp370
-tp371
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp372
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp373
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp374
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp375
-tp376
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp377
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp378
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp379
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp380
-tp381
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp382
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp383
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp384
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp385
-tp386
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp387
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp388
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp389
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp390
-tp391
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp392
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp393
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp394
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp395
-tp396
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp397
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp398
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp399
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp400
-tp401
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp402
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp403
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp404
-tp405
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp406
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp407
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp408
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp409
-tp410
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp411
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp412
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp413
-tp414
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp415
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp416
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp417
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp418
-tp419
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp420
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp421
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp422
-tp423
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp424
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp425
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp426
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp427
-tp428
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp429
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp430
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp431
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp432
-tp433
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp434
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp435
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp436
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp437
-tp438
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp439
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp440
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp441
-tp442
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp443
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp444
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp445
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp446
-tp447
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp448
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp449
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp450
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp451
-tp452
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp453
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp454
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp455
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp456
-tp457
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp458
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp459
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp460
-tp461
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp462
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp463
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp464
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp465
-tp466
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp467
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp468
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp469
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp470
-tp471
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp472
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp473
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp474
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp475
-tp476
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp477
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp478
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp479
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp480
-tp481
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp482
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp483
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp484
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp485
-tp486
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp487
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp488
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp489
-tp490
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp491
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp492
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp493
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp494
-tp495
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp496
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp497
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp498
-tp499
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp500
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp501
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp502
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp503
-tp504
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp505
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp506
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp507
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp508
-tp509
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp510
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp511
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp512
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp513
-tp514
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp515
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp516
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp517
-tp518
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp519
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp520
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp521
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp522
-tp523
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp524
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp525
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp526
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp527
-tp528
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp529
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp530
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp531
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp532
-tp533
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp534
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp535
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp536
-tp537
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp538
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp539
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp540
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp541
-tp542
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp543
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp544
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp545
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp546
-tp547
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp548
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp549
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp550
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp551
-tp552
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp553
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp554
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp555
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp556
-tp557
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp558
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp559
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp560
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp561
-tp562
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp563
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp564
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp565
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp566
-tp567
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp568
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp569
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp570
-tp571
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp572
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp573
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp574
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp575
-tp576
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp577
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp578
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp579
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp580
-tp581
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp582
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp583
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp584
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp585
-tp586
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp587
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp588
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp589
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp590
-tp591
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp592
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp593
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp594
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp595
-tp596
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp597
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp598
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp599
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp600
-tp601
-a(g11
-(g12
-S'8\x00\x00\x00\x00\x00\x00\x00'
-tRp602
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp603
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp604
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp605
-tp606
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp607
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp608
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp609
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp610
-tp611
-a(g11
-(g12
-S'6\x00\x00\x00\x00\x00\x00\x00'
-tRp612
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp613
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp614
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp615
-tp616
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp617
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp618
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp619
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp620
-tp621
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp622
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp623
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp624
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp625
-tp626
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp627
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp628
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp629
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp630
-tp631
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp632
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp633
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp634
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp635
-tp636
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp637
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp638
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp639
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp640
-tp641
-aa(lp642
-(g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp643
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp644
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp645
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp646
-tp647
-a(g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp648
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp649
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp650
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp651
-tp652
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp653
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp654
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp655
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp656
-tp657
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp658
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp659
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp660
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp661
-tp662
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp663
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp664
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp665
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp666
-tp667
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp668
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp669
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp670
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp671
-tp672
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp673
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp674
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp675
-tp676
-a(g11
-(g12
-S'8\x00\x00\x00\x00\x00\x00\x00'
-tRp677
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp678
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp679
-tp680
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp681
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp682
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp683
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp684
-tp685
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp686
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp687
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp688
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp689
-tp690
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp691
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp692
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp693
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp694
-tp695
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp696
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp697
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp698
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp699
-tp700
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp701
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp702
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp703
-tp704
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp705
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp706
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp707
-tp708
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp709
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp710
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp711
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp712
-tp713
-a(g11
-(g12
-S'8\x00\x00\x00\x00\x00\x00\x00'
-tRp714
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp715
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp716
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp717
-tp718
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp719
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp720
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp721
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp722
-tp723
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp724
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp725
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp726
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp727
-tp728
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp729
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp730
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp731
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp732
-tp733
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp734
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp735
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp736
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp737
-tp738
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp739
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp740
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp741
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp742
-tp743
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp744
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp745
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp746
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp747
-tp748
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp749
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp750
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp751
-tp752
-a(g11
-(g12
-S'8\x00\x00\x00\x00\x00\x00\x00'
-tRp753
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp754
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp755
-tp756
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp757
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp758
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp759
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp760
-tp761
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp762
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp763
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp764
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp765
-tp766
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp767
-g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp768
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp769
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp770
-tp771
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp772
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp773
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp774
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp775
-tp776
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp777
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp778
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp779
-tp780
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp781
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp782
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp783
-tp784
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp785
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp786
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp787
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp788
-tp789
-a(g11
-(g12
-S'8\x00\x00\x00\x00\x00\x00\x00'
-tRp790
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp791
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp792
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp793
-tp794
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp795
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp796
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp797
-g11
-(g12
-S'\\\x00\x00\x00\x00\x00\x00\x00'
-tRp798
-tp799
-a(g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp800
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp801
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp802
-g11
-(g12
-S'Z\x00\x00\x00\x00\x00\x00\x00'
-tRp803
-tp804
-a(g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp805
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp806
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp807
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp808
-tp809
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp810
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp811
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp812
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp813
-tp814
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp815
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp816
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp817
-tp818
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp819
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp820
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp821
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp822
-tp823
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp824
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp825
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp826
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp827
-tp828
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp829
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp830
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp831
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp832
-tp833
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp834
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp835
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp836
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp837
-tp838
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp839
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp840
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp841
-g11
-(g12
-S'Z\x00\x00\x00\x00\x00\x00\x00'
-tRp842
-tp843
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp844
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp845
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp846
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp847
-tp848
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp849
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp850
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp851
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp852
-tp853
-a(g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp854
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp855
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp856
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp857
-tp858
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp859
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp860
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp861
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp862
-tp863
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp864
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp865
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp866
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp867
-tp868
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp869
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp870
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp871
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp872
-tp873
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp874
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp875
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp876
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp877
-tp878
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp879
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp880
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp881
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp882
-tp883
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp884
-g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp885
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp886
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp887
-tp888
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp889
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp890
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp891
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp892
-tp893
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp894
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp895
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp896
-g11
-(g12
-S'Z\x00\x00\x00\x00\x00\x00\x00'
-tRp897
-tp898
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp899
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp900
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp901
-g11
-(g12
-S'\\\x00\x00\x00\x00\x00\x00\x00'
-tRp902
-tp903
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp904
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp905
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp906
-g11
-(g12
-S'Z\x00\x00\x00\x00\x00\x00\x00'
-tRp907
-tp908
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp909
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp910
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp911
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp912
-tp913
-a(g11
-(g12
-S'F\x00\x00\x00\x00\x00\x00\x00'
-tRp914
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp915
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp916
-g11
-(g12
-S'Z\x00\x00\x00\x00\x00\x00\x00'
-tRp917
-tp918
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp919
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp920
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp921
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp922
-tp923
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp924
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp925
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp926
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp927
-tp928
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp929
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp930
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp931
-g11
-(g12
-S'\\\x00\x00\x00\x00\x00\x00\x00'
-tRp932
-tp933
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp934
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp935
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp936
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp937
-tp938
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp939
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp940
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp941
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp942
-tp943
-a(g11
-(g12
-S'8\x00\x00\x00\x00\x00\x00\x00'
-tRp944
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp945
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp946
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp947
-tp948
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp949
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp950
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp951
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp952
-tp953
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp954
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp955
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp956
-g11
-(g12
-S'U\x00\x00\x00\x00\x00\x00\x00'
-tRp957
-tp958
-aa(lp959
-(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp960
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp961
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp962
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp963
-tp964
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp965
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp966
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp967
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp968
-tp969
-a(g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp970
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp971
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp972
-tp973
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp974
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp975
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp976
-g11
-(g12
-S']\x00\x00\x00\x00\x00\x00\x00'
-tRp977
-tp978
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp979
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp980
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp981
-g11
-(g12
-S']\x00\x00\x00\x00\x00\x00\x00'
-tRp982
-tp983
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp984
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp985
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp986
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp987
-tp988
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp989
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp990
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp991
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp992
-tp993
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp994
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp995
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp996
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp997
-tp998
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp999
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1000
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1001
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1002
-tp1003
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1004
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1005
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1006
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp1007
-tp1008
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1009
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1010
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1011
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1012
-tp1013
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1014
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1015
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1016
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1017
-tp1018
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1019
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1020
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1021
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1022
-tp1023
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1024
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1025
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1026
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1027
-tp1028
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1029
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1030
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1031
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1032
-tp1033
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1034
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1035
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1036
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1037
-tp1038
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1039
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1040
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1041
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1042
-tp1043
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1044
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1045
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1046
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1047
-tp1048
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1049
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1050
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1051
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1052
-tp1053
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1054
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1055
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1056
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1057
-tp1058
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1059
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1060
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1061
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1062
-tp1063
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1064
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1065
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1066
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1067
-tp1068
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1069
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1070
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1071
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1072
-tp1073
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1074
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1075
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1076
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1077
-tp1078
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1079
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1080
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1081
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1082
-tp1083
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1084
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1085
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1086
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1087
-tp1088
-a(g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1089
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1090
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1091
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp1092
-tp1093
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1094
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1095
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1096
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp1097
-tp1098
-a(g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1099
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1100
-g11
-(g12
-S'Z\x00\x00\x00\x00\x00\x00\x00'
-tRp1101
-tp1102
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1103
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1104
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1105
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp1106
-tp1107
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1108
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1109
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1110
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp1111
-tp1112
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1113
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1114
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1115
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp1116
-tp1117
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1118
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1119
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1120
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp1121
-tp1122
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1123
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1124
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1125
-g11
-(g12
-S']\x00\x00\x00\x00\x00\x00\x00'
-tRp1126
-tp1127
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1128
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1129
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1130
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp1131
-tp1132
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1133
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1134
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1135
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp1136
-tp1137
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1138
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp1139
-g11
-(g12
-S'W\x00\x00\x00\x00\x00\x00\x00'
-tRp1140
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1141
-tp1142
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1143
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1144
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1145
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp1146
-tp1147
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1148
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1149
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1150
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp1151
-tp1152
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1153
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1154
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1155
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp1156
-tp1157
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1158
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1159
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1160
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1161
-tp1162
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1163
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1164
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1165
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1166
-tp1167
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1168
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1169
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1170
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp1171
-tp1172
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1173
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1174
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1175
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1176
-tp1177
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1178
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1179
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1180
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1181
-tp1182
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1183
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1184
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1185
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1186
-tp1187
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1188
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1189
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1190
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1191
-tp1192
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1193
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1194
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1195
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1196
-tp1197
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1198
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1199
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1200
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1201
-tp1202
-aa(lp1203
-(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1204
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1205
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1206
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1207
-tp1208
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1209
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1210
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1211
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1212
-tp1213
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1214
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1215
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1216
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1217
-tp1218
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1219
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1220
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1221
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1222
-tp1223
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1224
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1225
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1226
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1227
-tp1228
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1229
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1230
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1231
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1232
-tp1233
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1234
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1235
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1236
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1237
-tp1238
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1239
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1240
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1241
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1242
-tp1243
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp1244
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1245
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1246
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1247
-tp1248
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1249
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1250
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1251
-tp1252
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1253
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1254
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1255
-tp1256
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1257
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1258
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1259
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1260
-tp1261
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1262
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1263
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1264
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1265
-tp1266
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1267
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1268
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1269
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1270
-tp1271
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1272
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1273
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1274
-tp1275
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1276
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1277
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1278
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1279
-tp1280
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1281
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1282
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1283
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1284
-tp1285
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1286
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1287
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1288
-tp1289
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1290
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1291
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1292
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1293
-tp1294
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1295
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1296
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1297
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1298
-tp1299
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1300
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1301
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1302
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1303
-tp1304
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1305
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1306
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1307
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1308
-tp1309
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1310
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1311
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1312
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1313
-tp1314
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1315
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1316
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1317
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1318
-tp1319
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1320
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1321
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1322
-tp1323
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1324
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1325
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1326
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1327
-tp1328
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1329
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1330
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1331
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp1332
-tp1333
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1334
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1335
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1336
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1337
-tp1338
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1339
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1340
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1341
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1342
-tp1343
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1344
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1345
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1346
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1347
-tp1348
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1349
-g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp1350
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1351
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1352
-tp1353
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1354
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1355
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1356
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1357
-tp1358
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1359
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1360
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1361
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1362
-tp1363
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1364
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1365
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1366
-tp1367
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1368
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1369
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1370
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1371
-tp1372
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1373
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1374
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1375
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1376
-tp1377
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1378
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1379
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1380
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp1381
-tp1382
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1383
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1384
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1385
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1386
-tp1387
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1388
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1389
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1390
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1391
-tp1392
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1393
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1394
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1395
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1396
-tp1397
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp1398
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1399
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1400
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1401
-tp1402
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1403
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1404
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1405
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1406
-tp1407
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1408
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1409
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp1410
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1411
-tp1412
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1413
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1414
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp1415
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp1416
-tp1417
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1418
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1419
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1420
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1421
-tp1422
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1423
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1424
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1425
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1426
-tp1427
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1428
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1429
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1430
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1431
-tp1432
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1433
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1434
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1435
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1436
-tp1437
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1438
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1439
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1440
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1441
-tp1442
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1443
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1444
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1445
-tp1446
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1447
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1448
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1449
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1450
-tp1451
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1452
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1453
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1454
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1455
-tp1456
-a(g11
-(g12
-S'8\x00\x00\x00\x00\x00\x00\x00'
-tRp1457
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1458
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1459
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1460
-tp1461
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1462
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1463
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1464
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1465
-tp1466
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1467
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1468
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1469
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1470
-tp1471
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1472
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1473
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1474
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1475
-tp1476
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1477
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1478
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1479
-tp1480
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1481
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1482
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1483
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1484
-tp1485
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1486
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1487
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1488
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1489
-tp1490
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp1491
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1492
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1493
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1494
-tp1495
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1496
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1497
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1498
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1499
-tp1500
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1501
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1502
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1503
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1504
-tp1505
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1506
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1507
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1508
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1509
-tp1510
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1511
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1512
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1513
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1514
-tp1515
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1516
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1517
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1518
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1519
-tp1520
-aa(lp1521
-(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp1522
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1523
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1524
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1525
-tp1526
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1527
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1528
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1529
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1530
-tp1531
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1532
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1533
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1534
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp1535
-tp1536
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1537
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1538
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1539
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1540
-tp1541
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1542
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1543
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1544
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1545
-tp1546
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1547
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1548
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1549
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1550
-tp1551
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1552
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1553
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1554
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1555
-tp1556
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1557
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1558
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1559
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1560
-tp1561
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1562
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1563
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1564
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1565
-tp1566
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1567
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1568
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1569
-tp1570
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1571
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1572
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1573
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1574
-tp1575
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1576
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1577
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1578
-tp1579
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1580
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1581
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1582
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1583
-tp1584
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1585
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1586
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1587
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1588
-tp1589
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1590
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1591
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1592
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1593
-tp1594
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1595
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1596
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1597
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1598
-tp1599
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1600
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1601
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1602
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1603
-tp1604
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp1605
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1606
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1607
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1608
-tp1609
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1610
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1611
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1612
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1613
-tp1614
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1615
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1616
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1617
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1618
-tp1619
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp1620
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1621
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1622
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1623
-tp1624
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1625
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1626
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1627
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1628
-tp1629
-a(g11
-(g12
-S'2\x00\x00\x00\x00\x00\x00\x00'
-tRp1630
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1631
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1632
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1633
-tp1634
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp1635
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1636
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1637
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1638
-tp1639
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1640
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1641
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1642
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1643
-tp1644
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp1645
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1646
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1647
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1648
-tp1649
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1650
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1651
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp1652
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1653
-tp1654
-a(g11
-(g12
-S'2\x00\x00\x00\x00\x00\x00\x00'
-tRp1655
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1656
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1657
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1658
-tp1659
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1660
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1661
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1662
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1663
-tp1664
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp1665
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1666
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1667
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1668
-tp1669
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1670
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1671
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1672
-tp1673
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp1674
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1675
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1676
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1677
-tp1678
-a(g11
-(g12
-S'2\x00\x00\x00\x00\x00\x00\x00'
-tRp1679
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1680
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1681
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1682
-tp1683
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp1684
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1685
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1686
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1687
-tp1688
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1689
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1690
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1691
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1692
-tp1693
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1694
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1695
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1696
-tp1697
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1698
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1699
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1700
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1701
-tp1702
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1703
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1704
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1705
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1706
-tp1707
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1708
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1709
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1710
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp1711
-tp1712
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1713
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1714
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1715
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1716
-tp1717
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1718
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1719
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1720
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1721
-tp1722
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1723
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1724
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1725
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1726
-tp1727
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1728
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1729
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1730
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1731
-tp1732
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1733
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1734
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1735
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1736
-tp1737
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1738
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1739
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1740
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1741
-tp1742
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1743
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1744
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1745
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1746
-tp1747
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp1748
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1749
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1750
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1751
-tp1752
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1753
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1754
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1755
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1756
-tp1757
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1758
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1759
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1760
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1761
-tp1762
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1763
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1764
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1765
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1766
-tp1767
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1768
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1769
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1770
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1771
-tp1772
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1773
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1774
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1775
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp1776
-tp1777
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1778
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1779
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1780
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp1781
-tp1782
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1783
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp1784
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1785
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1786
-tp1787
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1788
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1789
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1790
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1791
-tp1792
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1793
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1794
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1795
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1796
-tp1797
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1798
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1799
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1800
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1801
-tp1802
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1803
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1804
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1805
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1806
-tp1807
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1808
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1809
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1810
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1811
-tp1812
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1813
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1814
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1815
-tp1816
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1817
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1818
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1819
-tp1820
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1821
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1822
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1823
-tp1824
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1825
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp1826
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1827
-tp1828
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1829
-g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1830
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1831
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1832
-tp1833
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1834
-g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1835
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1836
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1837
-tp1838
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1839
-g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1840
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1841
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1842
-tp1843
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1844
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1845
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1846
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp1847
-tp1848
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1849
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1850
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp1851
-tp1852
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1853
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1854
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1855
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp1856
-tp1857
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp1858
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1859
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1860
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1861
-tp1862
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1863
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1864
-g11
-(g12
-S'I\x00\x00\x00\x00\x00\x00\x00'
-tRp1865
-tp1866
-a(g11
-(g12
-S'2\x00\x00\x00\x00\x00\x00\x00'
-tRp1867
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1868
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1869
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1870
-tp1871
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp1872
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1873
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1874
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1875
-tp1876
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1877
-g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1878
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1879
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1880
-tp1881
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1882
-g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp1883
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1884
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1885
-tp1886
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1887
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1888
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1889
-tp1890
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1891
-g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp1892
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1893
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1894
-tp1895
-a(g11
-(g12
-S'2\x00\x00\x00\x00\x00\x00\x00'
-tRp1896
-g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1897
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1898
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1899
-tp1900
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1901
-g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1902
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1903
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1904
-tp1905
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1906
-g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1907
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1908
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1909
-tp1910
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp1911
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1912
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1913
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1914
-tp1915
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1916
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1917
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1918
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1919
-tp1920
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1921
-g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp1922
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1923
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1924
-tp1925
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1926
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1927
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1928
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1929
-tp1930
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp1931
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1932
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1933
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1934
-tp1935
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1936
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1937
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1938
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1939
-tp1940
-a(g11
-(g12
-S'2\x00\x00\x00\x00\x00\x00\x00'
-tRp1941
-g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp1942
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1943
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1944
-tp1945
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1946
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1947
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1948
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1949
-tp1950
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1951
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1952
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1953
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp1954
-tp1955
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp1956
-g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1957
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp1958
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1959
-tp1960
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1961
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1962
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1963
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp1964
-tp1965
-a(g11
-(g12
-S'4\x00\x00\x00\x00\x00\x00\x00'
-tRp1966
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1967
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1968
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1969
-tp1970
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp1971
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1972
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1973
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp1974
-tp1975
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp1976
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1977
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1978
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1979
-tp1980
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp1981
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1982
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp1983
-tp1984
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp1985
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1986
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp1987
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1988
-tp1989
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp1990
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp1991
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1992
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp1993
-tp1994
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp1995
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp1996
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp1997
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp1998
-tp1999
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2000
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2001
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2002
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2003
-tp2004
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2005
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2006
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2007
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2008
-tp2009
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2010
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2011
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2012
-tp2013
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2014
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2015
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2016
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2017
-tp2018
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2019
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2020
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2021
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2022
-tp2023
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2024
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2025
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2026
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2027
-tp2028
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2029
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2030
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2031
-tp2032
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2033
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2034
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2035
-tp2036
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2037
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2038
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2039
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2040
-tp2041
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2042
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2043
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2044
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2045
-tp2046
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2047
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2048
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2049
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2050
-tp2051
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp2052
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2053
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2054
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2055
-tp2056
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2057
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2058
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2059
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2060
-tp2061
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp2062
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2063
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2064
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2065
-tp2066
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp2067
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2068
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2069
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2070
-tp2071
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp2072
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2073
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2074
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2075
-tp2076
-aa(lp2077
-(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2078
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2079
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2080
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2081
-tp2082
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2083
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2084
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2085
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2086
-tp2087
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2088
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2089
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2090
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2091
-tp2092
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2093
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2094
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2095
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2096
-tp2097
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2098
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2099
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2100
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2101
-tp2102
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2103
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2104
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2105
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2106
-tp2107
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2108
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2109
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2110
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2111
-tp2112
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2113
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2114
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2115
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2116
-tp2117
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2118
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2119
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2120
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2121
-tp2122
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2123
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2124
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2125
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2126
-tp2127
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp2128
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2129
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2130
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2131
-tp2132
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp2133
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2134
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2135
-tp2136
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2137
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2138
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2139
-tp2140
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2141
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2142
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2143
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2144
-tp2145
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2146
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2147
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2148
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2149
-tp2150
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2151
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2152
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2153
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2154
-tp2155
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2156
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2157
-g11
-(g12
-S'P\x00\x00\x00\x00\x00\x00\x00'
-tRp2158
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2159
-tp2160
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2161
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2162
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2163
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2164
-tp2165
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2166
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2167
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2168
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2169
-tp2170
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2171
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2172
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2173
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2174
-tp2175
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2176
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2177
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2178
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2179
-tp2180
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2181
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2182
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2183
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2184
-tp2185
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2186
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2187
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2188
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2189
-tp2190
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2191
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2192
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2193
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2194
-tp2195
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp2196
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2197
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2198
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2199
-tp2200
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2201
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2202
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2203
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp2204
-tp2205
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2206
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2207
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2208
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2209
-tp2210
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2211
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2212
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2213
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2214
-tp2215
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2216
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2217
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2218
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2219
-tp2220
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2221
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2222
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2223
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2224
-tp2225
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2226
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2227
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2228
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2229
-tp2230
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2231
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2232
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2233
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2234
-tp2235
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2236
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2237
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2238
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2239
-tp2240
-aa(lp2241
-(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2242
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2243
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2244
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2245
-tp2246
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp2247
-g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2248
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2249
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2250
-tp2251
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2252
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2253
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2254
-tp2255
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2256
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2257
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2258
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2259
-tp2260
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2261
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2262
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2263
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2264
-tp2265
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2266
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2267
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2268
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2269
-tp2270
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2271
-g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp2272
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2273
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2274
-tp2275
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2276
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2277
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2278
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2279
-tp2280
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2281
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2282
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2283
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2284
-tp2285
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2286
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2287
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2288
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2289
-tp2290
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2291
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2292
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2293
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2294
-tp2295
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2296
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2297
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2298
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2299
-tp2300
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2301
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2302
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2303
-tp2304
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2305
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2306
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2307
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2308
-tp2309
-a(g11
-(g12
-S'2\x00\x00\x00\x00\x00\x00\x00'
-tRp2310
-g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp2311
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2312
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2313
-tp2314
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2315
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2316
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2317
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2318
-tp2319
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2320
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2321
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2322
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2323
-tp2324
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp2325
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2326
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2327
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2328
-tp2329
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2330
-g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2331
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2332
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2333
-tp2334
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2335
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2336
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2337
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2338
-tp2339
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2340
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2341
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2342
-tp2343
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2344
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2345
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2346
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2347
-tp2348
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2349
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2350
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2351
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2352
-tp2353
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2354
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2355
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2356
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2357
-tp2358
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2359
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2360
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2361
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2362
-tp2363
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp2364
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2365
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2366
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2367
-tp2368
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp2369
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2370
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2371
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2372
-tp2373
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2374
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2375
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2376
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2377
-tp2378
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2379
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2380
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2381
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2382
-tp2383
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2384
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2385
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2386
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2387
-tp2388
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2389
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2390
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2391
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2392
-tp2393
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2394
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2395
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2396
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2397
-tp2398
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2399
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2400
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2401
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2402
-tp2403
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2404
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2405
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2406
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2407
-tp2408
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2409
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2410
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2411
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2412
-tp2413
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2414
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2415
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2416
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2417
-tp2418
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2419
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2420
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2421
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2422
-tp2423
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp2424
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2425
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2426
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2427
-tp2428
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2429
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2430
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2431
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2432
-tp2433
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2434
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2435
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2436
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2437
-tp2438
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2439
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2440
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2441
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2442
-tp2443
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2444
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2445
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2446
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2447
-tp2448
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2449
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2450
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2451
-tp2452
-a(g11
-(g12
-S'5\x00\x00\x00\x00\x00\x00\x00'
-tRp2453
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2454
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2455
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2456
-tp2457
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2458
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2459
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2460
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2461
-tp2462
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2463
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2464
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2465
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2466
-tp2467
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2468
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2469
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2470
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2471
-tp2472
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2473
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2474
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2475
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2476
-tp2477
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp2478
-g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp2479
-g11
-(g12
-S'K\x00\x00\x00\x00\x00\x00\x00'
-tRp2480
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2481
-tp2482
-a(g11
-(g12
-S'=\x00\x00\x00\x00\x00\x00\x00'
-tRp2483
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2484
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2485
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2486
-tp2487
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2488
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2489
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2490
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2491
-tp2492
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2493
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2494
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2495
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2496
-tp2497
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2498
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2499
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2500
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2501
-tp2502
-a(g11
-(g12
-S'6\x00\x00\x00\x00\x00\x00\x00'
-tRp2503
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2504
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2505
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2506
-tp2507
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2508
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2509
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2510
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2511
-tp2512
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2513
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2514
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2515
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2516
-tp2517
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2518
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2519
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2520
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2521
-tp2522
-aa(lp2523
-(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2524
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2525
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2526
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2527
-tp2528
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2529
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2530
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2531
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2532
-tp2533
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2534
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2535
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2536
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2537
-tp2538
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2539
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2540
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2541
-g11
-(g12
-S']\x00\x00\x00\x00\x00\x00\x00'
-tRp2542
-tp2543
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2544
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2545
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2546
-g11
-(g12
-S']\x00\x00\x00\x00\x00\x00\x00'
-tRp2547
-tp2548
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2549
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2550
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2551
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2552
-tp2553
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2554
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2555
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2556
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2557
-tp2558
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2559
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2560
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2561
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2562
-tp2563
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2564
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2565
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2566
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2567
-tp2568
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2569
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2570
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2571
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp2572
-tp2573
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2574
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2575
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2576
-tp2577
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2578
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2579
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2580
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2581
-tp2582
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2583
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2584
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2585
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2586
-tp2587
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2588
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2589
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2590
-tp2591
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2592
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2593
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2594
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2595
-tp2596
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2597
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2598
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2599
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2600
-tp2601
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2602
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2603
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2604
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2605
-tp2606
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2607
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2608
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2609
-tp2610
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2611
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2612
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2613
-tp2614
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2615
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2616
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2617
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2618
-tp2619
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2620
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2621
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2622
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2623
-tp2624
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2625
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2626
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2627
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2628
-tp2629
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2630
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2631
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2632
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2633
-tp2634
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2635
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2636
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2637
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2638
-tp2639
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2640
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2641
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2642
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2643
-tp2644
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2645
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2646
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2647
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2648
-tp2649
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2650
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2651
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2652
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2653
-tp2654
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2655
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2656
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2657
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2658
-tp2659
-a(g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2660
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2661
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2662
-g11
-(g12
-S'Z\x00\x00\x00\x00\x00\x00\x00'
-tRp2663
-tp2664
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2665
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2666
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2667
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2668
-tp2669
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2670
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2671
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2672
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2673
-tp2674
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2675
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2676
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2677
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2678
-tp2679
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2680
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2681
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2682
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2683
-tp2684
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2685
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2686
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2687
-g11
-(g12
-S']\x00\x00\x00\x00\x00\x00\x00'
-tRp2688
-tp2689
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2690
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2691
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2692
-g11
-(g12
-S'[\x00\x00\x00\x00\x00\x00\x00'
-tRp2693
-tp2694
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2695
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2696
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2697
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp2698
-tp2699
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2700
-g11
-(g12
-S'R\x00\x00\x00\x00\x00\x00\x00'
-tRp2701
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2702
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2703
-tp2704
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2705
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2706
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2707
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp2708
-tp2709
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2710
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2711
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2712
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp2713
-tp2714
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2715
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2716
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2717
-g11
-(g12
-S'Y\x00\x00\x00\x00\x00\x00\x00'
-tRp2718
-tp2719
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2720
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2721
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2722
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2723
-tp2724
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2725
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2726
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2727
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2728
-tp2729
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2730
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2731
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2732
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp2733
-tp2734
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2735
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2736
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2737
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2738
-tp2739
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2740
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2741
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2742
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2743
-tp2744
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2745
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2746
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2747
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2748
-tp2749
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2750
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2751
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2752
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2753
-tp2754
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2755
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2756
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2757
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2758
-tp2759
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2760
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2761
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2762
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2763
-tp2764
-aa(lp2765
-(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2766
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2767
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2768
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2769
-tp2770
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2771
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2772
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2773
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2774
-tp2775
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2776
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2777
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2778
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2779
-tp2780
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2781
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2782
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2783
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2784
-tp2785
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2786
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2787
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2788
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2789
-tp2790
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2791
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2792
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2793
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2794
-tp2795
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2796
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2797
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2798
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2799
-tp2800
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2801
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2802
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2803
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2804
-tp2805
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2806
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2807
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2808
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2809
-tp2810
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2811
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2812
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2813
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2814
-tp2815
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2816
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2817
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2818
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2819
-tp2820
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2821
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2822
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2823
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2824
-tp2825
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2826
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2827
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2828
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2829
-tp2830
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2831
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2832
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2833
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2834
-tp2835
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2836
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2837
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2838
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2839
-tp2840
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp2841
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2842
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2843
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2844
-tp2845
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2846
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2847
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2848
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2849
-tp2850
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2851
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2852
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2853
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2854
-tp2855
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2856
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2857
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2858
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2859
-tp2860
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2861
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2862
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2863
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2864
-tp2865
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2866
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp2867
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2868
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2869
-tp2870
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2871
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp2872
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2873
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2874
-tp2875
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2876
-g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp2877
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2878
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2879
-tp2880
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2881
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2882
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2883
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2884
-tp2885
-a(g11
-(g12
-S'?\x00\x00\x00\x00\x00\x00\x00'
-tRp2886
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2887
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2888
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2889
-tp2890
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2891
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2892
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2893
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2894
-tp2895
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp2896
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2897
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2898
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2899
-tp2900
-a(g11
-(g12
-S'D\x00\x00\x00\x00\x00\x00\x00'
-tRp2901
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2902
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2903
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2904
-tp2905
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2906
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2907
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2908
-tp2909
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp2910
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2911
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2912
-tp2913
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2914
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2915
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2916
-tp2917
-a(g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2918
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2919
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp2920
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2921
-tp2922
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp2923
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2924
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2925
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2926
-tp2927
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2928
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2929
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2930
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2931
-tp2932
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp2933
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp2934
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2935
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2936
-tp2937
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp2938
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2939
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2940
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2941
-tp2942
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2943
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2944
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2945
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2946
-tp2947
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2948
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2949
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2950
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2951
-tp2952
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2953
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2954
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2955
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2956
-tp2957
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2958
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp2959
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2960
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2961
-tp2962
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp2963
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2964
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2965
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2966
-tp2967
-a(g11
-(g12
-S'8\x00\x00\x00\x00\x00\x00\x00'
-tRp2968
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2969
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2970
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2971
-tp2972
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp2973
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp2974
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2975
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp2976
-tp2977
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp2978
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2979
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2980
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2981
-tp2982
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2983
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2984
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2985
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2986
-tp2987
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp2988
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2989
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp2990
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp2991
-tp2992
-a(g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2993
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp2994
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp2995
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp2996
-tp2997
-a(g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp2998
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp2999
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3000
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp3001
-tp3002
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3003
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3004
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3005
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp3006
-tp3007
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3008
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3009
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3010
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp3011
-tp3012
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3013
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3014
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3015
-g11
-(g12
-S'X\x00\x00\x00\x00\x00\x00\x00'
-tRp3016
-tp3017
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp3018
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp3019
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp3020
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp3021
-tp3022
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3023
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3024
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3025
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3026
-tp3027
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3028
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3029
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3030
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3031
-tp3032
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp3033
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3034
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3035
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3036
-tp3037
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3038
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3039
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3040
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3041
-tp3042
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp3043
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3044
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3045
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3046
-tp3047
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp3048
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3049
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3050
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3051
-tp3052
-a(g11
-(g12
-S'6\x00\x00\x00\x00\x00\x00\x00'
-tRp3053
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3054
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3055
-g11
-(g12
-S'V\x00\x00\x00\x00\x00\x00\x00'
-tRp3056
-tp3057
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp3058
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3059
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3060
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3061
-tp3062
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp3063
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3064
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3065
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3066
-tp3067
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp3068
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3069
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp3070
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3071
-tp3072
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3073
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3074
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3075
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp3076
-tp3077
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3078
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3079
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3080
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp3081
-tp3082
-aa(lp3083
-(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3084
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3085
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3086
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3087
-tp3088
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3089
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3090
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3091
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3092
-tp3093
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp3094
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp3095
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3096
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp3097
-tp3098
-a(g11
-(g12
-S';\x00\x00\x00\x00\x00\x00\x00'
-tRp3099
-g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp3100
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3101
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3102
-tp3103
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3104
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3105
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3106
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3107
-tp3108
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp3109
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp3110
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp3111
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp3112
-tp3113
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3114
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3115
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3116
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3117
-tp3118
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3119
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3120
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3121
-tp3122
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3123
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3124
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3125
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3126
-tp3127
-a(g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp3128
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3129
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3130
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3131
-tp3132
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3133
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3134
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3135
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3136
-tp3137
-a(g11
-(g12
-S'B\x00\x00\x00\x00\x00\x00\x00'
-tRp3138
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3139
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3140
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp3141
-tp3142
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3143
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3144
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3145
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3146
-tp3147
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3148
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3149
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3150
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3151
-tp3152
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp3153
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3154
-g11
-(g12
-S'N\x00\x00\x00\x00\x00\x00\x00'
-tRp3155
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3156
-tp3157
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp3158
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3159
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3160
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3161
-tp3162
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3163
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3164
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3165
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3166
-tp3167
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3168
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3169
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3170
-g11
-(g12
-S'T\x00\x00\x00\x00\x00\x00\x00'
-tRp3171
-tp3172
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp3173
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp3174
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3175
-g11
-(g12
-S'S\x00\x00\x00\x00\x00\x00\x00'
-tRp3176
-tp3177
-a(g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp3178
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp3179
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3180
-g11
-(g12
-S'Q\x00\x00\x00\x00\x00\x00\x00'
-tRp3181
-tp3182
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp3183
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp3184
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3185
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3186
-tp3187
-a(g11
-(g12
-S'>\x00\x00\x00\x00\x00\x00\x00'
-tRp3188
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp3189
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3190
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp3191
-tp3192
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3193
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3194
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3195
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3196
-tp3197
-a(g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3198
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3199
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3200
-tp3201
-a(g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3202
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3203
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3204
-g11
-(g12
-S'O\x00\x00\x00\x00\x00\x00\x00'
-tRp3205
-tp3206
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp3207
-g11
-(g12
-S'E\x00\x00\x00\x00\x00\x00\x00'
-tRp3208
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3209
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp3210
-tp3211
-a(g11
-(g12
-S'<\x00\x00\x00\x00\x00\x00\x00'
-tRp3212
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3213
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3214
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3215
-tp3216
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp3217
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3218
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3219
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3220
-tp3221
-a(g11
-(g12
-S'9\x00\x00\x00\x00\x00\x00\x00'
-tRp3222
-g11
-(g12
-S'A\x00\x00\x00\x00\x00\x00\x00'
-tRp3223
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3224
-g11
-(g12
-S'M\x00\x00\x00\x00\x00\x00\x00'
-tRp3225
-tp3226
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp3227
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3228
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3229
-g11
-(g12
-S'L\x00\x00\x00\x00\x00\x00\x00'
-tRp3230
-tp3231
-a(g11
-(g12
-S'7\x00\x00\x00\x00\x00\x00\x00'
-tRp3232
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3233
-g11
-(g12
-S'G\x00\x00\x00\x00\x00\x00\x00'
-tRp3234
-g11
-(g12
-S'J\x00\x00\x00\x00\x00\x00\x00'
-tRp3235
-tp3236
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp3237
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3238
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3239
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3240
-tp3241
-a(g11
-(g12
-S'0\x00\x00\x00\x00\x00\x00\x00'
-tRp3242
-g11
-(g12
-S'@\x00\x00\x00\x00\x00\x00\x00'
-tRp3243
-g11
-(g12
-S'C\x00\x00\x00\x00\x00\x00\x00'
-tRp3244
-g11
-(g12
-S'H\x00\x00\x00\x00\x00\x00\x00'
-tRp3245
-tp3246
-aas.
\ No newline at end of file
diff --git a/research/fivo/fivo/test_data/tiny_speech_dataset.tfrecord b/research/fivo/fivo/test_data/tiny_speech_dataset.tfrecord
deleted file mode 100644
index 93fe8791b631da35b9d03d37e6494cc7c50cb55d..0000000000000000000000000000000000000000
Binary files a/research/fivo/fivo/test_data/tiny_speech_dataset.tfrecord and /dev/null differ
diff --git a/research/fivo/fivo/test_utils.py b/research/fivo/fivo/test_utils.py
deleted file mode 100644
index 48bbd3d483c45457b82b12ac1587d4c314b79f49..0000000000000000000000000000000000000000
--- a/research/fivo/fivo/test_utils.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities for testing FIVO.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-from fivo.models import base
-from fivo.models import srnn
-from fivo.models import vrnn
-
-
-def create_vrnn(generative_class=base.ConditionalNormalDistribution,
- batch_size=2, data_size=3, rnn_hidden_size=4,
- latent_size=5, fcnet_hidden_size=7, encoded_data_size=9,
- encoded_latent_size=11, num_timesteps=7, data_lengths=(7, 4),
- use_tilt=False, random_seed=None):
- """Creates a VRNN and some dummy data to feed it for testing purposes.
-
- Args:
- generative_class: The class of the generative distribution.
- batch_size: The number of elements per batch.
- data_size: The dimension of the vectors that make up the data sequences.
- rnn_hidden_size: The hidden state dimension of the RNN that forms the
- deterministic part of this VRNN.
- latent_size: The size of the stochastic latent state of the VRNN.
- fcnet_hidden_size: The size of the hidden layer of the fully connected
- networks that parameterize the conditional probability distributions
- of the VRNN.
- encoded_data_size: The size of the output of the data encoding network.
- encoded_latent_size: The size of the output of the latent state encoding
- network.
- num_timesteps: The maximum number of timesteps in the data.
- data_lengths: A tuple of size batch_size that contains the desired lengths
- of each sequence in the dummy data.
- use_tilt: Use a tilting function.
- random_seed: A random seed to feed the VRNN, mainly useful for testing
- purposes.
-
- Returns:
- model: A VRNN object.
- inputs: A Tensor of shape [num_timesteps, batch_size, data_size], the inputs
- to the model, also known as the observations.
- targets: A Tensor of shape [num_timesteps, batch_size, data_size], the
- desired outputs of the model.
- lengths: A Tensor of shape [batch_size], the lengths of the sequences in the
- batch.
- """
-
- fcnet_hidden_sizes = [fcnet_hidden_size]
- initializers = {"w": tf.contrib.layers.xavier_initializer(seed=random_seed),
- "b": tf.zeros_initializer()}
- model = vrnn.create_vrnn(
- data_size,
- latent_size,
- generative_class,
- rnn_hidden_size=rnn_hidden_size,
- fcnet_hidden_sizes=fcnet_hidden_sizes,
- encoded_data_size=encoded_data_size,
- encoded_latent_size=encoded_latent_size,
- use_tilt=use_tilt,
- initializers=initializers,
- random_seed=random_seed)
- inputs = tf.random_uniform([num_timesteps, batch_size, data_size],
- seed=random_seed, dtype=tf.float32)
- targets = tf.random_uniform([num_timesteps, batch_size, data_size],
- seed=random_seed, dtype=tf.float32)
- lengths = tf.constant(data_lengths, dtype=tf.int32)
- return model, inputs, targets, lengths
-
-
-def create_srnn(generative_class=base.ConditionalNormalDistribution,
- batch_size=2, data_size=3, rnn_hidden_size=4,
- latent_size=5, fcnet_hidden_size=7, encoded_data_size=3,
- encoded_latent_size=2, num_timesteps=7, data_lengths=(7, 4),
- use_tilt=False, random_seed=None):
- """Creates a SRNN and some dummy data to feed it for testing purposes.
-
- Args:
- generative_class: The class of the generative distribution.
- batch_size: The number of elements per batch.
- data_size: The dimension of the vectors that make up the data sequences.
- rnn_hidden_size: The hidden state dimension of the RNN that forms the
- deterministic part of this SRNN.
- latent_size: The size of the stochastic latent state of the SRNN.
- fcnet_hidden_size: The size of the hidden layer of the fully connected
- networks that parameterize the conditional probability distributions
- of the SRNN.
- encoded_data_size: The size of the output of the data encoding network.
- encoded_latent_size: The size of the output of the latent state encoding
- network.
- num_timesteps: The maximum number of timesteps in the data.
- data_lengths: A tuple of size batch_size that contains the desired lengths
- of each sequence in the dummy data.
- use_tilt: Use a tilting function.
- random_seed: A random seed to feed the SRNN, mainly useful for testing
- purposes.
-
- Returns:
- model: A SRNN object.
- inputs: A Tensor of shape [num_timesteps, batch_size, data_size], the inputs
- to the model, also known as the observations.
- targets: A Tensor of shape [num_timesteps, batch_size, data_size], the
- desired outputs of the model.
- lengths: A Tensor of shape [batch_size], the lengths of the sequences in the
- batch.
- """
-
- fcnet_hidden_sizes = [fcnet_hidden_size]
- initializers = {"w": tf.contrib.layers.xavier_initializer(seed=random_seed),
- "b": tf.zeros_initializer()}
- model = srnn.create_srnn(
- data_size,
- latent_size,
- generative_class,
- rnn_hidden_size=rnn_hidden_size,
- fcnet_hidden_sizes=fcnet_hidden_sizes,
- encoded_data_size=encoded_data_size,
- encoded_latent_size=encoded_latent_size,
- use_tilt=use_tilt,
- initializers=initializers,
- random_seed=random_seed)
- inputs = tf.random_uniform([num_timesteps, batch_size, data_size],
- seed=random_seed, dtype=tf.float32)
- targets = tf.random_uniform([num_timesteps, batch_size, data_size],
- seed=random_seed, dtype=tf.float32)
- lengths = tf.constant(data_lengths, dtype=tf.int32)
- return model, inputs, targets, lengths
diff --git a/research/fivo/run_fivo.py b/research/fivo/run_fivo.py
deleted file mode 100644
index 1ca079421f09fb65439dae210b1c3760240b51ad..0000000000000000000000000000000000000000
--- a/research/fivo/run_fivo.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A script to run training for sequential latent variable models.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from fivo import ghmm_runners
-from fivo import runners
-
-# Shared flags.
-tf.app.flags.DEFINE_enum("mode", "train",
- ["train", "eval", "sample"],
- "The mode of the binary.")
-tf.app.flags.DEFINE_enum("model", "vrnn",
- ["vrnn", "ghmm", "srnn"],
- "Model choice.")
-tf.app.flags.DEFINE_integer("latent_size", 64,
- "The size of the latent state of the model.")
-tf.app.flags.DEFINE_enum("dataset_type", "pianoroll",
- ["pianoroll", "speech", "pose"],
- "The type of dataset.")
-tf.app.flags.DEFINE_string("dataset_path", "",
- "Path to load the dataset from.")
-tf.app.flags.DEFINE_integer("data_dimension", None,
- "The dimension of each vector in the data sequence. "
- "Defaults to 88 for pianoroll datasets and 200 for speech "
- "datasets. Should not need to be changed except for "
- "testing.")
-tf.app.flags.DEFINE_integer("batch_size", 4,
- "Batch size.")
-tf.app.flags.DEFINE_integer("num_samples", 4,
- "The number of samples (or particles) for multisample "
- "algorithms.")
-tf.app.flags.DEFINE_string("logdir", "/tmp/smc_vi",
- "The directory to keep checkpoints and summaries in.")
-tf.app.flags.DEFINE_integer("random_seed", None,
- "A random seed for seeding the TensorFlow graph.")
-tf.app.flags.DEFINE_integer("parallel_iterations", 30,
- "The number of parallel iterations to use for the while "
- "loop that computes the bounds.")
-
-# Training flags.
-tf.app.flags.DEFINE_enum("bound", "fivo",
- ["elbo", "iwae", "fivo", "fivo-aux"],
- "The bound to optimize.")
-tf.app.flags.DEFINE_boolean("normalize_by_seq_len", True,
- "If true, normalize the loss by the number of timesteps "
- "per sequence.")
-tf.app.flags.DEFINE_float("learning_rate", 0.0002,
- "The learning rate for ADAM.")
-tf.app.flags.DEFINE_integer("max_steps", int(1e9),
- "The number of gradient update steps to train for.")
-tf.app.flags.DEFINE_integer("summarize_every", 50,
- "The number of steps between summaries.")
-tf.app.flags.DEFINE_enum("resampling_type", "multinomial",
- ["multinomial", "relaxed"],
- "The resampling strategy to use for training.")
-tf.app.flags.DEFINE_float("relaxed_resampling_temperature", 0.5,
- "The relaxation temperature for relaxed resampling.")
-tf.app.flags.DEFINE_enum("proposal_type", "filtering",
- ["prior", "filtering", "smoothing",
- "true-filtering", "true-smoothing"],
- "The type of proposal to use. true-filtering and true-smoothing "
- "are only available for the GHMM. The specific implementation "
- "of each proposal type is left to model-writers.")
-
-# Distributed training flags.
-tf.app.flags.DEFINE_string("master", "",
- "The BNS name of the TensorFlow master to use.")
-tf.app.flags.DEFINE_integer("task", 0,
- "Task id of the replica running the training.")
-tf.app.flags.DEFINE_integer("ps_tasks", 0,
- "Number of tasks in the ps job. If 0 no ps job is used.")
-tf.app.flags.DEFINE_boolean("stagger_workers", True,
- "If true, bring one worker online every 1000 steps.")
-
-# Evaluation flags.
-tf.app.flags.DEFINE_enum("split", "train",
- ["train", "test", "valid"],
- "Split to evaluate the model on.")
-
-# Sampling flags.
-tf.app.flags.DEFINE_integer("sample_length", 50,
- "The number of timesteps to sample for.")
-tf.app.flags.DEFINE_integer("prefix_length", 25,
- "The number of timesteps to condition the model on "
- "before sampling.")
-tf.app.flags.DEFINE_string("sample_out_dir", None,
- "The directory to write the samples to. "
- "Defaults to logdir.")
-
-# GHMM flags.
-tf.app.flags.DEFINE_float("variance", 0.1,
- "The variance of the ghmm.")
-tf.app.flags.DEFINE_integer("num_timesteps", 5,
- "The number of timesteps to run the gmp for.")
-FLAGS = tf.app.flags.FLAGS
-
-PIANOROLL_DEFAULT_DATA_DIMENSION = 88
-SPEECH_DEFAULT_DATA_DIMENSION = 200
-
-
-def main(unused_argv):
- tf.logging.set_verbosity(tf.logging.INFO)
- if FLAGS.model in ["vrnn", "srnn"]:
- if FLAGS.data_dimension is None:
- if FLAGS.dataset_type == "pianoroll":
- FLAGS.data_dimension = PIANOROLL_DEFAULT_DATA_DIMENSION
- elif FLAGS.dataset_type == "speech":
- FLAGS.data_dimension = SPEECH_DEFAULT_DATA_DIMENSION
- if FLAGS.mode == "train":
- runners.run_train(FLAGS)
- elif FLAGS.mode == "eval":
- runners.run_eval(FLAGS)
- elif FLAGS.mode == "sample":
- runners.run_sample(FLAGS)
- elif FLAGS.model == "ghmm":
- if FLAGS.mode == "train":
- ghmm_runners.run_train(FLAGS)
- elif FLAGS.mode == "eval":
- ghmm_runners.run_eval(FLAGS)
-
-if __name__ == "__main__":
- tf.app.run(main)
diff --git a/research/global_objectives/README.md b/research/global_objectives/README.md
deleted file mode 100644
index f9a778c59d420f9bf5deccf4b2b45147636de582..0000000000000000000000000000000000000000
--- a/research/global_objectives/README.md
+++ /dev/null
@@ -1,152 +0,0 @@
-
-
-
-
-# Global Objectives
-The Global Objectives library provides TensorFlow loss functions that optimize
-directly for a variety of objectives including AUC, recall at precision, and
-more. The global objectives losses can be used as drop-in replacements for
-TensorFlow's standard multilabel loss functions:
-`tf.nn.sigmoid_cross_entropy_with_logits` and `tf.losses.sigmoid_cross_entropy`.
-
-Many machine learning classification models are optimized for classification
-accuracy, when the real objective the user cares about is different and can be
-precision at a fixed recall, precision-recall AUC, ROC AUC or similar metrics.
-These are referred to as "global objectives" because they depend on how the
-model classifies the dataset as a whole and do not decouple across data points
-as accuracy does.
-
-Because these objectives are combinatorial, discontinuous, and essentially
-intractable to optimize directly, the functions in this library approximate
-their corresponding objectives. This approximation approach follows the same
-pattern as optimizing for accuracy, where a surrogate objective such as
-cross-entropy or the hinge loss is used as an upper bound on the error rate.
-
-## Getting Started
-For a full example of how to use the loss functions in practice, see
-loss_layers_example.py.
-
-Briefly, global objective losses can be used to replace
-`tf.nn.sigmoid_cross_entropy_with_logits` by providing the relevant
-additional arguments. For example,
-
-``` python
-tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits)
-```
-
-could be replaced with
-
-``` python
-global_objectives.recall_at_precision_loss(
- labels=labels,
- logits=logits,
- target_precision=0.95)[0]
-```
-
-Just as minimizing the cross-entropy loss will maximize accuracy, the loss
-functions in loss_layers.py were written so that minimizing the loss will
-maximize the corresponding objective.
-
-The global objective losses have two return values -- the loss tensor and
-additional quantities for debugging and customization -- which is why the first
-value is used above. For more information, see
-[Visualization & Debugging](#visualization-debugging).
-
-## Binary Label Format
-Binary classification problems can be represented as a multi-class problem with
-two classes, or as a multi-label problem with one label. (Recall that multiclass
-problems have mutually exclusive classes, e.g. 'cat xor dog', and multilabel
-have classes which are not mutually exclusive, e.g. an image can contain a cat,
-a dog, both, or neither.) The softmax loss
-(`tf.nn.softmax_cross_entropy_with_logits`) is used for multi-class problems,
-while the sigmoid loss (`tf.nn.sigmoid_cross_entropy_with_logits`) is used for
-multi-label problems.
-
-A multiclass label format for binary classification might represent positives
-with the label [1, 0] and negatives with the label [0, 1], while the multilbel
-format for the same problem would use [1] and [0], respectively.
-
-All global objectives loss functions assume that the multilabel format is used.
-Accordingly, if your current loss function is softmax, the labels will have to
-be reformatted for the loss to work properly.
-
-## Dual Variables
-Global objectives losses (except for `roc_auc_loss`) use internal variables
-called dual variables or Lagrange multipliers to enforce the desired constraint
-(e.g. if optimzing for recall at precision, the constraint is on precision).
-
-These dual variables are created and initialized internally by the loss
-functions, and are updated during training by the same optimizer used for the
-model's other variables. To initialize the dual variables to a particular value,
-use the `lambdas_initializer` argument. The dual variables can be found under
-the key `lambdas` in the `other_outputs` dictionary returned by the losses.
-
-## Loss Function Arguments
-The following arguments are common to all loss functions in the library, and are
-either required or very important.
-
-* `labels`: Corresponds directly to the `labels` argument of
- `tf.nn.sigmoid_cross_entropy_with_logits`.
-* `logits`: Corresponds directly to the `logits` argument of
- `tf.nn.sigmoid_cross_entropy_with_logits`.
-* `dual_rate_factor`: A floating point value which controls the step size for
- the Lagrange multipliers. Setting this value less than 1.0 will cause the
- constraint to be enforced more gradually and will result in more stable
- training.
-
-In addition, the objectives with a single constraint (e.g.
-`recall_at_precision_loss`) have an argument (e.g. `target_precision`) used to
-specify the value of the constraint. The optional `precision_range` argument to
-`precision_recall_auc_loss` is used to specify the range of precision values
-over which to optimize the AUC, and defaults to the interval [0, 1].
-
-Optional arguments:
-
-* `weights`: A tensor which acts as coefficients for the loss. If a weight of x
- is provided for a datapoint and that datapoint is a true (false) positive
- (negative), it will be counted as x true (false) positives (negatives).
- Defaults to 1.0.
-* `label_priors`: A tensor specifying the fraction of positive datapoints for
- each label. If not provided, it will be computed inside the loss function.
-* `surrogate_type`: Either 'xent' or 'hinge', specifying which upper bound
- should be used for indicator functions.
-* `lambdas_initializer`: An initializer for the dual variables (Lagrange
- multipliers). See also the Dual Variables section.
-* `num_anchors` (precision_recall_auc_loss only): The number of grid points used
- when approximating the AUC as a Riemann sum.
-
-## Hyperparameters
-While the functional form of the global objectives losses allow them to be
-easily substituted in place of `sigmoid_cross_entropy_with_logits`, model
-hyperparameters such as learning rate, weight decay, etc. may need to be
-fine-tuned to the new loss. Fortunately, the amount of hyperparameter re-tuning
-is usually minor.
-
-The most important hyperparameters to modify are the learning rate and
-dual_rate_factor (see the section on Loss Function Arguments, above).
-
-## Visualization & Debugging
-The global objectives losses return two values. The first is a tensor
-representing the numerical value of the loss, which can be passed to an
-optimizer. The second is a dictionary of tensors created by the loss function
-which are not necessary for optimization but useful in debugging. These vary
-depending on the loss function, but usually include `lambdas` (the Lagrange
-multipliers) as well as the lower bound on true positives and upper bound on
-false positives.
-
-When visualizing the loss during training, note that the global objectives
-losses differ from standard losses in some important ways:
-
-* The global losses may be negative. This is because the value returned by the
- loss includes terms involving the Lagrange multipliers, which may be negative.
-* The global losses may not decrease over the course of training. To enforce the
- constraints in the objective, the loss changes over time and may increase.
-
-## More Info
-For more details, see the [Global Objectives paper](https://arxiv.org/abs/1608.04802).
-
-## Maintainers
-
-* Mariano Schain
-* Elad Eban
-* [Alan Mackey](https://github.com/mackeya-google)
diff --git a/research/global_objectives/loss_layers.py b/research/global_objectives/loss_layers.py
deleted file mode 100644
index eaea05398ef3771247060afda63be184ea76cdf0..0000000000000000000000000000000000000000
--- a/research/global_objectives/loss_layers.py
+++ /dev/null
@@ -1,930 +0,0 @@
-# Copyright 2018 The TensorFlow Global Objectives Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Loss functions for learning global objectives.
-
-These functions have two return values: a Tensor with the value of
-the loss, and a dictionary of internal quantities for customizability.
-"""
-
-# Dependency imports
-import numpy
-import tensorflow as tf
-
-from global_objectives import util
-
-
-def precision_recall_auc_loss(
- labels,
- logits,
- precision_range=(0.0, 1.0),
- num_anchors=20,
- weights=1.0,
- dual_rate_factor=0.1,
- label_priors=None,
- surrogate_type='xent',
- lambdas_initializer=tf.constant_initializer(1.0),
- reuse=None,
- variables_collections=None,
- trainable=True,
- scope=None):
- """Computes precision-recall AUC loss.
-
- The loss is based on a sum of losses for recall at a range of
- precision values (anchor points). This sum is a Riemann sum that
- approximates the area under the precision-recall curve.
-
- The per-example `weights` argument changes not only the coefficients of
- individual training examples, but how the examples are counted toward the
- constraint. If `label_priors` is given, it MUST take `weights` into account.
- That is,
- label_priors = P / (P + N)
- where
- P = sum_i (wt_i on positives)
- N = sum_i (wt_i on negatives).
-
- Args:
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- logits: A `Tensor` with the same shape as `labels`.
- precision_range: A length-two tuple, the range of precision values over
- which to compute AUC. The entries must be nonnegative, increasing, and
- less than or equal to 1.0.
- num_anchors: The number of grid points used to approximate the Riemann sum.
- weights: Coefficients for the loss. Must be a scalar or `Tensor` of shape
- [batch_size] or [batch_size, num_labels].
- dual_rate_factor: A floating point value which controls the step size for
- the Lagrange multipliers.
- label_priors: None, or a floating point `Tensor` of shape [num_labels]
- containing the prior probability of each label (i.e. the fraction of the
- training data consisting of positive examples). If None, the label
- priors are computed from `labels` with a moving average. See the notes
- above regarding the interaction with `weights` and do not set this unless
- you have a good reason to do so.
- surrogate_type: Either 'xent' or 'hinge', specifying which upper bound
- should be used for indicator functions.
- lambdas_initializer: An initializer for the Lagrange multipliers.
- reuse: Whether or not the layer and its variables should be reused. To be
- able to reuse the layer scope must be given.
- variables_collections: Optional list of collections for the variables.
- trainable: If `True` also add variables to the graph collection
- `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
- scope: Optional scope for `variable_scope`.
-
- Returns:
- loss: A `Tensor` of the same shape as `logits` with the component-wise
- loss.
- other_outputs: A dictionary of useful internal quantities for debugging. For
- more details, see http://arxiv.org/pdf/1608.04802.pdf.
- lambdas: A Tensor of shape [1, num_labels, num_anchors] consisting of the
- Lagrange multipliers.
- biases: A Tensor of shape [1, num_labels, num_anchors] consisting of the
- learned bias term for each.
- label_priors: A Tensor of shape [1, num_labels, 1] consisting of the prior
- probability of each label learned by the loss, if not provided.
- true_positives_lower_bound: Lower bound on the number of true positives
- given `labels` and `logits`. This is the same lower bound which is used
- in the loss expression to be optimized.
- false_positives_upper_bound: Upper bound on the number of false positives
- given `labels` and `logits`. This is the same upper bound which is used
- in the loss expression to be optimized.
-
- Raises:
- ValueError: If `surrogate_type` is not `xent` or `hinge`.
- """
- with tf.variable_scope(scope,
- 'precision_recall_auc',
- [labels, logits, label_priors],
- reuse=reuse):
- labels, logits, weights, original_shape = _prepare_labels_logits_weights(
- labels, logits, weights)
- num_labels = util.get_num_labels(logits)
-
- # Convert other inputs to tensors and standardize dtypes.
- dual_rate_factor = util.convert_and_cast(
- dual_rate_factor, 'dual_rate_factor', logits.dtype)
-
- # Create Tensor of anchor points and distance between anchors.
- precision_values, delta = _range_to_anchors_and_delta(
- precision_range, num_anchors, logits.dtype)
- # Create lambdas with shape [1, num_labels, num_anchors].
- lambdas, lambdas_variable = _create_dual_variable(
- 'lambdas',
- shape=[1, num_labels, num_anchors],
- dtype=logits.dtype,
- initializer=lambdas_initializer,
- collections=variables_collections,
- trainable=trainable,
- dual_rate_factor=dual_rate_factor)
- # Create biases with shape [1, num_labels, num_anchors].
- biases = tf.contrib.framework.model_variable(
- name='biases',
- shape=[1, num_labels, num_anchors],
- dtype=logits.dtype,
- initializer=tf.zeros_initializer(),
- collections=variables_collections,
- trainable=trainable)
- # Maybe create label_priors.
- label_priors = maybe_create_label_priors(
- label_priors, labels, weights, variables_collections)
- label_priors = tf.reshape(label_priors, [1, num_labels, 1])
-
- # Expand logits, labels, and weights to shape [batch_size, num_labels, 1].
- logits = tf.expand_dims(logits, 2)
- labels = tf.expand_dims(labels, 2)
- weights = tf.expand_dims(weights, 2)
-
- # Calculate weighted loss and other outputs. The log(2.0) term corrects for
- # logloss not being an upper bound on the indicator function.
- loss = weights * util.weighted_surrogate_loss(
- labels,
- logits + biases,
- surrogate_type=surrogate_type,
- positive_weights=1.0 + lambdas * (1.0 - precision_values),
- negative_weights=lambdas * precision_values)
- maybe_log2 = tf.log(2.0) if surrogate_type == 'xent' else 1.0
- maybe_log2 = tf.cast(maybe_log2, logits.dtype.base_dtype)
- lambda_term = lambdas * (1.0 - precision_values) * label_priors * maybe_log2
- per_anchor_loss = loss - lambda_term
- per_label_loss = delta * tf.reduce_sum(per_anchor_loss, 2)
- # Normalize the AUC such that a perfect score function will have AUC 1.0.
- # Because precision_range is discretized into num_anchors + 1 intervals
- # but only num_anchors terms are included in the Riemann sum, the
- # effective length of the integration interval is `delta` less than the
- # length of precision_range.
- scaled_loss = tf.div(per_label_loss,
- precision_range[1] - precision_range[0] - delta,
- name='AUC_Normalize')
- scaled_loss = tf.reshape(scaled_loss, original_shape)
-
- other_outputs = {
- 'lambdas': lambdas_variable,
- 'biases': biases,
- 'label_priors': label_priors,
- 'true_positives_lower_bound': true_positives_lower_bound(
- labels, logits, weights, surrogate_type),
- 'false_positives_upper_bound': false_positives_upper_bound(
- labels, logits, weights, surrogate_type)}
-
- return scaled_loss, other_outputs
-
-
-def roc_auc_loss(
- labels,
- logits,
- weights=1.0,
- surrogate_type='xent',
- scope=None):
- """Computes ROC AUC loss.
-
- The area under the ROC curve is the probability p that a randomly chosen
- positive example will be scored higher than a randomly chosen negative
- example. This loss approximates 1-p by using a surrogate (either hinge loss or
- cross entropy) for the indicator function. Specifically, the loss is:
-
- sum_i sum_j w_i*w_j*loss(logit_i - logit_j)
-
- where i ranges over the positive datapoints, j ranges over the negative
- datapoints, logit_k denotes the logit (or score) of the k-th datapoint, and
- loss is either the hinge or log loss given a positive label.
-
- Args:
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- logits: A `Tensor` with the same shape and dtype as `labels`.
- weights: Coefficients for the loss. Must be a scalar or `Tensor` of shape
- [batch_size] or [batch_size, num_labels].
- surrogate_type: Either 'xent' or 'hinge', specifying which upper bound
- should be used for the indicator function.
- scope: Optional scope for `name_scope`.
-
- Returns:
- loss: A `Tensor` of the same shape as `logits` with the component-wise loss.
- other_outputs: An empty dictionary, for consistency.
-
- Raises:
- ValueError: If `surrogate_type` is not `xent` or `hinge`.
- """
- with tf.name_scope(scope, 'roc_auc', [labels, logits, weights]):
- # Convert inputs to tensors and standardize dtypes.
- labels, logits, weights, original_shape = _prepare_labels_logits_weights(
- labels, logits, weights)
-
- # Create tensors of pairwise differences for logits and labels, and
- # pairwise products of weights. These have shape
- # [batch_size, batch_size, num_labels].
- logits_difference = tf.expand_dims(logits, 0) - tf.expand_dims(logits, 1)
- labels_difference = tf.expand_dims(labels, 0) - tf.expand_dims(labels, 1)
- weights_product = tf.expand_dims(weights, 0) * tf.expand_dims(weights, 1)
-
- signed_logits_difference = labels_difference * logits_difference
- raw_loss = util.weighted_surrogate_loss(
- labels=tf.ones_like(signed_logits_difference),
- logits=signed_logits_difference,
- surrogate_type=surrogate_type)
- weighted_loss = weights_product * raw_loss
-
- # Zero out entries of the loss where labels_difference zero (so loss is only
- # computed on pairs with different labels).
- loss = tf.reduce_mean(tf.abs(labels_difference) * weighted_loss, 0) * 0.5
- loss = tf.reshape(loss, original_shape)
- return loss, {}
-
-
-def recall_at_precision_loss(
- labels,
- logits,
- target_precision,
- weights=1.0,
- dual_rate_factor=0.1,
- label_priors=None,
- surrogate_type='xent',
- lambdas_initializer=tf.constant_initializer(1.0),
- reuse=None,
- variables_collections=None,
- trainable=True,
- scope=None):
- """Computes recall at precision loss.
-
- The loss is based on a surrogate of the form
- wt * w(+) * loss(+) + wt * w(-) * loss(-) - c * pi,
- where:
- - w(+) = 1 + lambdas * (1 - target_precision)
- - loss(+) is the cross-entropy loss on the positive examples
- - w(-) = lambdas * target_precision
- - loss(-) is the cross-entropy loss on the negative examples
- - wt is a scalar or tensor of per-example weights
- - c = lambdas * (1 - target_precision)
- - pi is the label_priors.
-
- The per-example weights change not only the coefficients of individual
- training examples, but how the examples are counted toward the constraint.
- If `label_priors` is given, it MUST take `weights` into account. That is,
- label_priors = P / (P + N)
- where
- P = sum_i (wt_i on positives)
- N = sum_i (wt_i on negatives).
-
- Args:
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- logits: A `Tensor` with the same shape as `labels`.
- target_precision: The precision at which to compute the loss. Can be a
- floating point value between 0 and 1 for a single precision value, or a
- `Tensor` of shape [num_labels], holding each label's target precision
- value.
- weights: Coefficients for the loss. Must be a scalar or `Tensor` of shape
- [batch_size] or [batch_size, num_labels].
- dual_rate_factor: A floating point value which controls the step size for
- the Lagrange multipliers.
- label_priors: None, or a floating point `Tensor` of shape [num_labels]
- containing the prior probability of each label (i.e. the fraction of the
- training data consisting of positive examples). If None, the label
- priors are computed from `labels` with a moving average. See the notes
- above regarding the interaction with `weights` and do not set this unless
- you have a good reason to do so.
- surrogate_type: Either 'xent' or 'hinge', specifying which upper bound
- should be used for indicator functions.
- lambdas_initializer: An initializer for the Lagrange multipliers.
- reuse: Whether or not the layer and its variables should be reused. To be
- able to reuse the layer scope must be given.
- variables_collections: Optional list of collections for the variables.
- trainable: If `True` also add variables to the graph collection
- `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
- scope: Optional scope for `variable_scope`.
-
- Returns:
- loss: A `Tensor` of the same shape as `logits` with the component-wise
- loss.
- other_outputs: A dictionary of useful internal quantities for debugging. For
- more details, see http://arxiv.org/pdf/1608.04802.pdf.
- lambdas: A Tensor of shape [num_labels] consisting of the Lagrange
- multipliers.
- label_priors: A Tensor of shape [num_labels] consisting of the prior
- probability of each label learned by the loss, if not provided.
- true_positives_lower_bound: Lower bound on the number of true positives
- given `labels` and `logits`. This is the same lower bound which is used
- in the loss expression to be optimized.
- false_positives_upper_bound: Upper bound on the number of false positives
- given `labels` and `logits`. This is the same upper bound which is used
- in the loss expression to be optimized.
-
- Raises:
- ValueError: If `logits` and `labels` do not have the same shape.
- """
- with tf.variable_scope(scope,
- 'recall_at_precision',
- [logits, labels, label_priors],
- reuse=reuse):
- labels, logits, weights, original_shape = _prepare_labels_logits_weights(
- labels, logits, weights)
- num_labels = util.get_num_labels(logits)
-
- # Convert other inputs to tensors and standardize dtypes.
- target_precision = util.convert_and_cast(
- target_precision, 'target_precision', logits.dtype)
- dual_rate_factor = util.convert_and_cast(
- dual_rate_factor, 'dual_rate_factor', logits.dtype)
-
- # Create lambdas.
- lambdas, lambdas_variable = _create_dual_variable(
- 'lambdas',
- shape=[num_labels],
- dtype=logits.dtype,
- initializer=lambdas_initializer,
- collections=variables_collections,
- trainable=trainable,
- dual_rate_factor=dual_rate_factor)
- # Maybe create label_priors.
- label_priors = maybe_create_label_priors(
- label_priors, labels, weights, variables_collections)
-
- # Calculate weighted loss and other outputs. The log(2.0) term corrects for
- # logloss not being an upper bound on the indicator function.
- weighted_loss = weights * util.weighted_surrogate_loss(
- labels,
- logits,
- surrogate_type=surrogate_type,
- positive_weights=1.0 + lambdas * (1.0 - target_precision),
- negative_weights=lambdas * target_precision)
- maybe_log2 = tf.log(2.0) if surrogate_type == 'xent' else 1.0
- maybe_log2 = tf.cast(maybe_log2, logits.dtype.base_dtype)
- lambda_term = lambdas * (1.0 - target_precision) * label_priors * maybe_log2
- loss = tf.reshape(weighted_loss - lambda_term, original_shape)
- other_outputs = {
- 'lambdas': lambdas_variable,
- 'label_priors': label_priors,
- 'true_positives_lower_bound': true_positives_lower_bound(
- labels, logits, weights, surrogate_type),
- 'false_positives_upper_bound': false_positives_upper_bound(
- labels, logits, weights, surrogate_type)}
-
- return loss, other_outputs
-
-
-def precision_at_recall_loss(
- labels,
- logits,
- target_recall,
- weights=1.0,
- dual_rate_factor=0.1,
- label_priors=None,
- surrogate_type='xent',
- lambdas_initializer=tf.constant_initializer(1.0),
- reuse=None,
- variables_collections=None,
- trainable=True,
- scope=None):
- """Computes precision at recall loss.
-
- The loss is based on a surrogate of the form
- wt * loss(-) + lambdas * (pi * (b - 1) + wt * loss(+))
- where:
- - loss(-) is the cross-entropy loss on the negative examples
- - loss(+) is the cross-entropy loss on the positive examples
- - wt is a scalar or tensor of per-example weights
- - b is the target recall
- - pi is the label_priors.
-
- The per-example weights change not only the coefficients of individual
- training examples, but how the examples are counted toward the constraint.
- If `label_priors` is given, it MUST take `weights` into account. That is,
- label_priors = P / (P + N)
- where
- P = sum_i (wt_i on positives)
- N = sum_i (wt_i on negatives).
-
- Args:
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- logits: A `Tensor` with the same shape as `labels`.
- target_recall: The recall at which to compute the loss. Can be a floating
- point value between 0 and 1 for a single target recall value, or a
- `Tensor` of shape [num_labels] holding each label's target recall value.
- weights: Coefficients for the loss. Must be a scalar or `Tensor` of shape
- [batch_size] or [batch_size, num_labels].
- dual_rate_factor: A floating point value which controls the step size for
- the Lagrange multipliers.
- label_priors: None, or a floating point `Tensor` of shape [num_labels]
- containing the prior probability of each label (i.e. the fraction of the
- training data consisting of positive examples). If None, the label
- priors are computed from `labels` with a moving average. See the notes
- above regarding the interaction with `weights` and do not set this unless
- you have a good reason to do so.
- surrogate_type: Either 'xent' or 'hinge', specifying which upper bound
- should be used for indicator functions.
- lambdas_initializer: An initializer for the Lagrange multipliers.
- reuse: Whether or not the layer and its variables should be reused. To be
- able to reuse the layer scope must be given.
- variables_collections: Optional list of collections for the variables.
- trainable: If `True` also add variables to the graph collection
- `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
- scope: Optional scope for `variable_scope`.
-
- Returns:
- loss: A `Tensor` of the same shape as `logits` with the component-wise
- loss.
- other_outputs: A dictionary of useful internal quantities for debugging. For
- more details, see http://arxiv.org/pdf/1608.04802.pdf.
- lambdas: A Tensor of shape [num_labels] consisting of the Lagrange
- multipliers.
- label_priors: A Tensor of shape [num_labels] consisting of the prior
- probability of each label learned by the loss, if not provided.
- true_positives_lower_bound: Lower bound on the number of true positives
- given `labels` and `logits`. This is the same lower bound which is used
- in the loss expression to be optimized.
- false_positives_upper_bound: Upper bound on the number of false positives
- given `labels` and `logits`. This is the same upper bound which is used
- in the loss expression to be optimized.
- """
- with tf.variable_scope(scope,
- 'precision_at_recall',
- [logits, labels, label_priors],
- reuse=reuse):
- labels, logits, weights, original_shape = _prepare_labels_logits_weights(
- labels, logits, weights)
- num_labels = util.get_num_labels(logits)
-
- # Convert other inputs to tensors and standardize dtypes.
- target_recall = util.convert_and_cast(
- target_recall, 'target_recall', logits.dtype)
- dual_rate_factor = util.convert_and_cast(
- dual_rate_factor, 'dual_rate_factor', logits.dtype)
-
- # Create lambdas.
- lambdas, lambdas_variable = _create_dual_variable(
- 'lambdas',
- shape=[num_labels],
- dtype=logits.dtype,
- initializer=lambdas_initializer,
- collections=variables_collections,
- trainable=trainable,
- dual_rate_factor=dual_rate_factor)
- # Maybe create label_priors.
- label_priors = maybe_create_label_priors(
- label_priors, labels, weights, variables_collections)
-
- # Calculate weighted loss and other outputs. The log(2.0) term corrects for
- # logloss not being an upper bound on the indicator function.
- weighted_loss = weights * util.weighted_surrogate_loss(
- labels,
- logits,
- surrogate_type,
- positive_weights=lambdas,
- negative_weights=1.0)
- maybe_log2 = tf.log(2.0) if surrogate_type == 'xent' else 1.0
- maybe_log2 = tf.cast(maybe_log2, logits.dtype.base_dtype)
- lambda_term = lambdas * label_priors * (target_recall - 1.0) * maybe_log2
- loss = tf.reshape(weighted_loss + lambda_term, original_shape)
- other_outputs = {
- 'lambdas': lambdas_variable,
- 'label_priors': label_priors,
- 'true_positives_lower_bound': true_positives_lower_bound(
- labels, logits, weights, surrogate_type),
- 'false_positives_upper_bound': false_positives_upper_bound(
- labels, logits, weights, surrogate_type)}
-
- return loss, other_outputs
-
-
-def false_positive_rate_at_true_positive_rate_loss(
- labels,
- logits,
- target_rate,
- weights=1.0,
- dual_rate_factor=0.1,
- label_priors=None,
- surrogate_type='xent',
- lambdas_initializer=tf.constant_initializer(1.0),
- reuse=None,
- variables_collections=None,
- trainable=True,
- scope=None):
- """Computes false positive rate at true positive rate loss.
-
- Note that `true positive rate` is a synonym for Recall, and that minimizing
- the false positive rate and maximizing precision are equivalent for a fixed
- Recall. Therefore, this function is identical to precision_at_recall_loss.
-
- The per-example weights change not only the coefficients of individual
- training examples, but how the examples are counted toward the constraint.
- If `label_priors` is given, it MUST take `weights` into account. That is,
- label_priors = P / (P + N)
- where
- P = sum_i (wt_i on positives)
- N = sum_i (wt_i on negatives).
-
- Args:
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- logits: A `Tensor` with the same shape as `labels`.
- target_rate: The true positive rate at which to compute the loss. Can be a
- floating point value between 0 and 1 for a single true positive rate, or
- a `Tensor` of shape [num_labels] holding each label's true positive rate.
- weights: Coefficients for the loss. Must be a scalar or `Tensor` of shape
- [batch_size] or [batch_size, num_labels].
- dual_rate_factor: A floating point value which controls the step size for
- the Lagrange multipliers.
- label_priors: None, or a floating point `Tensor` of shape [num_labels]
- containing the prior probability of each label (i.e. the fraction of the
- training data consisting of positive examples). If None, the label
- priors are computed from `labels` with a moving average. See the notes
- above regarding the interaction with `weights` and do not set this unless
- you have a good reason to do so.
- surrogate_type: Either 'xent' or 'hinge', specifying which upper bound
- should be used for indicator functions. 'xent' will use the cross-entropy
- loss surrogate, and 'hinge' will use the hinge loss.
- lambdas_initializer: An initializer op for the Lagrange multipliers.
- reuse: Whether or not the layer and its variables should be reused. To be
- able to reuse the layer scope must be given.
- variables_collections: Optional list of collections for the variables.
- trainable: If `True` also add variables to the graph collection
- `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
- scope: Optional scope for `variable_scope`.
-
- Returns:
- loss: A `Tensor` of the same shape as `logits` with the component-wise
- loss.
- other_outputs: A dictionary of useful internal quantities for debugging. For
- more details, see http://arxiv.org/pdf/1608.04802.pdf.
- lambdas: A Tensor of shape [num_labels] consisting of the Lagrange
- multipliers.
- label_priors: A Tensor of shape [num_labels] consisting of the prior
- probability of each label learned by the loss, if not provided.
- true_positives_lower_bound: Lower bound on the number of true positives
- given `labels` and `logits`. This is the same lower bound which is used
- in the loss expression to be optimized.
- false_positives_upper_bound: Upper bound on the number of false positives
- given `labels` and `logits`. This is the same upper bound which is used
- in the loss expression to be optimized.
-
- Raises:
- ValueError: If `surrogate_type` is not `xent` or `hinge`.
- """
- return precision_at_recall_loss(labels=labels,
- logits=logits,
- target_recall=target_rate,
- weights=weights,
- dual_rate_factor=dual_rate_factor,
- label_priors=label_priors,
- surrogate_type=surrogate_type,
- lambdas_initializer=lambdas_initializer,
- reuse=reuse,
- variables_collections=variables_collections,
- trainable=trainable,
- scope=scope)
-
-
-def true_positive_rate_at_false_positive_rate_loss(
- labels,
- logits,
- target_rate,
- weights=1.0,
- dual_rate_factor=0.1,
- label_priors=None,
- surrogate_type='xent',
- lambdas_initializer=tf.constant_initializer(1.0),
- reuse=None,
- variables_collections=None,
- trainable=True,
- scope=None):
- """Computes true positive rate at false positive rate loss.
-
- The loss is based on a surrogate of the form
- wt * loss(+) + lambdas * (wt * loss(-) - r * (1 - pi))
- where:
- - loss(-) is the loss on the negative examples
- - loss(+) is the loss on the positive examples
- - wt is a scalar or tensor of per-example weights
- - r is the target rate
- - pi is the label_priors.
-
- The per-example weights change not only the coefficients of individual
- training examples, but how the examples are counted toward the constraint.
- If `label_priors` is given, it MUST take `weights` into account. That is,
- label_priors = P / (P + N)
- where
- P = sum_i (wt_i on positives)
- N = sum_i (wt_i on negatives).
-
- Args:
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- logits: A `Tensor` with the same shape as `labels`.
- target_rate: The false positive rate at which to compute the loss. Can be a
- floating point value between 0 and 1 for a single false positive rate, or
- a `Tensor` of shape [num_labels] holding each label's false positive rate.
- weights: Coefficients for the loss. Must be a scalar or `Tensor` of shape
- [batch_size] or [batch_size, num_labels].
- dual_rate_factor: A floating point value which controls the step size for
- the Lagrange multipliers.
- label_priors: None, or a floating point `Tensor` of shape [num_labels]
- containing the prior probability of each label (i.e. the fraction of the
- training data consisting of positive examples). If None, the label
- priors are computed from `labels` with a moving average. See the notes
- above regarding the interaction with `weights` and do not set this unless
- you have a good reason to do so.
- surrogate_type: Either 'xent' or 'hinge', specifying which upper bound
- should be used for indicator functions. 'xent' will use the cross-entropy
- loss surrogate, and 'hinge' will use the hinge loss.
- lambdas_initializer: An initializer op for the Lagrange multipliers.
- reuse: Whether or not the layer and its variables should be reused. To be
- able to reuse the layer scope must be given.
- variables_collections: Optional list of collections for the variables.
- trainable: If `True` also add variables to the graph collection
- `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
- scope: Optional scope for `variable_scope`.
-
- Returns:
- loss: A `Tensor` of the same shape as `logits` with the component-wise
- loss.
- other_outputs: A dictionary of useful internal quantities for debugging. For
- more details, see http://arxiv.org/pdf/1608.04802.pdf.
- lambdas: A Tensor of shape [num_labels] consisting of the Lagrange
- multipliers.
- label_priors: A Tensor of shape [num_labels] consisting of the prior
- probability of each label learned by the loss, if not provided.
- true_positives_lower_bound: Lower bound on the number of true positives
- given `labels` and `logits`. This is the same lower bound which is used
- in the loss expression to be optimized.
- false_positives_upper_bound: Upper bound on the number of false positives
- given `labels` and `logits`. This is the same upper bound which is used
- in the loss expression to be optimized.
-
- Raises:
- ValueError: If `surrogate_type` is not `xent` or `hinge`.
- """
- with tf.variable_scope(scope,
- 'tpr_at_fpr',
- [labels, logits, label_priors],
- reuse=reuse):
- labels, logits, weights, original_shape = _prepare_labels_logits_weights(
- labels, logits, weights)
- num_labels = util.get_num_labels(logits)
-
- # Convert other inputs to tensors and standardize dtypes.
- target_rate = util.convert_and_cast(
- target_rate, 'target_rate', logits.dtype)
- dual_rate_factor = util.convert_and_cast(
- dual_rate_factor, 'dual_rate_factor', logits.dtype)
-
- # Create lambdas.
- lambdas, lambdas_variable = _create_dual_variable(
- 'lambdas',
- shape=[num_labels],
- dtype=logits.dtype,
- initializer=lambdas_initializer,
- collections=variables_collections,
- trainable=trainable,
- dual_rate_factor=dual_rate_factor)
- # Maybe create label_priors.
- label_priors = maybe_create_label_priors(
- label_priors, labels, weights, variables_collections)
-
- # Loss op and other outputs. The log(2.0) term corrects for
- # logloss not being an upper bound on the indicator function.
- weighted_loss = weights * util.weighted_surrogate_loss(
- labels,
- logits,
- surrogate_type=surrogate_type,
- positive_weights=1.0,
- negative_weights=lambdas)
- maybe_log2 = tf.log(2.0) if surrogate_type == 'xent' else 1.0
- maybe_log2 = tf.cast(maybe_log2, logits.dtype.base_dtype)
- lambda_term = lambdas * target_rate * (1.0 - label_priors) * maybe_log2
- loss = tf.reshape(weighted_loss - lambda_term, original_shape)
- other_outputs = {
- 'lambdas': lambdas_variable,
- 'label_priors': label_priors,
- 'true_positives_lower_bound': true_positives_lower_bound(
- labels, logits, weights, surrogate_type),
- 'false_positives_upper_bound': false_positives_upper_bound(
- labels, logits, weights, surrogate_type)}
-
- return loss, other_outputs
-
-
-def _prepare_labels_logits_weights(labels, logits, weights):
- """Validates labels, logits, and weights.
-
- Converts inputs to tensors, checks shape compatibility, and casts dtype if
- necessary.
-
- Args:
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- logits: A `Tensor` with the same shape as `labels`.
- weights: Either `None` or a `Tensor` with shape broadcastable to `logits`.
-
- Returns:
- labels: Same as `labels` arg after possible conversion to tensor, cast, and
- reshape.
- logits: Same as `logits` arg after possible conversion to tensor and
- reshape.
- weights: Same as `weights` arg after possible conversion, cast, and reshape.
- original_shape: Shape of `labels` and `logits` before reshape.
-
- Raises:
- ValueError: If `labels` and `logits` do not have the same shape.
- """
- # Convert `labels` and `logits` to Tensors and standardize dtypes.
- logits = tf.convert_to_tensor(logits, name='logits')
- labels = util.convert_and_cast(labels, 'labels', logits.dtype.base_dtype)
- weights = util.convert_and_cast(weights, 'weights', logits.dtype.base_dtype)
-
- try:
- labels.get_shape().merge_with(logits.get_shape())
- except ValueError:
- raise ValueError('logits and labels must have the same shape (%s vs %s)' %
- (logits.get_shape(), labels.get_shape()))
-
- original_shape = labels.get_shape().as_list()
- if labels.get_shape().ndims > 0:
- original_shape[0] = -1
- if labels.get_shape().ndims <= 1:
- labels = tf.reshape(labels, [-1, 1])
- logits = tf.reshape(logits, [-1, 1])
-
- if weights.get_shape().ndims == 1:
- # Weights has shape [batch_size]. Reshape to [batch_size, 1].
- weights = tf.reshape(weights, [-1, 1])
- if weights.get_shape().ndims == 0:
- # Weights is a scalar. Change shape of weights to match logits.
- weights *= tf.ones_like(logits)
-
- return labels, logits, weights, original_shape
-
-
-def _range_to_anchors_and_delta(precision_range, num_anchors, dtype):
- """Calculates anchor points from precision range.
-
- Args:
- precision_range: As required in precision_recall_auc_loss.
- num_anchors: int, number of equally spaced anchor points.
- dtype: Data type of returned tensors.
-
- Returns:
- precision_values: A `Tensor` of data type dtype with equally spaced values
- in the interval precision_range.
- delta: The spacing between the values in precision_values.
-
- Raises:
- ValueError: If precision_range is invalid.
- """
- # Validate precision_range.
- if not 0 <= precision_range[0] <= precision_range[-1] <= 1:
- raise ValueError('precision values must obey 0 <= %f <= %f <= 1' %
- (precision_range[0], precision_range[-1]))
- if not 0 < len(precision_range) < 3:
- raise ValueError('length of precision_range (%d) must be 1 or 2' %
- len(precision_range))
-
- # Sets precision_values uniformly between min_precision and max_precision.
- values = numpy.linspace(start=precision_range[0],
- stop=precision_range[1],
- num=num_anchors+2)[1:-1]
- precision_values = util.convert_and_cast(
- values, 'precision_values', dtype)
- delta = util.convert_and_cast(
- values[0] - precision_range[0], 'delta', dtype)
- # Makes precision_values [1, 1, num_anchors].
- precision_values = util.expand_outer(precision_values, 3)
- return precision_values, delta
-
-
-def _create_dual_variable(name, shape, dtype, initializer, collections,
- trainable, dual_rate_factor):
- """Creates a new dual variable.
-
- Dual variables are required to be nonnegative. If trainable, their gradient
- is reversed so that they are maximized (rather than minimized) by the
- optimizer.
-
- Args:
- name: A string, the name for the new variable.
- shape: Shape of the new variable.
- dtype: Data type for the new variable.
- initializer: Initializer for the new variable.
- collections: List of graph collections keys. The new variable is added to
- these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`.
- trainable: If `True`, the default, also adds the variable to the graph
- collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as
- the default list of variables to use by the `Optimizer` classes.
- dual_rate_factor: A floating point value or `Tensor`. The learning rate for
- the dual variable is scaled by this factor.
-
- Returns:
- dual_value: An op that computes the absolute value of the dual variable
- and reverses its gradient.
- dual_variable: The underlying variable itself.
- """
- # We disable partitioning while constructing dual variables because they will
- # be updated with assign, which is not available for partitioned variables.
- partitioner = tf.get_variable_scope().partitioner
- try:
- tf.get_variable_scope().set_partitioner(None)
- dual_variable = tf.contrib.framework.model_variable(
- name=name,
- shape=shape,
- dtype=dtype,
- initializer=initializer,
- collections=collections,
- trainable=trainable)
- finally:
- tf.get_variable_scope().set_partitioner(partitioner)
- # Using the absolute value enforces nonnegativity.
- dual_value = tf.abs(dual_variable)
-
- if trainable:
- # To reverse the gradient on the dual variable, multiply the gradient by
- # -dual_rate_factor
- dual_value = (tf.stop_gradient((1.0 + dual_rate_factor) * dual_value)
- - dual_rate_factor * dual_value)
- return dual_value, dual_variable
-
-
-def maybe_create_label_priors(label_priors,
- labels,
- weights,
- variables_collections):
- """Creates moving average ops to track label priors, if necessary.
-
- Args:
- label_priors: As required in e.g. precision_recall_auc_loss.
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- weights: As required in e.g. precision_recall_auc_loss.
- variables_collections: Optional list of collections for the variables, if
- any must be created.
-
- Returns:
- label_priors: A Tensor of shape [num_labels] consisting of the
- weighted label priors, after updating with moving average ops if created.
- """
- if label_priors is not None:
- label_priors = util.convert_and_cast(
- label_priors, name='label_priors', dtype=labels.dtype.base_dtype)
- return tf.squeeze(label_priors)
-
- label_priors = util.build_label_priors(
- labels,
- weights,
- variables_collections=variables_collections)
- return label_priors
-
-
-def true_positives_lower_bound(labels, logits, weights, surrogate_type):
- """Calculate a lower bound on the number of true positives.
-
- This lower bound on the number of true positives given `logits` and `labels`
- is the same one used in the global objectives loss functions.
-
- Args:
- labels: A `Tensor` of shape [batch_size] or [batch_size, num_labels].
- logits: A `Tensor` of shape [batch_size, num_labels] or
- [batch_size, num_labels, num_anchors]. If the third dimension is present,
- the lower bound is computed on each slice [:, :, k] independently.
- weights: Per-example loss coefficients, with shape broadcast-compatible with
- that of `labels`.
- surrogate_type: Either 'xent' or 'hinge', specifying which upper bound
- should be used for indicator functions.
-
- Returns:
- A `Tensor` of shape [num_labels] or [num_labels, num_anchors].
- """
- maybe_log2 = tf.log(2.0) if surrogate_type == 'xent' else 1.0
- maybe_log2 = tf.cast(maybe_log2, logits.dtype.base_dtype)
- if logits.get_shape().ndims == 3 and labels.get_shape().ndims < 3:
- labels = tf.expand_dims(labels, 2)
- loss_on_positives = util.weighted_surrogate_loss(
- labels, logits, surrogate_type, negative_weights=0.0) / maybe_log2
- return tf.reduce_sum(weights * (labels - loss_on_positives), 0)
-
-
-def false_positives_upper_bound(labels, logits, weights, surrogate_type):
- """Calculate an upper bound on the number of false positives.
-
- This upper bound on the number of false positives given `logits` and `labels`
- is the same one used in the global objectives loss functions.
-
- Args:
- labels: A `Tensor` of shape [batch_size, num_labels]
- logits: A `Tensor` of shape [batch_size, num_labels] or
- [batch_size, num_labels, num_anchors]. If the third dimension is present,
- the lower bound is computed on each slice [:, :, k] independently.
- weights: Per-example loss coefficients, with shape broadcast-compatible with
- that of `labels`.
- surrogate_type: Either 'xent' or 'hinge', specifying which upper bound
- should be used for indicator functions.
-
- Returns:
- A `Tensor` of shape [num_labels] or [num_labels, num_anchors].
- """
- maybe_log2 = tf.log(2.0) if surrogate_type == 'xent' else 1.0
- maybe_log2 = tf.cast(maybe_log2, logits.dtype.base_dtype)
- loss_on_negatives = util.weighted_surrogate_loss(
- labels, logits, surrogate_type, positive_weights=0.0) / maybe_log2
- return tf.reduce_sum(weights * loss_on_negatives, 0)
diff --git a/research/global_objectives/loss_layers_example.py b/research/global_objectives/loss_layers_example.py
deleted file mode 100644
index 2323cb0762e7f4eade8f283162be61cc45513d49..0000000000000000000000000000000000000000
--- a/research/global_objectives/loss_layers_example.py
+++ /dev/null
@@ -1,211 +0,0 @@
-# Copyright 2018 The TensorFlow Global Objectives Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Example for using global objectives.
-
-Illustrate, using synthetic data, how using the precision_at_recall loss
-significanly improves the performace of a linear classifier.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-import numpy as np
-from sklearn.metrics import precision_score
-import tensorflow as tf
-from global_objectives import loss_layers
-
-# When optimizing using global_objectives, if set to True then the saddle point
-# optimization steps are performed internally by the Tensorflow optimizer,
-# otherwise by dedicated saddle-point steps as part of the optimization loop.
-USE_GO_SADDLE_POINT_OPT = False
-
-TARGET_RECALL = 0.98
-TRAIN_ITERATIONS = 150
-LEARNING_RATE = 1.0
-GO_DUAL_RATE_FACTOR = 15.0
-NUM_CHECKPOINTS = 6
-
-EXPERIMENT_DATA_CONFIG = {
- 'positives_centers': [[0, 1.0], [1, -0.5]],
- 'negatives_centers': [[0, -0.5], [1, 1.0]],
- 'positives_variances': [0.15, 0.1],
- 'negatives_variances': [0.15, 0.1],
- 'positives_counts': [500, 50],
- 'negatives_counts': [3000, 100]
-}
-
-
-def create_training_and_eval_data_for_experiment(**data_config):
- """Creates train and eval data sets.
-
- Note: The synthesized binary-labeled data is a mixture of four Gaussians - two
- positives and two negatives. The centers, variances, and sizes for each of
- the two positives and negatives mixtures are passed in the respective keys
- of data_config:
-
- Args:
- **data_config: Dictionary with Array entries as follows:
- positives_centers - float [2,2] two centers of positives data sets.
- negatives_centers - float [2,2] two centers of negatives data sets.
- positives_variances - float [2] Variances for the positives sets.
- negatives_variances - float [2] Variances for the negatives sets.
- positives_counts - int [2] Counts for each of the two positives sets.
- negatives_counts - int [2] Counts for each of the two negatives sets.
-
- Returns:
- A dictionary with two shuffled data sets created - one for training and one
- for eval. The dictionary keys are 'train_data', 'train_labels', 'eval_data',
- and 'eval_labels'. The data points are two-dimentional floats, and the
- labels are in {0,1}.
- """
- def data_points(is_positives, index):
- variance = data_config['positives_variances'
- if is_positives else 'negatives_variances'][index]
- center = data_config['positives_centers'
- if is_positives else 'negatives_centers'][index]
- count = data_config['positives_counts'
- if is_positives else 'negatives_counts'][index]
- return variance*np.random.randn(count, 2) + np.array([center])
-
- def create_data():
- return np.concatenate([data_points(False, 0),
- data_points(True, 0),
- data_points(True, 1),
- data_points(False, 1)], axis=0)
-
- def create_labels():
- """Creates an array of 0.0 or 1.0 labels for the data_config batches."""
- return np.array([0.0]*data_config['negatives_counts'][0] +
- [1.0]*data_config['positives_counts'][0] +
- [1.0]*data_config['positives_counts'][1] +
- [0.0]*data_config['negatives_counts'][1])
-
- permutation = np.random.permutation(
- sum(data_config['positives_counts'] + data_config['negatives_counts']))
-
- train_data = create_data()[permutation, :]
- eval_data = create_data()[permutation, :]
- train_labels = create_labels()[permutation]
- eval_labels = create_labels()[permutation]
-
- return {
- 'train_data': train_data,
- 'train_labels': train_labels,
- 'eval_data': eval_data,
- 'eval_labels': eval_labels
- }
-
-
-def train_model(data, use_global_objectives):
- """Trains a linear model for maximal accuracy or precision at given recall."""
-
- def precision_at_recall(scores, labels, target_recall):
- """Computes precision - at target recall - over data."""
- positive_scores = scores[labels == 1.0]
- threshold = np.percentile(positive_scores, 100 - target_recall*100)
- predicted = scores >= threshold
- return precision_score(labels, predicted)
-
- w = tf.Variable(tf.constant([-1.0, -1.0], shape=[2, 1]), trainable=True,
- name='weights', dtype=tf.float32)
- b = tf.Variable(tf.zeros([1]), trainable=True, name='biases',
- dtype=tf.float32)
-
- logits = tf.matmul(tf.cast(data['train_data'], tf.float32), w) + b
-
- labels = tf.constant(
- data['train_labels'],
- shape=[len(data['train_labels']), 1],
- dtype=tf.float32)
-
- if use_global_objectives:
- loss, other_outputs = loss_layers.precision_at_recall_loss(
- labels, logits,
- TARGET_RECALL,
- dual_rate_factor=GO_DUAL_RATE_FACTOR)
- loss = tf.reduce_mean(loss)
- else:
- loss = tf.reduce_mean(
- tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits))
-
- global_step = tf.Variable(0, trainable=False)
-
- learning_rate = tf.train.polynomial_decay(
- LEARNING_RATE,
- global_step,
- TRAIN_ITERATIONS, (LEARNING_RATE / TRAIN_ITERATIONS),
- power=1.0,
- cycle=False,
- name='learning_rate')
-
- optimizer = tf.train.GradientDescentOptimizer(learning_rate)
-
- if (not use_global_objectives) or USE_GO_SADDLE_POINT_OPT:
- training_op = optimizer.minimize(loss, global_step=global_step)
- else:
- lambdas = other_outputs['lambdas']
- primal_update_op = optimizer.minimize(loss, var_list=[w, b])
- dual_update_op = optimizer.minimize(
- loss, global_step=global_step, var_list=[lambdas])
-
- # Training loop:
- with tf.Session() as sess:
- checkpoint_step = TRAIN_ITERATIONS // NUM_CHECKPOINTS
- sess.run(tf.global_variables_initializer())
- step = sess.run(global_step)
-
- while step <= TRAIN_ITERATIONS:
- if (not use_global_objectives) or USE_GO_SADDLE_POINT_OPT:
- _, step, loss_value, w_value, b_value = sess.run(
- [training_op, global_step, loss, w, b])
- else:
- _, w_value, b_value = sess.run([primal_update_op, w, b])
- _, loss_value, step = sess.run([dual_update_op, loss, global_step])
-
- if use_global_objectives:
- go_outputs = sess.run(other_outputs.values())
-
- if step % checkpoint_step == 0:
- precision = precision_at_recall(
- np.dot(data['train_data'], w_value) + b_value,
- data['train_labels'], TARGET_RECALL)
-
- tf.logging.info('Loss = %f Precision = %f', loss_value, precision)
- if use_global_objectives:
- for i, output_name in enumerate(other_outputs.keys()):
- tf.logging.info('\t%s = %f', output_name, go_outputs[i])
-
- w_value, b_value = sess.run([w, b])
- return precision_at_recall(np.dot(data['eval_data'], w_value) + b_value,
- data['eval_labels'],
- TARGET_RECALL)
-
-
-def main(unused_argv):
- del unused_argv
- experiment_data = create_training_and_eval_data_for_experiment(
- **EXPERIMENT_DATA_CONFIG)
- global_objectives_loss_precision = train_model(experiment_data, True)
- tf.logging.info('global_objectives precision at requested recall is %f',
- global_objectives_loss_precision)
- cross_entropy_loss_precision = train_model(experiment_data, False)
- tf.logging.info('cross_entropy precision at requested recall is %f',
- cross_entropy_loss_precision)
-
-
-if __name__ == '__main__':
- tf.logging.set_verbosity(tf.logging.INFO)
- tf.app.run()
diff --git a/research/global_objectives/loss_layers_test.py b/research/global_objectives/loss_layers_test.py
deleted file mode 100644
index 3f91c80deec16a34f5271cdfadbd0d364c3a8cea..0000000000000000000000000000000000000000
--- a/research/global_objectives/loss_layers_test.py
+++ /dev/null
@@ -1,1379 +0,0 @@
-# Copyright 2018 The TensorFlow Global Objectives Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for global objectives loss layers."""
-
-# Dependency imports
-from absl.testing import parameterized
-import numpy
-import tensorflow as tf
-
-from global_objectives import loss_layers
-from global_objectives import util
-
-
-# TODO: Include weights in the lagrange multiplier update tests.
-class PrecisionRecallAUCLossTest(parameterized.TestCase, tf.test.TestCase):
-
- @parameterized.named_parameters(
- ('_xent', 'xent', 0.7),
- ('_hinge', 'hinge', 0.7),
- ('_hinge_2', 'hinge', 0.5)
- )
- def testSinglePointAUC(self, surrogate_type, target_precision):
- # Tests a case with only one anchor point, where the loss should equal
- # recall_at_precision_loss
- batch_shape = [10, 2]
- logits = tf.Variable(tf.random_normal(batch_shape))
- labels = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
-
- auc_loss, _ = loss_layers.precision_recall_auc_loss(
- labels,
- logits,
- precision_range=(target_precision - 0.01, target_precision + 0.01),
- num_anchors=1,
- surrogate_type=surrogate_type)
- point_loss, _ = loss_layers.recall_at_precision_loss(
- labels, logits, target_precision=target_precision,
- surrogate_type=surrogate_type)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(auc_loss.eval(), point_loss.eval())
-
- def testThreePointAUC(self):
- # Tests a case with three anchor points against a weighted sum of recall
- # at precision losses.
- batch_shape = [11, 3]
- logits = tf.Variable(tf.random_normal(batch_shape))
- labels = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
-
- # TODO: Place the hing/xent loss in a for loop.
- auc_loss, _ = loss_layers.precision_recall_auc_loss(
- labels, logits, num_anchors=1)
- first_point_loss, _ = loss_layers.recall_at_precision_loss(
- labels, logits, target_precision=0.25)
- second_point_loss, _ = loss_layers.recall_at_precision_loss(
- labels, logits, target_precision=0.5)
- third_point_loss, _ = loss_layers.recall_at_precision_loss(
- labels, logits, target_precision=0.75)
- expected_loss = (first_point_loss + second_point_loss +
- third_point_loss) / 3
-
- auc_loss_hinge, _ = loss_layers.precision_recall_auc_loss(
- labels, logits, num_anchors=1, surrogate_type='hinge')
- first_point_hinge, _ = loss_layers.recall_at_precision_loss(
- labels, logits, target_precision=0.25, surrogate_type='hinge')
- second_point_hinge, _ = loss_layers.recall_at_precision_loss(
- labels, logits, target_precision=0.5, surrogate_type='hinge')
- third_point_hinge, _ = loss_layers.recall_at_precision_loss(
- labels, logits, target_precision=0.75, surrogate_type='hinge')
- expected_hinge = (first_point_hinge + second_point_hinge +
- third_point_hinge) / 3
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(auc_loss.eval(), expected_loss.eval())
- self.assertAllClose(auc_loss_hinge.eval(), expected_hinge.eval())
-
- def testLagrangeMultiplierUpdateDirection(self):
- for target_precision in [0.35, 0.65]:
- precision_range = (target_precision - 0.01, target_precision + 0.01)
-
- for surrogate_type in ['xent', 'hinge']:
- kwargs = {'precision_range': precision_range,
- 'num_anchors': 1,
- 'surrogate_type': surrogate_type,
- 'scope': 'pr-auc_{}_{}'.format(target_precision,
- surrogate_type)}
- run_lagrange_multiplier_test(
- global_objective=loss_layers.precision_recall_auc_loss,
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=loss_layers.precision_recall_auc_loss,
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
-
-class ROCAUCLossTest(parameterized.TestCase, tf.test.TestCase):
-
- def testSimpleScores(self):
- # Tests the loss on data with only one negative example with score zero.
- # In this case, the loss should equal the surrogate loss on the scores with
- # positive labels.
- num_positives = 10
- scores_positives = tf.constant(3.0 * numpy.random.randn(num_positives),
- shape=[num_positives, 1])
- labels = tf.constant([0.0] + [1.0] * num_positives,
- shape=[num_positives + 1, 1])
- scores = tf.concat([[[0.0]], scores_positives], 0)
-
- loss = tf.reduce_sum(
- loss_layers.roc_auc_loss(labels, scores, surrogate_type='hinge')[0])
- expected_loss = tf.reduce_sum(
- tf.maximum(1.0 - scores_positives, 0)) / (num_positives + 1)
- with self.test_session():
- self.assertAllClose(expected_loss.eval(), loss.eval())
-
- def testRandomROCLoss(self):
- # Checks that random Bernoulli scores and labels has ~25% swaps.
- shape = [1000, 30]
- scores = tf.constant(
- numpy.random.randint(0, 2, size=shape), shape=shape, dtype=tf.float32)
- labels = tf.constant(
- numpy.random.randint(0, 2, size=shape), shape=shape, dtype=tf.float32)
- loss = tf.reduce_mean(loss_layers.roc_auc_loss(
- labels, scores, surrogate_type='hinge')[0])
- with self.test_session():
- self.assertAllClose(0.25, loss.eval(), 1e-2)
-
- @parameterized.named_parameters(
- ('_zero_hinge', 'xent',
- [0.0, 0.0, 0.0, 1.0, 1.0, 1.0],
- [-5.0, -7.0, -9.0, 8.0, 10.0, 14.0],
- 0.0),
- ('_zero_xent', 'hinge',
- [0.0, 0.0, 0.0, 1.0, 1.0, 1.0],
- [-0.2, 0, -0.1, 1.0, 1.1, 1.0],
- 0.0),
- ('_xent', 'xent',
- [0.0, 0.0, 0.0, 1.0, 1.0, 1.0],
- [0.0, -17.0, -19.0, 1.0, 14.0, 14.0],
- numpy.log(1.0 + numpy.exp(-1.0)) / 6),
- ('_hinge', 'hinge',
- [0.0, 0.0, 0.0, 1.0, 1.0, 1.0],
- [-0.2, -0.05, 0.0, 0.95, 0.8, 1.0],
- 0.4 / 6)
- )
- def testManualROCLoss(self, surrogate_type, labels, logits, expected_value):
- labels = tf.constant(labels)
- logits = tf.constant(logits)
- loss, _ = loss_layers.roc_auc_loss(
- labels=labels, logits=logits, surrogate_type=surrogate_type)
-
- with self.test_session():
- self.assertAllClose(expected_value, tf.reduce_sum(loss).eval())
-
- def testMultiLabelROCLoss(self):
- # Tests the loss on multi-label data against manually computed loss.
- targets = numpy.array([[0.0, 0.0, 1.0, 1.0], [0.0, 0.0, 1.0, 1.0]])
- scores = numpy.array([[0.1, 1.0, 1.1, 1.0], [1.0, 0.0, 1.3, 1.1]])
- class_1_auc = tf.reduce_sum(
- loss_layers.roc_auc_loss(targets[0], scores[0])[0])
- class_2_auc = tf.reduce_sum(
- loss_layers.roc_auc_loss(targets[1], scores[1])[0])
- total_auc = tf.reduce_sum(loss_layers.roc_auc_loss(
- targets.transpose(), scores.transpose())[0])
-
- with self.test_session():
- self.assertAllClose(total_auc.eval(),
- class_1_auc.eval() + class_2_auc.eval())
-
- def testWeights(self):
- # Test the loss with per-example weights.
- # The logits_negatives below are repeated, so that setting half their
- # weights to 2 and the other half to 0 should leave the loss unchanged.
- logits_positives = tf.constant([2.54321, -0.26, 3.334334], shape=[3, 1])
- logits_negatives = tf.constant([-0.6, 1, -1.3, -1.3, -0.6, 1], shape=[6, 1])
- logits = tf.concat([logits_positives, logits_negatives], 0)
- targets = tf.constant([1, 1, 1, 0, 0, 0, 0, 0, 0],
- shape=[9, 1], dtype=tf.float32)
- weights = tf.constant([1, 1, 1, 0, 0, 0, 2, 2, 2],
- shape=[9, 1], dtype=tf.float32)
-
- loss = tf.reduce_sum(loss_layers.roc_auc_loss(targets, logits)[0])
- weighted_loss = tf.reduce_sum(
- loss_layers.roc_auc_loss(targets, logits, weights)[0])
-
- with self.test_session():
- self.assertAllClose(loss.eval(), weighted_loss.eval())
-
-
-class RecallAtPrecisionTest(tf.test.TestCase):
-
- def testEqualWeightLoss(self):
- # Tests a special case where the loss should equal cross entropy loss.
- target_precision = 1.0
- num_labels = 5
- batch_shape = [20, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.7)))
- label_priors = tf.constant(0.34, shape=[num_labels])
-
- loss, _ = loss_layers.recall_at_precision_loss(
- targets, logits, target_precision, label_priors=label_priors)
- expected_loss = (
- tf.contrib.nn.deprecated_flipped_sigmoid_cross_entropy_with_logits(
- logits, targets))
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- loss_val, expected_val = session.run([loss, expected_loss])
- self.assertAllClose(loss_val, expected_val)
-
- def testEqualWeightLossWithMultiplePrecisions(self):
- """Tests a case where the loss equals xent loss with multiple precisions."""
- target_precision = [1.0, 1.0]
- num_labels = 2
- batch_size = 20
- target_shape = [batch_size, num_labels]
- logits = tf.Variable(tf.random_normal(target_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(target_shape), 0.7)))
- label_priors = tf.constant([0.34], shape=[num_labels])
-
- loss, _ = loss_layers.recall_at_precision_loss(
- targets,
- logits,
- target_precision,
- label_priors=label_priors,
- surrogate_type='xent',
- )
-
- expected_loss = (
- tf.contrib.nn.deprecated_flipped_sigmoid_cross_entropy_with_logits(
- logits, targets))
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- loss_val, expected_val = session.run([loss, expected_loss])
- self.assertAllClose(loss_val, expected_val)
-
- def testPositivesOnlyLoss(self):
- # Tests a special case where the loss should equal cross entropy loss
- # on the negatives only.
- target_precision = 1.0
- num_labels = 3
- batch_shape = [30, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
- label_priors = tf.constant(0.45, shape=[num_labels])
-
- loss, _ = loss_layers.recall_at_precision_loss(
- targets, logits, target_precision, label_priors=label_priors,
- lambdas_initializer=tf.zeros_initializer())
- expected_loss = util.weighted_sigmoid_cross_entropy_with_logits(
- targets,
- logits,
- positive_weights=1.0,
- negative_weights=0.0)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- loss_val, expected_val = session.run([loss, expected_loss])
- self.assertAllClose(loss_val, expected_val)
-
- def testEquivalenceBetweenSingleAndMultiplePrecisions(self):
- """Checks recall at precision with different precision values.
-
- Runs recall at precision with multiple precision values, and runs each label
- seperately with its own precision value as a scalar. Validates that the
- returned loss values are the same.
- """
- target_precision = [0.2, 0.9, 0.4]
- num_labels = 3
- batch_shape = [30, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
- label_priors = tf.constant([0.45, 0.8, 0.3], shape=[num_labels])
-
- multi_label_loss, _ = loss_layers.recall_at_precision_loss(
- targets, logits, target_precision, label_priors=label_priors,
- )
-
- single_label_losses = [
- loss_layers.recall_at_precision_loss(
- tf.expand_dims(targets[:, i], -1),
- tf.expand_dims(logits[:, i], -1),
- target_precision[i],
- label_priors=label_priors[i])[0]
- for i in range(num_labels)
- ]
-
- single_label_losses = tf.concat(single_label_losses, 1)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- multi_label_loss_val, single_label_loss_val = session.run(
- [multi_label_loss, single_label_losses])
- self.assertAllClose(multi_label_loss_val, single_label_loss_val)
-
- def testEquivalenceBetweenSingleAndEqualMultiplePrecisions(self):
- """Compares single and multiple target precisions with the same value.
-
- Checks that using a single target precision and multiple target precisions
- with the same value would result in the same loss value.
- """
- num_labels = 2
- target_shape = [20, num_labels]
- logits = tf.Variable(tf.random_normal(target_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(target_shape), 0.7)))
- label_priors = tf.constant([0.34], shape=[num_labels])
-
- multi_precision_loss, _ = loss_layers.recall_at_precision_loss(
- targets,
- logits,
- [0.75, 0.75],
- label_priors=label_priors,
- surrogate_type='xent',
- )
-
- single_precision_loss, _ = loss_layers.recall_at_precision_loss(
- targets,
- logits,
- 0.75,
- label_priors=label_priors,
- surrogate_type='xent',
- )
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- multi_precision_loss_val, single_precision_loss_val = session.run(
- [multi_precision_loss, single_precision_loss])
- self.assertAllClose(multi_precision_loss_val, single_precision_loss_val)
-
- def testLagrangeMultiplierUpdateDirection(self):
- for target_precision in [0.35, 0.65]:
- for surrogate_type in ['xent', 'hinge']:
- kwargs = {'target_precision': target_precision,
- 'surrogate_type': surrogate_type,
- 'scope': 'r-at-p_{}_{}'.format(target_precision,
- surrogate_type)}
- run_lagrange_multiplier_test(
- global_objective=loss_layers.recall_at_precision_loss,
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=loss_layers.recall_at_precision_loss,
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
- def testLagrangeMultiplierUpdateDirectionWithMultiplePrecisions(self):
- """Runs Lagrange multiplier test with multiple precision values."""
- target_precision = [0.65, 0.35]
-
- for surrogate_type in ['xent', 'hinge']:
- scope_str = 'r-at-p_{}_{}'.format(
- '_'.join([str(precision) for precision in target_precision]),
- surrogate_type)
- kwargs = {
- 'target_precision': target_precision,
- 'surrogate_type': surrogate_type,
- 'scope': scope_str,
- }
- run_lagrange_multiplier_test(
- global_objective=loss_layers.recall_at_precision_loss,
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=loss_layers.recall_at_precision_loss,
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
-
-class PrecisionAtRecallTest(tf.test.TestCase):
-
- def testCrossEntropyEquivalence(self):
- # Checks a special case where the loss should equal cross-entropy loss.
- target_recall = 1.0
- num_labels = 3
- batch_shape = [10, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
-
- loss, _ = loss_layers.precision_at_recall_loss(
- targets, logits, target_recall,
- lambdas_initializer=tf.constant_initializer(1.0))
- expected_loss = util.weighted_sigmoid_cross_entropy_with_logits(
- targets, logits)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(loss.eval(), expected_loss.eval())
-
- def testNegativesOnlyLoss(self):
- # Checks a special case where the loss should equal the loss on
- # the negative examples only.
- target_recall = 0.61828
- num_labels = 4
- batch_shape = [8, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.6)))
-
- loss, _ = loss_layers.precision_at_recall_loss(
- targets,
- logits,
- target_recall,
- surrogate_type='hinge',
- lambdas_initializer=tf.constant_initializer(0.0),
- scope='negatives_only_test')
- expected_loss = util.weighted_hinge_loss(
- targets, logits, positive_weights=0.0, negative_weights=1.0)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(expected_loss.eval(), loss.eval())
-
- def testLagrangeMultiplierUpdateDirection(self):
- for target_recall in [0.34, 0.66]:
- for surrogate_type in ['xent', 'hinge']:
- kwargs = {'target_recall': target_recall,
- 'dual_rate_factor': 1.0,
- 'surrogate_type': surrogate_type,
- 'scope': 'p-at-r_{}_{}'.format(target_recall, surrogate_type)}
-
- run_lagrange_multiplier_test(
- global_objective=loss_layers.precision_at_recall_loss,
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=loss_layers.precision_at_recall_loss,
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
- def testCrossEntropyEquivalenceWithMultipleRecalls(self):
- """Checks a case where the loss equals xent loss with multiple recalls."""
- num_labels = 3
- target_recall = [1.0] * num_labels
- batch_shape = [10, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
-
- loss, _ = loss_layers.precision_at_recall_loss(
- targets, logits, target_recall,
- lambdas_initializer=tf.constant_initializer(1.0))
- expected_loss = util.weighted_sigmoid_cross_entropy_with_logits(
- targets, logits)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(loss.eval(), expected_loss.eval())
-
- def testNegativesOnlyLossWithMultipleRecalls(self):
- """Tests a case where the loss equals the loss on the negative examples.
-
- Checks this special case using multiple target recall values.
- """
- num_labels = 4
- target_recall = [0.61828] * num_labels
- batch_shape = [8, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.6)))
-
- loss, _ = loss_layers.precision_at_recall_loss(
- targets,
- logits,
- target_recall,
- surrogate_type='hinge',
- lambdas_initializer=tf.constant_initializer(0.0),
- scope='negatives_only_test')
- expected_loss = util.weighted_hinge_loss(
- targets, logits, positive_weights=0.0, negative_weights=1.0)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(expected_loss.eval(), loss.eval())
-
- def testLagrangeMultiplierUpdateDirectionWithMultipleRecalls(self):
- """Runs Lagrange multiplier test with multiple recall values."""
- target_recall = [0.34, 0.66]
- for surrogate_type in ['xent', 'hinge']:
- scope_str = 'p-at-r_{}_{}'.format(
- '_'.join([str(recall) for recall in target_recall]),
- surrogate_type)
- kwargs = {'target_recall': target_recall,
- 'dual_rate_factor': 1.0,
- 'surrogate_type': surrogate_type,
- 'scope': scope_str}
-
- run_lagrange_multiplier_test(
- global_objective=loss_layers.precision_at_recall_loss,
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=loss_layers.precision_at_recall_loss,
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
- def testEquivalenceBetweenSingleAndMultipleRecalls(self):
- """Checks precision at recall with multiple different recall values.
-
- Runs precision at recall with multiple recall values, and runs each label
- seperately with its own recall value as a scalar. Validates that the
- returned loss values are the same.
- """
- target_precision = [0.7, 0.9, 0.4]
- num_labels = 3
- batch_shape = [30, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
- label_priors = tf.constant(0.45, shape=[num_labels])
-
- multi_label_loss, _ = loss_layers.precision_at_recall_loss(
- targets, logits, target_precision, label_priors=label_priors
- )
-
- single_label_losses = [
- loss_layers.precision_at_recall_loss(
- tf.expand_dims(targets[:, i], -1),
- tf.expand_dims(logits[:, i], -1),
- target_precision[i],
- label_priors=label_priors[i])[0]
- for i in range(num_labels)
- ]
-
- single_label_losses = tf.concat(single_label_losses, 1)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- multi_label_loss_val, single_label_loss_val = session.run(
- [multi_label_loss, single_label_losses])
- self.assertAllClose(multi_label_loss_val, single_label_loss_val)
-
- def testEquivalenceBetweenSingleAndEqualMultipleRecalls(self):
- """Compares single and multiple target recalls of the same value.
-
- Checks that using a single target recall and multiple recalls with the
- same value would result in the same loss value.
- """
- num_labels = 2
- target_shape = [20, num_labels]
- logits = tf.Variable(tf.random_normal(target_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(target_shape), 0.7)))
- label_priors = tf.constant([0.34], shape=[num_labels])
-
- multi_precision_loss, _ = loss_layers.precision_at_recall_loss(
- targets,
- logits,
- [0.75, 0.75],
- label_priors=label_priors,
- surrogate_type='xent',
- )
-
- single_precision_loss, _ = loss_layers.precision_at_recall_loss(
- targets,
- logits,
- 0.75,
- label_priors=label_priors,
- surrogate_type='xent',
- )
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- multi_precision_loss_val, single_precision_loss_val = session.run(
- [multi_precision_loss, single_precision_loss])
- self.assertAllClose(multi_precision_loss_val, single_precision_loss_val)
-
-
-class FalsePositiveRateAtTruePositiveRateTest(tf.test.TestCase):
-
- def testNegativesOnlyLoss(self):
- # Checks a special case where the loss returned should be the loss on the
- # negative examples.
- target_recall = 0.6
- num_labels = 3
- batch_shape = [3, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
- label_priors = tf.constant(numpy.random.uniform(size=[num_labels]),
- dtype=tf.float32)
-
- xent_loss, _ = loss_layers.false_positive_rate_at_true_positive_rate_loss(
- targets, logits, target_recall, label_priors=label_priors,
- lambdas_initializer=tf.constant_initializer(0.0))
- xent_expected = util.weighted_sigmoid_cross_entropy_with_logits(
- targets,
- logits,
- positive_weights=0.0,
- negative_weights=1.0)
- hinge_loss, _ = loss_layers.false_positive_rate_at_true_positive_rate_loss(
- targets, logits, target_recall, label_priors=label_priors,
- lambdas_initializer=tf.constant_initializer(0.0),
- surrogate_type='hinge')
- hinge_expected = util.weighted_hinge_loss(
- targets,
- logits,
- positive_weights=0.0,
- negative_weights=1.0)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- xent_val, xent_expected = session.run([xent_loss, xent_expected])
- self.assertAllClose(xent_val, xent_expected)
- hinge_val, hinge_expected = session.run([hinge_loss, hinge_expected])
- self.assertAllClose(hinge_val, hinge_expected)
-
- def testPositivesOnlyLoss(self):
- # Checks a special case where the loss returned should be the loss on the
- # positive examples only.
- target_recall = 1.0
- num_labels = 5
- batch_shape = [5, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.ones_like(logits)
- label_priors = tf.constant(numpy.random.uniform(size=[num_labels]),
- dtype=tf.float32)
-
- loss, _ = loss_layers.false_positive_rate_at_true_positive_rate_loss(
- targets, logits, target_recall, label_priors=label_priors)
- expected_loss = tf.nn.sigmoid_cross_entropy_with_logits(
- labels=targets, logits=logits)
- hinge_loss, _ = loss_layers.false_positive_rate_at_true_positive_rate_loss(
- targets, logits, target_recall, label_priors=label_priors,
- surrogate_type='hinge')
- expected_hinge = util.weighted_hinge_loss(
- targets, logits)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(loss.eval(), expected_loss.eval())
- self.assertAllClose(hinge_loss.eval(), expected_hinge.eval())
-
- def testEqualWeightLoss(self):
- # Checks a special case where the loss returned should be proportional to
- # the ordinary loss.
- target_recall = 1.0
- num_labels = 4
- batch_shape = [40, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.6)))
- label_priors = tf.constant(0.5, shape=[num_labels])
-
- loss, _ = loss_layers.false_positive_rate_at_true_positive_rate_loss(
- targets, logits, target_recall, label_priors=label_priors)
- expected_loss = tf.nn.sigmoid_cross_entropy_with_logits(
- labels=targets, logits=logits)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(loss.eval(), expected_loss.eval())
-
- def testLagrangeMultiplierUpdateDirection(self):
- for target_rate in [0.35, 0.65]:
- for surrogate_type in ['xent', 'hinge']:
- kwargs = {'target_rate': target_rate,
- 'surrogate_type': surrogate_type,
- 'scope': 'fpr-at-tpr_{}_{}'.format(target_rate,
- surrogate_type)}
- # True positive rate is a synonym for recall, so we use the
- # recall constraint data.
- run_lagrange_multiplier_test(
- global_objective=(
- loss_layers.false_positive_rate_at_true_positive_rate_loss),
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=(
- loss_layers.false_positive_rate_at_true_positive_rate_loss),
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
- def testLagrangeMultiplierUpdateDirectionWithMultipleRates(self):
- """Runs Lagrange multiplier test with multiple target rates."""
- target_rate = [0.35, 0.65]
- for surrogate_type in ['xent', 'hinge']:
- kwargs = {'target_rate': target_rate,
- 'surrogate_type': surrogate_type,
- 'scope': 'fpr-at-tpr_{}_{}'.format(
- '_'.join([str(target) for target in target_rate]),
- surrogate_type)}
- # True positive rate is a synonym for recall, so we use the
- # recall constraint data.
- run_lagrange_multiplier_test(
- global_objective=(
- loss_layers.false_positive_rate_at_true_positive_rate_loss),
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=(
- loss_layers.false_positive_rate_at_true_positive_rate_loss),
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
- def testEquivalenceBetweenSingleAndEqualMultipleRates(self):
- """Compares single and multiple target rates of the same value.
-
- Checks that using a single target rate and multiple rates with the
- same value would result in the same loss value.
- """
- num_labels = 2
- target_shape = [20, num_labels]
- logits = tf.Variable(tf.random_normal(target_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(target_shape), 0.7)))
- label_priors = tf.constant([0.34], shape=[num_labels])
-
- multi_label_loss, _ = (
- loss_layers.false_positive_rate_at_true_positive_rate_loss(
- targets, logits, [0.75, 0.75], label_priors=label_priors))
-
- single_label_loss, _ = (
- loss_layers.false_positive_rate_at_true_positive_rate_loss(
- targets, logits, 0.75, label_priors=label_priors))
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- multi_label_loss_val, single_label_loss_val = session.run(
- [multi_label_loss, single_label_loss])
- self.assertAllClose(multi_label_loss_val, single_label_loss_val)
-
- def testEquivalenceBetweenSingleAndMultipleRates(self):
- """Compares single and multiple target rates of different values.
-
- Runs false_positive_rate_at_true_positive_rate_loss with multiple target
- rates, and runs each label seperately with its own target rate as a
- scalar. Validates that the returned loss values are the same.
- """
- target_precision = [0.7, 0.9, 0.4]
- num_labels = 3
- batch_shape = [30, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
- label_priors = tf.constant(0.45, shape=[num_labels])
-
- multi_label_loss, _ = (
- loss_layers.false_positive_rate_at_true_positive_rate_loss(
- targets, logits, target_precision, label_priors=label_priors))
-
- single_label_losses = [
- loss_layers.false_positive_rate_at_true_positive_rate_loss(
- tf.expand_dims(targets[:, i], -1),
- tf.expand_dims(logits[:, i], -1),
- target_precision[i],
- label_priors=label_priors[i])[0]
- for i in range(num_labels)
- ]
-
- single_label_losses = tf.concat(single_label_losses, 1)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- multi_label_loss_val, single_label_loss_val = session.run(
- [multi_label_loss, single_label_losses])
- self.assertAllClose(multi_label_loss_val, single_label_loss_val)
-
-
-class TruePositiveRateAtFalsePositiveRateTest(tf.test.TestCase):
-
- def testPositivesOnlyLoss(self):
- # A special case where the loss should equal the loss on the positive
- # examples.
- target_rate = numpy.random.uniform()
- num_labels = 3
- batch_shape = [20, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.6)))
- label_priors = tf.constant(numpy.random.uniform(size=[num_labels]),
- dtype=tf.float32)
-
- xent_loss, _ = loss_layers.true_positive_rate_at_false_positive_rate_loss(
- targets, logits, target_rate, label_priors=label_priors,
- lambdas_initializer=tf.constant_initializer(0.0))
- xent_expected = util.weighted_sigmoid_cross_entropy_with_logits(
- targets,
- logits,
- positive_weights=1.0,
- negative_weights=0.0)
- hinge_loss, _ = loss_layers.true_positive_rate_at_false_positive_rate_loss(
- targets, logits, target_rate, label_priors=label_priors,
- lambdas_initializer=tf.constant_initializer(0.0),
- surrogate_type='hinge')
- hinge_expected = util.weighted_hinge_loss(
- targets,
- logits,
- positive_weights=1.0,
- negative_weights=0.0)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(xent_expected.eval(), xent_loss.eval())
- self.assertAllClose(hinge_expected.eval(), hinge_loss.eval())
-
- def testNegativesOnlyLoss(self):
- # A special case where the loss should equal the loss on the negative
- # examples, minus target_rate * (1 - label_priors) * maybe_log2.
- target_rate = numpy.random.uniform()
- num_labels = 3
- batch_shape = [25, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.zeros_like(logits)
- label_priors = tf.constant(numpy.random.uniform(size=[num_labels]),
- dtype=tf.float32)
-
- xent_loss, _ = loss_layers.true_positive_rate_at_false_positive_rate_loss(
- targets, logits, target_rate, label_priors=label_priors)
- xent_expected = tf.subtract(
- util.weighted_sigmoid_cross_entropy_with_logits(targets,
- logits,
- positive_weights=0.0,
- negative_weights=1.0),
- target_rate * (1.0 - label_priors) * numpy.log(2))
- hinge_loss, _ = loss_layers.true_positive_rate_at_false_positive_rate_loss(
- targets, logits, target_rate, label_priors=label_priors,
- surrogate_type='hinge')
- hinge_expected = util.weighted_hinge_loss(
- targets, logits) - target_rate * (1.0 - label_priors)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(xent_expected.eval(), xent_loss.eval())
- self.assertAllClose(hinge_expected.eval(), hinge_loss.eval())
-
- def testLagrangeMultiplierUpdateDirection(self):
- for target_rate in [0.35, 0.65]:
- for surrogate_type in ['xent', 'hinge']:
- kwargs = {'target_rate': target_rate,
- 'surrogate_type': surrogate_type,
- 'scope': 'tpr-at-fpr_{}_{}'.format(target_rate,
- surrogate_type)}
- run_lagrange_multiplier_test(
- global_objective=(
- loss_layers.true_positive_rate_at_false_positive_rate_loss),
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=(
- loss_layers.true_positive_rate_at_false_positive_rate_loss),
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
- def testLagrangeMultiplierUpdateDirectionWithMultipleRates(self):
- """Runs Lagrange multiplier test with multiple target rates."""
- target_rate = [0.35, 0.65]
- for surrogate_type in ['xent', 'hinge']:
- kwargs = {'target_rate': target_rate,
- 'surrogate_type': surrogate_type,
- 'scope': 'tpr-at-fpr_{}_{}'.format(
- '_'.join([str(target) for target in target_rate]),
- surrogate_type)}
- run_lagrange_multiplier_test(
- global_objective=(
- loss_layers.true_positive_rate_at_false_positive_rate_loss),
- objective_kwargs=kwargs,
- data_builder=_multilabel_data,
- test_object=self)
- kwargs['scope'] = 'other-' + kwargs['scope']
- run_lagrange_multiplier_test(
- global_objective=(
- loss_layers.true_positive_rate_at_false_positive_rate_loss),
- objective_kwargs=kwargs,
- data_builder=_other_multilabel_data(surrogate_type),
- test_object=self)
-
- def testEquivalenceBetweenSingleAndEqualMultipleRates(self):
- """Compares single and multiple target rates of the same value.
-
- Checks that using a single target rate and multiple rates with the
- same value would result in the same loss value.
- """
- num_labels = 2
- target_shape = [20, num_labels]
- logits = tf.Variable(tf.random_normal(target_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(target_shape), 0.7)))
- label_priors = tf.constant([0.34], shape=[num_labels])
-
- multi_label_loss, _ = (
- loss_layers.true_positive_rate_at_false_positive_rate_loss(
- targets, logits, [0.75, 0.75], label_priors=label_priors))
-
- single_label_loss, _ = (
- loss_layers.true_positive_rate_at_false_positive_rate_loss(
- targets, logits, 0.75, label_priors=label_priors))
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- multi_label_loss_val, single_label_loss_val = session.run(
- [multi_label_loss, single_label_loss])
- self.assertAllClose(multi_label_loss_val, single_label_loss_val)
-
- def testEquivalenceBetweenSingleAndMultipleRates(self):
- """Compares single and multiple target rates of different values.
-
- Runs true_positive_rate_at_false_positive_rate_loss with multiple target
- rates, and runs each label seperately with its own target rate as a
- scalar. Validates that the returned loss values are the same.
- """
- target_precision = [0.7, 0.9, 0.4]
- num_labels = 3
- batch_shape = [30, num_labels]
- logits = tf.Variable(tf.random_normal(batch_shape))
- targets = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
- label_priors = tf.constant(0.45, shape=[num_labels])
-
- multi_label_loss, _ = (
- loss_layers.true_positive_rate_at_false_positive_rate_loss(
- targets, logits, target_precision, label_priors=label_priors))
-
- single_label_losses = [
- loss_layers.true_positive_rate_at_false_positive_rate_loss(
- tf.expand_dims(targets[:, i], -1),
- tf.expand_dims(logits[:, i], -1),
- target_precision[i],
- label_priors=label_priors[i])[0]
- for i in range(num_labels)
- ]
-
- single_label_losses = tf.concat(single_label_losses, 1)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- multi_label_loss_val, single_label_loss_val = session.run(
- [multi_label_loss, single_label_losses])
- self.assertAllClose(multi_label_loss_val, single_label_loss_val)
-
-
-class UtilityFunctionsTest(tf.test.TestCase):
-
- def testTrainableDualVariable(self):
- # Confirm correct behavior of a trainable dual variable.
- x = tf.get_variable('primal', dtype=tf.float32, initializer=2.0)
- y_value, y = loss_layers._create_dual_variable(
- 'dual', shape=None, dtype=tf.float32, initializer=1.0, collections=None,
- trainable=True, dual_rate_factor=0.3)
- optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
- update = optimizer.minimize(0.5 * tf.square(x - y_value))
-
- with self.test_session():
- tf.global_variables_initializer().run()
- update.run()
- self.assertAllClose(0.7, y.eval())
-
- def testUntrainableDualVariable(self):
- # Confirm correct behavior of dual variable which is not trainable.
- x = tf.get_variable('primal', dtype=tf.float32, initializer=-2.0)
- y_value, y = loss_layers._create_dual_variable(
- 'dual', shape=None, dtype=tf.float32, initializer=1.0, collections=None,
- trainable=False, dual_rate_factor=0.8)
- optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
- update = optimizer.minimize(tf.square(x) * y_value + tf.exp(y_value))
-
- with self.test_session():
- tf.global_variables_initializer().run()
- update.run()
- self.assertAllClose(1.0, y.eval())
-
-
-class BoundTest(parameterized.TestCase, tf.test.TestCase):
-
- @parameterized.named_parameters(
- ('_xent', 'xent', 1.0, [2.0, 1.0]),
- ('_xent_weighted', 'xent',
- numpy.array([0, 2, 0.5, 1, 2, 3]).reshape(6, 1), [2.5, 0]),
- ('_hinge', 'hinge', 1.0, [2.0, 1.0]),
- ('_hinge_weighted', 'hinge',
- numpy.array([1.0, 2, 3, 4, 5, 6]).reshape(6, 1), [5.0, 1]))
- def testLowerBoundMultilabel(self, surrogate_type, weights, expected):
- labels, logits, _ = _multilabel_data()
- lower_bound = loss_layers.true_positives_lower_bound(
- labels, logits, weights, surrogate_type)
-
- with self.test_session():
- self.assertAllClose(lower_bound.eval(), expected)
-
- @parameterized.named_parameters(
- ('_xent', 'xent'), ('_hinge', 'hinge'))
- def testLowerBoundOtherMultilabel(self, surrogate_type):
- labels, logits, _ = _other_multilabel_data(surrogate_type)()
- lower_bound = loss_layers.true_positives_lower_bound(
- labels, logits, 1.0, surrogate_type)
-
- with self.test_session():
- self.assertAllClose(lower_bound.eval(), [4.0, 2.0], atol=1e-5)
-
- @parameterized.named_parameters(
- ('_xent', 'xent', 1.0, [1.0, 2.0]),
- ('_xent_weighted', 'xent',
- numpy.array([3.0, 2, 1, 0, 1, 2]).reshape(6, 1), [2.0, 1.0]),
- ('_hinge', 'hinge', 1.0, [1.0, 2.0]),
- ('_hinge_weighted', 'hinge',
- numpy.array([13, 12, 11, 0.5, 0, 0.5]).reshape(6, 1), [0.5, 0.5]))
- def testUpperBoundMultilabel(self, surrogate_type, weights, expected):
- labels, logits, _ = _multilabel_data()
- upper_bound = loss_layers.false_positives_upper_bound(
- labels, logits, weights, surrogate_type)
-
- with self.test_session():
- self.assertAllClose(upper_bound.eval(), expected)
-
- @parameterized.named_parameters(
- ('_xent', 'xent'), ('_hinge', 'hinge'))
- def testUpperBoundOtherMultilabel(self, surrogate_type):
- labels, logits, _ = _other_multilabel_data(surrogate_type)()
- upper_bound = loss_layers.false_positives_upper_bound(
- labels, logits, 1.0, surrogate_type)
-
- with self.test_session():
- self.assertAllClose(upper_bound.eval(), [2.0, 4.0], atol=1e-5)
-
- @parameterized.named_parameters(
- ('_lower', 'lower'), ('_upper', 'upper'))
- def testThreeDimensionalLogits(self, bound):
- bound_function = loss_layers.false_positives_upper_bound
- if bound == 'lower':
- bound_function = loss_layers.true_positives_lower_bound
- random_labels = numpy.float32(numpy.random.uniform(size=[2, 3]) > 0.5)
- random_logits = numpy.float32(numpy.random.randn(2, 3, 2))
- first_slice_logits = random_logits[:, :, 0].reshape(2, 3)
- second_slice_logits = random_logits[:, :, 1].reshape(2, 3)
-
- full_bound = bound_function(
- tf.constant(random_labels), tf.constant(random_logits), 1.0, 'xent')
- first_slice_bound = bound_function(tf.constant(random_labels),
- tf.constant(first_slice_logits),
- 1.0,
- 'xent')
- second_slice_bound = bound_function(tf.constant(random_labels),
- tf.constant(second_slice_logits),
- 1.0,
- 'xent')
- stacked_bound = tf.stack([first_slice_bound, second_slice_bound], axis=1)
-
- with self.test_session():
- self.assertAllClose(full_bound.eval(), stacked_bound.eval())
-
-
-def run_lagrange_multiplier_test(global_objective,
- objective_kwargs,
- data_builder,
- test_object):
- """Runs a test for the Lagrange multiplier update of `global_objective`.
-
- The test checks that the constraint for `global_objective` is satisfied on
- the first label of the data produced by `data_builder` but not the second.
-
- Args:
- global_objective: One of the global objectives.
- objective_kwargs: A dictionary of keyword arguments to pass to
- `global_objective`. Must contain an entry for the constraint argument
- of `global_objective`, e.g. 'target_rate' or 'target_precision'.
- data_builder: A function which returns tensors corresponding to labels,
- logits, and label priors.
- test_object: An instance of tf.test.TestCase.
- """
- # Construct global objective kwargs from a copy of `objective_kwargs`.
- kwargs = dict(objective_kwargs)
- targets, logits, priors = data_builder()
- kwargs['labels'] = targets
- kwargs['logits'] = logits
- kwargs['label_priors'] = priors
-
- loss, output_dict = global_objective(**kwargs)
- lambdas = tf.squeeze(output_dict['lambdas'])
- opt = tf.train.GradientDescentOptimizer(learning_rate=0.1)
- update_op = opt.minimize(loss, var_list=[output_dict['lambdas']])
-
- with test_object.test_session() as session:
- tf.global_variables_initializer().run()
- lambdas_before = session.run(lambdas)
- session.run(update_op)
- lambdas_after = session.run(lambdas)
- test_object.assertLess(lambdas_after[0], lambdas_before[0])
- test_object.assertGreater(lambdas_after[1], lambdas_before[1])
-
-
-class CrossFunctionTest(parameterized.TestCase, tf.test.TestCase):
-
- @parameterized.named_parameters(
- ('_auc01xent', loss_layers.precision_recall_auc_loss, {
- 'precision_range': (0.0, 1.0), 'surrogate_type': 'xent'
- }),
- ('_auc051xent', loss_layers.precision_recall_auc_loss, {
- 'precision_range': (0.5, 1.0), 'surrogate_type': 'xent'
- }),
- ('_auc01)hinge', loss_layers.precision_recall_auc_loss, {
- 'precision_range': (0.0, 1.0), 'surrogate_type': 'hinge'
- }),
- ('_ratp04', loss_layers.recall_at_precision_loss, {
- 'target_precision': 0.4, 'surrogate_type': 'xent'
- }),
- ('_ratp066', loss_layers.recall_at_precision_loss, {
- 'target_precision': 0.66, 'surrogate_type': 'xent'
- }),
- ('_ratp07_hinge', loss_layers.recall_at_precision_loss, {
- 'target_precision': 0.7, 'surrogate_type': 'hinge'
- }),
- ('_fpattp066', loss_layers.false_positive_rate_at_true_positive_rate_loss,
- {'target_rate': 0.66, 'surrogate_type': 'xent'}),
- ('_fpattp046', loss_layers.false_positive_rate_at_true_positive_rate_loss,
- {
- 'target_rate': 0.46, 'surrogate_type': 'xent'
- }),
- ('_fpattp076_hinge',
- loss_layers.false_positive_rate_at_true_positive_rate_loss, {
- 'target_rate': 0.76, 'surrogate_type': 'hinge'
- }),
- ('_fpattp036_hinge',
- loss_layers.false_positive_rate_at_true_positive_rate_loss, {
- 'target_rate': 0.36, 'surrogate_type': 'hinge'
- }),
- )
- def testWeigtedGlobalObjective(self,
- global_objective,
- objective_kwargs):
- """Runs a test of `global_objective` with per-example weights.
-
- Args:
- global_objective: One of the global objectives.
- objective_kwargs: A dictionary of keyword arguments to pass to
- `global_objective`. Must contain keys 'surrogate_type', and the keyword
- for the constraint argument of `global_objective`, e.g. 'target_rate' or
- 'target_precision'.
- """
- logits_positives = tf.constant([1, -0.5, 3], shape=[3, 1])
- logits_negatives = tf.constant([-0.5, 1, -1, -1, -0.5, 1], shape=[6, 1])
-
- # Dummy tensor is used to compute the gradients.
- dummy = tf.constant(1.0)
- logits = tf.concat([logits_positives, logits_negatives], 0)
- logits = tf.multiply(logits, dummy)
- targets = tf.constant([1, 1, 1, 0, 0, 0, 0, 0, 0],
- shape=[9, 1], dtype=tf.float32)
- priors = tf.constant(1.0/3.0, shape=[1])
- weights = tf.constant([1, 1, 1, 0, 0, 0, 2, 2, 2],
- shape=[9, 1], dtype=tf.float32)
-
- # Construct global objective kwargs.
- objective_kwargs['labels'] = targets
- objective_kwargs['logits'] = logits
- objective_kwargs['label_priors'] = priors
-
- scope = 'weighted_test'
- # Unweighted loss.
- objective_kwargs['scope'] = scope + '_plain'
- raw_loss, update = global_objective(**objective_kwargs)
- loss = tf.reduce_sum(raw_loss)
-
- # Weighted loss.
- objective_kwargs['weights'] = weights
- objective_kwargs['scope'] = scope + '_weighted'
- raw_weighted_loss, weighted_update = global_objective(**objective_kwargs)
- weighted_loss = tf.reduce_sum(raw_weighted_loss)
-
- lambdas = tf.contrib.framework.get_unique_variable(scope + '_plain/lambdas')
- weighted_lambdas = tf.contrib.framework.get_unique_variable(
- scope + '_weighted/lambdas')
- logits_gradient = tf.gradients(loss, dummy)
- weighted_logits_gradient = tf.gradients(weighted_loss, dummy)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- self.assertAllClose(loss.eval(), weighted_loss.eval())
-
- logits_grad, weighted_logits_grad = session.run(
- [logits_gradient, weighted_logits_gradient])
- self.assertAllClose(logits_grad, weighted_logits_grad)
-
- session.run([update, weighted_update])
- lambdas_value, weighted_lambdas_value = session.run(
- [lambdas, weighted_lambdas])
- self.assertAllClose(lambdas_value, weighted_lambdas_value)
-
- @parameterized.named_parameters(
- ('_prauc051xent', loss_layers.precision_recall_auc_loss, {
- 'precision_range': (0.5, 1.0), 'surrogate_type': 'xent'
- }),
- ('_prauc01hinge', loss_layers.precision_recall_auc_loss, {
- 'precision_range': (0.0, 1.0), 'surrogate_type': 'hinge'
- }),
- ('_rocxent', loss_layers.roc_auc_loss, {'surrogate_type': 'xent'}),
- ('_rochinge', loss_layers.roc_auc_loss, {'surrogate_type': 'xent'}),
- ('_ratp04', loss_layers.recall_at_precision_loss, {
- 'target_precision': 0.4, 'surrogate_type': 'xent'
- }),
- ('_ratp07_hinge', loss_layers.recall_at_precision_loss, {
- 'target_precision': 0.7, 'surrogate_type': 'hinge'
- }),
- ('_patr05', loss_layers.precision_at_recall_loss, {
- 'target_recall': 0.4, 'surrogate_type': 'xent'
- }),
- ('_patr08_hinge', loss_layers.precision_at_recall_loss, {
- 'target_recall': 0.7, 'surrogate_type': 'hinge'
- }),
- ('_fpattp046', loss_layers.false_positive_rate_at_true_positive_rate_loss,
- {
- 'target_rate': 0.46, 'surrogate_type': 'xent'
- }),
- ('_fpattp036_hinge',
- loss_layers.false_positive_rate_at_true_positive_rate_loss, {
- 'target_rate': 0.36, 'surrogate_type': 'hinge'
- }),
- ('_tpatfp076', loss_layers.true_positive_rate_at_false_positive_rate_loss,
- {
- 'target_rate': 0.76, 'surrogate_type': 'xent'
- }),
- ('_tpatfp036_hinge',
- loss_layers.true_positive_rate_at_false_positive_rate_loss, {
- 'target_rate': 0.36, 'surrogate_type': 'hinge'
- }),
- )
- def testVectorAndMatrixLabelEquivalence(self,
- global_objective,
- objective_kwargs):
- """Tests equivalence between label shape [batch_size] or [batch_size, 1]."""
- vector_labels = tf.constant([1.0, 1.0, 0.0, 0.0], shape=[4])
- vector_logits = tf.constant([1.0, 0.1, 0.1, -1.0], shape=[4])
-
- # Construct vector global objective kwargs and loss.
- vector_kwargs = objective_kwargs.copy()
- vector_kwargs['labels'] = vector_labels
- vector_kwargs['logits'] = vector_logits
- vector_loss, _ = global_objective(**vector_kwargs)
- vector_loss_sum = tf.reduce_sum(vector_loss)
-
- # Construct matrix global objective kwargs and loss.
- matrix_kwargs = objective_kwargs.copy()
- matrix_kwargs['labels'] = tf.expand_dims(vector_labels, 1)
- matrix_kwargs['logits'] = tf.expand_dims(vector_logits, 1)
- matrix_loss, _ = global_objective(**matrix_kwargs)
- matrix_loss_sum = tf.reduce_sum(matrix_loss)
-
- self.assertEqual(1, vector_loss.get_shape().ndims)
- self.assertEqual(2, matrix_loss.get_shape().ndims)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(vector_loss_sum.eval(), matrix_loss_sum.eval())
-
- @parameterized.named_parameters(
- ('_prauc', loss_layers.precision_recall_auc_loss, None),
- ('_roc', loss_layers.roc_auc_loss, None),
- ('_rap', loss_layers.recall_at_precision_loss, {'target_precision': 0.8}),
- ('_patr', loss_layers.precision_at_recall_loss, {'target_recall': 0.7}),
- ('_fpattp', loss_layers.false_positive_rate_at_true_positive_rate_loss,
- {'target_rate': 0.9}),
- ('_tpatfp', loss_layers.true_positive_rate_at_false_positive_rate_loss,
- {'target_rate': 0.1})
- )
- def testUnknownBatchSize(self, global_objective, objective_kwargs):
- # Tests that there are no errors when the batch size is not known.
- batch_shape = [5, 2]
- logits = tf.placeholder(tf.float32)
- logits_feed = numpy.random.randn(*batch_shape)
- labels = tf.placeholder(tf.float32)
- labels_feed = logits_feed > 0.1
- logits.set_shape([None, 2])
- labels.set_shape([None, 2])
-
- if objective_kwargs is None:
- objective_kwargs = {}
-
- placeholder_kwargs = objective_kwargs.copy()
- placeholder_kwargs['labels'] = labels
- placeholder_kwargs['logits'] = logits
- placeholder_loss, _ = global_objective(**placeholder_kwargs)
-
- kwargs = objective_kwargs.copy()
- kwargs['labels'] = labels_feed
- kwargs['logits'] = logits_feed
- loss, _ = global_objective(**kwargs)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
- feed_loss_val = session.run(placeholder_loss,
- feed_dict={logits: logits_feed,
- labels: labels_feed})
- loss_val = session.run(loss)
- self.assertAllClose(feed_loss_val, loss_val)
-
-
-# Both sets of logits below are designed so that the surrogate precision and
-# recall (true positive rate) of class 1 is ~ 2/3, and the same surrogates for
-# class 2 are ~ 1/3. The false positive rate surrogates are ~ 1/3 and 2/3.
-def _multilabel_data():
- targets = tf.constant([1.0, 1.0, 1.0, 0.0, 0.0, 0.0], shape=[6, 1])
- targets = tf.concat([targets, targets], 1)
- logits_positives = tf.constant([[0.0, 15],
- [16, 0.0],
- [14, 0.0]], shape=[3, 2])
- logits_negatives = tf.constant([[-17, 0.0],
- [-15, 0.0],
- [0.0, -101]], shape=[3, 2])
- logits = tf.concat([logits_positives, logits_negatives], 0)
- priors = tf.constant(0.5, shape=[2])
-
- return targets, logits, priors
-
-
-def _other_multilabel_data(surrogate_type):
- targets = tf.constant(
- [1.0] * 6 + [0.0] * 6, shape=[12, 1])
- targets = tf.concat([targets, targets], 1)
- logits_positives = tf.constant([[0.0, 13],
- [12, 0.0],
- [15, 0.0],
- [0.0, 30],
- [13, 0.0],
- [18, 0.0]], shape=[6, 2])
- # A score of cost_2 incurs a loss of ~2.0.
- cost_2 = 1.0 if surrogate_type == 'hinge' else 1.09861229
- logits_negatives = tf.constant([[-16, cost_2],
- [-15, cost_2],
- [cost_2, -111],
- [-133, -14,],
- [-14.0100101, -16,],
- [-19.888828882, -101]], shape=[6, 2])
- logits = tf.concat([logits_positives, logits_negatives], 0)
- priors = tf.constant(0.5, shape=[2])
-
- def builder():
- return targets, logits, priors
-
- return builder
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/global_objectives/test_all.py b/research/global_objectives/test_all.py
deleted file mode 100644
index d7e439e219840a9ec5c65382c6bc392b1d68b447..0000000000000000000000000000000000000000
--- a/research/global_objectives/test_all.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright 2018 The TensorFlow Global Objectives Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Runs all unit tests in the Global Objectives package.
-
-Requires that TensorFlow and abseil (https://github.com/abseil/abseil-py) be
-installed on your machine. Command to run the tests:
-python test_all.py
-
-"""
-
-import os
-import sys
-import unittest
-
-this_file = os.path.realpath(__file__)
-start_dir = os.path.dirname(this_file)
-parent_dir = os.path.dirname(start_dir)
-
-sys.path.append(parent_dir)
-loader = unittest.TestLoader()
-suite = loader.discover(start_dir, pattern='*_test.py')
-
-runner = unittest.TextTestRunner(verbosity=2)
-runner.run(suite)
diff --git a/research/global_objectives/util.py b/research/global_objectives/util.py
deleted file mode 100644
index e2b287a90bd743e5466b875c933c3872868f4a5f..0000000000000000000000000000000000000000
--- a/research/global_objectives/util.py
+++ /dev/null
@@ -1,348 +0,0 @@
-# Copyright 2018 The TensorFlow Global Objectives Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Contains utility functions for the global objectives library."""
-
-# Dependency imports
-import tensorflow as tf
-
-
-def weighted_sigmoid_cross_entropy_with_logits(labels,
- logits,
- positive_weights=1.0,
- negative_weights=1.0,
- name=None):
- """Computes a weighting of sigmoid cross entropy given `logits`.
-
- Measures the weighted probability error in discrete classification tasks in
- which classes are independent and not mutually exclusive. For instance, one
- could perform multilabel classification where a picture can contain both an
- elephant and a dog at the same time. The class weight multiplies the
- different types of errors.
- For brevity, let `x = logits`, `z = labels`, `c = positive_weights`,
- `d = negative_weights` The
- weighed logistic loss is
-
- ```
- c * z * -log(sigmoid(x)) + d * (1 - z) * -log(1 - sigmoid(x))
- = c * z * -log(1 / (1 + exp(-x))) - d * (1 - z) * log(exp(-x) / (1 + exp(-x)))
- = c * z * log(1 + exp(-x)) + d * (1 - z) * (-log(exp(-x)) + log(1 + exp(-x)))
- = c * z * log(1 + exp(-x)) + d * (1 - z) * (x + log(1 + exp(-x)))
- = (1 - z) * x * d + (1 - z + c * z ) * log(1 + exp(-x))
- = - d * x * z + d * x + (d - d * z + c * z ) * log(1 + exp(-x))
- ```
-
- To ensure stability and avoid overflow, the implementation uses the identity
- log(1 + exp(-x)) = max(0,-x) + log(1 + exp(-abs(x)))
- and the result is computed as
-
- ```
- = -d * x * z + d * x
- + (d - d * z + c * z ) * (max(0,-x) + log(1 + exp(-abs(x))))
- ```
-
- Note that the loss is NOT an upper bound on the 0-1 loss, unless it is divided
- by log(2).
-
- Args:
- labels: A `Tensor` of type `float32` or `float64`. `labels` can be a 2D
- tensor with shape [batch_size, num_labels] or a 3D tensor with shape
- [batch_size, num_labels, K].
- logits: A `Tensor` of the same type and shape as `labels`. If `logits` has
- shape [batch_size, num_labels, K], the loss is computed separately on each
- slice [:, :, k] of `logits`.
- positive_weights: A `Tensor` that holds positive weights and has the
- following semantics according to its shape:
- scalar - A global positive weight.
- 1D tensor - must be of size K, a weight for each 'attempt'
- 2D tensor - of size [num_labels, K'] where K' is either K or 1.
- The `positive_weights` will be expanded to the left to match the
- dimensions of logits and labels.
- negative_weights: A `Tensor` that holds positive weight and has the
- semantics identical to positive_weights.
- name: A name for the operation (optional).
-
- Returns:
- A `Tensor` of the same shape as `logits` with the componentwise
- weighted logistic losses.
- """
- with tf.name_scope(
- name,
- 'weighted_logistic_loss',
- [logits, labels, positive_weights, negative_weights]) as name:
- labels, logits, positive_weights, negative_weights = prepare_loss_args(
- labels, logits, positive_weights, negative_weights)
-
- softplus_term = tf.add(tf.maximum(-logits, 0.0),
- tf.log(1.0 + tf.exp(-tf.abs(logits))))
- weight_dependent_factor = (
- negative_weights + (positive_weights - negative_weights) * labels)
- return (negative_weights * (logits - labels * logits) +
- weight_dependent_factor * softplus_term)
-
-
-def weighted_hinge_loss(labels,
- logits,
- positive_weights=1.0,
- negative_weights=1.0,
- name=None):
- """Computes weighted hinge loss given logits `logits`.
-
- The loss applies to multi-label classification tasks where labels are
- independent and not mutually exclusive. See also
- `weighted_sigmoid_cross_entropy_with_logits`.
-
- Args:
- labels: A `Tensor` of type `float32` or `float64`. Each entry must be
- either 0 or 1. `labels` can be a 2D tensor with shape
- [batch_size, num_labels] or a 3D tensor with shape
- [batch_size, num_labels, K].
- logits: A `Tensor` of the same type and shape as `labels`. If `logits` has
- shape [batch_size, num_labels, K], the loss is computed separately on each
- slice [:, :, k] of `logits`.
- positive_weights: A `Tensor` that holds positive weights and has the
- following semantics according to its shape:
- scalar - A global positive weight.
- 1D tensor - must be of size K, a weight for each 'attempt'
- 2D tensor - of size [num_labels, K'] where K' is either K or 1.
- The `positive_weights` will be expanded to the left to match the
- dimensions of logits and labels.
- negative_weights: A `Tensor` that holds positive weight and has the
- semantics identical to positive_weights.
- name: A name for the operation (optional).
-
- Returns:
- A `Tensor` of the same shape as `logits` with the componentwise
- weighted hinge loss.
- """
- with tf.name_scope(
- name, 'weighted_hinge_loss',
- [logits, labels, positive_weights, negative_weights]) as name:
- labels, logits, positive_weights, negative_weights = prepare_loss_args(
- labels, logits, positive_weights, negative_weights)
-
- positives_term = positive_weights * labels * tf.maximum(1.0 - logits, 0)
- negatives_term = (negative_weights * (1.0 - labels)
- * tf.maximum(1.0 + logits, 0))
- return positives_term + negatives_term
-
-
-def weighted_surrogate_loss(labels,
- logits,
- surrogate_type='xent',
- positive_weights=1.0,
- negative_weights=1.0,
- name=None):
- """Returns either weighted cross-entropy or hinge loss.
-
- For example `surrogate_type` is 'xent' returns the weighted cross
- entropy loss.
-
- Args:
- labels: A `Tensor` of type `float32` or `float64`. Each entry must be
- between 0 and 1. `labels` can be a 2D tensor with shape
- [batch_size, num_labels] or a 3D tensor with shape
- [batch_size, num_labels, K].
- logits: A `Tensor` of the same type and shape as `labels`. If `logits` has
- shape [batch_size, num_labels, K], each slice [:, :, k] represents an
- 'attempt' to predict `labels` and the loss is computed per slice.
- surrogate_type: A string that determines which loss to return, supports
- 'xent' for cross-entropy and 'hinge' for hinge loss.
- positive_weights: A `Tensor` that holds positive weights and has the
- following semantics according to its shape:
- scalar - A global positive weight.
- 1D tensor - must be of size K, a weight for each 'attempt'
- 2D tensor - of size [num_labels, K'] where K' is either K or 1.
- The `positive_weights` will be expanded to the left to match the
- dimensions of logits and labels.
- negative_weights: A `Tensor` that holds positive weight and has the
- semantics identical to positive_weights.
- name: A name for the operation (optional).
-
- Returns:
- The weigthed loss.
-
- Raises:
- ValueError: If value of `surrogate_type` is not supported.
- """
- with tf.name_scope(
- name, 'weighted_loss',
- [logits, labels, surrogate_type, positive_weights,
- negative_weights]) as name:
- if surrogate_type == 'xent':
- return weighted_sigmoid_cross_entropy_with_logits(
- logits=logits,
- labels=labels,
- positive_weights=positive_weights,
- negative_weights=negative_weights,
- name=name)
- elif surrogate_type == 'hinge':
- return weighted_hinge_loss(
- logits=logits,
- labels=labels,
- positive_weights=positive_weights,
- negative_weights=negative_weights,
- name=name)
- raise ValueError('surrogate_type %s not supported.' % surrogate_type)
-
-
-def expand_outer(tensor, rank):
- """Expands the given `Tensor` outwards to a target rank.
-
- For example if rank = 3 and tensor.shape is [3, 4], this function will expand
- to such that the resulting shape will be [1, 3, 4].
-
- Args:
- tensor: The tensor to expand.
- rank: The target dimension.
-
- Returns:
- The expanded tensor.
-
- Raises:
- ValueError: If rank of `tensor` is unknown, or if `rank` is smaller than
- the rank of `tensor`.
- """
- if tensor.get_shape().ndims is None:
- raise ValueError('tensor dimension must be known.')
- if len(tensor.get_shape()) > rank:
- raise ValueError(
- '`rank` must be at least the current tensor dimension: (%s vs %s).' %
- (rank, len(tensor.get_shape())))
- while len(tensor.get_shape()) < rank:
- tensor = tf.expand_dims(tensor, 0)
- return tensor
-
-
-def build_label_priors(labels,
- weights=None,
- positive_pseudocount=1.0,
- negative_pseudocount=1.0,
- variables_collections=None):
- """Creates an op to maintain and update label prior probabilities.
-
- For each label, the label priors are estimated as
- (P + sum_i w_i y_i) / (P + N + sum_i w_i),
- where y_i is the ith label, w_i is the ith weight, P is a pseudo-count of
- positive labels, and N is a pseudo-count of negative labels. The index i
- ranges over all labels observed during all evaluations of the returned op.
-
- Args:
- labels: A `Tensor` with shape [batch_size, num_labels]. Entries should be
- in [0, 1].
- weights: Coefficients representing the weight of each label. Must be either
- a Tensor of shape [batch_size, num_labels] or `None`, in which case each
- weight is treated as 1.0.
- positive_pseudocount: Number of positive labels used to initialize the label
- priors.
- negative_pseudocount: Number of negative labels used to initialize the label
- priors.
- variables_collections: Optional list of collections for created variables.
-
- Returns:
- label_priors: An op to update the weighted label_priors. Gives the
- current value of the label priors when evaluated.
- """
- dtype = labels.dtype.base_dtype
- num_labels = get_num_labels(labels)
-
- if weights is None:
- weights = tf.ones_like(labels)
-
- # We disable partitioning while constructing dual variables because they will
- # be updated with assign, which is not available for partitioned variables.
- partitioner = tf.get_variable_scope().partitioner
- try:
- tf.get_variable_scope().set_partitioner(None)
- # Create variable and update op for weighted label counts.
- weighted_label_counts = tf.contrib.framework.model_variable(
- name='weighted_label_counts',
- shape=[num_labels],
- dtype=dtype,
- initializer=tf.constant_initializer(
- [positive_pseudocount] * num_labels, dtype=dtype),
- collections=variables_collections,
- trainable=False)
- weighted_label_counts_update = weighted_label_counts.assign_add(
- tf.reduce_sum(weights * labels, 0))
-
- # Create variable and update op for the sum of the weights.
- weight_sum = tf.contrib.framework.model_variable(
- name='weight_sum',
- shape=[num_labels],
- dtype=dtype,
- initializer=tf.constant_initializer(
- [positive_pseudocount + negative_pseudocount] * num_labels,
- dtype=dtype),
- collections=variables_collections,
- trainable=False)
- weight_sum_update = weight_sum.assign_add(tf.reduce_sum(weights, 0))
-
- finally:
- tf.get_variable_scope().set_partitioner(partitioner)
-
- label_priors = tf.div(
- weighted_label_counts_update,
- weight_sum_update)
- return label_priors
-
-
-def convert_and_cast(value, name, dtype):
- """Convert input to tensor and cast to dtype.
-
- Args:
- value: An object whose type has a registered Tensor conversion function,
- e.g. python numerical type or numpy array.
- name: Name to use for the new Tensor, if one is created.
- dtype: Optional element type for the returned tensor.
-
- Returns:
- A tensor.
- """
- return tf.cast(tf.convert_to_tensor(value, name=name), dtype=dtype)
-
-
-def prepare_loss_args(labels, logits, positive_weights, negative_weights):
- """Prepare arguments for weighted loss functions.
-
- If needed, will convert given arguments to appropriate type and shape.
-
- Args:
- labels: labels or labels of the loss function.
- logits: Logits of the loss function.
- positive_weights: Weight on the positive examples.
- negative_weights: Weight on the negative examples.
-
- Returns:
- Converted labels, logits, positive_weights, negative_weights.
- """
- logits = tf.convert_to_tensor(logits, name='logits')
- labels = convert_and_cast(labels, 'labels', logits.dtype)
- if len(labels.get_shape()) == 2 and len(logits.get_shape()) == 3:
- labels = tf.expand_dims(labels, [2])
-
- positive_weights = convert_and_cast(positive_weights, 'positive_weights',
- logits.dtype)
- positive_weights = expand_outer(positive_weights, logits.get_shape().ndims)
- negative_weights = convert_and_cast(negative_weights, 'negative_weights',
- logits.dtype)
- negative_weights = expand_outer(negative_weights, logits.get_shape().ndims)
- return labels, logits, positive_weights, negative_weights
-
-
-def get_num_labels(labels_or_logits):
- """Returns the number of labels inferred from labels_or_logits."""
- if labels_or_logits.get_shape().ndims <= 1:
- return 1
- return labels_or_logits.get_shape()[1].value
diff --git a/research/global_objectives/util_test.py b/research/global_objectives/util_test.py
deleted file mode 100644
index 195252a53eb1d0a50735d2f987b0882681b0544a..0000000000000000000000000000000000000000
--- a/research/global_objectives/util_test.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# Copyright 2018 The TensorFlow Global Objectives Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for global objectives util functions."""
-
-# Dependency imports
-from absl.testing import parameterized
-import numpy as np
-import tensorflow as tf
-
-from global_objectives import util
-
-
-def weighted_sigmoid_cross_entropy(targets, logits, weight):
- return (weight * targets * np.log(1.0 + np.exp(-logits)) + (
- (1.0 - targets) * np.log(1.0 + 1.0 / np.exp(-logits))))
-
-
-def hinge_loss(labels, logits):
- # Mostly copied from tensorflow.python.ops.losses but with loss per datapoint.
- labels = tf.to_float(labels)
- all_ones = tf.ones_like(labels)
- labels = tf.subtract(2 * labels, all_ones)
- return tf.nn.relu(tf.subtract(all_ones, tf.multiply(labels, logits)))
-
-
-class WeightedSigmoidCrossEntropyTest(parameterized.TestCase, tf.test.TestCase):
-
- def testTrivialCompatibilityWithSigmoidCrossEntropy(self):
- """Tests compatibility with unweighted function with weight 1.0."""
- x_shape = [300, 10]
- targets = np.random.random_sample(x_shape).astype(np.float32)
- logits = np.random.randn(*x_shape).astype(np.float32)
- weighted_loss = util.weighted_sigmoid_cross_entropy_with_logits(
- targets,
- logits)
- expected_loss = (
- tf.contrib.nn.deprecated_flipped_sigmoid_cross_entropy_with_logits(
- logits, targets))
- with self.test_session():
- self.assertAllClose(expected_loss.eval(),
- weighted_loss.eval(),
- atol=0.000001)
-
- def testNonTrivialCompatibilityWithSigmoidCrossEntropy(self):
- """Tests use of an arbitrary weight (4.12)."""
- x_shape = [300, 10]
- targets = np.random.random_sample(x_shape).astype(np.float32)
- logits = np.random.randn(*x_shape).astype(np.float32)
- weight = 4.12
- weighted_loss = util.weighted_sigmoid_cross_entropy_with_logits(
- targets,
- logits,
- weight,
- weight)
- expected_loss = (
- weight *
- tf.contrib.nn.deprecated_flipped_sigmoid_cross_entropy_with_logits(
- logits, targets))
- with self.test_session():
- self.assertAllClose(expected_loss.eval(),
- weighted_loss.eval(),
- atol=0.000001)
-
- def testDifferentSizeWeightedSigmoidCrossEntropy(self):
- """Tests correctness on 3D tensors.
-
- Tests that the function works as expected when logits is a 3D tensor and
- targets is a 2D tensor.
- """
- targets_shape = [30, 4]
- logits_shape = [targets_shape[0], targets_shape[1], 3]
- targets = np.random.random_sample(targets_shape).astype(np.float32)
- logits = np.random.randn(*logits_shape).astype(np.float32)
-
- weight_vector = [2.0, 3.0, 13.0]
- loss = util.weighted_sigmoid_cross_entropy_with_logits(targets,
- logits,
- weight_vector)
-
- with self.test_session():
- loss = loss.eval()
- for i in range(0, len(weight_vector)):
- expected = weighted_sigmoid_cross_entropy(targets, logits[:, :, i],
- weight_vector[i])
- self.assertAllClose(loss[:, :, i], expected, atol=0.000001)
-
- @parameterized.parameters((300, 10, 0.3), (20, 4, 2.0), (30, 4, 3.9))
- def testWeightedSigmoidCrossEntropy(self, batch_size, num_labels, weight):
- """Tests thats the tf and numpy functions agree on many instances."""
- x_shape = [batch_size, num_labels]
- targets = np.random.random_sample(x_shape).astype(np.float32)
- logits = np.random.randn(*x_shape).astype(np.float32)
-
- with self.test_session():
- loss = util.weighted_sigmoid_cross_entropy_with_logits(
- targets,
- logits,
- weight,
- 1.0,
- name='weighted-loss')
- expected = weighted_sigmoid_cross_entropy(targets, logits, weight)
- self.assertAllClose(expected, loss.eval(), atol=0.000001)
-
- def testGradients(self):
- """Tests that weighted loss gradients behave as expected."""
- dummy_tensor = tf.constant(1.0)
-
- positives_shape = [10, 1]
- positives_logits = dummy_tensor * tf.Variable(
- tf.random_normal(positives_shape) + 1.0)
- positives_targets = tf.ones(positives_shape)
- positives_weight = 4.6
- positives_loss = (
- tf.contrib.nn.deprecated_flipped_sigmoid_cross_entropy_with_logits(
- positives_logits, positives_targets) * positives_weight)
-
- negatives_shape = [190, 1]
- negatives_logits = dummy_tensor * tf.Variable(
- tf.random_normal(negatives_shape))
- negatives_targets = tf.zeros(negatives_shape)
- negatives_weight = 0.9
- negatives_loss = (
- tf.contrib.nn.deprecated_flipped_sigmoid_cross_entropy_with_logits(
- negatives_logits, negatives_targets) * negatives_weight)
-
- all_logits = tf.concat([positives_logits, negatives_logits], 0)
- all_targets = tf.concat([positives_targets, negatives_targets], 0)
- weighted_loss = tf.reduce_sum(
- util.weighted_sigmoid_cross_entropy_with_logits(
- all_targets, all_logits, positives_weight, negatives_weight))
- weighted_gradients = tf.gradients(weighted_loss, dummy_tensor)
-
- expected_loss = tf.add(
- tf.reduce_sum(positives_loss),
- tf.reduce_sum(negatives_loss))
- expected_gradients = tf.gradients(expected_loss, dummy_tensor)
-
- with tf.Session() as session:
- tf.global_variables_initializer().run()
- grad, expected_grad = session.run(
- [weighted_gradients, expected_gradients])
- self.assertAllClose(grad, expected_grad)
-
- def testDtypeFlexibility(self):
- """Tests the loss on inputs of varying data types."""
- shape = [20, 3]
- logits = np.random.randn(*shape)
- targets = tf.truncated_normal(shape)
- positive_weights = tf.constant(3, dtype=tf.int64)
- negative_weights = 1
-
- loss = util.weighted_sigmoid_cross_entropy_with_logits(
- targets, logits, positive_weights, negative_weights)
-
- with self.test_session():
- self.assertEqual(loss.eval().dtype, np.float)
-
-
-class WeightedHingeLossTest(tf.test.TestCase):
-
- def testTrivialCompatibilityWithHinge(self):
- # Tests compatibility with unweighted hinge loss.
- x_shape = [55, 10]
- logits = tf.constant(np.random.randn(*x_shape).astype(np.float32))
- targets = tf.to_float(tf.constant(np.random.random_sample(x_shape) > 0.3))
- weighted_loss = util.weighted_hinge_loss(targets, logits)
- expected_loss = hinge_loss(targets, logits)
- with self.test_session():
- self.assertAllClose(expected_loss.eval(), weighted_loss.eval())
-
- def testLessTrivialCompatibilityWithHinge(self):
- # Tests compatibility with a constant weight for positives and negatives.
- x_shape = [56, 11]
- logits = tf.constant(np.random.randn(*x_shape).astype(np.float32))
- targets = tf.to_float(tf.constant(np.random.random_sample(x_shape) > 0.7))
- weight = 1.0 + 1.0/2 + 1.0/3 + 1.0/4 + 1.0/5 + 1.0/6 + 1.0/7
- weighted_loss = util.weighted_hinge_loss(targets, logits, weight, weight)
- expected_loss = hinge_loss(targets, logits) * weight
- with self.test_session():
- self.assertAllClose(expected_loss.eval(), weighted_loss.eval())
-
- def testNontrivialCompatibilityWithHinge(self):
- # Tests compatibility with different positive and negative weights.
- x_shape = [23, 8]
- logits_positives = tf.constant(np.random.randn(*x_shape).astype(np.float32))
- logits_negatives = tf.constant(np.random.randn(*x_shape).astype(np.float32))
- targets_positives = tf.ones(x_shape)
- targets_negatives = tf.zeros(x_shape)
- logits = tf.concat([logits_positives, logits_negatives], 0)
- targets = tf.concat([targets_positives, targets_negatives], 0)
-
- raw_loss = util.weighted_hinge_loss(targets,
- logits,
- positive_weights=3.4,
- negative_weights=1.2)
- loss = tf.reduce_sum(raw_loss, 0)
- positives_hinge = hinge_loss(targets_positives, logits_positives)
- negatives_hinge = hinge_loss(targets_negatives, logits_negatives)
- expected = tf.add(tf.reduce_sum(3.4 * positives_hinge, 0),
- tf.reduce_sum(1.2 * negatives_hinge, 0))
-
- with self.test_session():
- self.assertAllClose(loss.eval(), expected.eval())
-
- def test3DLogitsAndTargets(self):
- # Tests correctness when logits is 3D and targets is 2D.
- targets_shape = [30, 4]
- logits_shape = [targets_shape[0], targets_shape[1], 3]
- targets = tf.to_float(
- tf.constant(np.random.random_sample(targets_shape) > 0.7))
- logits = tf.constant(np.random.randn(*logits_shape).astype(np.float32))
- weight_vector = [1.0, 1.0, 1.0]
- loss = util.weighted_hinge_loss(targets, logits, weight_vector)
-
- with self.test_session():
- loss_value = loss.eval()
- for i in range(len(weight_vector)):
- expected = hinge_loss(targets, logits[:, :, i]).eval()
- self.assertAllClose(loss_value[:, :, i], expected)
-
-
-class BuildLabelPriorsTest(tf.test.TestCase):
-
- def testLabelPriorConsistency(self):
- # Checks that, with zero pseudocounts, the returned label priors reproduce
- # label frequencies in the batch.
- batch_shape = [4, 10]
- labels = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.678)))
-
- label_priors_update = util.build_label_priors(
- labels=labels, positive_pseudocount=0, negative_pseudocount=0)
- expected_priors = tf.reduce_mean(labels, 0)
-
- with self.test_session():
- tf.global_variables_initializer().run()
- self.assertAllClose(label_priors_update.eval(), expected_priors.eval())
-
- def testLabelPriorsUpdate(self):
- # Checks that the update of label priors behaves as expected.
- batch_shape = [1, 5]
- labels = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.4)))
- label_priors_update = util.build_label_priors(labels)
-
- label_sum = np.ones(shape=batch_shape)
- weight_sum = 2.0 * np.ones(shape=batch_shape)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
-
- for _ in range(3):
- label_sum += labels.eval()
- weight_sum += np.ones(shape=batch_shape)
- expected_posteriors = label_sum / weight_sum
- label_priors = label_priors_update.eval().reshape(batch_shape)
- self.assertAllClose(label_priors, expected_posteriors)
-
- # Re-initialize labels to get a new random sample.
- session.run(labels.initializer)
-
- def testLabelPriorsUpdateWithWeights(self):
- # Checks the update of label priors with per-example weights.
- batch_size = 6
- num_labels = 5
- batch_shape = [batch_size, num_labels]
- labels = tf.Variable(
- tf.to_float(tf.greater(tf.random_uniform(batch_shape), 0.6)))
- weights = tf.Variable(tf.random_uniform(batch_shape) * 6.2)
-
- update_op = util.build_label_priors(labels, weights=weights)
-
- expected_weighted_label_counts = 1.0 + tf.reduce_sum(weights * labels, 0)
- expected_weight_sum = 2.0 + tf.reduce_sum(weights, 0)
- expected_label_posteriors = tf.divide(expected_weighted_label_counts,
- expected_weight_sum)
-
- with self.test_session() as session:
- tf.global_variables_initializer().run()
-
- updated_priors, expected_posteriors = session.run(
- [update_op, expected_label_posteriors])
- self.assertAllClose(updated_priors, expected_posteriors)
-
-
-class WeightedSurrogateLossTest(parameterized.TestCase, tf.test.TestCase):
-
- @parameterized.parameters(
- ('hinge', util.weighted_hinge_loss),
- ('xent', util.weighted_sigmoid_cross_entropy_with_logits))
- def testCompatibilityLoss(self, loss_name, loss_fn):
- x_shape = [28, 4]
- logits = tf.constant(np.random.randn(*x_shape).astype(np.float32))
- targets = tf.to_float(tf.constant(np.random.random_sample(x_shape) > 0.5))
- positive_weights = 0.66
- negative_weights = 11.1
- expected_loss = loss_fn(
- targets,
- logits,
- positive_weights=positive_weights,
- negative_weights=negative_weights)
- computed_loss = util.weighted_surrogate_loss(
- targets,
- logits,
- loss_name,
- positive_weights=positive_weights,
- negative_weights=negative_weights)
- with self.test_session():
- self.assertAllClose(expected_loss.eval(), computed_loss.eval())
-
- def testSurrogatgeError(self):
- x_shape = [7, 3]
- logits = tf.constant(np.random.randn(*x_shape).astype(np.float32))
- targets = tf.to_float(tf.constant(np.random.random_sample(x_shape) > 0.5))
-
- with self.assertRaises(ValueError):
- util.weighted_surrogate_loss(logits, targets, 'bug')
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/im2txt/.gitignore b/research/im2txt/.gitignore
deleted file mode 100644
index fb46913cc7a5994c4324de50829c95d7858c30f4..0000000000000000000000000000000000000000
--- a/research/im2txt/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
-/bazel-bin
-/bazel-ci_build-cache
-/bazel-genfiles
-/bazel-out
-/bazel-im2txt
-/bazel-testlogs
-/bazel-tf
diff --git a/research/im2txt/README.md b/research/im2txt/README.md
deleted file mode 100644
index 2eb72822a39e3959a5a9370f26a9cc5c12be0fda..0000000000000000000000000000000000000000
--- a/research/im2txt/README.md
+++ /dev/null
@@ -1,342 +0,0 @@
-
-
-
-
-# Show and Tell: A Neural Image Caption Generator
-
-A TensorFlow implementation of the image-to-text model described in the paper:
-
-"Show and Tell: Lessons learned from the 2015 MSCOCO Image Captioning
-Challenge."
-
-Oriol Vinyals, Alexander Toshev, Samy Bengio, Dumitru Erhan.
-
-*IEEE transactions on pattern analysis and machine intelligence (2016).*
-
-Full text available at: http://arxiv.org/abs/1609.06647
-
-## Contact
-***Author:*** Chris Shallue
-
-***Pull requests and issues:*** @cshallue
-
-## Contents
-* [Model Overview](#model-overview)
- * [Introduction](#introduction)
- * [Architecture](#architecture)
-* [Getting Started](#getting-started)
- * [A Note on Hardware and Training Time](#a-note-on-hardware-and-training-time)
- * [Install Required Packages](#install-required-packages)
- * [Prepare the Training Data](#prepare-the-training-data)
- * [Download the Inception v3 Checkpoint](#download-the-inception-v3-checkpoint)
-* [Training a Model](#training-a-model)
- * [Initial Training](#initial-training)
- * [Fine Tune the Inception v3 Model](#fine-tune-the-inception-v3-model)
-* [Generating Captions](#generating-captions)
-
-## Model Overview
-
-### Introduction
-
-The *Show and Tell* model is a deep neural network that learns how to describe
-the content of images. For example:
-
-
-
-### Architecture
-
-The *Show and Tell* model is an example of an *encoder-decoder* neural network.
-It works by first "encoding" an image into a fixed-length vector representation,
-and then "decoding" the representation into a natural language description.
-
-The image encoder is a deep convolutional neural network. This type of
-network is widely used for image tasks and is currently state-of-the-art for
-object recognition and detection. Our particular choice of network is the
-[*Inception v3*](http://arxiv.org/abs/1512.00567) image recognition model
-pretrained on the
-[ILSVRC-2012-CLS](http://www.image-net.org/challenges/LSVRC/2012/) image
-classification dataset.
-
-The decoder is a long short-term memory (LSTM) network. This type of network is
-commonly used for sequence modeling tasks such as language modeling and machine
-translation. In the *Show and Tell* model, the LSTM network is trained as a
-language model conditioned on the image encoding.
-
-Words in the captions are represented with an embedding model. Each word in the
-vocabulary is associated with a fixed-length vector representation that is
-learned during training.
-
-The following diagram illustrates the model architecture.
-
-
-
-In this diagram, \{*s*0, *s*1, ..., *s**N*-1\}
-are the words of the caption and \{*w**e**s*0,
-*w**e**s*1, ..., *w**e**s**N*-1\}
-are their corresponding word embedding vectors. The outputs \{*p*1,
-*p*2, ..., *p**N*\} of the LSTM are probability
-distributions generated by the model for the next word in the sentence. The
-terms \{log *p*1(*s*1),
-log *p*2(*s*2), ...,
-log *p**N*(*s**N*)\} are the log-likelihoods of the
-correct word at each step; the negated sum of these terms is the minimization
-objective of the model.
-
-During the first phase of training the parameters of the *Inception v3* model
-are kept fixed: it is simply a static image encoder function. A single trainable
-layer is added on top of the *Inception v3* model to transform the image
-embedding into the word embedding vector space. The model is trained with
-respect to the parameters of the word embeddings, the parameters of the layer on
-top of *Inception v3* and the parameters of the LSTM. In the second phase of
-training, all parameters - including the parameters of *Inception v3* - are
-trained to jointly fine-tune the image encoder and the LSTM.
-
-Given a trained model and an image we use *beam search* to generate captions for
-that image. Captions are generated word-by-word, where at each step *t* we use
-the set of sentences already generated with length *t* - 1 to generate a new set
-of sentences with length *t*. We keep only the top *k* candidates at each step,
-where the hyperparameter *k* is called the *beam size*. We have found the best
-performance with *k* = 3.
-
-## Getting Started
-
-### A Note on Hardware and Training Time
-
-The time required to train the *Show and Tell* model depends on your specific
-hardware and computational capacity. In this guide we assume you will be running
-training on a single machine with a GPU. In our experience on an NVIDIA Tesla
-K20m GPU the initial training phase takes 1-2 weeks. The second training phase
-may take several additional weeks to achieve peak performance (but you can stop
-this phase early and still get reasonable results).
-
-It is possible to achieve a speed-up by implementing distributed training across
-a cluster of machines with GPUs, but that is not covered in this guide.
-
-Whilst it is possible to run this code on a CPU, beware that this may be
-approximately 10 times slower.
-
-### Install Required Packages
-First ensure that you have installed the following required packages:
-
-* **Bazel** ([instructions](http://bazel.io/docs/install.html))
-* **Python 2.7**
-* **TensorFlow** 1.0 or greater ([instructions](https://www.tensorflow.org/install/))
-* **NumPy** ([instructions](http://www.scipy.org/install.html))
-* **Natural Language Toolkit (NLTK)**:
- * First install NLTK ([instructions](http://www.nltk.org/install.html))
- * Then install the NLTK data package "punkt" ([instructions](http://www.nltk.org/data.html))
-* **Unzip**
-### Prepare the Training Data
-
-To train the model you will need to provide training data in native TFRecord
-format. The TFRecord format consists of a set of sharded files containing
-serialized `tf.SequenceExample` protocol buffers. Each `tf.SequenceExample`
-proto contains an image (JPEG format), a caption and metadata such as the image
-id.
-
-Each caption is a list of words. During preprocessing, a dictionary is created
-that assigns each word in the vocabulary to an integer-valued id. Each caption
-is encoded as a list of integer word ids in the `tf.SequenceExample` protos.
-
-We have provided a script to download and preprocess the [MSCOCO](http://mscoco.org/) image captioning data set into this format. Downloading
-and preprocessing the data may take several hours depending on your network and
-computer speed. Please be patient.
-
-Before running the script, ensure that your hard disk has at least 150GB of
-available space for storing the downloaded and processed data.
-
-```shell
-# Location to save the MSCOCO data.
-MSCOCO_DIR="${HOME}/im2txt/data/mscoco"
-
-# Build the preprocessing script.
-cd research/im2txt
-bazel build //im2txt:download_and_preprocess_mscoco
-
-# Run the preprocessing script.
-bazel-bin/im2txt/download_and_preprocess_mscoco "${MSCOCO_DIR}"
-```
-
-The final line of the output should read:
-
-```
-2016-09-01 16:47:47.296630: Finished processing all 20267 image-caption pairs in data set 'test'.
-```
-
-When the script finishes you will find 256 training, 4 validation and 8 testing
-files in `DATA_DIR`. The files will match the patterns `train-?????-of-00256`,
-`val-?????-of-00004` and `test-?????-of-00008`, respectively.
-
-### Download the Inception v3 Checkpoint
-
-The *Show and Tell* model requires a pretrained *Inception v3* checkpoint file
-to initialize the parameters of its image encoder submodel.
-
-This checkpoint file is provided by the
-[TensorFlow-Slim image classification library](https://github.com/tensorflow/models/tree/master/research/slim#tensorflow-slim-image-classification-library)
-which provides a suite of pre-trained image classification models. You can read
-more about the models provided by the library
-[here](https://github.com/tensorflow/models/tree/master/research/slim#pre-trained-models).
-
-
-Run the following commands to download the *Inception v3* checkpoint.
-
-```shell
-# Location to save the Inception v3 checkpoint.
-INCEPTION_DIR="${HOME}/im2txt/data"
-mkdir -p ${INCEPTION_DIR}
-
-wget "http://download.tensorflow.org/models/inception_v3_2016_08_28.tar.gz"
-tar -xvf "inception_v3_2016_08_28.tar.gz" -C ${INCEPTION_DIR}
-rm "inception_v3_2016_08_28.tar.gz"
-```
-
-Note that the *Inception v3* checkpoint will only be used for initializing the
-parameters of the *Show and Tell* model. Once the *Show and Tell* model starts
-training it will save its own checkpoint files containing the values of all its
-parameters (including copies of the *Inception v3* parameters). If training is
-stopped and restarted, the parameter values will be restored from the latest
-*Show and Tell* checkpoint and the *Inception v3* checkpoint will be ignored. In
-other words, the *Inception v3* checkpoint is only used in the 0-th global step
-(initialization) of training the *Show and Tell* model.
-
-## Training a Model
-
-### Initial Training
-
-Run the training script.
-
-```shell
-# Directory containing preprocessed MSCOCO data.
-MSCOCO_DIR="${HOME}/im2txt/data/mscoco"
-
-# Inception v3 checkpoint file.
-INCEPTION_CHECKPOINT="${HOME}/im2txt/data/inception_v3.ckpt"
-
-# Directory to save the model.
-MODEL_DIR="${HOME}/im2txt/model"
-
-# Build the model.
-cd research/im2txt
-bazel build -c opt //im2txt/...
-
-# Run the training script.
-bazel-bin/im2txt/train \
- --input_file_pattern="${MSCOCO_DIR}/train-?????-of-00256" \
- --inception_checkpoint_file="${INCEPTION_CHECKPOINT}" \
- --train_dir="${MODEL_DIR}/train" \
- --train_inception=false \
- --number_of_steps=1000000
-```
-
-Run the evaluation script in a separate process. This will log evaluation
-metrics to TensorBoard which allows training progress to be monitored in
-real-time.
-
-Note that you may run out of memory if you run the evaluation script on the same
-GPU as the training script. You can run the command
-`export CUDA_VISIBLE_DEVICES=""` to force the evaluation script to run on CPU.
-If evaluation runs too slowly on CPU, you can decrease the value of
-`--num_eval_examples`.
-
-```shell
-MSCOCO_DIR="${HOME}/im2txt/data/mscoco"
-MODEL_DIR="${HOME}/im2txt/model"
-
-# Ignore GPU devices (only necessary if your GPU is currently memory
-# constrained, for example, by running the training script).
-export CUDA_VISIBLE_DEVICES=""
-
-# Run the evaluation script. This will run in a loop, periodically loading the
-# latest model checkpoint file and computing evaluation metrics.
-bazel-bin/im2txt/evaluate \
- --input_file_pattern="${MSCOCO_DIR}/val-?????-of-00004" \
- --checkpoint_dir="${MODEL_DIR}/train" \
- --eval_dir="${MODEL_DIR}/eval"
-```
-
-Run a TensorBoard server in a separate process for real-time monitoring of
-training progress and evaluation metrics.
-
-```shell
-MODEL_DIR="${HOME}/im2txt/model"
-
-# Run a TensorBoard server.
-tensorboard --logdir="${MODEL_DIR}"
-```
-
-### Fine Tune the Inception v3 Model
-
-Your model will already be able to generate reasonable captions after the first
-phase of training. Try it out! (See [Generating Captions](#generating-captions)).
-
-You can further improve the performance of the model by running a
-second training phase to jointly fine-tune the parameters of the *Inception v3*
-image submodel and the LSTM.
-
-```shell
-# Restart the training script with --train_inception=true.
-bazel-bin/im2txt/train \
- --input_file_pattern="${MSCOCO_DIR}/train-?????-of-00256" \
- --train_dir="${MODEL_DIR}/train" \
- --train_inception=true \
- --number_of_steps=3000000 # Additional 2M steps (assuming 1M in initial training).
-```
-
-Note that training will proceed much slower now, and the model will continue to
-improve by a small amount for a long time. We have found that it will improve
-slowly for an additional 2-2.5 million steps before it begins to overfit. This
-may take several weeks on a single GPU. If you don't care about absolutely
-optimal performance then feel free to halt training sooner by stopping the
-training script or passing a smaller value to the flag `--number_of_steps`. Your
-model will still work reasonably well.
-
-## Generating Captions
-
-Your trained *Show and Tell* model can generate captions for any JPEG image! The
-following command line will generate captions for an image from the test set.
-
-```shell
-# Path to checkpoint file or a directory containing checkpoint files. Passing
-# a directory will only work if there is also a file named 'checkpoint' which
-# lists the available checkpoints in the directory. It will not work if you
-# point to a directory with just a copy of a model checkpoint: in that case,
-# you will need to pass the checkpoint path explicitly.
-CHECKPOINT_PATH="${HOME}/im2txt/model/train"
-
-# Vocabulary file generated by the preprocessing script.
-VOCAB_FILE="${HOME}/im2txt/data/mscoco/word_counts.txt"
-
-# JPEG image file to caption.
-IMAGE_FILE="${HOME}/im2txt/data/mscoco/raw-data/val2014/COCO_val2014_000000224477.jpg"
-
-# Build the inference binary.
-cd research/im2txt
-bazel build -c opt //im2txt:run_inference
-
-# Ignore GPU devices (only necessary if your GPU is currently memory
-# constrained, for example, by running the training script).
-export CUDA_VISIBLE_DEVICES=""
-
-# Run inference to generate captions.
-bazel-bin/im2txt/run_inference \
- --checkpoint_path=${CHECKPOINT_PATH} \
- --vocab_file=${VOCAB_FILE} \
- --input_files=${IMAGE_FILE}
-```
-
-Example output:
-
-```
-Captions for image COCO_val2014_000000224477.jpg:
- 0) a man riding a wave on top of a surfboard . (p=0.040413)
- 1) a person riding a surf board on a wave (p=0.017452)
- 2) a man riding a wave on a surfboard in the ocean . (p=0.005743)
-```
-
-Note: you may get different results. Some variation between different models is
-expected.
-
-Here is the image:
-
-
diff --git a/research/im2txt/WORKSPACE b/research/im2txt/WORKSPACE
deleted file mode 100644
index 22da718b06f9c61be4ffdf45e48919ed4a5f17ae..0000000000000000000000000000000000000000
--- a/research/im2txt/WORKSPACE
+++ /dev/null
@@ -1 +0,0 @@
-workspace(name = "im2txt")
diff --git a/research/im2txt/conda-env/ubuntu-18-04-environment.yaml b/research/im2txt/conda-env/ubuntu-18-04-environment.yaml
deleted file mode 100644
index 332ff2a47f8f49fcdde7b769c29ff84cf5a5ff9d..0000000000000000000000000000000000000000
--- a/research/im2txt/conda-env/ubuntu-18-04-environment.yaml
+++ /dev/null
@@ -1,142 +0,0 @@
-name: im2txt
-channels:
- - defaults
-dependencies:
- - _tflow_select=2.3.0=mkl
- - absl-py=0.5.0=py27_0
- - astor=0.7.1=py27_0
- - backports=1.0=py27_1
- - backports.functools_lru_cache=1.5=py27_1
- - backports.shutil_get_terminal_size=1.0.0=py27_2
- - backports.weakref=1.0.post1=py27_0
- - backports_abc=0.5=py27_0
- - blas=1.0=mkl
- - bleach=3.0.2=py27_0
- - ca-certificates=2018.03.07=0
- - certifi=2018.10.15=py27_0
- - configparser=3.5.0=py27_0
- - cycler=0.10.0=py27_0
- - dbus=1.13.2=h714fa37_1
- - decorator=4.3.0=py27_0
- - entrypoints=0.2.3=py27_2
- - enum34=1.1.6=py27_1
- - expat=2.2.6=he6710b0_0
- - fastcache=1.0.2=py27h14c3975_2
- - fontconfig=2.13.0=h9420a91_0
- - freetype=2.9.1=h8a8886c_1
- - funcsigs=1.0.2=py27_0
- - functools32=3.2.3.2=py27_1
- - futures=3.2.0=py27_0
- - gast=0.2.0=py27_0
- - glib=2.56.2=hd408876_0
- - gmp=6.1.2=h6c8ec71_1
- - gmpy2=2.0.8=py27h10f8cd9_2
- - grpcio=1.12.1=py27hdbcaa40_0
- - gst-plugins-base=1.14.0=hbbd80ab_1
- - gstreamer=1.14.0=hb453b48_1
- - h5py=2.8.0=py27h989c5e5_3
- - hdf5=1.10.2=hba1933b_1
- - icu=58.2=h9c2bf20_1
- - intel-openmp=2019.0=118
- - ipaddress=1.0.22=py27_0
- - ipykernel=4.10.0=py27_0
- - ipython=5.8.0=py27_0
- - ipython_genutils=0.2.0=py27_0
- - ipywidgets=7.4.2=py27_0
- - jinja2=2.10=py27_0
- - jpeg=9b=h024ee3a_2
- - jsonschema=2.6.0=py27_0
- - jupyter=1.0.0=py27_7
- - jupyter_client=5.2.3=py27_0
- - jupyter_console=5.2.0=py27_1
- - jupyter_core=4.4.0=py27_0
- - keras-applications=1.0.6=py27_0
- - keras-preprocessing=1.0.5=py27_0
- - kiwisolver=1.0.1=py27hf484d3e_0
- - libedit=3.1.20170329=h6b74fdf_2
- - libffi=3.2.1=hd88cf55_4
- - libgcc-ng=8.2.0=hdf63c60_1
- - libgfortran-ng=7.3.0=hdf63c60_0
- - libpng=1.6.35=hbc83047_0
- - libprotobuf=3.6.0=hdbcaa40_0
- - libsodium=1.0.16=h1bed415_0
- - libstdcxx-ng=8.2.0=hdf63c60_1
- - libuuid=1.0.3=h1bed415_2
- - libxcb=1.13=h1bed415_1
- - libxml2=2.9.8=h26e45fe_1
- - linecache2=1.0.0=py27_0
- - markdown=3.0.1=py27_0
- - markupsafe=1.0=py27h14c3975_1
- - matplotlib=2.2.3=py27hb69df0a_0
- - mistune=0.8.4=py27h7b6447c_0
- - mkl=2019.0=118
- - mkl_fft=1.0.6=py27h7dd41cf_0
- - mkl_random=1.0.1=py27h4414c95_1
- - mock=2.0.0=py27_0
- - mpc=1.1.0=h10f8cd9_1
- - mpfr=4.0.1=hdf1c602_3
- - mpmath=1.0.0=py27_2
- - nbconvert=5.3.1=py27_0
- - nbformat=4.4.0=py27_0
- - ncurses=6.1=hf484d3e_0
- - nltk=3.3.0=py27_0
- - nose=1.3.7=py27_2
- - notebook=5.7.0=py27_0
- - numpy=1.15.3=py27h1d66e8a_0
- - numpy-base=1.15.3=py27h81de0dd_0
- - openssl=1.0.2p=h14c3975_0
- - pandas=0.23.4=py27h04863e7_0
- - pandoc=2.2.3.2=0
- - pandocfilters=1.4.2=py27_1
- - pathlib2=2.3.2=py27_0
- - pbr=4.3.0=py27_0
- - pcre=8.42=h439df22_0
- - pexpect=4.6.0=py27_0
- - pickleshare=0.7.5=py27_0
- - pip=10.0.1=py27_0
- - prometheus_client=0.4.2=py27_0
- - prompt_toolkit=1.0.15=py27_0
- - protobuf=3.6.0=py27hf484d3e_0
- - ptyprocess=0.6.0=py27_0
- - pygments=2.2.0=py27_0
- - pyparsing=2.2.2=py27_0
- - pyqt=5.9.2=py27h05f1152_2
- - python=2.7.15=h77bded6_2
- - python-dateutil=2.7.3=py27_0
- - pytz=2018.5=py27_0
- - pyzmq=17.1.2=py27h14c3975_0
- - qt=5.9.6=h8703b6f_2
- - qtconsole=4.4.2=py27_0
- - readline=7.0=h7b6447c_5
- - scandir=1.9.0=py27h14c3975_0
- - scipy=1.1.0=py27hfa4b5c9_1
- - send2trash=1.5.0=py27_0
- - setuptools=40.4.3=py27_0
- - simplegeneric=0.8.1=py27_2
- - singledispatch=3.4.0.3=py27_0
- - sip=4.19.8=py27hf484d3e_0
- - six=1.11.0=py27_1
- - sqlite=3.25.2=h7b6447c_0
- - subprocess32=3.5.3=py27h7b6447c_0
- - sympy=1.3=py27_0
- - tensorboard=1.11.0=py27hf484d3e_0
- - tensorflow=1.11.0=mkl_py27h25e0b76_0
- - tensorflow-base=1.11.0=mkl_py27h3c3e929_0
- - termcolor=1.1.0=py27_1
- - terminado=0.8.1=py27_1
- - testpath=0.4.2=py27_0
- - tk=8.6.8=hbc83047_0
- - tornado=5.1.1=py27h7b6447c_0
- - traceback2=1.4.0=py27_0
- - traitlets=4.3.2=py27_0
- - unittest2=1.1.0=py27_0
- - wcwidth=0.1.7=py27_0
- - webencodings=0.5.1=py27_1
- - werkzeug=0.14.1=py27_0
- - wheel=0.32.2=py27_0
- - widgetsnbextension=3.4.2=py27_0
- - xz=5.2.4=h14c3975_4
- - zeromq=4.2.5=hf484d3e_1
- - zlib=1.2.11=ha838bed_2
-prefix: /home/arinto_murdopo/anaconda3/envs/im2txt
-
diff --git a/research/im2txt/g3doc/COCO_val2014_000000224477.jpg b/research/im2txt/g3doc/COCO_val2014_000000224477.jpg
deleted file mode 100644
index 8976fa84b40b04c5bf1205a49c8d236b747f8f9b..0000000000000000000000000000000000000000
Binary files a/research/im2txt/g3doc/COCO_val2014_000000224477.jpg and /dev/null differ
diff --git a/research/im2txt/g3doc/example_captions.jpg b/research/im2txt/g3doc/example_captions.jpg
deleted file mode 100644
index b3a8f43247e5c9c39a3f93daaf1ad34837959ec5..0000000000000000000000000000000000000000
Binary files a/research/im2txt/g3doc/example_captions.jpg and /dev/null differ
diff --git a/research/im2txt/g3doc/show_and_tell_architecture.png b/research/im2txt/g3doc/show_and_tell_architecture.png
deleted file mode 100644
index 984590d54ba4aa089b5740fd69f6dc6216b9047f..0000000000000000000000000000000000000000
Binary files a/research/im2txt/g3doc/show_and_tell_architecture.png and /dev/null differ
diff --git a/research/im2txt/im2txt/BUILD b/research/im2txt/im2txt/BUILD
deleted file mode 100644
index 8c403171153c36ee43cde2788dbfcaf9c7bf4293..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/BUILD
+++ /dev/null
@@ -1,96 +0,0 @@
-package(default_visibility = [":internal"])
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-package_group(
- name = "internal",
- packages = [
- "//im2txt/...",
- ],
-)
-
-py_binary(
- name = "build_mscoco_data",
- srcs = [
- "data/build_mscoco_data.py",
- ],
-)
-
-sh_binary(
- name = "download_and_preprocess_mscoco",
- srcs = ["data/download_and_preprocess_mscoco.sh"],
- data = [
- ":build_mscoco_data",
- ],
-)
-
-py_library(
- name = "configuration",
- srcs = ["configuration.py"],
- srcs_version = "PY2AND3",
-)
-
-py_library(
- name = "show_and_tell_model",
- srcs = ["show_and_tell_model.py"],
- srcs_version = "PY2AND3",
- deps = [
- "//im2txt/ops:image_embedding",
- "//im2txt/ops:image_processing",
- "//im2txt/ops:inputs",
- ],
-)
-
-py_test(
- name = "show_and_tell_model_test",
- size = "large",
- srcs = ["show_and_tell_model_test.py"],
- deps = [
- ":configuration",
- ":show_and_tell_model",
- ],
-)
-
-py_library(
- name = "inference_wrapper",
- srcs = ["inference_wrapper.py"],
- srcs_version = "PY2AND3",
- deps = [
- ":show_and_tell_model",
- "//im2txt/inference_utils:inference_wrapper_base",
- ],
-)
-
-py_binary(
- name = "train",
- srcs = ["train.py"],
- srcs_version = "PY2AND3",
- deps = [
- ":configuration",
- ":show_and_tell_model",
- ],
-)
-
-py_binary(
- name = "evaluate",
- srcs = ["evaluate.py"],
- srcs_version = "PY2AND3",
- deps = [
- ":configuration",
- ":show_and_tell_model",
- ],
-)
-
-py_binary(
- name = "run_inference",
- srcs = ["run_inference.py"],
- srcs_version = "PY2AND3",
- deps = [
- ":configuration",
- ":inference_wrapper",
- "//im2txt/inference_utils:caption_generator",
- "//im2txt/inference_utils:vocabulary",
- ],
-)
diff --git a/research/im2txt/im2txt/configuration.py b/research/im2txt/im2txt/configuration.py
deleted file mode 100644
index 3b664eb9f0cd963fb26929d019ec9cdb3282d0a8..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/configuration.py
+++ /dev/null
@@ -1,104 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Image-to-text model and training configurations."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-class ModelConfig(object):
- """Wrapper class for model hyperparameters."""
-
- def __init__(self):
- """Sets the default model hyperparameters."""
- # File pattern of sharded TFRecord file containing SequenceExample protos.
- # Must be provided in training and evaluation modes.
- self.input_file_pattern = None
-
- # Image format ("jpeg" or "png").
- self.image_format = "jpeg"
-
- # Approximate number of values per input shard. Used to ensure sufficient
- # mixing between shards in training.
- self.values_per_input_shard = 2300
- # Minimum number of shards to keep in the input queue.
- self.input_queue_capacity_factor = 2
- # Number of threads for prefetching SequenceExample protos.
- self.num_input_reader_threads = 1
-
- # Name of the SequenceExample context feature containing image data.
- self.image_feature_name = "image/data"
- # Name of the SequenceExample feature list containing integer captions.
- self.caption_feature_name = "image/caption_ids"
-
- # Number of unique words in the vocab (plus 1, for ).
- # The default value is larger than the expected actual vocab size to allow
- # for differences between tokenizer versions used in preprocessing. There is
- # no harm in using a value greater than the actual vocab size, but using a
- # value less than the actual vocab size will result in an error.
- self.vocab_size = 12000
-
- # Number of threads for image preprocessing. Should be a multiple of 2.
- self.num_preprocess_threads = 4
-
- # Batch size.
- self.batch_size = 32
-
- # File containing an Inception v3 checkpoint to initialize the variables
- # of the Inception model. Must be provided when starting training for the
- # first time.
- self.inception_checkpoint_file = None
-
- # Dimensions of Inception v3 input images.
- self.image_height = 299
- self.image_width = 299
-
- # Scale used to initialize model variables.
- self.initializer_scale = 0.08
-
- # LSTM input and output dimensionality, respectively.
- self.embedding_size = 512
- self.num_lstm_units = 512
-
- # If < 1.0, the dropout keep probability applied to LSTM variables.
- self.lstm_dropout_keep_prob = 0.7
-
-
-class TrainingConfig(object):
- """Wrapper class for training hyperparameters."""
-
- def __init__(self):
- """Sets the default training hyperparameters."""
- # Number of examples per epoch of training data.
- self.num_examples_per_epoch = 586363
-
- # Optimizer for training the model.
- self.optimizer = "SGD"
-
- # Learning rate for the initial phase of training.
- self.initial_learning_rate = 2.0
- self.learning_rate_decay_factor = 0.5
- self.num_epochs_per_decay = 8.0
-
- # Learning rate when fine tuning the Inception v3 parameters.
- self.train_inception_learning_rate = 0.0005
-
- # If not None, clip gradients to this value.
- self.clip_gradients = 5.0
-
- # How many model checkpoints to keep.
- self.max_checkpoints_to_keep = 5
diff --git a/research/im2txt/im2txt/data/build_mscoco_data.py b/research/im2txt/im2txt/data/build_mscoco_data.py
deleted file mode 100644
index 2c3e9d977669bf63d8e39128336319b48c0432dd..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/data/build_mscoco_data.py
+++ /dev/null
@@ -1,483 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Converts MSCOCO data to TFRecord file format with SequenceExample protos.
-
-The MSCOCO images are expected to reside in JPEG files located in the following
-directory structure:
-
- train_image_dir/COCO_train2014_000000000151.jpg
- train_image_dir/COCO_train2014_000000000260.jpg
- ...
-
-and
-
- val_image_dir/COCO_val2014_000000000042.jpg
- val_image_dir/COCO_val2014_000000000073.jpg
- ...
-
-The MSCOCO annotations JSON files are expected to reside in train_captions_file
-and val_captions_file respectively.
-
-This script converts the combined MSCOCO data into sharded data files consisting
-of 256, 4 and 8 TFRecord files, respectively:
-
- output_dir/train-00000-of-00256
- output_dir/train-00001-of-00256
- ...
- output_dir/train-00255-of-00256
-
-and
-
- output_dir/val-00000-of-00004
- ...
- output_dir/val-00003-of-00004
-
-and
-
- output_dir/test-00000-of-00008
- ...
- output_dir/test-00007-of-00008
-
-Each TFRecord file contains ~2300 records. Each record within the TFRecord file
-is a serialized SequenceExample proto consisting of precisely one image-caption
-pair. Note that each image has multiple captions (usually 5) and therefore each
-image is replicated multiple times in the TFRecord files.
-
-The SequenceExample proto contains the following fields:
-
- context:
- image/image_id: integer MSCOCO image identifier
- image/data: string containing JPEG encoded image in RGB colorspace
-
- feature_lists:
- image/caption: list of strings containing the (tokenized) caption words
- image/caption_ids: list of integer ids corresponding to the caption words
-
-The captions are tokenized using the NLTK (http://www.nltk.org/) word tokenizer.
-The vocabulary of word identifiers is constructed from the sorted list (by
-descending frequency) of word tokens in the training set. Only tokens appearing
-at least 4 times are considered; all other words get the "unknown" word id.
-
-NOTE: This script will consume around 100GB of disk space because each image
-in the MSCOCO dataset is replicated ~5 times (once per caption) in the output.
-This is done for two reasons:
- 1. In order to better shuffle the training data.
- 2. It makes it easier to perform asynchronous preprocessing of each image in
- TensorFlow.
-
-Running this script using 16 threads may take around 1 hour on a HP Z420.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import Counter
-from collections import namedtuple
-from datetime import datetime
-import json
-import os.path
-import random
-import sys
-import threading
-
-
-
-import nltk.tokenize
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-tf.flags.DEFINE_string("train_image_dir", "/tmp/train2014/",
- "Training image directory.")
-tf.flags.DEFINE_string("val_image_dir", "/tmp/val2014",
- "Validation image directory.")
-
-tf.flags.DEFINE_string("train_captions_file", "/tmp/captions_train2014.json",
- "Training captions JSON file.")
-tf.flags.DEFINE_string("val_captions_file", "/tmp/captions_val2014.json",
- "Validation captions JSON file.")
-
-tf.flags.DEFINE_string("output_dir", "/tmp/", "Output data directory.")
-
-tf.flags.DEFINE_integer("train_shards", 256,
- "Number of shards in training TFRecord files.")
-tf.flags.DEFINE_integer("val_shards", 4,
- "Number of shards in validation TFRecord files.")
-tf.flags.DEFINE_integer("test_shards", 8,
- "Number of shards in testing TFRecord files.")
-
-tf.flags.DEFINE_string("start_word", "",
- "Special word added to the beginning of each sentence.")
-tf.flags.DEFINE_string("end_word", "",
- "Special word added to the end of each sentence.")
-tf.flags.DEFINE_string("unknown_word", "",
- "Special word meaning 'unknown'.")
-tf.flags.DEFINE_integer("min_word_count", 4,
- "The minimum number of occurrences of each word in the "
- "training set for inclusion in the vocabulary.")
-tf.flags.DEFINE_string("word_counts_output_file", "/tmp/word_counts.txt",
- "Output vocabulary file of word counts.")
-
-tf.flags.DEFINE_integer("num_threads", 8,
- "Number of threads to preprocess the images.")
-
-FLAGS = tf.flags.FLAGS
-
-ImageMetadata = namedtuple("ImageMetadata",
- ["image_id", "filename", "captions"])
-
-
-class Vocabulary(object):
- """Simple vocabulary wrapper."""
-
- def __init__(self, vocab, unk_id):
- """Initializes the vocabulary.
-
- Args:
- vocab: A dictionary of word to word_id.
- unk_id: Id of the special 'unknown' word.
- """
- self._vocab = vocab
- self._unk_id = unk_id
-
- def word_to_id(self, word):
- """Returns the integer id of a word string."""
- if word in self._vocab:
- return self._vocab[word]
- else:
- return self._unk_id
-
-
-class ImageDecoder(object):
- """Helper class for decoding images in TensorFlow."""
-
- def __init__(self):
- # Create a single TensorFlow Session for all image decoding calls.
- self._sess = tf.Session()
-
- # TensorFlow ops for JPEG decoding.
- self._encoded_jpeg = tf.placeholder(dtype=tf.string)
- self._decode_jpeg = tf.image.decode_jpeg(self._encoded_jpeg, channels=3)
-
- def decode_jpeg(self, encoded_jpeg):
- image = self._sess.run(self._decode_jpeg,
- feed_dict={self._encoded_jpeg: encoded_jpeg})
- assert len(image.shape) == 3
- assert image.shape[2] == 3
- return image
-
-
-def _int64_feature(value):
- """Wrapper for inserting an int64 Feature into a SequenceExample proto."""
- return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-
-
-def _bytes_feature(value):
- """Wrapper for inserting a bytes Feature into a SequenceExample proto."""
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[str(value)]))
-
-
-def _int64_feature_list(values):
- """Wrapper for inserting an int64 FeatureList into a SequenceExample proto."""
- return tf.train.FeatureList(feature=[_int64_feature(v) for v in values])
-
-
-def _bytes_feature_list(values):
- """Wrapper for inserting a bytes FeatureList into a SequenceExample proto."""
- return tf.train.FeatureList(feature=[_bytes_feature(v) for v in values])
-
-
-def _to_sequence_example(image, decoder, vocab):
- """Builds a SequenceExample proto for an image-caption pair.
-
- Args:
- image: An ImageMetadata object.
- decoder: An ImageDecoder object.
- vocab: A Vocabulary object.
-
- Returns:
- A SequenceExample proto.
- """
- with tf.gfile.FastGFile(image.filename, "r") as f:
- encoded_image = f.read()
-
- try:
- decoder.decode_jpeg(encoded_image)
- except (tf.errors.InvalidArgumentError, AssertionError):
- print("Skipping file with invalid JPEG data: %s" % image.filename)
- return
-
- context = tf.train.Features(feature={
- "image/image_id": _int64_feature(image.image_id),
- "image/data": _bytes_feature(encoded_image),
- })
-
- assert len(image.captions) == 1
- caption = image.captions[0]
- caption_ids = [vocab.word_to_id(word) for word in caption]
- feature_lists = tf.train.FeatureLists(feature_list={
- "image/caption": _bytes_feature_list(caption),
- "image/caption_ids": _int64_feature_list(caption_ids)
- })
- sequence_example = tf.train.SequenceExample(
- context=context, feature_lists=feature_lists)
-
- return sequence_example
-
-
-def _process_image_files(thread_index, ranges, name, images, decoder, vocab,
- num_shards):
- """Processes and saves a subset of images as TFRecord files in one thread.
-
- Args:
- thread_index: Integer thread identifier within [0, len(ranges)].
- ranges: A list of pairs of integers specifying the ranges of the dataset to
- process in parallel.
- name: Unique identifier specifying the dataset.
- images: List of ImageMetadata.
- decoder: An ImageDecoder object.
- vocab: A Vocabulary object.
- num_shards: Integer number of shards for the output files.
- """
- # Each thread produces N shards where N = num_shards / num_threads. For
- # instance, if num_shards = 128, and num_threads = 2, then the first thread
- # would produce shards [0, 64).
- num_threads = len(ranges)
- assert not num_shards % num_threads
- num_shards_per_batch = int(num_shards / num_threads)
-
- shard_ranges = np.linspace(ranges[thread_index][0], ranges[thread_index][1],
- num_shards_per_batch + 1).astype(int)
- num_images_in_thread = ranges[thread_index][1] - ranges[thread_index][0]
-
- counter = 0
- for s in xrange(num_shards_per_batch):
- # Generate a sharded version of the file name, e.g. 'train-00002-of-00010'
- shard = thread_index * num_shards_per_batch + s
- output_filename = "%s-%.5d-of-%.5d" % (name, shard, num_shards)
- output_file = os.path.join(FLAGS.output_dir, output_filename)
- writer = tf.python_io.TFRecordWriter(output_file)
-
- shard_counter = 0
- images_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int)
- for i in images_in_shard:
- image = images[i]
-
- sequence_example = _to_sequence_example(image, decoder, vocab)
- if sequence_example is not None:
- writer.write(sequence_example.SerializeToString())
- shard_counter += 1
- counter += 1
-
- if not counter % 1000:
- print("%s [thread %d]: Processed %d of %d items in thread batch." %
- (datetime.now(), thread_index, counter, num_images_in_thread))
- sys.stdout.flush()
-
- writer.close()
- print("%s [thread %d]: Wrote %d image-caption pairs to %s" %
- (datetime.now(), thread_index, shard_counter, output_file))
- sys.stdout.flush()
- shard_counter = 0
- print("%s [thread %d]: Wrote %d image-caption pairs to %d shards." %
- (datetime.now(), thread_index, counter, num_shards_per_batch))
- sys.stdout.flush()
-
-
-def _process_dataset(name, images, vocab, num_shards):
- """Processes a complete data set and saves it as a TFRecord.
-
- Args:
- name: Unique identifier specifying the dataset.
- images: List of ImageMetadata.
- vocab: A Vocabulary object.
- num_shards: Integer number of shards for the output files.
- """
- # Break up each image into a separate entity for each caption.
- images = [ImageMetadata(image.image_id, image.filename, [caption])
- for image in images for caption in image.captions]
-
- # Shuffle the ordering of images. Make the randomization repeatable.
- random.seed(12345)
- random.shuffle(images)
-
- # Break the images into num_threads batches. Batch i is defined as
- # images[ranges[i][0]:ranges[i][1]].
- num_threads = min(num_shards, FLAGS.num_threads)
- spacing = np.linspace(0, len(images), num_threads + 1).astype(np.int)
- ranges = []
- threads = []
- for i in xrange(len(spacing) - 1):
- ranges.append([spacing[i], spacing[i + 1]])
-
- # Create a mechanism for monitoring when all threads are finished.
- coord = tf.train.Coordinator()
-
- # Create a utility for decoding JPEG images to run sanity checks.
- decoder = ImageDecoder()
-
- # Launch a thread for each batch.
- print("Launching %d threads for spacings: %s" % (num_threads, ranges))
- for thread_index in xrange(len(ranges)):
- args = (thread_index, ranges, name, images, decoder, vocab, num_shards)
- t = threading.Thread(target=_process_image_files, args=args)
- t.start()
- threads.append(t)
-
- # Wait for all the threads to terminate.
- coord.join(threads)
- print("%s: Finished processing all %d image-caption pairs in data set '%s'." %
- (datetime.now(), len(images), name))
-
-
-def _create_vocab(captions):
- """Creates the vocabulary of word to word_id.
-
- The vocabulary is saved to disk in a text file of word counts. The id of each
- word in the file is its corresponding 0-based line number.
-
- Args:
- captions: A list of lists of strings.
-
- Returns:
- A Vocabulary object.
- """
- print("Creating vocabulary.")
- counter = Counter()
- for c in captions:
- counter.update(c)
- print("Total words:", len(counter))
-
- # Filter uncommon words and sort by descending count.
- word_counts = [x for x in counter.items() if x[1] >= FLAGS.min_word_count]
- word_counts.sort(key=lambda x: x[1], reverse=True)
- print("Words in vocabulary:", len(word_counts))
-
- # Write out the word counts file.
- with tf.gfile.FastGFile(FLAGS.word_counts_output_file, "w") as f:
- f.write("\n".join(["%s %d" % (w, c) for w, c in word_counts]))
- print("Wrote vocabulary file:", FLAGS.word_counts_output_file)
-
- # Create the vocabulary dictionary.
- reverse_vocab = [x[0] for x in word_counts]
- unk_id = len(reverse_vocab)
- vocab_dict = dict([(x, y) for (y, x) in enumerate(reverse_vocab)])
- vocab = Vocabulary(vocab_dict, unk_id)
-
- return vocab
-
-
-def _process_caption(caption):
- """Processes a caption string into a list of tonenized words.
-
- Args:
- caption: A string caption.
-
- Returns:
- A list of strings; the tokenized caption.
- """
- tokenized_caption = [FLAGS.start_word]
- tokenized_caption.extend(nltk.tokenize.word_tokenize(caption.lower()))
- tokenized_caption.append(FLAGS.end_word)
- return tokenized_caption
-
-
-def _load_and_process_metadata(captions_file, image_dir):
- """Loads image metadata from a JSON file and processes the captions.
-
- Args:
- captions_file: JSON file containing caption annotations.
- image_dir: Directory containing the image files.
-
- Returns:
- A list of ImageMetadata.
- """
- with tf.gfile.FastGFile(captions_file, "r") as f:
- caption_data = json.load(f)
-
- # Extract the filenames.
- id_to_filename = [(x["id"], x["file_name"]) for x in caption_data["images"]]
-
- # Extract the captions. Each image_id is associated with multiple captions.
- id_to_captions = {}
- for annotation in caption_data["annotations"]:
- image_id = annotation["image_id"]
- caption = annotation["caption"]
- id_to_captions.setdefault(image_id, [])
- id_to_captions[image_id].append(caption)
-
- assert len(id_to_filename) == len(id_to_captions)
- assert set([x[0] for x in id_to_filename]) == set(id_to_captions.keys())
- print("Loaded caption metadata for %d images from %s" %
- (len(id_to_filename), captions_file))
-
- # Process the captions and combine the data into a list of ImageMetadata.
- print("Processing captions.")
- image_metadata = []
- num_captions = 0
- for image_id, base_filename in id_to_filename:
- filename = os.path.join(image_dir, base_filename)
- captions = [_process_caption(c) for c in id_to_captions[image_id]]
- image_metadata.append(ImageMetadata(image_id, filename, captions))
- num_captions += len(captions)
- print("Finished processing %d captions for %d images in %s" %
- (num_captions, len(id_to_filename), captions_file))
-
- return image_metadata
-
-
-def main(unused_argv):
- def _is_valid_num_shards(num_shards):
- """Returns True if num_shards is compatible with FLAGS.num_threads."""
- return num_shards < FLAGS.num_threads or not num_shards % FLAGS.num_threads
-
- assert _is_valid_num_shards(FLAGS.train_shards), (
- "Please make the FLAGS.num_threads commensurate with FLAGS.train_shards")
- assert _is_valid_num_shards(FLAGS.val_shards), (
- "Please make the FLAGS.num_threads commensurate with FLAGS.val_shards")
- assert _is_valid_num_shards(FLAGS.test_shards), (
- "Please make the FLAGS.num_threads commensurate with FLAGS.test_shards")
-
- if not tf.gfile.IsDirectory(FLAGS.output_dir):
- tf.gfile.MakeDirs(FLAGS.output_dir)
-
- # Load image metadata from caption files.
- mscoco_train_dataset = _load_and_process_metadata(FLAGS.train_captions_file,
- FLAGS.train_image_dir)
- mscoco_val_dataset = _load_and_process_metadata(FLAGS.val_captions_file,
- FLAGS.val_image_dir)
-
- # Redistribute the MSCOCO data as follows:
- # train_dataset = 100% of mscoco_train_dataset + 85% of mscoco_val_dataset.
- # val_dataset = 5% of mscoco_val_dataset (for validation during training).
- # test_dataset = 10% of mscoco_val_dataset (for final evaluation).
- train_cutoff = int(0.85 * len(mscoco_val_dataset))
- val_cutoff = int(0.90 * len(mscoco_val_dataset))
- train_dataset = mscoco_train_dataset + mscoco_val_dataset[0:train_cutoff]
- val_dataset = mscoco_val_dataset[train_cutoff:val_cutoff]
- test_dataset = mscoco_val_dataset[val_cutoff:]
-
- # Create vocabulary from the training captions.
- train_captions = [c for image in train_dataset for c in image.captions]
- vocab = _create_vocab(train_captions)
-
- _process_dataset("train", train_dataset, vocab, FLAGS.train_shards)
- _process_dataset("val", val_dataset, vocab, FLAGS.val_shards)
- _process_dataset("test", test_dataset, vocab, FLAGS.test_shards)
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/im2txt/im2txt/data/download_and_preprocess_mscoco.sh b/research/im2txt/im2txt/data/download_and_preprocess_mscoco.sh
deleted file mode 100755
index ab3ff28d576adcbf1992de4c00dfa350dd93b1c3..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/data/download_and_preprocess_mscoco.sh
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/bin/bash
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Script to download and preprocess the MSCOCO data set.
-#
-# The outputs of this script are sharded TFRecord files containing serialized
-# SequenceExample protocol buffers. See build_mscoco_data.py for details of how
-# the SequenceExample protocol buffers are constructed.
-#
-# usage:
-# ./download_and_preprocess_mscoco.sh
-set -e
-
-if [ -z "$1" ]; then
- echo "usage download_and_preproces_mscoco.sh [data dir]"
- exit
-fi
-
-if [ "$(uname)" == "Darwin" ]; then
- UNZIP="tar -xf"
-else
- UNZIP="unzip -nq"
-fi
-
-# Create the output directories.
-OUTPUT_DIR="${1%/}"
-SCRATCH_DIR="${OUTPUT_DIR}/raw-data"
-mkdir -p "${OUTPUT_DIR}"
-mkdir -p "${SCRATCH_DIR}"
-CURRENT_DIR=$(pwd)
-WORK_DIR="$0.runfiles/im2txt/im2txt"
-
-# Helper function to download and unpack a .zip file.
-function download_and_unzip() {
- local BASE_URL=${1}
- local FILENAME=${2}
-
- if [ ! -f ${FILENAME} ]; then
- echo "Downloading ${FILENAME} to $(pwd)"
- wget -nd -c "${BASE_URL}/${FILENAME}"
- else
- echo "Skipping download of ${FILENAME}"
- fi
- echo "Unzipping ${FILENAME}"
- ${UNZIP} ${FILENAME}
-}
-
-cd ${SCRATCH_DIR}
-
-# Download the images.
-BASE_IMAGE_URL="http://msvocds.blob.core.windows.net/coco2014"
-
-TRAIN_IMAGE_FILE="train2014.zip"
-download_and_unzip ${BASE_IMAGE_URL} ${TRAIN_IMAGE_FILE}
-TRAIN_IMAGE_DIR="${SCRATCH_DIR}/train2014"
-
-VAL_IMAGE_FILE="val2014.zip"
-download_and_unzip ${BASE_IMAGE_URL} ${VAL_IMAGE_FILE}
-VAL_IMAGE_DIR="${SCRATCH_DIR}/val2014"
-
-# Download the captions.
-BASE_CAPTIONS_URL="http://msvocds.blob.core.windows.net/annotations-1-0-3"
-CAPTIONS_FILE="captions_train-val2014.zip"
-download_and_unzip ${BASE_CAPTIONS_URL} ${CAPTIONS_FILE}
-TRAIN_CAPTIONS_FILE="${SCRATCH_DIR}/annotations/captions_train2014.json"
-VAL_CAPTIONS_FILE="${SCRATCH_DIR}/annotations/captions_val2014.json"
-
-# Build TFRecords of the image data.
-cd "${CURRENT_DIR}"
-BUILD_SCRIPT="${WORK_DIR}/build_mscoco_data"
-"${BUILD_SCRIPT}" \
- --train_image_dir="${TRAIN_IMAGE_DIR}" \
- --val_image_dir="${VAL_IMAGE_DIR}" \
- --train_captions_file="${TRAIN_CAPTIONS_FILE}" \
- --val_captions_file="${VAL_CAPTIONS_FILE}" \
- --output_dir="${OUTPUT_DIR}" \
- --word_counts_output_file="${OUTPUT_DIR}/word_counts.txt" \
diff --git a/research/im2txt/im2txt/evaluate.py b/research/im2txt/im2txt/evaluate.py
deleted file mode 100644
index 0c81a59dab56626cb2c6a19433544f4d239cbd9d..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/evaluate.py
+++ /dev/null
@@ -1,198 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Evaluate the model.
-
-This script should be run concurrently with training so that summaries show up
-in TensorBoard.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-import os.path
-import time
-
-
-import numpy as np
-import tensorflow as tf
-
-from im2txt import configuration
-from im2txt import show_and_tell_model
-
-FLAGS = tf.flags.FLAGS
-
-tf.flags.DEFINE_string("input_file_pattern", "",
- "File pattern of sharded TFRecord input files.")
-tf.flags.DEFINE_string("checkpoint_dir", "",
- "Directory containing model checkpoints.")
-tf.flags.DEFINE_string("eval_dir", "", "Directory to write event logs.")
-
-tf.flags.DEFINE_integer("eval_interval_secs", 600,
- "Interval between evaluation runs.")
-tf.flags.DEFINE_integer("num_eval_examples", 10132,
- "Number of examples for evaluation.")
-
-tf.flags.DEFINE_integer("min_global_step", 5000,
- "Minimum global step to run evaluation.")
-
-tf.logging.set_verbosity(tf.logging.INFO)
-
-
-def evaluate_model(sess, model, global_step, summary_writer, summary_op):
- """Computes perplexity-per-word over the evaluation dataset.
-
- Summaries and perplexity-per-word are written out to the eval directory.
-
- Args:
- sess: Session object.
- model: Instance of ShowAndTellModel; the model to evaluate.
- global_step: Integer; global step of the model checkpoint.
- summary_writer: Instance of FileWriter.
- summary_op: Op for generating model summaries.
- """
- # Log model summaries on a single batch.
- summary_str = sess.run(summary_op)
- summary_writer.add_summary(summary_str, global_step)
-
- # Compute perplexity over the entire dataset.
- num_eval_batches = int(
- math.ceil(FLAGS.num_eval_examples / model.config.batch_size))
-
- start_time = time.time()
- sum_losses = 0.
- sum_weights = 0.
- for i in range(num_eval_batches):
- cross_entropy_losses, weights = sess.run([
- model.target_cross_entropy_losses,
- model.target_cross_entropy_loss_weights
- ])
- sum_losses += np.sum(cross_entropy_losses * weights)
- sum_weights += np.sum(weights)
- if not i % 100:
- tf.logging.info("Computed losses for %d of %d batches.", i + 1,
- num_eval_batches)
- eval_time = time.time() - start_time
-
- perplexity = math.exp(sum_losses / sum_weights)
- tf.logging.info("Perplexity = %f (%.2g sec)", perplexity, eval_time)
-
- # Log perplexity to the FileWriter.
- summary = tf.Summary()
- value = summary.value.add()
- value.simple_value = perplexity
- value.tag = "Perplexity"
- summary_writer.add_summary(summary, global_step)
-
- # Write the Events file to the eval directory.
- summary_writer.flush()
- tf.logging.info("Finished processing evaluation at global step %d.",
- global_step)
-
-
-def run_once(model, saver, summary_writer, summary_op):
- """Evaluates the latest model checkpoint.
-
- Args:
- model: Instance of ShowAndTellModel; the model to evaluate.
- saver: Instance of tf.train.Saver for restoring model Variables.
- summary_writer: Instance of FileWriter.
- summary_op: Op for generating model summaries.
- """
- model_path = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
- if not model_path:
- tf.logging.info("Skipping evaluation. No checkpoint found in: %s",
- FLAGS.checkpoint_dir)
- return
-
- with tf.Session() as sess:
- # Load model from checkpoint.
- tf.logging.info("Loading model from checkpoint: %s", model_path)
- saver.restore(sess, model_path)
- global_step = tf.train.global_step(sess, model.global_step.name)
- tf.logging.info("Successfully loaded %s at global step = %d.",
- os.path.basename(model_path), global_step)
- if global_step < FLAGS.min_global_step:
- tf.logging.info("Skipping evaluation. Global step = %d < %d", global_step,
- FLAGS.min_global_step)
- return
-
- # Start the queue runners.
- coord = tf.train.Coordinator()
- threads = tf.train.start_queue_runners(coord=coord)
-
- # Run evaluation on the latest checkpoint.
- try:
- evaluate_model(
- sess=sess,
- model=model,
- global_step=global_step,
- summary_writer=summary_writer,
- summary_op=summary_op)
- except Exception as e: # pylint: disable=broad-except
- tf.logging.error("Evaluation failed.")
- coord.request_stop(e)
-
- coord.request_stop()
- coord.join(threads, stop_grace_period_secs=10)
-
-
-def run():
- """Runs evaluation in a loop, and logs summaries to TensorBoard."""
- # Create the evaluation directory if it doesn't exist.
- eval_dir = FLAGS.eval_dir
- if not tf.gfile.IsDirectory(eval_dir):
- tf.logging.info("Creating eval directory: %s", eval_dir)
- tf.gfile.MakeDirs(eval_dir)
-
- g = tf.Graph()
- with g.as_default():
- # Build the model for evaluation.
- model_config = configuration.ModelConfig()
- model_config.input_file_pattern = FLAGS.input_file_pattern
- model = show_and_tell_model.ShowAndTellModel(model_config, mode="eval")
- model.build()
-
- # Create the Saver to restore model Variables.
- saver = tf.train.Saver()
-
- # Create the summary operation and the summary writer.
- summary_op = tf.summary.merge_all()
- summary_writer = tf.summary.FileWriter(eval_dir)
-
- g.finalize()
-
- # Run a new evaluation run every eval_interval_secs.
- while True:
- start = time.time()
- tf.logging.info("Starting evaluation at " + time.strftime(
- "%Y-%m-%d-%H:%M:%S", time.localtime()))
- run_once(model, saver, summary_writer, summary_op)
- time_to_next_eval = start + FLAGS.eval_interval_secs - time.time()
- if time_to_next_eval > 0:
- time.sleep(time_to_next_eval)
-
-
-def main(unused_argv):
- assert FLAGS.input_file_pattern, "--input_file_pattern is required"
- assert FLAGS.checkpoint_dir, "--checkpoint_dir is required"
- assert FLAGS.eval_dir, "--eval_dir is required"
- run()
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/im2txt/im2txt/inference_utils/BUILD b/research/im2txt/im2txt/inference_utils/BUILD
deleted file mode 100644
index 82a15fd3ca487e542c41ab337404f8caa63b8c63..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/inference_utils/BUILD
+++ /dev/null
@@ -1,31 +0,0 @@
-package(default_visibility = ["//im2txt:internal"])
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-py_library(
- name = "inference_wrapper_base",
- srcs = ["inference_wrapper_base.py"],
- srcs_version = "PY2AND3",
-)
-
-py_library(
- name = "vocabulary",
- srcs = ["vocabulary.py"],
- srcs_version = "PY2AND3",
-)
-
-py_library(
- name = "caption_generator",
- srcs = ["caption_generator.py"],
- srcs_version = "PY2AND3",
-)
-
-py_test(
- name = "caption_generator_test",
- srcs = ["caption_generator_test.py"],
- deps = [
- ":caption_generator",
- ],
-)
diff --git a/research/im2txt/im2txt/inference_utils/caption_generator.py b/research/im2txt/im2txt/inference_utils/caption_generator.py
deleted file mode 100644
index f158d3d2330e8f839efdad4cbc4d38811b58d826..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/inference_utils/caption_generator.py
+++ /dev/null
@@ -1,213 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Class for generating captions from an image-to-text model."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import heapq
-import math
-
-
-import numpy as np
-
-
-class Caption(object):
- """Represents a complete or partial caption."""
-
- def __init__(self, sentence, state, logprob, score, metadata=None):
- """Initializes the Caption.
-
- Args:
- sentence: List of word ids in the caption.
- state: Model state after generating the previous word.
- logprob: Log-probability of the caption.
- score: Score of the caption.
- metadata: Optional metadata associated with the partial sentence. If not
- None, a list of strings with the same length as 'sentence'.
- """
- self.sentence = sentence
- self.state = state
- self.logprob = logprob
- self.score = score
- self.metadata = metadata
-
- def __cmp__(self, other):
- """Compares Captions by score."""
- assert isinstance(other, Caption)
- if self.score == other.score:
- return 0
- elif self.score < other.score:
- return -1
- else:
- return 1
-
- # For Python 3 compatibility (__cmp__ is deprecated).
- def __lt__(self, other):
- assert isinstance(other, Caption)
- return self.score < other.score
-
- # Also for Python 3 compatibility.
- def __eq__(self, other):
- assert isinstance(other, Caption)
- return self.score == other.score
-
-
-class TopN(object):
- """Maintains the top n elements of an incrementally provided set."""
-
- def __init__(self, n):
- self._n = n
- self._data = []
-
- def size(self):
- assert self._data is not None
- return len(self._data)
-
- def push(self, x):
- """Pushes a new element."""
- assert self._data is not None
- if len(self._data) < self._n:
- heapq.heappush(self._data, x)
- else:
- heapq.heappushpop(self._data, x)
-
- def extract(self, sort=False):
- """Extracts all elements from the TopN. This is a destructive operation.
-
- The only method that can be called immediately after extract() is reset().
-
- Args:
- sort: Whether to return the elements in descending sorted order.
-
- Returns:
- A list of data; the top n elements provided to the set.
- """
- assert self._data is not None
- data = self._data
- self._data = None
- if sort:
- data.sort(reverse=True)
- return data
-
- def reset(self):
- """Returns the TopN to an empty state."""
- self._data = []
-
-
-class CaptionGenerator(object):
- """Class to generate captions from an image-to-text model."""
-
- def __init__(self,
- model,
- vocab,
- beam_size=3,
- max_caption_length=20,
- length_normalization_factor=0.0):
- """Initializes the generator.
-
- Args:
- model: Object encapsulating a trained image-to-text model. Must have
- methods feed_image() and inference_step(). For example, an instance of
- InferenceWrapperBase.
- vocab: A Vocabulary object.
- beam_size: Beam size to use when generating captions.
- max_caption_length: The maximum caption length before stopping the search.
- length_normalization_factor: If != 0, a number x such that captions are
- scored by logprob/length^x, rather than logprob. This changes the
- relative scores of captions depending on their lengths. For example, if
- x > 0 then longer captions will be favored.
- """
- self.vocab = vocab
- self.model = model
-
- self.beam_size = beam_size
- self.max_caption_length = max_caption_length
- self.length_normalization_factor = length_normalization_factor
-
- def beam_search(self, sess, encoded_image):
- """Runs beam search caption generation on a single image.
-
- Args:
- sess: TensorFlow Session object.
- encoded_image: An encoded image string.
-
- Returns:
- A list of Caption sorted by descending score.
- """
- # Feed in the image to get the initial state.
- initial_state = self.model.feed_image(sess, encoded_image)
-
- initial_beam = Caption(
- sentence=[self.vocab.start_id],
- state=initial_state[0],
- logprob=0.0,
- score=0.0,
- metadata=[""])
- partial_captions = TopN(self.beam_size)
- partial_captions.push(initial_beam)
- complete_captions = TopN(self.beam_size)
-
- # Run beam search.
- for _ in range(self.max_caption_length - 1):
- partial_captions_list = partial_captions.extract()
- partial_captions.reset()
- input_feed = np.array([c.sentence[-1] for c in partial_captions_list])
- state_feed = np.array([c.state for c in partial_captions_list])
-
- softmax, new_states, metadata = self.model.inference_step(sess,
- input_feed,
- state_feed)
-
- for i, partial_caption in enumerate(partial_captions_list):
- word_probabilities = softmax[i]
- state = new_states[i]
- # For this partial caption, get the beam_size most probable next words.
- # Sort the indexes with numpy, select the last self.beam_size
- # (3 by default) (ie, the most likely) and then reverse the sorted
- # indexes with [::-1] to sort them from higher to lower.
- most_likely_words = np.argsort(word_probabilities)[:-self.beam_size][::-1]
-
- for w in most_likely_words:
- p = word_probabilities[w]
- if p < 1e-12:
- continue # Avoid log(0).
- sentence = partial_caption.sentence + [w]
- logprob = partial_caption.logprob + math.log(p)
- score = logprob
- if metadata:
- metadata_list = partial_caption.metadata + [metadata[i]]
- else:
- metadata_list = None
- if w == self.vocab.end_id:
- if self.length_normalization_factor > 0:
- score /= len(sentence)**self.length_normalization_factor
- beam = Caption(sentence, state, logprob, score, metadata_list)
- complete_captions.push(beam)
- else:
- beam = Caption(sentence, state, logprob, score, metadata_list)
- partial_captions.push(beam)
- if partial_captions.size() == 0:
- # We have run out of partial candidates; happens when beam_size = 1.
- break
-
- # If we have no complete captions then fall back to the partial captions.
- # But never output a mixture of complete and partial captions because a
- # partial caption could have a higher score than all the complete captions.
- if not complete_captions.size():
- complete_captions = partial_captions
-
- return complete_captions.extract(sort=True)
diff --git a/research/im2txt/im2txt/inference_utils/caption_generator_test.py b/research/im2txt/im2txt/inference_utils/caption_generator_test.py
deleted file mode 100644
index bbd069313ac4ddb10a8463d166ab282b68b2e24d..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/inference_utils/caption_generator_test.py
+++ /dev/null
@@ -1,178 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Unit tests for CaptionGenerator."""
-
-import math
-
-
-
-import numpy as np
-import tensorflow as tf
-
-from im2txt.inference_utils import caption_generator
-
-
-class FakeVocab(object):
- """Fake Vocabulary for testing purposes."""
-
- def __init__(self):
- self.start_id = 0 # Word id denoting sentence start.
- self.end_id = 1 # Word id denoting sentence end.
-
-
-class FakeModel(object):
- """Fake model for testing purposes."""
-
- def __init__(self):
- # Number of words in the vocab.
- self._vocab_size = 12
-
- # Dimensionality of the nominal model state.
- self._state_size = 1
-
- # Map of previous word to the probability distribution of the next word.
- self._probabilities = {
- 0: {1: 0.1,
- 2: 0.2,
- 3: 0.3,
- 4: 0.4},
- 2: {5: 0.1,
- 6: 0.9},
- 3: {1: 0.1,
- 7: 0.4,
- 8: 0.5},
- 4: {1: 0.3,
- 9: 0.3,
- 10: 0.4},
- 5: {1: 1.0},
- 6: {1: 1.0},
- 7: {1: 1.0},
- 8: {1: 1.0},
- 9: {1: 0.5,
- 11: 0.5},
- 10: {1: 1.0},
- 11: {1: 1.0},
- }
-
- # pylint: disable=unused-argument
-
- def feed_image(self, sess, encoded_image):
- # Return a nominal model state.
- return np.zeros([1, self._state_size])
-
- def inference_step(self, sess, input_feed, state_feed):
- # Compute the matrix of softmax distributions for the next batch of words.
- batch_size = input_feed.shape[0]
- softmax_output = np.zeros([batch_size, self._vocab_size])
- for batch_index, word_id in enumerate(input_feed):
- for next_word, probability in self._probabilities[word_id].items():
- softmax_output[batch_index, next_word] = probability
-
- # Nominal state and metadata.
- new_state = np.zeros([batch_size, self._state_size])
- metadata = None
-
- return softmax_output, new_state, metadata
-
- # pylint: enable=unused-argument
-
-
-class CaptionGeneratorTest(tf.test.TestCase):
-
- def _assertExpectedCaptions(self,
- expected_captions,
- beam_size=3,
- max_caption_length=20,
- length_normalization_factor=0):
- """Tests that beam search generates the expected captions.
-
- Args:
- expected_captions: A sequence of pairs (sentence, probability), where
- sentence is a list of integer ids and probability is a float in [0, 1].
- beam_size: Parameter passed to beam_search().
- max_caption_length: Parameter passed to beam_search().
- length_normalization_factor: Parameter passed to beam_search().
- """
- expected_sentences = [c[0] for c in expected_captions]
- expected_probabilities = [c[1] for c in expected_captions]
-
- # Generate captions.
- generator = caption_generator.CaptionGenerator(
- model=FakeModel(),
- vocab=FakeVocab(),
- beam_size=beam_size,
- max_caption_length=max_caption_length,
- length_normalization_factor=length_normalization_factor)
- actual_captions = generator.beam_search(sess=None, encoded_image=None)
-
- actual_sentences = [c.sentence for c in actual_captions]
- actual_probabilities = [math.exp(c.logprob) for c in actual_captions]
-
- self.assertEqual(expected_sentences, actual_sentences)
- self.assertAllClose(expected_probabilities, actual_probabilities)
-
- def testBeamSize(self):
- # Beam size = 1.
- expected = [([0, 4, 10, 1], 0.16)]
- self._assertExpectedCaptions(expected, beam_size=1)
-
- # Beam size = 2.
- expected = [([0, 4, 10, 1], 0.16), ([0, 3, 8, 1], 0.15)]
- self._assertExpectedCaptions(expected, beam_size=2)
-
- # Beam size = 3.
- expected = [
- ([0, 2, 6, 1], 0.18), ([0, 4, 10, 1], 0.16), ([0, 3, 8, 1], 0.15)
- ]
- self._assertExpectedCaptions(expected, beam_size=3)
-
- def testMaxLength(self):
- # Max length = 1.
- expected = [([0], 1.0)]
- self._assertExpectedCaptions(expected, max_caption_length=1)
-
- # Max length = 2.
- # There are no complete sentences, so partial sentences are returned.
- expected = [([0, 4], 0.4), ([0, 3], 0.3), ([0, 2], 0.2)]
- self._assertExpectedCaptions(expected, max_caption_length=2)
-
- # Max length = 3.
- # There is at least one complete sentence, so only complete sentences are
- # returned.
- expected = [([0, 4, 1], 0.12), ([0, 3, 1], 0.03)]
- self._assertExpectedCaptions(expected, max_caption_length=3)
-
- # Max length = 4.
- expected = [
- ([0, 2, 6, 1], 0.18), ([0, 4, 10, 1], 0.16), ([0, 3, 8, 1], 0.15)
- ]
- self._assertExpectedCaptions(expected, max_caption_length=4)
-
- def testLengthNormalization(self):
- # Length normalization factor = 3.
- # The longest caption is returned first, despite having low probability,
- # because it has the highest log(probability)/length**3.
- expected = [
- ([0, 4, 9, 11, 1], 0.06),
- ([0, 2, 6, 1], 0.18),
- ([0, 4, 10, 1], 0.16),
- ([0, 3, 8, 1], 0.15),
- ]
- self._assertExpectedCaptions(
- expected, beam_size=4, length_normalization_factor=3)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/im2txt/im2txt/inference_utils/inference_wrapper_base.py b/research/im2txt/im2txt/inference_utils/inference_wrapper_base.py
deleted file mode 100644
index e94cd6af474488e4b8175fc959e1dbe33cca18c9..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/inference_utils/inference_wrapper_base.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Base wrapper class for performing inference with an image-to-text model.
-
-Subclasses must implement the following methods:
-
- build_model():
- Builds the model for inference and returns the model object.
-
- feed_image():
- Takes an encoded image and returns the initial model state, where "state"
- is a numpy array whose specifics are defined by the subclass, e.g.
- concatenated LSTM state. It's assumed that feed_image() will be called
- precisely once at the start of inference for each image. Subclasses may
- compute and/or save per-image internal context in this method.
-
- inference_step():
- Takes a batch of inputs and states at a single time-step. Returns the
- softmax output corresponding to the inputs, and the new states of the batch.
- Optionally also returns metadata about the current inference step, e.g. a
- serialized numpy array containing activations from a particular model layer.
-
-Client usage:
- 1. Build the model inference graph via build_graph_from_config() or
- build_graph_from_proto().
- 2. Call the resulting restore_fn to load the model checkpoint.
- 3. For each image in a batch of images:
- a) Call feed_image() once to get the initial state.
- b) For each step of caption generation, call inference_step().
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os.path
-
-
-import tensorflow as tf
-
-# pylint: disable=unused-argument
-
-
-class InferenceWrapperBase(object):
- """Base wrapper class for performing inference with an image-to-text model."""
-
- def __init__(self):
- pass
-
- def build_model(self, model_config):
- """Builds the model for inference.
-
- Args:
- model_config: Object containing configuration for building the model.
-
- Returns:
- model: The model object.
- """
- tf.logging.fatal("Please implement build_model in subclass")
-
- def _create_restore_fn(self, checkpoint_path, saver):
- """Creates a function that restores a model from checkpoint.
-
- Args:
- checkpoint_path: Checkpoint file or a directory containing a checkpoint
- file.
- saver: Saver for restoring variables from the checkpoint file.
-
- Returns:
- restore_fn: A function such that restore_fn(sess) loads model variables
- from the checkpoint file.
-
- Raises:
- ValueError: If checkpoint_path does not refer to a checkpoint file or a
- directory containing a checkpoint file.
- """
- if tf.gfile.IsDirectory(checkpoint_path):
- checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
- if not checkpoint_path:
- raise ValueError("No checkpoint file found in: %s" % checkpoint_path)
-
- def _restore_fn(sess):
- tf.logging.info("Loading model from checkpoint: %s", checkpoint_path)
- saver.restore(sess, checkpoint_path)
- tf.logging.info("Successfully loaded checkpoint: %s",
- os.path.basename(checkpoint_path))
-
- return _restore_fn
-
- def build_graph_from_config(self, model_config, checkpoint_path):
- """Builds the inference graph from a configuration object.
-
- Args:
- model_config: Object containing configuration for building the model.
- checkpoint_path: Checkpoint file or a directory containing a checkpoint
- file.
-
- Returns:
- restore_fn: A function such that restore_fn(sess) loads model variables
- from the checkpoint file.
- """
- tf.logging.info("Building model.")
- self.build_model(model_config)
- saver = tf.train.Saver()
-
- return self._create_restore_fn(checkpoint_path, saver)
-
- def build_graph_from_proto(self, graph_def_file, saver_def_file,
- checkpoint_path):
- """Builds the inference graph from serialized GraphDef and SaverDef protos.
-
- Args:
- graph_def_file: File containing a serialized GraphDef proto.
- saver_def_file: File containing a serialized SaverDef proto.
- checkpoint_path: Checkpoint file or a directory containing a checkpoint
- file.
-
- Returns:
- restore_fn: A function such that restore_fn(sess) loads model variables
- from the checkpoint file.
- """
- # Load the Graph.
- tf.logging.info("Loading GraphDef from file: %s", graph_def_file)
- graph_def = tf.GraphDef()
- with tf.gfile.FastGFile(graph_def_file, "rb") as f:
- graph_def.ParseFromString(f.read())
- tf.import_graph_def(graph_def, name="")
-
- # Load the Saver.
- tf.logging.info("Loading SaverDef from file: %s", saver_def_file)
- saver_def = tf.train.SaverDef()
- with tf.gfile.FastGFile(saver_def_file, "rb") as f:
- saver_def.ParseFromString(f.read())
- saver = tf.train.Saver(saver_def=saver_def)
-
- return self._create_restore_fn(checkpoint_path, saver)
-
- def feed_image(self, sess, encoded_image):
- """Feeds an image and returns the initial model state.
-
- See comments at the top of file.
-
- Args:
- sess: TensorFlow Session object.
- encoded_image: An encoded image string.
-
- Returns:
- state: A numpy array of shape [1, state_size].
- """
- tf.logging.fatal("Please implement feed_image in subclass")
-
- def inference_step(self, sess, input_feed, state_feed):
- """Runs one step of inference.
-
- Args:
- sess: TensorFlow Session object.
- input_feed: A numpy array of shape [batch_size].
- state_feed: A numpy array of shape [batch_size, state_size].
-
- Returns:
- softmax_output: A numpy array of shape [batch_size, vocab_size].
- new_state: A numpy array of shape [batch_size, state_size].
- metadata: Optional. If not None, a string containing metadata about the
- current inference step (e.g. serialized numpy array containing
- activations from a particular model layer.).
- """
- tf.logging.fatal("Please implement inference_step in subclass")
-
-# pylint: enable=unused-argument
diff --git a/research/im2txt/im2txt/inference_utils/vocabulary.py b/research/im2txt/im2txt/inference_utils/vocabulary.py
deleted file mode 100644
index ecf0ada9c2242cb32c2ea9a300d16411f5e83fab..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/inference_utils/vocabulary.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Vocabulary class for an image-to-text model."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-
-class Vocabulary(object):
- """Vocabulary class for an image-to-text model."""
-
- def __init__(self,
- vocab_file,
- start_word="",
- end_word="",
- unk_word=""):
- """Initializes the vocabulary.
-
- Args:
- vocab_file: File containing the vocabulary, where the words are the first
- whitespace-separated token on each line (other tokens are ignored) and
- the word ids are the corresponding line numbers.
- start_word: Special word denoting sentence start.
- end_word: Special word denoting sentence end.
- unk_word: Special word denoting unknown words.
- """
- if not tf.gfile.Exists(vocab_file):
- tf.logging.fatal("Vocab file %s not found.", vocab_file)
- tf.logging.info("Initializing vocabulary from file: %s", vocab_file)
-
- with tf.gfile.GFile(vocab_file, mode="r") as f:
- reverse_vocab = list(f.readlines())
- reverse_vocab = [line.split()[0] for line in reverse_vocab]
- assert start_word in reverse_vocab
- assert end_word in reverse_vocab
- if unk_word not in reverse_vocab:
- reverse_vocab.append(unk_word)
- vocab = dict([(x, y) for (y, x) in enumerate(reverse_vocab)])
-
- tf.logging.info("Created vocabulary with %d words" % len(vocab))
-
- self.vocab = vocab # vocab[word] = id
- self.reverse_vocab = reverse_vocab # reverse_vocab[id] = word
-
- # Save special word ids.
- self.start_id = vocab[start_word]
- self.end_id = vocab[end_word]
- self.unk_id = vocab[unk_word]
-
- def word_to_id(self, word):
- """Returns the integer word id of a word string."""
- if word in self.vocab:
- return self.vocab[word]
- else:
- return self.unk_id
-
- def id_to_word(self, word_id):
- """Returns the word string of an integer word id."""
- if word_id >= len(self.reverse_vocab):
- return self.reverse_vocab[self.unk_id]
- else:
- return self.reverse_vocab[word_id]
diff --git a/research/im2txt/im2txt/inference_wrapper.py b/research/im2txt/im2txt/inference_wrapper.py
deleted file mode 100644
index a047a9c8d084fd9e69c937915cea8553c2d51817..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/inference_wrapper.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Model wrapper class for performing inference with a ShowAndTellModel."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-
-from im2txt import show_and_tell_model
-from im2txt.inference_utils import inference_wrapper_base
-
-
-class InferenceWrapper(inference_wrapper_base.InferenceWrapperBase):
- """Model wrapper class for performing inference with a ShowAndTellModel."""
-
- def __init__(self):
- super(InferenceWrapper, self).__init__()
-
- def build_model(self, model_config):
- model = show_and_tell_model.ShowAndTellModel(model_config, mode="inference")
- model.build()
- return model
-
- def feed_image(self, sess, encoded_image):
- initial_state = sess.run(fetches="lstm/initial_state:0",
- feed_dict={"image_feed:0": encoded_image})
- return initial_state
-
- def inference_step(self, sess, input_feed, state_feed):
- softmax_output, state_output = sess.run(
- fetches=["softmax:0", "lstm/state:0"],
- feed_dict={
- "input_feed:0": input_feed,
- "lstm/state_feed:0": state_feed,
- })
- return softmax_output, state_output, None
diff --git a/research/im2txt/im2txt/ops/BUILD b/research/im2txt/im2txt/ops/BUILD
deleted file mode 100644
index 7d48bf3938c7ecfc94ac6498386e7ce214b8be92..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/ops/BUILD
+++ /dev/null
@@ -1,32 +0,0 @@
-package(default_visibility = ["//im2txt:internal"])
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-py_library(
- name = "image_processing",
- srcs = ["image_processing.py"],
- srcs_version = "PY2AND3",
-)
-
-py_library(
- name = "image_embedding",
- srcs = ["image_embedding.py"],
- srcs_version = "PY2AND3",
-)
-
-py_test(
- name = "image_embedding_test",
- size = "small",
- srcs = ["image_embedding_test.py"],
- deps = [
- ":image_embedding",
- ],
-)
-
-py_library(
- name = "inputs",
- srcs = ["inputs.py"],
- srcs_version = "PY2AND3",
-)
diff --git a/research/im2txt/im2txt/ops/image_embedding.py b/research/im2txt/im2txt/ops/image_embedding.py
deleted file mode 100644
index 58e3ddaa95fa799f245fe2a46f2e948be7d9ebf2..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/ops/image_embedding.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Image embedding ops."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-from tensorflow.contrib.slim.python.slim.nets.inception_v3 import inception_v3_base
-
-slim = tf.contrib.slim
-
-
-def inception_v3(images,
- trainable=True,
- is_training=True,
- weight_decay=0.00004,
- stddev=0.1,
- dropout_keep_prob=0.8,
- use_batch_norm=True,
- batch_norm_params=None,
- add_summaries=True,
- scope="InceptionV3"):
- """Builds an Inception V3 subgraph for image embeddings.
-
- Args:
- images: A float32 Tensor of shape [batch, height, width, channels].
- trainable: Whether the inception submodel should be trainable or not.
- is_training: Boolean indicating training mode or not.
- weight_decay: Coefficient for weight regularization.
- stddev: The standard deviation of the trunctated normal weight initializer.
- dropout_keep_prob: Dropout keep probability.
- use_batch_norm: Whether to use batch normalization.
- batch_norm_params: Parameters for batch normalization. See
- tf.contrib.layers.batch_norm for details.
- add_summaries: Whether to add activation summaries.
- scope: Optional Variable scope.
-
- Returns:
- end_points: A dictionary of activations from inception_v3 layers.
- """
- # Only consider the inception model to be in training mode if it's trainable.
- is_inception_model_training = trainable and is_training
-
- if use_batch_norm:
- # Default parameters for batch normalization.
- if not batch_norm_params:
- batch_norm_params = {
- "is_training": is_inception_model_training,
- "trainable": trainable,
- # Decay for the moving averages.
- "decay": 0.9997,
- # Epsilon to prevent 0s in variance.
- "epsilon": 0.001,
- # Collection containing the moving mean and moving variance.
- "variables_collections": {
- "beta": None,
- "gamma": None,
- "moving_mean": ["moving_vars"],
- "moving_variance": ["moving_vars"],
- }
- }
- else:
- batch_norm_params = None
-
- if trainable:
- weights_regularizer = tf.contrib.layers.l2_regularizer(weight_decay)
- else:
- weights_regularizer = None
-
- with tf.variable_scope(scope, "InceptionV3", [images]) as scope:
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_regularizer=weights_regularizer,
- trainable=trainable):
- with slim.arg_scope(
- [slim.conv2d],
- weights_initializer=tf.truncated_normal_initializer(stddev=stddev),
- activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm,
- normalizer_params=batch_norm_params):
- net, end_points = inception_v3_base(images, scope=scope)
- with tf.variable_scope("logits"):
- shape = net.get_shape()
- net = slim.avg_pool2d(net, shape[1:3], padding="VALID", scope="pool")
- net = slim.dropout(
- net,
- keep_prob=dropout_keep_prob,
- is_training=is_inception_model_training,
- scope="dropout")
- net = slim.flatten(net, scope="flatten")
-
- # Add summaries.
- if add_summaries:
- for v in end_points.values():
- tf.contrib.layers.summaries.summarize_activation(v)
-
- return net
diff --git a/research/im2txt/im2txt/ops/image_embedding_test.py b/research/im2txt/im2txt/ops/image_embedding_test.py
deleted file mode 100644
index 66324d68eee0ec9c450375c25229d80283fc909f..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/ops/image_embedding_test.py
+++ /dev/null
@@ -1,136 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for tensorflow_models.im2txt.ops.image_embedding."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-from im2txt.ops import image_embedding
-
-
-class InceptionV3Test(tf.test.TestCase):
-
- def setUp(self):
- super(InceptionV3Test, self).setUp()
-
- batch_size = 4
- height = 299
- width = 299
- num_channels = 3
- self._images = tf.placeholder(tf.float32,
- [batch_size, height, width, num_channels])
- self._batch_size = batch_size
-
- def _countInceptionParameters(self):
- """Counts the number of parameters in the inception model at top scope."""
- counter = {}
- for v in tf.global_variables():
- name_tokens = v.op.name.split("/")
- if name_tokens[0] == "InceptionV3":
- name = "InceptionV3/" + name_tokens[1]
- num_params = v.get_shape().num_elements()
- assert num_params
- counter[name] = counter.get(name, 0) + num_params
- return counter
-
- def _verifyParameterCounts(self):
- """Verifies the number of parameters in the inception model."""
- param_counts = self._countInceptionParameters()
- expected_param_counts = {
- "InceptionV3/Conv2d_1a_3x3": 960,
- "InceptionV3/Conv2d_2a_3x3": 9312,
- "InceptionV3/Conv2d_2b_3x3": 18624,
- "InceptionV3/Conv2d_3b_1x1": 5360,
- "InceptionV3/Conv2d_4a_3x3": 138816,
- "InceptionV3/Mixed_5b": 256368,
- "InceptionV3/Mixed_5c": 277968,
- "InceptionV3/Mixed_5d": 285648,
- "InceptionV3/Mixed_6a": 1153920,
- "InceptionV3/Mixed_6b": 1298944,
- "InceptionV3/Mixed_6c": 1692736,
- "InceptionV3/Mixed_6d": 1692736,
- "InceptionV3/Mixed_6e": 2143872,
- "InceptionV3/Mixed_7a": 1699584,
- "InceptionV3/Mixed_7b": 5047872,
- "InceptionV3/Mixed_7c": 6080064,
- }
- self.assertDictEqual(expected_param_counts, param_counts)
-
- def _assertCollectionSize(self, expected_size, collection):
- actual_size = len(tf.get_collection(collection))
- if expected_size != actual_size:
- self.fail("Found %d items in collection %s (expected %d)." %
- (actual_size, collection, expected_size))
-
- def testTrainableTrueIsTrainingTrue(self):
- embeddings = image_embedding.inception_v3(
- self._images, trainable=True, is_training=True)
- self.assertEqual([self._batch_size, 2048], embeddings.get_shape().as_list())
-
- self._verifyParameterCounts()
- self._assertCollectionSize(376, tf.GraphKeys.GLOBAL_VARIABLES)
- self._assertCollectionSize(188, tf.GraphKeys.TRAINABLE_VARIABLES)
- self._assertCollectionSize(188, tf.GraphKeys.UPDATE_OPS)
- self._assertCollectionSize(94, tf.GraphKeys.REGULARIZATION_LOSSES)
- self._assertCollectionSize(0, tf.GraphKeys.LOSSES)
- self._assertCollectionSize(23, tf.GraphKeys.SUMMARIES)
-
- def testTrainableTrueIsTrainingFalse(self):
- embeddings = image_embedding.inception_v3(
- self._images, trainable=True, is_training=False)
- self.assertEqual([self._batch_size, 2048], embeddings.get_shape().as_list())
-
- self._verifyParameterCounts()
- self._assertCollectionSize(376, tf.GraphKeys.GLOBAL_VARIABLES)
- self._assertCollectionSize(188, tf.GraphKeys.TRAINABLE_VARIABLES)
- self._assertCollectionSize(0, tf.GraphKeys.UPDATE_OPS)
- self._assertCollectionSize(94, tf.GraphKeys.REGULARIZATION_LOSSES)
- self._assertCollectionSize(0, tf.GraphKeys.LOSSES)
- self._assertCollectionSize(23, tf.GraphKeys.SUMMARIES)
-
- def testTrainableFalseIsTrainingTrue(self):
- embeddings = image_embedding.inception_v3(
- self._images, trainable=False, is_training=True)
- self.assertEqual([self._batch_size, 2048], embeddings.get_shape().as_list())
-
- self._verifyParameterCounts()
- self._assertCollectionSize(376, tf.GraphKeys.GLOBAL_VARIABLES)
- self._assertCollectionSize(0, tf.GraphKeys.TRAINABLE_VARIABLES)
- self._assertCollectionSize(0, tf.GraphKeys.UPDATE_OPS)
- self._assertCollectionSize(0, tf.GraphKeys.REGULARIZATION_LOSSES)
- self._assertCollectionSize(0, tf.GraphKeys.LOSSES)
- self._assertCollectionSize(23, tf.GraphKeys.SUMMARIES)
-
- def testTrainableFalseIsTrainingFalse(self):
- embeddings = image_embedding.inception_v3(
- self._images, trainable=False, is_training=False)
- self.assertEqual([self._batch_size, 2048], embeddings.get_shape().as_list())
-
- self._verifyParameterCounts()
- self._assertCollectionSize(376, tf.GraphKeys.GLOBAL_VARIABLES)
- self._assertCollectionSize(0, tf.GraphKeys.TRAINABLE_VARIABLES)
- self._assertCollectionSize(0, tf.GraphKeys.UPDATE_OPS)
- self._assertCollectionSize(0, tf.GraphKeys.REGULARIZATION_LOSSES)
- self._assertCollectionSize(0, tf.GraphKeys.LOSSES)
- self._assertCollectionSize(23, tf.GraphKeys.SUMMARIES)
-
-
-if __name__ == "__main__":
- tf.test.main()
diff --git a/research/im2txt/im2txt/ops/image_processing.py b/research/im2txt/im2txt/ops/image_processing.py
deleted file mode 100644
index 6a7545547d5507febaabebf642ee81b6f94319f6..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/ops/image_processing.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Helper functions for image preprocessing."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-
-def distort_image(image, thread_id):
- """Perform random distortions on an image.
-
- Args:
- image: A float32 Tensor of shape [height, width, 3] with values in [0, 1).
- thread_id: Preprocessing thread id used to select the ordering of color
- distortions. There should be a multiple of 2 preprocessing threads.
-
- Returns:
- distorted_image: A float32 Tensor of shape [height, width, 3] with values in
- [0, 1].
- """
- # Randomly flip horizontally.
- with tf.name_scope("flip_horizontal", values=[image]):
- image = tf.image.random_flip_left_right(image)
-
- # Randomly distort the colors based on thread id.
- color_ordering = thread_id % 2
- with tf.name_scope("distort_color", values=[image]):
- if color_ordering == 0:
- image = tf.image.random_brightness(image, max_delta=32. / 255.)
- image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
- image = tf.image.random_hue(image, max_delta=0.032)
- image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
- elif color_ordering == 1:
- image = tf.image.random_brightness(image, max_delta=32. / 255.)
- image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
- image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
- image = tf.image.random_hue(image, max_delta=0.032)
-
- # The random_* ops do not necessarily clamp.
- image = tf.clip_by_value(image, 0.0, 1.0)
-
- return image
-
-
-def process_image(encoded_image,
- is_training,
- height,
- width,
- resize_height=346,
- resize_width=346,
- thread_id=0,
- image_format="jpeg"):
- """Decode an image, resize and apply random distortions.
-
- In training, images are distorted slightly differently depending on thread_id.
-
- Args:
- encoded_image: String Tensor containing the image.
- is_training: Boolean; whether preprocessing for training or eval.
- height: Height of the output image.
- width: Width of the output image.
- resize_height: If > 0, resize height before crop to final dimensions.
- resize_width: If > 0, resize width before crop to final dimensions.
- thread_id: Preprocessing thread id used to select the ordering of color
- distortions. There should be a multiple of 2 preprocessing threads.
- image_format: "jpeg" or "png".
-
- Returns:
- A float32 Tensor of shape [height, width, 3] with values in [-1, 1].
-
- Raises:
- ValueError: If image_format is invalid.
- """
- # Helper function to log an image summary to the visualizer. Summaries are
- # only logged in thread 0.
- def image_summary(name, image):
- if not thread_id:
- tf.summary.image(name, tf.expand_dims(image, 0))
-
- # Decode image into a float32 Tensor of shape [?, ?, 3] with values in [0, 1).
- with tf.name_scope("decode", values=[encoded_image]):
- if image_format == "jpeg":
- image = tf.image.decode_jpeg(encoded_image, channels=3)
- elif image_format == "png":
- image = tf.image.decode_png(encoded_image, channels=3)
- else:
- raise ValueError("Invalid image format: %s" % image_format)
- image = tf.image.convert_image_dtype(image, dtype=tf.float32)
- image_summary("original_image", image)
-
- # Resize image.
- assert (resize_height > 0) == (resize_width > 0)
- if resize_height:
- image = tf.image.resize_images(image,
- size=[resize_height, resize_width],
- method=tf.image.ResizeMethod.BILINEAR)
-
- # Crop to final dimensions.
- if is_training:
- image = tf.random_crop(image, [height, width, 3])
- else:
- # Central crop, assuming resize_height > height, resize_width > width.
- image = tf.image.resize_image_with_crop_or_pad(image, height, width)
-
- image_summary("resized_image", image)
-
- # Randomly distort the image.
- if is_training:
- image = distort_image(image, thread_id)
-
- image_summary("final_image", image)
-
- # Rescale to [-1,1] instead of [0, 1]
- image = tf.subtract(image, 0.5)
- image = tf.multiply(image, 2.0)
- return image
diff --git a/research/im2txt/im2txt/ops/inputs.py b/research/im2txt/im2txt/ops/inputs.py
deleted file mode 100644
index 5dc90c0ce5dfd5c30fe0e0e543999bb15cc13a8c..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/ops/inputs.py
+++ /dev/null
@@ -1,204 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Input ops."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-
-def parse_sequence_example(serialized, image_feature, caption_feature):
- """Parses a tensorflow.SequenceExample into an image and caption.
-
- Args:
- serialized: A scalar string Tensor; a single serialized SequenceExample.
- image_feature: Name of SequenceExample context feature containing image
- data.
- caption_feature: Name of SequenceExample feature list containing integer
- captions.
-
- Returns:
- encoded_image: A scalar string Tensor containing a JPEG encoded image.
- caption: A 1-D uint64 Tensor with dynamically specified length.
- """
- context, sequence = tf.parse_single_sequence_example(
- serialized,
- context_features={
- image_feature: tf.FixedLenFeature([], dtype=tf.string)
- },
- sequence_features={
- caption_feature: tf.FixedLenSequenceFeature([], dtype=tf.int64),
- })
-
- encoded_image = context[image_feature]
- caption = sequence[caption_feature]
- return encoded_image, caption
-
-
-def prefetch_input_data(reader,
- file_pattern,
- is_training,
- batch_size,
- values_per_shard,
- input_queue_capacity_factor=16,
- num_reader_threads=1,
- shard_queue_name="filename_queue",
- value_queue_name="input_queue"):
- """Prefetches string values from disk into an input queue.
-
- In training the capacity of the queue is important because a larger queue
- means better mixing of training examples between shards. The minimum number of
- values kept in the queue is values_per_shard * input_queue_capacity_factor,
- where input_queue_memory factor should be chosen to trade-off better mixing
- with memory usage.
-
- Args:
- reader: Instance of tf.ReaderBase.
- file_pattern: Comma-separated list of file patterns (e.g.
- /tmp/train_data-?????-of-00100).
- is_training: Boolean; whether prefetching for training or eval.
- batch_size: Model batch size used to determine queue capacity.
- values_per_shard: Approximate number of values per shard.
- input_queue_capacity_factor: Minimum number of values to keep in the queue
- in multiples of values_per_shard. See comments above.
- num_reader_threads: Number of reader threads to fill the queue.
- shard_queue_name: Name for the shards filename queue.
- value_queue_name: Name for the values input queue.
-
- Returns:
- A Queue containing prefetched string values.
- """
- data_files = []
- for pattern in file_pattern.split(","):
- data_files.extend(tf.gfile.Glob(pattern))
- if not data_files:
- tf.logging.fatal("Found no input files matching %s", file_pattern)
- else:
- tf.logging.info("Prefetching values from %d files matching %s",
- len(data_files), file_pattern)
-
- if is_training:
- filename_queue = tf.train.string_input_producer(
- data_files, shuffle=True, capacity=16, name=shard_queue_name)
- min_queue_examples = values_per_shard * input_queue_capacity_factor
- capacity = min_queue_examples + 100 * batch_size
- values_queue = tf.RandomShuffleQueue(
- capacity=capacity,
- min_after_dequeue=min_queue_examples,
- dtypes=[tf.string],
- name="random_" + value_queue_name)
- else:
- filename_queue = tf.train.string_input_producer(
- data_files, shuffle=False, capacity=1, name=shard_queue_name)
- capacity = values_per_shard + 3 * batch_size
- values_queue = tf.FIFOQueue(
- capacity=capacity, dtypes=[tf.string], name="fifo_" + value_queue_name)
-
- enqueue_ops = []
- for _ in range(num_reader_threads):
- _, value = reader.read(filename_queue)
- enqueue_ops.append(values_queue.enqueue([value]))
- tf.train.queue_runner.add_queue_runner(tf.train.queue_runner.QueueRunner(
- values_queue, enqueue_ops))
- tf.summary.scalar(
- "queue/%s/fraction_of_%d_full" % (values_queue.name, capacity),
- tf.cast(values_queue.size(), tf.float32) * (1. / capacity))
-
- return values_queue
-
-
-def batch_with_dynamic_pad(images_and_captions,
- batch_size,
- queue_capacity,
- add_summaries=True):
- """Batches input images and captions.
-
- This function splits the caption into an input sequence and a target sequence,
- where the target sequence is the input sequence right-shifted by 1. Input and
- target sequences are batched and padded up to the maximum length of sequences
- in the batch. A mask is created to distinguish real words from padding words.
-
- Example:
- Actual captions in the batch ('-' denotes padded character):
- [
- [ 1 2 3 4 5 ],
- [ 1 2 3 4 - ],
- [ 1 2 3 - - ],
- ]
-
- input_seqs:
- [
- [ 1 2 3 4 ],
- [ 1 2 3 - ],
- [ 1 2 - - ],
- ]
-
- target_seqs:
- [
- [ 2 3 4 5 ],
- [ 2 3 4 - ],
- [ 2 3 - - ],
- ]
-
- mask:
- [
- [ 1 1 1 1 ],
- [ 1 1 1 0 ],
- [ 1 1 0 0 ],
- ]
-
- Args:
- images_and_captions: A list of pairs [image, caption], where image is a
- Tensor of shape [height, width, channels] and caption is a 1-D Tensor of
- any length. Each pair will be processed and added to the queue in a
- separate thread.
- batch_size: Batch size.
- queue_capacity: Queue capacity.
- add_summaries: If true, add caption length summaries.
-
- Returns:
- images: A Tensor of shape [batch_size, height, width, channels].
- input_seqs: An int32 Tensor of shape [batch_size, padded_length].
- target_seqs: An int32 Tensor of shape [batch_size, padded_length].
- mask: An int32 0/1 Tensor of shape [batch_size, padded_length].
- """
- enqueue_list = []
- for image, caption in images_and_captions:
- caption_length = tf.shape(caption)[0]
- input_length = tf.expand_dims(tf.subtract(caption_length, 1), 0)
-
- input_seq = tf.slice(caption, [0], input_length)
- target_seq = tf.slice(caption, [1], input_length)
- indicator = tf.ones(input_length, dtype=tf.int32)
- enqueue_list.append([image, input_seq, target_seq, indicator])
-
- images, input_seqs, target_seqs, mask = tf.train.batch_join(
- enqueue_list,
- batch_size=batch_size,
- capacity=queue_capacity,
- dynamic_pad=True,
- name="batch_and_pad")
-
- if add_summaries:
- lengths = tf.add(tf.reduce_sum(mask, 1), 1)
- tf.summary.scalar("caption_length/batch_min", tf.reduce_min(lengths))
- tf.summary.scalar("caption_length/batch_max", tf.reduce_max(lengths))
- tf.summary.scalar("caption_length/batch_mean", tf.reduce_mean(lengths))
-
- return images, input_seqs, target_seqs, mask
diff --git a/research/im2txt/im2txt/run_inference.py b/research/im2txt/im2txt/run_inference.py
deleted file mode 100644
index 9848522df162e52394ee8349dab1f5220aeb88f6..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/run_inference.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-r"""Generate captions for images using default beam search parameters."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-import os
-
-
-import tensorflow as tf
-
-from im2txt import configuration
-from im2txt import inference_wrapper
-from im2txt.inference_utils import caption_generator
-from im2txt.inference_utils import vocabulary
-
-FLAGS = tf.flags.FLAGS
-
-tf.flags.DEFINE_string("checkpoint_path", "",
- "Model checkpoint file or directory containing a "
- "model checkpoint file.")
-tf.flags.DEFINE_string("vocab_file", "", "Text file containing the vocabulary.")
-tf.flags.DEFINE_string("input_files", "",
- "File pattern or comma-separated list of file patterns "
- "of image files.")
-
-tf.logging.set_verbosity(tf.logging.INFO)
-
-
-def main(_):
- # Build the inference graph.
- g = tf.Graph()
- with g.as_default():
- model = inference_wrapper.InferenceWrapper()
- restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
- FLAGS.checkpoint_path)
- g.finalize()
-
- # Create the vocabulary.
- vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
-
- filenames = []
- for file_pattern in FLAGS.input_files.split(","):
- filenames.extend(tf.gfile.Glob(file_pattern))
- tf.logging.info("Running caption generation on %d files matching %s",
- len(filenames), FLAGS.input_files)
-
- with tf.Session(graph=g) as sess:
- # Load the model from checkpoint.
- restore_fn(sess)
-
- # Prepare the caption generator. Here we are implicitly using the default
- # beam search parameters. See caption_generator.py for a description of the
- # available beam search parameters.
- generator = caption_generator.CaptionGenerator(model, vocab)
-
- for filename in filenames:
- with tf.gfile.GFile(filename, "rb") as f:
- image = f.read()
- captions = generator.beam_search(sess, image)
- print("Captions for image %s:" % os.path.basename(filename))
- for i, caption in enumerate(captions):
- # Ignore begin and end words.
- sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
- sentence = " ".join(sentence)
- print(" %d) %s (p=%f)" % (i, sentence, math.exp(caption.logprob)))
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/im2txt/im2txt/show_and_tell_model.py b/research/im2txt/im2txt/show_and_tell_model.py
deleted file mode 100644
index 0ac29e7fdb80fbefe3594eabc972648a3fb32312..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/show_and_tell_model.py
+++ /dev/null
@@ -1,358 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Image-to-text implementation based on http://arxiv.org/abs/1411.4555.
-
-"Show and Tell: A Neural Image Caption Generator"
-Oriol Vinyals, Alexander Toshev, Samy Bengio, Dumitru Erhan
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-from im2txt.ops import image_embedding
-from im2txt.ops import image_processing
-from im2txt.ops import inputs as input_ops
-
-
-class ShowAndTellModel(object):
- """Image-to-text implementation based on http://arxiv.org/abs/1411.4555.
-
- "Show and Tell: A Neural Image Caption Generator"
- Oriol Vinyals, Alexander Toshev, Samy Bengio, Dumitru Erhan
- """
-
- def __init__(self, config, mode, train_inception=False):
- """Basic setup.
-
- Args:
- config: Object containing configuration parameters.
- mode: "train", "eval" or "inference".
- train_inception: Whether the inception submodel variables are trainable.
- """
- assert mode in ["train", "eval", "inference"]
- self.config = config
- self.mode = mode
- self.train_inception = train_inception
-
- # Reader for the input data.
- self.reader = tf.TFRecordReader()
-
- # To match the "Show and Tell" paper we initialize all variables with a
- # random uniform initializer.
- self.initializer = tf.random_uniform_initializer(
- minval=-self.config.initializer_scale,
- maxval=self.config.initializer_scale)
-
- # A float32 Tensor with shape [batch_size, height, width, channels].
- self.images = None
-
- # An int32 Tensor with shape [batch_size, padded_length].
- self.input_seqs = None
-
- # An int32 Tensor with shape [batch_size, padded_length].
- self.target_seqs = None
-
- # An int32 0/1 Tensor with shape [batch_size, padded_length].
- self.input_mask = None
-
- # A float32 Tensor with shape [batch_size, embedding_size].
- self.image_embeddings = None
-
- # A float32 Tensor with shape [batch_size, padded_length, embedding_size].
- self.seq_embeddings = None
-
- # A float32 scalar Tensor; the total loss for the trainer to optimize.
- self.total_loss = None
-
- # A float32 Tensor with shape [batch_size * padded_length].
- self.target_cross_entropy_losses = None
-
- # A float32 Tensor with shape [batch_size * padded_length].
- self.target_cross_entropy_loss_weights = None
-
- # Collection of variables from the inception submodel.
- self.inception_variables = []
-
- # Function to restore the inception submodel from checkpoint.
- self.init_fn = None
-
- # Global step Tensor.
- self.global_step = None
-
- def is_training(self):
- """Returns true if the model is built for training mode."""
- return self.mode == "train"
-
- def process_image(self, encoded_image, thread_id=0):
- """Decodes and processes an image string.
-
- Args:
- encoded_image: A scalar string Tensor; the encoded image.
- thread_id: Preprocessing thread id used to select the ordering of color
- distortions.
-
- Returns:
- A float32 Tensor of shape [height, width, 3]; the processed image.
- """
- return image_processing.process_image(encoded_image,
- is_training=self.is_training(),
- height=self.config.image_height,
- width=self.config.image_width,
- thread_id=thread_id,
- image_format=self.config.image_format)
-
- def build_inputs(self):
- """Input prefetching, preprocessing and batching.
-
- Outputs:
- self.images
- self.input_seqs
- self.target_seqs (training and eval only)
- self.input_mask (training and eval only)
- """
- if self.mode == "inference":
- # In inference mode, images and inputs are fed via placeholders.
- image_feed = tf.placeholder(dtype=tf.string, shape=[], name="image_feed")
- input_feed = tf.placeholder(dtype=tf.int64,
- shape=[None], # batch_size
- name="input_feed")
-
- # Process image and insert batch dimensions.
- images = tf.expand_dims(self.process_image(image_feed), 0)
- input_seqs = tf.expand_dims(input_feed, 1)
-
- # No target sequences or input mask in inference mode.
- target_seqs = None
- input_mask = None
- else:
- # Prefetch serialized SequenceExample protos.
- input_queue = input_ops.prefetch_input_data(
- self.reader,
- self.config.input_file_pattern,
- is_training=self.is_training(),
- batch_size=self.config.batch_size,
- values_per_shard=self.config.values_per_input_shard,
- input_queue_capacity_factor=self.config.input_queue_capacity_factor,
- num_reader_threads=self.config.num_input_reader_threads)
-
- # Image processing and random distortion. Split across multiple threads
- # with each thread applying a slightly different distortion.
- assert self.config.num_preprocess_threads % 2 == 0
- images_and_captions = []
- for thread_id in range(self.config.num_preprocess_threads):
- serialized_sequence_example = input_queue.dequeue()
- encoded_image, caption = input_ops.parse_sequence_example(
- serialized_sequence_example,
- image_feature=self.config.image_feature_name,
- caption_feature=self.config.caption_feature_name)
- image = self.process_image(encoded_image, thread_id=thread_id)
- images_and_captions.append([image, caption])
-
- # Batch inputs.
- queue_capacity = (2 * self.config.num_preprocess_threads *
- self.config.batch_size)
- images, input_seqs, target_seqs, input_mask = (
- input_ops.batch_with_dynamic_pad(images_and_captions,
- batch_size=self.config.batch_size,
- queue_capacity=queue_capacity))
-
- self.images = images
- self.input_seqs = input_seqs
- self.target_seqs = target_seqs
- self.input_mask = input_mask
-
- def build_image_embeddings(self):
- """Builds the image model subgraph and generates image embeddings.
-
- Inputs:
- self.images
-
- Outputs:
- self.image_embeddings
- """
- inception_output = image_embedding.inception_v3(
- self.images,
- trainable=self.train_inception,
- is_training=self.is_training())
- self.inception_variables = tf.get_collection(
- tf.GraphKeys.GLOBAL_VARIABLES, scope="InceptionV3")
-
- # Map inception output into embedding space.
- with tf.variable_scope("image_embedding") as scope:
- image_embeddings = tf.contrib.layers.fully_connected(
- inputs=inception_output,
- num_outputs=self.config.embedding_size,
- activation_fn=None,
- weights_initializer=self.initializer,
- biases_initializer=None,
- scope=scope)
-
- # Save the embedding size in the graph.
- tf.constant(self.config.embedding_size, name="embedding_size")
-
- self.image_embeddings = image_embeddings
-
- def build_seq_embeddings(self):
- """Builds the input sequence embeddings.
-
- Inputs:
- self.input_seqs
-
- Outputs:
- self.seq_embeddings
- """
- with tf.variable_scope("seq_embedding"), tf.device("/cpu:0"):
- embedding_map = tf.get_variable(
- name="map",
- shape=[self.config.vocab_size, self.config.embedding_size],
- initializer=self.initializer)
- seq_embeddings = tf.nn.embedding_lookup(embedding_map, self.input_seqs)
-
- self.seq_embeddings = seq_embeddings
-
- def build_model(self):
- """Builds the model.
-
- Inputs:
- self.image_embeddings
- self.seq_embeddings
- self.target_seqs (training and eval only)
- self.input_mask (training and eval only)
-
- Outputs:
- self.total_loss (training and eval only)
- self.target_cross_entropy_losses (training and eval only)
- self.target_cross_entropy_loss_weights (training and eval only)
- """
- # This LSTM cell has biases and outputs tanh(new_c) * sigmoid(o), but the
- # modified LSTM in the "Show and Tell" paper has no biases and outputs
- # new_c * sigmoid(o).
- lstm_cell = tf.contrib.rnn.BasicLSTMCell(
- num_units=self.config.num_lstm_units, state_is_tuple=True)
- if self.mode == "train":
- lstm_cell = tf.contrib.rnn.DropoutWrapper(
- lstm_cell,
- input_keep_prob=self.config.lstm_dropout_keep_prob,
- output_keep_prob=self.config.lstm_dropout_keep_prob)
-
- with tf.variable_scope("lstm", initializer=self.initializer) as lstm_scope:
- # Feed the image embeddings to set the initial LSTM state.
- zero_state = lstm_cell.zero_state(
- batch_size=self.image_embeddings.get_shape()[0], dtype=tf.float32)
- _, initial_state = lstm_cell(self.image_embeddings, zero_state)
-
- # Allow the LSTM variables to be reused.
- lstm_scope.reuse_variables()
-
- if self.mode == "inference":
- # In inference mode, use concatenated states for convenient feeding and
- # fetching.
- tf.concat(axis=1, values=initial_state, name="initial_state")
-
- # Placeholder for feeding a batch of concatenated states.
- state_feed = tf.placeholder(dtype=tf.float32,
- shape=[None, sum(lstm_cell.state_size)],
- name="state_feed")
- state_tuple = tf.split(value=state_feed, num_or_size_splits=2, axis=1)
-
- # Run a single LSTM step.
- lstm_outputs, state_tuple = lstm_cell(
- inputs=tf.squeeze(self.seq_embeddings, axis=[1]),
- state=state_tuple)
-
- # Concatentate the resulting state.
- tf.concat(axis=1, values=state_tuple, name="state")
- else:
- # Run the batch of sequence embeddings through the LSTM.
- sequence_length = tf.reduce_sum(self.input_mask, 1)
- lstm_outputs, _ = tf.nn.dynamic_rnn(cell=lstm_cell,
- inputs=self.seq_embeddings,
- sequence_length=sequence_length,
- initial_state=initial_state,
- dtype=tf.float32,
- scope=lstm_scope)
-
- # Stack batches vertically.
- lstm_outputs = tf.reshape(lstm_outputs, [-1, lstm_cell.output_size])
-
- with tf.variable_scope("logits") as logits_scope:
- logits = tf.contrib.layers.fully_connected(
- inputs=lstm_outputs,
- num_outputs=self.config.vocab_size,
- activation_fn=None,
- weights_initializer=self.initializer,
- scope=logits_scope)
-
- if self.mode == "inference":
- tf.nn.softmax(logits, name="softmax")
- else:
- targets = tf.reshape(self.target_seqs, [-1])
- weights = tf.to_float(tf.reshape(self.input_mask, [-1]))
-
- # Compute losses.
- losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=targets,
- logits=logits)
- batch_loss = tf.div(tf.reduce_sum(tf.multiply(losses, weights)),
- tf.reduce_sum(weights),
- name="batch_loss")
- tf.losses.add_loss(batch_loss)
- total_loss = tf.losses.get_total_loss()
-
- # Add summaries.
- tf.summary.scalar("losses/batch_loss", batch_loss)
- tf.summary.scalar("losses/total_loss", total_loss)
- for var in tf.trainable_variables():
- tf.summary.histogram("parameters/" + var.op.name, var)
-
- self.total_loss = total_loss
- self.target_cross_entropy_losses = losses # Used in evaluation.
- self.target_cross_entropy_loss_weights = weights # Used in evaluation.
-
- def setup_inception_initializer(self):
- """Sets up the function to restore inception variables from checkpoint."""
- if self.mode != "inference":
- # Restore inception variables only.
- saver = tf.train.Saver(self.inception_variables)
-
- def restore_fn(sess):
- tf.logging.info("Restoring Inception variables from checkpoint file %s",
- self.config.inception_checkpoint_file)
- saver.restore(sess, self.config.inception_checkpoint_file)
-
- self.init_fn = restore_fn
-
- def setup_global_step(self):
- """Sets up the global step Tensor."""
- global_step = tf.Variable(
- initial_value=0,
- name="global_step",
- trainable=False,
- collections=[tf.GraphKeys.GLOBAL_STEP, tf.GraphKeys.GLOBAL_VARIABLES])
-
- self.global_step = global_step
-
- def build(self):
- """Creates all ops for training and evaluation."""
- self.build_inputs()
- self.build_image_embeddings()
- self.build_seq_embeddings()
- self.build_model()
- self.setup_inception_initializer()
- self.setup_global_step()
diff --git a/research/im2txt/im2txt/show_and_tell_model_test.py b/research/im2txt/im2txt/show_and_tell_model_test.py
deleted file mode 100644
index 0bdfb6e1a3ae3c15bd1c8daf005fe2542436ca8e..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/show_and_tell_model_test.py
+++ /dev/null
@@ -1,200 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for tensorflow_models.im2txt.show_and_tell_model."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import numpy as np
-import tensorflow as tf
-
-from im2txt import configuration
-from im2txt import show_and_tell_model
-
-
-class ShowAndTellModel(show_and_tell_model.ShowAndTellModel):
- """Subclass of ShowAndTellModel without the disk I/O."""
-
- def build_inputs(self):
- if self.mode == "inference":
- # Inference mode doesn't read from disk, so defer to parent.
- return super(ShowAndTellModel, self).build_inputs()
- else:
- # Replace disk I/O with random Tensors.
- self.images = tf.random_uniform(
- shape=[self.config.batch_size, self.config.image_height,
- self.config.image_width, 3],
- minval=-1,
- maxval=1)
- self.input_seqs = tf.random_uniform(
- [self.config.batch_size, 15],
- minval=0,
- maxval=self.config.vocab_size,
- dtype=tf.int64)
- self.target_seqs = tf.random_uniform(
- [self.config.batch_size, 15],
- minval=0,
- maxval=self.config.vocab_size,
- dtype=tf.int64)
- self.input_mask = tf.ones_like(self.input_seqs)
-
-
-class ShowAndTellModelTest(tf.test.TestCase):
-
- def setUp(self):
- super(ShowAndTellModelTest, self).setUp()
- self._model_config = configuration.ModelConfig()
-
- def _countModelParameters(self):
- """Counts the number of parameters in the model at top level scope."""
- counter = {}
- for v in tf.global_variables():
- name = v.op.name.split("/")[0]
- num_params = v.get_shape().num_elements()
- assert num_params
- counter[name] = counter.get(name, 0) + num_params
- return counter
-
- def _checkModelParameters(self):
- """Verifies the number of parameters in the model."""
- param_counts = self._countModelParameters()
- expected_param_counts = {
- "InceptionV3": 21802784,
- # inception_output_size * embedding_size
- "image_embedding": 1048576,
- # vocab_size * embedding_size
- "seq_embedding": 6144000,
- # (embedding_size + num_lstm_units + 1) * 4 * num_lstm_units
- "lstm": 2099200,
- # (num_lstm_units + 1) * vocab_size
- "logits": 6156000,
- "global_step": 1,
- }
- self.assertDictEqual(expected_param_counts, param_counts)
-
- def _checkOutputs(self, expected_shapes, feed_dict=None):
- """Verifies that the model produces expected outputs.
-
- Args:
- expected_shapes: A dict mapping Tensor or Tensor name to expected output
- shape.
- feed_dict: Values of Tensors to feed into Session.run().
- """
- fetches = expected_shapes.keys()
-
- with self.test_session() as sess:
- sess.run(tf.global_variables_initializer())
- outputs = sess.run(fetches, feed_dict)
-
- for index, output in enumerate(outputs):
- tensor = fetches[index]
- expected = expected_shapes[tensor]
- actual = output.shape
- if expected != actual:
- self.fail("Tensor %s has shape %s (expected %s)." %
- (tensor, actual, expected))
-
- def testBuildForTraining(self):
- model = ShowAndTellModel(self._model_config, mode="train")
- model.build()
-
- self._checkModelParameters()
-
- expected_shapes = {
- # [batch_size, image_height, image_width, 3]
- model.images: (32, 299, 299, 3),
- # [batch_size, sequence_length]
- model.input_seqs: (32, 15),
- # [batch_size, sequence_length]
- model.target_seqs: (32, 15),
- # [batch_size, sequence_length]
- model.input_mask: (32, 15),
- # [batch_size, embedding_size]
- model.image_embeddings: (32, 512),
- # [batch_size, sequence_length, embedding_size]
- model.seq_embeddings: (32, 15, 512),
- # Scalar
- model.total_loss: (),
- # [batch_size * sequence_length]
- model.target_cross_entropy_losses: (480,),
- # [batch_size * sequence_length]
- model.target_cross_entropy_loss_weights: (480,),
- }
- self._checkOutputs(expected_shapes)
-
- def testBuildForEval(self):
- model = ShowAndTellModel(self._model_config, mode="eval")
- model.build()
-
- self._checkModelParameters()
-
- expected_shapes = {
- # [batch_size, image_height, image_width, 3]
- model.images: (32, 299, 299, 3),
- # [batch_size, sequence_length]
- model.input_seqs: (32, 15),
- # [batch_size, sequence_length]
- model.target_seqs: (32, 15),
- # [batch_size, sequence_length]
- model.input_mask: (32, 15),
- # [batch_size, embedding_size]
- model.image_embeddings: (32, 512),
- # [batch_size, sequence_length, embedding_size]
- model.seq_embeddings: (32, 15, 512),
- # Scalar
- model.total_loss: (),
- # [batch_size * sequence_length]
- model.target_cross_entropy_losses: (480,),
- # [batch_size * sequence_length]
- model.target_cross_entropy_loss_weights: (480,),
- }
- self._checkOutputs(expected_shapes)
-
- def testBuildForInference(self):
- model = ShowAndTellModel(self._model_config, mode="inference")
- model.build()
-
- self._checkModelParameters()
-
- # Test feeding an image to get the initial LSTM state.
- images_feed = np.random.rand(1, 299, 299, 3)
- feed_dict = {model.images: images_feed}
- expected_shapes = {
- # [batch_size, embedding_size]
- model.image_embeddings: (1, 512),
- # [batch_size, 2 * num_lstm_units]
- "lstm/initial_state:0": (1, 1024),
- }
- self._checkOutputs(expected_shapes, feed_dict)
-
- # Test feeding a batch of inputs and LSTM states to get softmax output and
- # LSTM states.
- input_feed = np.random.randint(0, 10, size=3)
- state_feed = np.random.rand(3, 1024)
- feed_dict = {"input_feed:0": input_feed, "lstm/state_feed:0": state_feed}
- expected_shapes = {
- # [batch_size, 2 * num_lstm_units]
- "lstm/state:0": (3, 1024),
- # [batch_size, vocab_size]
- "softmax:0": (3, 12000),
- }
- self._checkOutputs(expected_shapes, feed_dict)
-
-
-if __name__ == "__main__":
- tf.test.main()
diff --git a/research/im2txt/im2txt/train.py b/research/im2txt/im2txt/train.py
deleted file mode 100644
index db602735ba11e7f540a4e985333d8a457512c977..0000000000000000000000000000000000000000
--- a/research/im2txt/im2txt/train.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Train the model."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-from im2txt import configuration
-from im2txt import show_and_tell_model
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.flags.DEFINE_string("input_file_pattern", "",
- "File pattern of sharded TFRecord input files.")
-tf.flags.DEFINE_string("inception_checkpoint_file", "",
- "Path to a pretrained inception_v3 model.")
-tf.flags.DEFINE_string("train_dir", "",
- "Directory for saving and loading model checkpoints.")
-tf.flags.DEFINE_boolean("train_inception", False,
- "Whether to train inception submodel variables.")
-tf.flags.DEFINE_integer("number_of_steps", 1000000, "Number of training steps.")
-tf.flags.DEFINE_integer("log_every_n_steps", 1,
- "Frequency at which loss and global step are logged.")
-
-tf.logging.set_verbosity(tf.logging.INFO)
-
-
-def main(unused_argv):
- assert FLAGS.input_file_pattern, "--input_file_pattern is required"
- assert FLAGS.train_dir, "--train_dir is required"
-
- model_config = configuration.ModelConfig()
- model_config.input_file_pattern = FLAGS.input_file_pattern
- model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file
- training_config = configuration.TrainingConfig()
-
- # Create training directory.
- train_dir = FLAGS.train_dir
- if not tf.gfile.IsDirectory(train_dir):
- tf.logging.info("Creating training directory: %s", train_dir)
- tf.gfile.MakeDirs(train_dir)
-
- # Build the TensorFlow graph.
- g = tf.Graph()
- with g.as_default():
- # Build the model.
- model = show_and_tell_model.ShowAndTellModel(
- model_config, mode="train", train_inception=FLAGS.train_inception)
- model.build()
-
- # Set up the learning rate.
- learning_rate_decay_fn = None
- if FLAGS.train_inception:
- learning_rate = tf.constant(training_config.train_inception_learning_rate)
- else:
- learning_rate = tf.constant(training_config.initial_learning_rate)
- if training_config.learning_rate_decay_factor > 0:
- num_batches_per_epoch = (training_config.num_examples_per_epoch /
- model_config.batch_size)
- decay_steps = int(num_batches_per_epoch *
- training_config.num_epochs_per_decay)
-
- def _learning_rate_decay_fn(learning_rate, global_step):
- return tf.train.exponential_decay(
- learning_rate,
- global_step,
- decay_steps=decay_steps,
- decay_rate=training_config.learning_rate_decay_factor,
- staircase=True)
-
- learning_rate_decay_fn = _learning_rate_decay_fn
-
- # Set up the training ops.
- train_op = tf.contrib.layers.optimize_loss(
- loss=model.total_loss,
- global_step=model.global_step,
- learning_rate=learning_rate,
- optimizer=training_config.optimizer,
- clip_gradients=training_config.clip_gradients,
- learning_rate_decay_fn=learning_rate_decay_fn)
-
- # Set up the Saver for saving and restoring model checkpoints.
- saver = tf.train.Saver(max_to_keep=training_config.max_checkpoints_to_keep)
-
- # Run training.
- tf.contrib.slim.learning.train(
- train_op,
- train_dir,
- log_every_n_steps=FLAGS.log_every_n_steps,
- graph=g,
- global_step=model.global_step,
- number_of_steps=FLAGS.number_of_steps,
- init_fn=model.init_fn,
- saver=saver)
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/inception/.gitignore b/research/inception/.gitignore
deleted file mode 100644
index 58cbf2f4e0d5d39a0e3910d6993508546dad429f..0000000000000000000000000000000000000000
--- a/research/inception/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
-/bazel-bin
-/bazel-ci_build-cache
-/bazel-genfiles
-/bazel-out
-/bazel-inception
-/bazel-testlogs
-/bazel-tf
diff --git a/research/inception/README.md b/research/inception/README.md
deleted file mode 100644
index beed66cf5cd83a6843ec39b28b5dbd88f1c0d3d0..0000000000000000000000000000000000000000
--- a/research/inception/README.md
+++ /dev/null
@@ -1,858 +0,0 @@
-
-
-
-
-**NOTE: For the most part, you will find a newer version of this code at [models/research/slim](https://github.com/tensorflow/models/tree/master/research/slim).** In particular:
-
-* `inception_train.py` and `imagenet_train.py` should no longer be used. The slim editions for running on multiple GPUs are the current best examples.
-* `inception_distributed_train.py` and `imagenet_distributed_train.py` are still valid examples of distributed training.
-
-For performance benchmarking, please see https://www.tensorflow.org/performance/benchmarks.
-
----
-
-# Inception in TensorFlow
-
-[ImageNet](http://www.image-net.org/) is a common academic data set in machine
-learning for training an image recognition system. Code in this directory
-demonstrates how to use TensorFlow to train and evaluate a type of convolutional
-neural network (CNN) on this academic data set. In particular, we demonstrate
-how to train the Inception v3 architecture as specified in:
-
-_Rethinking the Inception Architecture for Computer Vision_
-
-Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, Zbigniew
-Wojna
-
-http://arxiv.org/abs/1512.00567
-
-This network achieves 21.2% top-1 and 5.6% top-5 error for single frame
-evaluation with a computational cost of 5 billion multiply-adds per inference
-and with using less than 25 million parameters. Below is a visualization of the
-model architecture.
-
-
-
-## Description of Code
-
-The code base provides three core binaries for:
-
-* Training an Inception v3 network from scratch across multiple GPUs and/or
- multiple machines using the ImageNet 2012 Challenge training data set.
-* Evaluating an Inception v3 network using the ImageNet 2012 Challenge
- validation data set.
-* Retraining an Inception v3 network on a novel task and back-propagating the
- errors to fine tune the network weights.
-
-The training procedure employs synchronous stochastic gradient descent across
-multiple GPUs. The user may specify the number of GPUs they wish to harness. The
-synchronous training performs *batch-splitting* by dividing a given batch across
-multiple GPUs.
-
-The training set up is nearly identical to the section [Training a Model Using
-Multiple GPU Cards](https://www.tensorflow.org/tutorials/deep_cnn/index.html#launching_and_training_the_model_on_multiple_gpu_cards)
-where we have substituted the CIFAR-10 model architecture with Inception v3. The
-primary differences with that setup are:
-
-* Calculate and update the batch-norm statistics during training so that they
- may be substituted in during evaluation.
-* Specify the model architecture using a (still experimental) higher level
- language called TensorFlow-Slim.
-
-For more details about TensorFlow-Slim, please see the [Slim README](inception/slim/README.md). Please note that this higher-level language is still
-*experimental* and the API may change over time depending on usage and
-subsequent research.
-
-## Getting Started
-
-Before you run the training script for the first time, you will need to download
-and convert the ImageNet data to native TFRecord format. The TFRecord format
-consists of a set of sharded files where each entry is a serialized `tf.Example`
-proto. Each `tf.Example` proto contains the ImageNet image (JPEG encoded) as
-well as metadata such as label and bounding box information. See
-[`parse_example_proto`](inception/image_processing.py) for details.
-
-We provide a single [script](inception/data/download_and_preprocess_imagenet.sh) for
-downloading and converting ImageNet data to TFRecord format. Downloading and
-preprocessing the data may take several hours (up to half a day) depending on
-your network and computer speed. Please be patient.
-
-To begin, you will need to sign up for an account with [ImageNet](http://image-net.org) to gain access to the data. Look for the sign up page,
-create an account and request an access key to download the data.
-
-After you have `USERNAME` and `PASSWORD`, you are ready to run our script. Make
-sure that your hard disk has at least 500 GB of free space for downloading and
-storing the data. Here we select `DATA_DIR=$HOME/imagenet-data` as such a
-location but feel free to edit accordingly.
-
-When you run the below script, please enter *USERNAME* and *PASSWORD* when
-prompted. This will occur at the very beginning. Once these values are entered,
-you will not need to interact with the script again.
-
-```shell
-# location of where to place the ImageNet data
-DATA_DIR=$HOME/imagenet-data
-
-# build the preprocessing script.
-cd tensorflow-models/inception
-bazel build //inception:download_and_preprocess_imagenet
-
-# run it
-bazel-bin/inception/download_and_preprocess_imagenet "${DATA_DIR}"
-```
-
-The final line of the output script should read:
-
-```shell
-2016-02-17 14:30:17.287989: Finished writing all 1281167 images in data set.
-```
-
-When the script finishes, you will find 1024 training files and 128 validation
-files in the `DATA_DIR`. The files will match the patterns
-`train-?????-of-01024` and `validation-?????-of-00128`, respectively.
-
-[Congratulations!](https://www.youtube.com/watch?v=9bZkp7q19f0) You are now
-ready to train or evaluate with the ImageNet data set.
-
-## How to Train from Scratch
-
-**WARNING** Training an Inception v3 network from scratch is a computationally
-intensive task and depending on your compute setup may take several days or even
-weeks.
-
-*Before proceeding* please read the [Convolutional Neural Networks](https://www.tensorflow.org/tutorials/deep_cnn/index.html) tutorial; in
-particular, focus on [Training a Model Using Multiple GPU Cards](https://www.tensorflow.org/tutorials/deep_cnn/index.html#launching_and_training_the_model_on_multiple_gpu_cards). The model training method is nearly identical to that described in the
-CIFAR-10 multi-GPU model training. Briefly, the model training
-
-* Places an individual model replica on each GPU.
-* Splits the batch across the GPUs.
-* Updates model parameters synchronously by waiting for all GPUs to finish
- processing a batch of data.
-
-The training procedure is encapsulated by this diagram of how operations and
-variables are placed on CPU and GPUs respectively.
-
-
-
-
-
-Each tower computes the gradients for a portion of the batch and the gradients
-are combined and averaged across the multiple towers in order to provide a
-single update of the Variables stored on the CPU.
-
-A crucial aspect of training a network of this size is *training speed* in terms
-of wall-clock time. The training speed is dictated by many factors -- most
-importantly the batch size and the learning rate schedule. Both of these
-parameters are heavily coupled to the hardware set up.
-
-Generally speaking, a batch size is a difficult parameter to tune as it requires
-balancing memory demands of the model, memory available on the GPU and speed of
-computation. Generally speaking, employing larger batch sizes leads to more
-efficient computation and potentially more efficient training steps.
-
-We have tested several hardware setups for training this model from scratch but
-we emphasize that depending your hardware set up, you may need to adapt the
-batch size and learning rate schedule.
-
-Please see the comments in `inception_train.py` for a few selected learning rate
-plans based on some selected hardware setups.
-
-To train this model, you simply need to specify the following:
-
-```shell
-# Build the model. Note that we need to make sure the TensorFlow is ready to
-# use before this as this command will not build TensorFlow.
-cd tensorflow-models/inception
-bazel build //inception:imagenet_train
-
-# run it
-bazel-bin/inception/imagenet_train --num_gpus=1 --batch_size=32 --train_dir=/tmp/imagenet_train --data_dir=/tmp/imagenet_data
-```
-
-The model reads in the ImageNet training data from `--data_dir`. If you followed
-the instructions in [Getting Started](#getting-started), then set
-`--data_dir="${DATA_DIR}"`. The script assumes that there exists a set of
-sharded TFRecord files containing the ImageNet data. If you have not created
-TFRecord files, please refer to [Getting Started](#getting-started)
-
-Here is the output of the above command line when running on a Tesla K40c:
-
-```shell
-2016-03-07 12:24:59.922898: step 0, loss = 13.11 (5.3 examples/sec; 6.064 sec/batch)
-2016-03-07 12:25:55.206783: step 10, loss = 13.71 (9.4 examples/sec; 3.394 sec/batch)
-2016-03-07 12:26:28.905231: step 20, loss = 14.81 (9.5 examples/sec; 3.380 sec/batch)
-2016-03-07 12:27:02.699719: step 30, loss = 14.45 (9.5 examples/sec; 3.378 sec/batch)
-2016-03-07 12:27:36.515699: step 40, loss = 13.98 (9.5 examples/sec; 3.376 sec/batch)
-2016-03-07 12:28:10.220956: step 50, loss = 13.92 (9.6 examples/sec; 3.327 sec/batch)
-2016-03-07 12:28:43.658223: step 60, loss = 13.28 (9.6 examples/sec; 3.350 sec/batch)
-...
-```
-
-In this example, a log entry is printed every 10 step and the line includes the
-total loss (starts around 13.0-14.0) and the speed of processing in terms of
-throughput (examples / sec) and batch speed (sec/batch).
-
-The number of GPU devices is specified by `--num_gpus` (which defaults to 1).
-Specifying `--num_gpus` greater then 1 splits the batch evenly split across the
-GPU cards.
-
-```shell
-# Build the model. Note that we need to make sure the TensorFlow is ready to
-# use before this as this command will not build TensorFlow.
-cd tensorflow-models/inception
-bazel build //inception:imagenet_train
-
-# run it
-bazel-bin/inception/imagenet_train --num_gpus=2 --batch_size=64 --train_dir=/tmp/imagenet_train
-```
-
-This model splits the batch of 64 images across 2 GPUs and calculates the
-average gradient by waiting for both GPUs to finish calculating the gradients
-from their respective data (See diagram above). Generally speaking, using larger
-numbers of GPUs leads to higher throughput as well as the opportunity to use
-larger batch sizes. In turn, larger batch sizes imply better estimates of the
-gradient enabling the usage of higher learning rates. In summary, using more
-GPUs results in simply faster training speed.
-
-Note that selecting a batch size is a difficult parameter to tune as it requires
-balancing memory demands of the model, memory available on the GPU and speed of
-computation. Generally speaking, employing larger batch sizes leads to more
-efficient computation and potentially more efficient training steps.
-
-Note that there is considerable noise in the loss function on individual steps
-in the previous log. Because of this noise, it is difficult to discern how well
-a model is learning. The solution to the last problem is to launch TensorBoard
-pointing to the directory containing the events log.
-
-```shell
-tensorboard --logdir=/tmp/imagenet_train
-```
-
-TensorBoard has access to the many Summaries produced by the model that describe
-multitudes of statistics tracking the model behavior and the quality of the
-learned model. In particular, TensorBoard tracks a exponentially smoothed
-version of the loss. In practice, it is far easier to judge how well a model
-learns by monitoring the smoothed version of the loss.
-
-## How to Train from Scratch in a Distributed Setting
-
-**NOTE** Distributed TensorFlow requires version 0.8 or later.
-
-Distributed TensorFlow lets us use multiple machines to train a model faster.
-This is quite different from the training with multiple GPU towers on a single
-machine where all parameters and gradients computation are in the same place. We
-coordinate the computation across multiple machines by employing a centralized
-repository for parameters that maintains a unified, single copy of model
-parameters. Each individual machine sends gradient updates to the centralized
-parameter repository which coordinates these updates and sends back updated
-parameters to the individual machines running the model training.
-
-We term each machine that runs a copy of the training a `worker` or `replica`.
-We term each machine that maintains model parameters a `ps`, short for
-`parameter server`. Note that we might have more than one machine acting as a
-`ps` as the model parameters may be sharded across multiple machines.
-
-Variables may be updated with synchronous or asynchronous gradient updates. One
-may construct a an [`Optimizer`](https://www.tensorflow.org/api_docs/python/train.html#optimizers) in TensorFlow
-that constructs the necessary graph for either case diagrammed below from the
-TensorFlow [Whitepaper](http://download.tensorflow.org/paper/whitepaper2015.pdf):
-
-
-
-
-
-In [a recent paper](https://arxiv.org/abs/1604.00981), synchronous gradient
-updates have demonstrated to reach higher accuracy in a shorter amount of time.
-In this distributed Inception example we employ synchronous gradient updates.
-
-Note that in this example each replica has a single tower that uses one GPU.
-
-The command-line flags `worker_hosts` and `ps_hosts` specify available servers.
-The same binary will be used for both the `worker` jobs and the `ps` jobs.
-Command line flag `job_name` will be used to specify what role a task will be
-playing and `task_id` will be used to identify which one of the jobs it is
-running. Several things to note here:
-
-* The numbers of `ps` and `worker` tasks are inferred from the lists of hosts
- specified in the flags. The `task_id` should be within the range `[0,
- num_ps_tasks)` for `ps` tasks and `[0, num_worker_tasks)` for `worker`
- tasks.
-* `ps` and `worker` tasks can run on the same machine, as long as that machine
- has sufficient resources to handle both tasks. Note that the `ps` task does
- not benefit from a GPU, so it should not attempt to use one (see below).
-* Multiple `worker` tasks can run on the same machine with multiple GPUs so
- machine_A with 2 GPUs may have 2 workers while machine_B with 1 GPU just has
- 1 worker.
-* The default learning rate schedule works well for a wide range of number of
- replicas [25, 50, 100] but feel free to tune it for even better results.
-* The command line of both `ps` and `worker` tasks should include the complete
- list of `ps_hosts` and `worker_hosts`.
-* There is a chief `worker` among all workers which defaults to `worker` 0.
- The chief will be in charge of initializing all the parameters, writing out
- the summaries and the checkpoint. The checkpoint and summary will be in the
- `train_dir` of the host for `worker` 0.
-* Each worker processes a batch_size number of examples but each gradient
- update is computed from all replicas. Hence, the effective batch size of
- this model is batch_size * num_workers.
-
-```shell
-# Build the model. Note that we need to make sure the TensorFlow is ready to
-# use before this as this command will not build TensorFlow.
-cd tensorflow-models/inception
-bazel build //inception:imagenet_distributed_train
-
-# To start worker 0, go to the worker0 host and run the following (Note that
-# task_id should be in the range [0, num_worker_tasks):
-bazel-bin/inception/imagenet_distributed_train \
---batch_size=32 \
---data_dir=$HOME/imagenet-data \
---job_name='worker' \
---task_id=0 \
---ps_hosts='ps0.example.com:2222' \
---worker_hosts='worker0.example.com:2222,worker1.example.com:2222'
-
-# To start worker 1, go to the worker1 host and run the following (Note that
-# task_id should be in the range [0, num_worker_tasks):
-bazel-bin/inception/imagenet_distributed_train \
---batch_size=32 \
---data_dir=$HOME/imagenet-data \
---job_name='worker' \
---task_id=1 \
---ps_hosts='ps0.example.com:2222' \
---worker_hosts='worker0.example.com:2222,worker1.example.com:2222'
-
-# To start the parameter server (ps), go to the ps host and run the following (Note
-# that task_id should be in the range [0, num_ps_tasks):
-bazel-bin/inception/imagenet_distributed_train \
---job_name='ps' \
---task_id=0 \
---ps_hosts='ps0.example.com:2222' \
---worker_hosts='worker0.example.com:2222,worker1.example.com:2222'
-```
-
-If you have installed a GPU-compatible version of TensorFlow, the `ps` will also
-try to allocate GPU memory although it is not helpful. This could potentially
-crash the worker on the same machine as it has little to no GPU memory to
-allocate. To avoid this, you can prepend the previous command to start `ps`
-with: `CUDA_VISIBLE_DEVICES=''`
-
-```shell
-CUDA_VISIBLE_DEVICES='' bazel-bin/inception/imagenet_distributed_train \
---job_name='ps' \
---task_id=0 \
---ps_hosts='ps0.example.com:2222' \
---worker_hosts='worker0.example.com:2222,worker1.example.com:2222'
-```
-
-If you have run everything correctly, you should see a log in each `worker` job
-that looks like the following. Note the training speed varies depending on your
-hardware and the first several steps could take much longer.
-
-```shell
-INFO:tensorflow:PS hosts are: ['ps0.example.com:2222', 'ps1.example.com:2222']
-INFO:tensorflow:Worker hosts are: ['worker0.example.com:2222', 'worker1.example.com:2222']
-I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job ps -> {ps0.example.com:2222, ps1.example.com:2222}
-I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job worker -> {localhost:2222, worker1.example.com:2222}
-I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222
-INFO:tensorflow:Created variable global_step:0 with shape () and init
-
-...
-
-INFO:tensorflow:Created variable logits/logits/biases:0 with shape (1001,) and init
-INFO:tensorflow:SyncReplicas enabled: replicas_to_aggregate=2; total_num_replicas=2
-INFO:tensorflow:2016-04-13 01:56:26.405639 Supervisor
-INFO:tensorflow:Started 2 queues for processing input data.
-INFO:tensorflow:global_step/sec: 0
-INFO:tensorflow:Worker 0: 2016-04-13 01:58:40.342404: step 0, loss = 12.97(0.0 examples/sec; 65.428 sec/batch)
-INFO:tensorflow:global_step/sec: 0.0172907
-...
-```
-
-and a log in each `ps` job that looks like the following:
-
-```shell
-INFO:tensorflow:PS hosts are: ['ps0.example.com:2222', 'ps1.example.com:2222']
-INFO:tensorflow:Worker hosts are: ['worker0.example.com:2222', 'worker1.example.com:2222']
-I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job ps -> {localhost:2222, ps1.example.com:2222}
-I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job worker -> {worker0.example.com:2222, worker1.example.com:2222}
-I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222
-```
-
-If you compiled TensorFlow (from v1.1-rc3) with VERBS support and you have the
-required device and IB verbs SW stack, you can specify --protocol='grpc+verbs'
-In order to use Verbs RDMA for Tensor passing between workers and ps.
-Need to add the the --protocol flag in all tasks (ps and workers).
-The default protocol is the TensorFlow default protocol of grpc.
-
-
-[Congratulations!](https://www.youtube.com/watch?v=9bZkp7q19f0) You are now
-training Inception in a distributed manner.
-
-## How to Evaluate
-
-Evaluating an Inception v3 model on the ImageNet 2012 validation data set
-requires running a separate binary.
-
-The evaluation procedure is nearly identical to [Evaluating a Model](https://www.tensorflow.org/tutorials/deep_cnn/index.html#evaluating_a_model)
-described in the [Convolutional Neural Network](https://www.tensorflow.org/tutorials/deep_cnn/index.html) tutorial.
-
-**WARNING** Be careful not to run the evaluation and training binary on the same
-GPU or else you might run out of memory. Consider running the evaluation on a
-separate GPU if available or suspending the training binary while running the
-evaluation on the same GPU.
-
-Briefly, one can evaluate the model by running:
-
-```shell
-# Build the model. Note that we need to make sure the TensorFlow is ready to
-# use before this as this command will not build TensorFlow.
-cd tensorflow-models/inception
-bazel build //inception:imagenet_eval
-
-# run it
-bazel-bin/inception/imagenet_eval --checkpoint_dir=/tmp/imagenet_train --eval_dir=/tmp/imagenet_eval
-```
-
-Note that we point `--checkpoint_dir` to the location of the checkpoints saved
-by `inception_train.py` above. Running the above command results in the
-following output:
-
-```shell
-2016-02-17 22:32:50.391206: precision @ 1 = 0.735
-...
-```
-
-The script calculates the precision @ 1 over the entire validation data
-periodically. The precision @ 1 measures the how often the highest scoring
-prediction from the model matched the ImageNet label -- in this case, 73.5%. If
-you wish to run the eval just once and not periodically, append the `--run_once`
-option.
-
-Much like the training script, `imagenet_eval.py` also exports summaries that
-may be visualized in TensorBoard. These summaries calculate additional
-statistics on the predictions (e.g. recall @ 5) as well as monitor the
-statistics of the model activations and weights during evaluation.
-
-## How to Fine-Tune a Pre-Trained Model on a New Task
-
-### Getting Started
-
-Much like training the ImageNet model we must first convert a new data set to
-the sharded TFRecord format which each entry is a serialized `tf.Example` proto.
-
-We have provided a script demonstrating how to do this for small data set of of
-a few thousand flower images spread across 5 labels:
-
-```shell
-daisy, dandelion, roses, sunflowers, tulips
-```
-
-There is a single automated script that downloads the data set and converts it
-to the TFRecord format. Much like the ImageNet data set, each record in the
-TFRecord format is a serialized `tf.Example` proto whose entries include a
-JPEG-encoded string and an integer label. Please see [`parse_example_proto`](inception/image_processing.py) for details.
-
-The script just takes a few minutes to run depending your network connection
-speed for downloading and processing the images. Your hard disk requires 200MB
-of free storage. Here we select `DATA_DIR=/tmp/flowers-data/` as such a location
-but feel free to edit accordingly.
-
-```shell
-# location of where to place the flowers data
-FLOWERS_DATA_DIR=/tmp/flowers-data/
-
-# build the preprocessing script.
-cd tensorflow-models/inception
-bazel build //inception:download_and_preprocess_flowers
-
-# run it
-bazel-bin/inception/download_and_preprocess_flowers "${FLOWERS_DATA_DIR}"
-```
-
-If the script runs successfully, the final line of the terminal output should
-look like:
-
-```shell
-2016-02-24 20:42:25.067551: Finished writing all 3170 images in data set.
-```
-
-When the script finishes you will find 2 shards for the training and validation
-files in the `DATA_DIR`. The files will match the patterns `train-?????-of-00002`
-and `validation-?????-of-00002`, respectively.
-
-**NOTE** If you wish to prepare a custom image data set for transfer learning,
-you will need to invoke [`build_image_data.py`](inception/data/build_image_data.py) on
-your custom data set. Please see the associated options and assumptions behind
-this script by reading the comments section of [`build_image_data.py`](inception/data/build_image_data.py). Also, if your custom data has a different
-number of examples or classes, you need to change the appropriate values in
-[`imagenet_data.py`](inception/imagenet_data.py).
-
-The second piece you will need is a trained Inception v3 image model. You have
-the option of either training one yourself (See [How to Train from Scratch](#how-to-train-from-scratch) for details) or you can download a pre-trained
-model like so:
-
-```shell
-# location of where to place the Inception v3 model
-INCEPTION_MODEL_DIR=$HOME/inception-v3-model
-mkdir -p ${INCEPTION_MODEL_DIR}
-cd ${INCEPTION_MODEL_DIR}
-
-# download the Inception v3 model
-curl -O http://download.tensorflow.org/models/image/imagenet/inception-v3-2016-03-01.tar.gz
-tar xzf inception-v3-2016-03-01.tar.gz
-
-# this will create a directory called inception-v3 which contains the following files.
-> ls inception-v3
-README.txt
-checkpoint
-model.ckpt-157585
-```
-
-[Congratulations!](https://www.youtube.com/watch?v=9bZkp7q19f0) You are now
-ready to fine-tune your pre-trained Inception v3 model with the flower data set.
-
-### How to Retrain a Trained Model on the Flowers Data
-
-We are now ready to fine-tune a pre-trained Inception-v3 model on the flowers
-data set. This requires two distinct changes to our training procedure:
-
-1. Build the exact same model as previously except we change the number of
- labels in the final classification layer.
-
-2. Restore all weights from the pre-trained Inception-v3 except for the final
- classification layer; this will get randomly initialized instead.
-
-We can perform these two operations by specifying two flags:
-`--pretrained_model_checkpoint_path` and `--fine_tune`. The first flag is a
-string that points to the path of a pre-trained Inception-v3 model. If this flag
-is specified, it will load the entire model from the checkpoint before the
-script begins training.
-
-The second flag `--fine_tune` is a boolean that indicates whether the last
-classification layer should be randomly initialized or restored. You may set
-this flag to false if you wish to continue training a pre-trained model from a
-checkpoint. If you set this flag to true, you can train a new classification
-layer from scratch.
-
-In order to understand how `--fine_tune` works, please see the discussion on
-`Variables` in the TensorFlow-Slim [`README.md`](inception/slim/README.md).
-
-Putting this all together you can retrain a pre-trained Inception-v3 model on
-the flowers data set with the following command.
-
-```shell
-# Build the model. Note that we need to make sure the TensorFlow is ready to
-# use before this as this command will not build TensorFlow.
-cd tensorflow-models/inception
-bazel build //inception:flowers_train
-
-# Path to the downloaded Inception-v3 model.
-MODEL_PATH="${INCEPTION_MODEL_DIR}/inception-v3/model.ckpt-157585"
-
-# Directory where the flowers data resides.
-FLOWERS_DATA_DIR=/tmp/flowers-data/
-
-# Directory where to save the checkpoint and events files.
-TRAIN_DIR=/tmp/flowers_train/
-
-# Run the fine-tuning on the flowers data set starting from the pre-trained
-# Imagenet-v3 model.
-bazel-bin/inception/flowers_train \
- --train_dir="${TRAIN_DIR}" \
- --data_dir="${FLOWERS_DATA_DIR}" \
- --pretrained_model_checkpoint_path="${MODEL_PATH}" \
- --fine_tune=True \
- --initial_learning_rate=0.001 \
- --input_queue_memory_factor=1
-```
-
-We have added a few extra options to the training procedure.
-
-* Fine-tuning a model a separate data set requires significantly lowering the
- initial learning rate. We set the initial learning rate to 0.001.
-* The flowers data set is quite small so we shrink the size of the shuffling
- queue of examples. See [Adjusting Memory Demands](#adjusting-memory-demands)
- for more details.
-
-The training script will only reports the loss. To evaluate the quality of the
-fine-tuned model, you will need to run `flowers_eval`:
-
-```shell
-# Build the model. Note that we need to make sure the TensorFlow is ready to
-# use before this as this command will not build TensorFlow.
-cd tensorflow-models/inception
-bazel build //inception:flowers_eval
-
-# Directory where we saved the fine-tuned checkpoint and events files.
-TRAIN_DIR=/tmp/flowers_train/
-
-# Directory where the flowers data resides.
-FLOWERS_DATA_DIR=/tmp/flowers-data/
-
-# Directory where to save the evaluation events files.
-EVAL_DIR=/tmp/flowers_eval/
-
-# Evaluate the fine-tuned model on a hold-out of the flower data set.
-bazel-bin/inception/flowers_eval \
- --eval_dir="${EVAL_DIR}" \
- --data_dir="${FLOWERS_DATA_DIR}" \
- --subset=validation \
- --num_examples=500 \
- --checkpoint_dir="${TRAIN_DIR}" \
- --input_queue_memory_factor=1 \
- --run_once
-```
-
-We find that the evaluation arrives at roughly 93.4% precision@1 after the model
-has been running for 2000 steps.
-
-```shell
-Successfully loaded model from /tmp/flowers/model.ckpt-1999 at step=1999.
-2016-03-01 16:52:51.761219: starting evaluation on (validation).
-2016-03-01 16:53:05.450419: [20 batches out of 20] (36.5 examples/sec; 0.684sec/batch)
-2016-03-01 16:53:05.450471: precision @ 1 = 0.9340 recall @ 5 = 0.9960 [500 examples]
-```
-
-## How to Construct a New Dataset for Retraining
-
-One can use the existing scripts supplied with this model to build a new dataset
-for training or fine-tuning. The main script to employ is
-[`build_image_data.py`](inception/data/build_image_data.py). Briefly, this script takes a
-structured directory of images and converts it to a sharded `TFRecord` that can
-be read by the Inception model.
-
-In particular, you will need to create a directory of training images that
-reside within `$TRAIN_DIR` and `$VALIDATION_DIR` arranged as such:
-
-```shell
- $TRAIN_DIR/dog/image0.jpeg
- $TRAIN_DIR/dog/image1.jpg
- $TRAIN_DIR/dog/image2.png
- ...
- $TRAIN_DIR/cat/weird-image.jpeg
- $TRAIN_DIR/cat/my-image.jpeg
- $TRAIN_DIR/cat/my-image.JPG
- ...
- $VALIDATION_DIR/dog/imageA.jpeg
- $VALIDATION_DIR/dog/imageB.jpg
- $VALIDATION_DIR/dog/imageC.png
- ...
- $VALIDATION_DIR/cat/weird-image.PNG
- $VALIDATION_DIR/cat/that-image.jpg
- $VALIDATION_DIR/cat/cat.JPG
- ...
-```
-**NOTE**: This script will append an extra background class indexed at 0, so
-your class labels will range from 0 to num_labels. Using the example above, the
-corresponding class labels generated from `build_image_data.py` will be as
-follows:
-```shell
-0
-1 dog
-2 cat
-```
-
-Each sub-directory in `$TRAIN_DIR` and `$VALIDATION_DIR` corresponds to a unique
-label for the images that reside within that sub-directory. The images may be
-JPEG or PNG images. We do not support other images types currently.
-
-Once the data is arranged in this directory structure, we can run
-`build_image_data.py` on the data to generate the sharded `TFRecord` dataset.
-Each entry of the `TFRecord` is a serialized `tf.Example` protocol buffer. A
-complete list of information contained in the `tf.Example` is described in the
-comments of `build_image_data.py`.
-
-To run `build_image_data.py`, you can run the following command line:
-
-```shell
-# location to where to save the TFRecord data.
-OUTPUT_DIRECTORY=$HOME/my-custom-data/
-
-# build the preprocessing script.
-cd tensorflow-models/inception
-bazel build //inception:build_image_data
-
-# convert the data.
-bazel-bin/inception/build_image_data \
- --train_directory="${TRAIN_DIR}" \
- --validation_directory="${VALIDATION_DIR}" \
- --output_directory="${OUTPUT_DIRECTORY}" \
- --labels_file="${LABELS_FILE}" \
- --train_shards=128 \
- --validation_shards=24 \
- --num_threads=8
-```
-
-where the `$OUTPUT_DIRECTORY` is the location of the sharded `TFRecords`. The
-`$LABELS_FILE` will be a text file that is read by the script that provides
-a list of all of the labels. For instance, in the case flowers data set, the
-`$LABELS_FILE` contained the following data:
-
-```shell
-daisy
-dandelion
-roses
-sunflowers
-tulips
-```
-
-Note that each row of each label corresponds with the entry in the final
-classifier in the model. That is, the `daisy` corresponds to the classifier for
-entry `1`; `dandelion` is entry `2`, etc. We skip label `0` as a background
-class.
-
-After running this script produces files that look like the following:
-
-```shell
- $TRAIN_DIR/train-00000-of-00128
- $TRAIN_DIR/train-00001-of-00128
- ...
- $TRAIN_DIR/train-00127-of-00128
-
-and
-
- $VALIDATION_DIR/validation-00000-of-00024
- $VALIDATION_DIR/validation-00001-of-00024
- ...
- $VALIDATION_DIR/validation-00023-of-00024
-```
-
-where 128 and 24 are the number of shards specified for each dataset,
-respectively. Generally speaking, we aim for selecting the number of shards such
-that roughly 1024 images reside in each shard. Once this data set is built, you
-are ready to train or fine-tune an Inception model on this data set.
-
-Note, if you are piggy backing on the flowers retraining scripts, be sure to
-update `num_classes()` and `num_examples_per_epoch()` in `flowers_data.py`
-to correspond with your data.
-
-## Practical Considerations for Training a Model
-
-The model architecture and training procedure is heavily dependent on the
-hardware used to train the model. If you wish to train or fine-tune this model
-on your machine **you will need to adjust and empirically determine a good set
-of training hyper-parameters for your setup**. What follows are some general
-considerations for novices.
-
-### Finding Good Hyperparameters
-
-Roughly 5-10 hyper-parameters govern the speed at which a network is trained. In
-addition to `--batch_size` and `--num_gpus`, there are several constants defined
-in [inception_train.py](inception/inception_train.py) which dictate the learning
-schedule.
-
-```shell
-RMSPROP_DECAY = 0.9 # Decay term for RMSProp.
-MOMENTUM = 0.9 # Momentum in RMSProp.
-RMSPROP_EPSILON = 1.0 # Epsilon term for RMSProp.
-INITIAL_LEARNING_RATE = 0.1 # Initial learning rate.
-NUM_EPOCHS_PER_DECAY = 30.0 # Epochs after which learning rate decays.
-LEARNING_RATE_DECAY_FACTOR = 0.16 # Learning rate decay factor.
-```
-
-There are many papers that discuss the various tricks and trade-offs associated
-with training a model with stochastic gradient descent. For those new to the
-field, some great references are:
-
-* Y Bengio, [Practical recommendations for gradient-based training of deep
- architectures](http://arxiv.org/abs/1206.5533)
-* I Goodfellow, Y Bengio and A Courville, [Deep Learning]
- (http://www.deeplearningbook.org/)
-
-What follows is a summary of some general advice for identifying appropriate
-model hyper-parameters in the context of this particular model training setup.
-Namely, this library provides *synchronous* updates to model parameters based on
-batch-splitting the model across multiple GPUs.
-
-* Higher learning rates leads to faster training. Too high of learning rate
- leads to instability and will cause model parameters to diverge to infinity
- or NaN.
-
-* Larger batch sizes lead to higher quality estimates of the gradient and
- permit training the model with higher learning rates.
-
-* Often the GPU memory is a bottleneck that prevents employing larger batch
- sizes. Employing more GPUs allows one to use larger batch sizes because
- this model splits the batch across the GPUs.
-
-**NOTE** If one wishes to train this model with *asynchronous* gradient updates,
-one will need to substantially alter this model and new considerations need to
-be factored into hyperparameter tuning. See [Large Scale Distributed Deep
-Networks](http://research.google.com/archive/large_deep_networks_nips2012.html)
-for a discussion in this domain.
-
-### Adjusting Memory Demands
-
-Training this model has large memory demands in terms of the CPU and GPU. Let's
-discuss each item in turn.
-
-GPU memory is relatively small compared to CPU memory. Two items dictate the
-amount of GPU memory employed -- model architecture and batch size. Assuming
-that you keep the model architecture fixed, the sole parameter governing the GPU
-demand is the batch size. A good rule of thumb is to try employ as large of
-batch size as will fit on the GPU.
-
-If you run out of GPU memory, either lower the `--batch_size` or employ more
-GPUs on your desktop. The model performs batch-splitting across GPUs, thus N
-GPUs can handle N times the batch size of 1 GPU.
-
-The model requires a large amount of CPU memory as well. We have tuned the model
-to employ about ~20GB of CPU memory. Thus, having access to about 40 GB of CPU
-memory would be ideal.
-
-If that is not possible, you can tune down the memory demands of the model via
-lowering `--input_queue_memory_factor`. Images are preprocessed asynchronously
-with respect to the main training across `--num_preprocess_threads` threads. The
-preprocessed images are stored in shuffling queue in which each GPU performs a
-dequeue operation in order to receive a `batch_size` worth of images.
-
-In order to guarantee good shuffling across the data, we maintain a large
-shuffling queue of 1024 x `input_queue_memory_factor` images. For the current
-model architecture, this corresponds to about 4GB of CPU memory. You may lower
-`input_queue_memory_factor` in order to decrease the memory footprint. Keep in
-mind though that lowering this value drastically may result in a model with
-slightly lower predictive accuracy when training from scratch. Please see
-comments in [`image_processing.py`](inception/image_processing.py) for more details.
-
-## Troubleshooting
-
-#### The model runs out of CPU memory.
-
-In lieu of buying more CPU memory, an easy fix is to decrease
-`--input_queue_memory_factor`. See [Adjusting Memory Demands](#adjusting-memory-demands).
-
-#### The model runs out of GPU memory.
-
-The data is not able to fit on the GPU card. The simplest solution is to
-decrease the batch size of the model. Otherwise, you will need to think about a
-more sophisticated method for specifying the training which cuts up the model
-across multiple `session.run()` calls or partitions the model across multiple
-GPUs. See [Using GPUs](https://www.tensorflow.org/how_tos/using_gpu/index.html)
-and [Adjusting Memory Demands](#adjusting-memory-demands) for more information.
-
-#### The model training results in NaN's.
-
-The learning rate of the model is too high. Turn down your learning rate.
-
-#### I wish to train a model with a different image size.
-
-The simplest solution is to artificially resize your images to `299x299` pixels.
-See [Images](https://www.tensorflow.org/api_docs/python/image.html) section for
-many resizing, cropping and padding methods. Note that the entire model
-architecture is predicated on a `299x299` image, thus if you wish to change the
-input image size, then you may need to redesign the entire model architecture.
-
-#### What hardware specification are these hyper-parameters targeted for?
-
-We targeted a desktop with 128GB of CPU ram connected to 8 NVIDIA Tesla K40 GPU
-cards but we have run this on desktops with 32GB of CPU ram and 1 NVIDIA Tesla
-K40. You can get a sense of the various training configurations we tested by
-reading the comments in [`inception_train.py`](inception/inception_train.py).
-
-#### How do I continue training from a checkpoint in distributed setting?
-
-You only need to make sure that the checkpoint is in a location that can be
-reached by all of the `ps` tasks. By specifying the checkpoint location with
-`--train_dir` , the `ps` servers will load the checkpoint before commencing
-training.
diff --git a/research/inception/WORKSPACE b/research/inception/WORKSPACE
deleted file mode 100644
index 2d7b4fb254a0fcebe695cb3fd3685af29a02e0b0..0000000000000000000000000000000000000000
--- a/research/inception/WORKSPACE
+++ /dev/null
@@ -1 +0,0 @@
-workspace(name = "inception")
diff --git a/research/inception/g3doc/inception_v3_architecture.png b/research/inception/g3doc/inception_v3_architecture.png
deleted file mode 100644
index 91fb734a104b2f63114ade7c8f9b2f95ce6334a6..0000000000000000000000000000000000000000
Binary files a/research/inception/g3doc/inception_v3_architecture.png and /dev/null differ
diff --git a/research/inception/inception/BUILD b/research/inception/inception/BUILD
deleted file mode 100644
index 21fc27aa57c14f6a72359cf15d446787c8ea6c2e..0000000000000000000000000000000000000000
--- a/research/inception/inception/BUILD
+++ /dev/null
@@ -1,198 +0,0 @@
-# Description:
-# Example TensorFlow models for ImageNet.
-
-package(default_visibility = [":internal"])
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-package_group(
- name = "internal",
- packages = ["//inception/..."],
-)
-
-py_library(
- name = "dataset",
- srcs = [
- "dataset.py",
- ],
-)
-
-py_library(
- name = "imagenet_data",
- srcs = [
- "imagenet_data.py",
- ],
- deps = [
- ":dataset",
- ],
-)
-
-py_library(
- name = "flowers_data",
- srcs = [
- "flowers_data.py",
- ],
- deps = [
- ":dataset",
- ],
-)
-
-py_library(
- name = "image_processing",
- srcs = [
- "image_processing.py",
- ],
-)
-
-py_library(
- name = "inception",
- srcs = [
- "inception_model.py",
- ],
- visibility = ["//visibility:public"],
- deps = [
- ":dataset",
- "//inception/slim",
- ],
-)
-
-py_binary(
- name = "imagenet_eval",
- srcs = [
- "imagenet_eval.py",
- ],
- deps = [
- ":imagenet_data",
- ":inception_eval",
- ],
-)
-
-py_binary(
- name = "flowers_eval",
- srcs = [
- "flowers_eval.py",
- ],
- deps = [
- ":flowers_data",
- ":inception_eval",
- ],
-)
-
-py_library(
- name = "inception_eval",
- srcs = [
- "inception_eval.py",
- ],
- deps = [
- ":image_processing",
- ":inception",
- ],
-)
-
-py_binary(
- name = "imagenet_train",
- srcs = [
- "imagenet_train.py",
- ],
- deps = [
- ":imagenet_data",
- ":inception_train",
- ],
-)
-
-py_binary(
- name = "imagenet_distributed_train",
- srcs = [
- "imagenet_distributed_train.py",
- ],
- deps = [
- ":imagenet_data",
- ":inception_distributed_train",
- ],
-)
-
-py_binary(
- name = "flowers_train",
- srcs = [
- "flowers_train.py",
- ],
- deps = [
- ":flowers_data",
- ":inception_train",
- ],
-)
-
-py_library(
- name = "inception_train",
- srcs = [
- "inception_train.py",
- ],
- deps = [
- ":image_processing",
- ":inception",
- ],
-)
-
-py_library(
- name = "inception_distributed_train",
- srcs = [
- "inception_distributed_train.py",
- ],
- deps = [
- ":image_processing",
- ":inception",
- ],
-)
-
-py_binary(
- name = "build_image_data",
- srcs = ["data/build_image_data.py"],
-)
-
-sh_binary(
- name = "download_and_preprocess_flowers",
- srcs = ["data/download_and_preprocess_flowers.sh"],
- data = [
- ":build_image_data",
- ],
-)
-
-sh_binary(
- name = "download_and_preprocess_imagenet",
- srcs = ["data/download_and_preprocess_imagenet.sh"],
- data = [
- "data/download_imagenet.sh",
- "data/imagenet_2012_validation_synset_labels.txt",
- "data/imagenet_lsvrc_2015_synsets.txt",
- "data/imagenet_metadata.txt",
- "data/preprocess_imagenet_validation_data.py",
- "data/process_bounding_boxes.py",
- ":build_imagenet_data",
- ],
-)
-
-py_binary(
- name = "build_imagenet_data",
- srcs = ["data/build_imagenet_data.py"],
-)
-
-filegroup(
- name = "srcs",
- srcs = glob(
- [
- "**/*.py",
- "BUILD",
- ],
- ),
-)
-
-filegroup(
- name = "imagenet_metadata",
- srcs = [
- "data/imagenet_lsvrc_2015_synsets.txt",
- "data/imagenet_metadata.txt",
- ],
- visibility = ["//visibility:public"],
-)
diff --git a/research/inception/inception/data/build_image_data.py b/research/inception/inception/data/build_image_data.py
deleted file mode 100755
index 894388b7f758a46746870f2f0d55d1df7d3fe29b..0000000000000000000000000000000000000000
--- a/research/inception/inception/data/build_image_data.py
+++ /dev/null
@@ -1,436 +0,0 @@
-#!/usr/bin/python
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Converts image data to TFRecords file format with Example protos.
-
-The image data set is expected to reside in JPEG files located in the
-following directory structure.
-
- data_dir/label_0/image0.jpeg
- data_dir/label_0/image1.jpg
- ...
- data_dir/label_1/weird-image.jpeg
- data_dir/label_1/my-image.jpeg
- ...
-
-where the sub-directory is the unique label associated with these images.
-
-This TensorFlow script converts the training and evaluation data into
-a sharded data set consisting of TFRecord files
-
- train_directory/train-00000-of-01024
- train_directory/train-00001-of-01024
- ...
- train_directory/train-01023-of-01024
-
-and
-
- validation_directory/validation-00000-of-00128
- validation_directory/validation-00001-of-00128
- ...
- validation_directory/validation-00127-of-00128
-
-where we have selected 1024 and 128 shards for each data set. Each record
-within the TFRecord file is a serialized Example proto. The Example proto
-contains the following fields:
-
- image/encoded: string containing JPEG encoded image in RGB colorspace
- image/height: integer, image height in pixels
- image/width: integer, image width in pixels
- image/colorspace: string, specifying the colorspace, always 'RGB'
- image/channels: integer, specifying the number of channels, always 3
- image/format: string, specifying the format, always 'JPEG'
-
- image/filename: string containing the basename of the image file
- e.g. 'n01440764_10026.JPEG' or 'ILSVRC2012_val_00000293.JPEG'
- image/class/label: integer specifying the index in a classification layer.
- The label ranges from [0, num_labels] where 0 is unused and left as
- the background class.
- image/class/text: string specifying the human-readable version of the label
- e.g. 'dog'
-
-If your data set involves bounding boxes, please look at build_imagenet_data.py.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from datetime import datetime
-import os
-import random
-import sys
-import threading
-
-import numpy as np
-import tensorflow as tf
-
-tf.app.flags.DEFINE_string('train_directory', '/tmp/',
- 'Training data directory')
-tf.app.flags.DEFINE_string('validation_directory', '/tmp/',
- 'Validation data directory')
-tf.app.flags.DEFINE_string('output_directory', '/tmp/',
- 'Output data directory')
-
-tf.app.flags.DEFINE_integer('train_shards', 2,
- 'Number of shards in training TFRecord files.')
-tf.app.flags.DEFINE_integer('validation_shards', 2,
- 'Number of shards in validation TFRecord files.')
-
-tf.app.flags.DEFINE_integer('num_threads', 2,
- 'Number of threads to preprocess the images.')
-
-# The labels file contains a list of valid labels are held in this file.
-# Assumes that the file contains entries as such:
-# dog
-# cat
-# flower
-# where each line corresponds to a label. We map each label contained in
-# the file to an integer corresponding to the line number starting from 0.
-tf.app.flags.DEFINE_string('labels_file', '', 'Labels file')
-
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def _int64_feature(value):
- """Wrapper for inserting int64 features into Example proto."""
- if not isinstance(value, list):
- value = [value]
- return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
-
-
-def _bytes_feature(value):
- """Wrapper for inserting bytes features into Example proto."""
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
-
-def _convert_to_example(filename, image_buffer, label, text, height, width):
- """Build an Example proto for an example.
-
- Args:
- filename: string, path to an image file, e.g., '/path/to/example.JPG'
- image_buffer: string, JPEG encoding of RGB image
- label: integer, identifier for the ground truth for the network
- text: string, unique human-readable, e.g. 'dog'
- height: integer, image height in pixels
- width: integer, image width in pixels
- Returns:
- Example proto
- """
-
- colorspace = 'RGB'
- channels = 3
- image_format = 'JPEG'
-
- example = tf.train.Example(features=tf.train.Features(feature={
- 'image/height': _int64_feature(height),
- 'image/width': _int64_feature(width),
- 'image/colorspace': _bytes_feature(tf.compat.as_bytes(colorspace)),
- 'image/channels': _int64_feature(channels),
- 'image/class/label': _int64_feature(label),
- 'image/class/text': _bytes_feature(tf.compat.as_bytes(text)),
- 'image/format': _bytes_feature(tf.compat.as_bytes(image_format)),
- 'image/filename': _bytes_feature(tf.compat.as_bytes(os.path.basename(filename))),
- 'image/encoded': _bytes_feature(tf.compat.as_bytes(image_buffer))}))
- return example
-
-
-class ImageCoder(object):
- """Helper class that provides TensorFlow image coding utilities."""
-
- def __init__(self):
- # Create a single Session to run all image coding calls.
- self._sess = tf.Session()
-
- # Initializes function that converts PNG to JPEG data.
- self._png_data = tf.placeholder(dtype=tf.string)
- image = tf.image.decode_png(self._png_data, channels=3)
- self._png_to_jpeg = tf.image.encode_jpeg(image, format='rgb', quality=100)
-
- # Initializes function that decodes RGB JPEG data.
- self._decode_jpeg_data = tf.placeholder(dtype=tf.string)
- self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3)
-
- def png_to_jpeg(self, image_data):
- return self._sess.run(self._png_to_jpeg,
- feed_dict={self._png_data: image_data})
-
- def decode_jpeg(self, image_data):
- image = self._sess.run(self._decode_jpeg,
- feed_dict={self._decode_jpeg_data: image_data})
- assert len(image.shape) == 3
- assert image.shape[2] == 3
- return image
-
-
-def _is_png(filename):
- """Determine if a file contains a PNG format image.
-
- Args:
- filename: string, path of the image file.
-
- Returns:
- boolean indicating if the image is a PNG.
- """
- return filename.endswith('.png')
-
-
-def _process_image(filename, coder):
- """Process a single image file.
-
- Args:
- filename: string, path to an image file e.g., '/path/to/example.JPG'.
- coder: instance of ImageCoder to provide TensorFlow image coding utils.
- Returns:
- image_buffer: string, JPEG encoding of RGB image.
- height: integer, image height in pixels.
- width: integer, image width in pixels.
- """
- # Read the image file.
- with tf.gfile.FastGFile(filename, 'rb') as f:
- image_data = f.read()
-
- # Convert any PNG to JPEG's for consistency.
- if _is_png(filename):
- print('Converting PNG to JPEG for %s' % filename)
- image_data = coder.png_to_jpeg(image_data)
-
- # Decode the RGB JPEG.
- image = coder.decode_jpeg(image_data)
-
- # Check that image converted to RGB
- assert len(image.shape) == 3
- height = image.shape[0]
- width = image.shape[1]
- assert image.shape[2] == 3
-
- return image_data, height, width
-
-
-def _process_image_files_batch(coder, thread_index, ranges, name, filenames,
- texts, labels, num_shards):
- """Processes and saves list of images as TFRecord in 1 thread.
-
- Args:
- coder: instance of ImageCoder to provide TensorFlow image coding utils.
- thread_index: integer, unique batch to run index is within [0, len(ranges)).
- ranges: list of pairs of integers specifying ranges of each batches to
- analyze in parallel.
- name: string, unique identifier specifying the data set
- filenames: list of strings; each string is a path to an image file
- texts: list of strings; each string is human readable, e.g. 'dog'
- labels: list of integer; each integer identifies the ground truth
- num_shards: integer number of shards for this data set.
- """
- # Each thread produces N shards where N = int(num_shards / num_threads).
- # For instance, if num_shards = 128, and the num_threads = 2, then the first
- # thread would produce shards [0, 64).
- num_threads = len(ranges)
- assert not num_shards % num_threads
- num_shards_per_batch = int(num_shards / num_threads)
-
- shard_ranges = np.linspace(ranges[thread_index][0],
- ranges[thread_index][1],
- num_shards_per_batch + 1).astype(int)
- num_files_in_thread = ranges[thread_index][1] - ranges[thread_index][0]
-
- counter = 0
- for s in range(num_shards_per_batch):
- # Generate a sharded version of the file name, e.g. 'train-00002-of-00010'
- shard = thread_index * num_shards_per_batch + s
- output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards)
- output_file = os.path.join(FLAGS.output_directory, output_filename)
- writer = tf.python_io.TFRecordWriter(output_file)
-
- shard_counter = 0
- files_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int)
- for i in files_in_shard:
- filename = filenames[i]
- label = labels[i]
- text = texts[i]
-
- try:
- image_buffer, height, width = _process_image(filename, coder)
- except Exception as e:
- print(e)
- print('SKIPPED: Unexpected error while decoding %s.' % filename)
- continue
-
- example = _convert_to_example(filename, image_buffer, label,
- text, height, width)
- writer.write(example.SerializeToString())
- shard_counter += 1
- counter += 1
-
- if not counter % 1000:
- print('%s [thread %d]: Processed %d of %d images in thread batch.' %
- (datetime.now(), thread_index, counter, num_files_in_thread))
- sys.stdout.flush()
-
- writer.close()
- print('%s [thread %d]: Wrote %d images to %s' %
- (datetime.now(), thread_index, shard_counter, output_file))
- sys.stdout.flush()
- shard_counter = 0
- print('%s [thread %d]: Wrote %d images to %d shards.' %
- (datetime.now(), thread_index, counter, num_files_in_thread))
- sys.stdout.flush()
-
-
-def _process_image_files(name, filenames, texts, labels, num_shards):
- """Process and save list of images as TFRecord of Example protos.
-
- Args:
- name: string, unique identifier specifying the data set
- filenames: list of strings; each string is a path to an image file
- texts: list of strings; each string is human readable, e.g. 'dog'
- labels: list of integer; each integer identifies the ground truth
- num_shards: integer number of shards for this data set.
- """
- assert len(filenames) == len(texts)
- assert len(filenames) == len(labels)
-
- # Break all images into batches with a [ranges[i][0], ranges[i][1]].
- spacing = np.linspace(0, len(filenames), FLAGS.num_threads + 1).astype(np.int)
- ranges = []
- for i in range(len(spacing) - 1):
- ranges.append([spacing[i], spacing[i + 1]])
-
- # Launch a thread for each batch.
- print('Launching %d threads for spacings: %s' % (FLAGS.num_threads, ranges))
- sys.stdout.flush()
-
- # Create a mechanism for monitoring when all threads are finished.
- coord = tf.train.Coordinator()
-
- # Create a generic TensorFlow-based utility for converting all image codings.
- coder = ImageCoder()
-
- threads = []
- for thread_index in range(len(ranges)):
- args = (coder, thread_index, ranges, name, filenames,
- texts, labels, num_shards)
- t = threading.Thread(target=_process_image_files_batch, args=args)
- t.start()
- threads.append(t)
-
- # Wait for all the threads to terminate.
- coord.join(threads)
- print('%s: Finished writing all %d images in data set.' %
- (datetime.now(), len(filenames)))
- sys.stdout.flush()
-
-
-def _find_image_files(data_dir, labels_file):
- """Build a list of all images files and labels in the data set.
-
- Args:
- data_dir: string, path to the root directory of images.
-
- Assumes that the image data set resides in JPEG files located in
- the following directory structure.
-
- data_dir/dog/another-image.JPEG
- data_dir/dog/my-image.jpg
-
- where 'dog' is the label associated with these images.
-
- labels_file: string, path to the labels file.
-
- The list of valid labels are held in this file. Assumes that the file
- contains entries as such:
- dog
- cat
- flower
- where each line corresponds to a label. We map each label contained in
- the file to an integer starting with the integer 0 corresponding to the
- label contained in the first line.
-
- Returns:
- filenames: list of strings; each string is a path to an image file.
- texts: list of strings; each string is the class, e.g. 'dog'
- labels: list of integer; each integer identifies the ground truth.
- """
- print('Determining list of input files and labels from %s.' % data_dir)
- unique_labels = [l.strip() for l in tf.gfile.FastGFile(
- labels_file, 'r').readlines()]
-
- labels = []
- filenames = []
- texts = []
-
- # Leave label index 0 empty as a background class.
- label_index = 1
-
- # Construct the list of JPEG files and labels.
- for text in unique_labels:
- jpeg_file_path = '%s/%s/*' % (data_dir, text)
- matching_files = tf.gfile.Glob(jpeg_file_path)
-
- labels.extend([label_index] * len(matching_files))
- texts.extend([text] * len(matching_files))
- filenames.extend(matching_files)
-
- if not label_index % 100:
- print('Finished finding files in %d of %d classes.' % (
- label_index, len(labels)))
- label_index += 1
-
- # Shuffle the ordering of all image files in order to guarantee
- # random ordering of the images with respect to label in the
- # saved TFRecord files. Make the randomization repeatable.
- shuffled_index = list(range(len(filenames)))
- random.seed(12345)
- random.shuffle(shuffled_index)
-
- filenames = [filenames[i] for i in shuffled_index]
- texts = [texts[i] for i in shuffled_index]
- labels = [labels[i] for i in shuffled_index]
-
- print('Found %d JPEG files across %d labels inside %s.' %
- (len(filenames), len(unique_labels), data_dir))
- return filenames, texts, labels
-
-
-def _process_dataset(name, directory, num_shards, labels_file):
- """Process a complete data set and save it as a TFRecord.
-
- Args:
- name: string, unique identifier specifying the data set.
- directory: string, root path to the data set.
- num_shards: integer number of shards for this data set.
- labels_file: string, path to the labels file.
- """
- filenames, texts, labels = _find_image_files(directory, labels_file)
- _process_image_files(name, filenames, texts, labels, num_shards)
-
-
-def main(unused_argv):
- assert not FLAGS.train_shards % FLAGS.num_threads, (
- 'Please make the FLAGS.num_threads commensurate with FLAGS.train_shards')
- assert not FLAGS.validation_shards % FLAGS.num_threads, (
- 'Please make the FLAGS.num_threads commensurate with '
- 'FLAGS.validation_shards')
- print('Saving results to %s' % FLAGS.output_directory)
-
- # Run it!
- _process_dataset('validation', FLAGS.validation_directory,
- FLAGS.validation_shards, FLAGS.labels_file)
- _process_dataset('train', FLAGS.train_directory,
- FLAGS.train_shards, FLAGS.labels_file)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/inception/inception/data/build_imagenet_data.py b/research/inception/inception/data/build_imagenet_data.py
deleted file mode 100644
index c054735e782297f990451e29ff4383af24bbe802..0000000000000000000000000000000000000000
--- a/research/inception/inception/data/build_imagenet_data.py
+++ /dev/null
@@ -1,707 +0,0 @@
-#!/usr/bin/python
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Converts ImageNet data to TFRecords file format with Example protos.
-
-The raw ImageNet data set is expected to reside in JPEG files located in the
-following directory structure.
-
- data_dir/n01440764/ILSVRC2012_val_00000293.JPEG
- data_dir/n01440764/ILSVRC2012_val_00000543.JPEG
- ...
-
-where 'n01440764' is the unique synset label associated with
-these images.
-
-The training data set consists of 1000 sub-directories (i.e. labels)
-each containing 1200 JPEG images for a total of 1.2M JPEG images.
-
-The evaluation data set consists of 1000 sub-directories (i.e. labels)
-each containing 50 JPEG images for a total of 50K JPEG images.
-
-This TensorFlow script converts the training and evaluation data into
-a sharded data set consisting of 1024 and 128 TFRecord files, respectively.
-
- train_directory/train-00000-of-01024
- train_directory/train-00001-of-01024
- ...
- train_directory/train-01023-of-01024
-
-and
-
- validation_directory/validation-00000-of-00128
- validation_directory/validation-00001-of-00128
- ...
- validation_directory/validation-00127-of-00128
-
-Each validation TFRecord file contains ~390 records. Each training TFREcord
-file contains ~1250 records. Each record within the TFRecord file is a
-serialized Example proto. The Example proto contains the following fields:
-
- image/encoded: string containing JPEG encoded image in RGB colorspace
- image/height: integer, image height in pixels
- image/width: integer, image width in pixels
- image/colorspace: string, specifying the colorspace, always 'RGB'
- image/channels: integer, specifying the number of channels, always 3
- image/format: string, specifying the format, always 'JPEG'
-
- image/filename: string containing the basename of the image file
- e.g. 'n01440764_10026.JPEG' or 'ILSVRC2012_val_00000293.JPEG'
- image/class/label: integer specifying the index in a classification layer.
- The label ranges from [1, 1000] where 0 is not used.
- image/class/synset: string specifying the unique ID of the label,
- e.g. 'n01440764'
- image/class/text: string specifying the human-readable version of the label
- e.g. 'red fox, Vulpes vulpes'
-
- image/object/bbox/xmin: list of integers specifying the 0+ human annotated
- bounding boxes
- image/object/bbox/xmax: list of integers specifying the 0+ human annotated
- bounding boxes
- image/object/bbox/ymin: list of integers specifying the 0+ human annotated
- bounding boxes
- image/object/bbox/ymax: list of integers specifying the 0+ human annotated
- bounding boxes
- image/object/bbox/label: integer specifying the index in a classification
- layer. The label ranges from [1, 1000] where 0 is not used. Note this is
- always identical to the image label.
-
-Note that the length of xmin is identical to the length of xmax, ymin and ymax
-for each example.
-
-Running this script using 16 threads may take around ~2.5 hours on an HP Z420.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from datetime import datetime
-import os
-import random
-import sys
-import threading
-
-import numpy as np
-import six
-import tensorflow as tf
-
-tf.app.flags.DEFINE_string('train_directory', '/tmp/',
- 'Training data directory')
-tf.app.flags.DEFINE_string('validation_directory', '/tmp/',
- 'Validation data directory')
-tf.app.flags.DEFINE_string('output_directory', '/tmp/',
- 'Output data directory')
-
-tf.app.flags.DEFINE_integer('train_shards', 1024,
- 'Number of shards in training TFRecord files.')
-tf.app.flags.DEFINE_integer('validation_shards', 128,
- 'Number of shards in validation TFRecord files.')
-
-tf.app.flags.DEFINE_integer('num_threads', 8,
- 'Number of threads to preprocess the images.')
-
-# The labels file contains a list of valid labels are held in this file.
-# Assumes that the file contains entries as such:
-# n01440764
-# n01443537
-# n01484850
-# where each line corresponds to a label expressed as a synset. We map
-# each synset contained in the file to an integer (based on the alphabetical
-# ordering). See below for details.
-tf.app.flags.DEFINE_string('labels_file',
- 'imagenet_lsvrc_2015_synsets.txt',
- 'Labels file')
-
-# This file containing mapping from synset to human-readable label.
-# Assumes each line of the file looks like:
-#
-# n02119247 black fox
-# n02119359 silver fox
-# n02119477 red fox, Vulpes fulva
-#
-# where each line corresponds to a unique mapping. Note that each line is
-# formatted as \t.
-tf.app.flags.DEFINE_string('imagenet_metadata_file',
- 'imagenet_metadata.txt',
- 'ImageNet metadata file')
-
-# This file is the output of process_bounding_box.py
-# Assumes each line of the file looks like:
-#
-# n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940
-#
-# where each line corresponds to one bounding box annotation associated
-# with an image. Each line can be parsed as:
-#
-# , , , ,
-#
-# Note that there might exist mulitple bounding box annotations associated
-# with an image file.
-tf.app.flags.DEFINE_string('bounding_box_file',
- './imagenet_2012_bounding_boxes.csv',
- 'Bounding box file')
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def _int64_feature(value):
- """Wrapper for inserting int64 features into Example proto."""
- if not isinstance(value, list):
- value = [value]
- return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
-
-
-def _float_feature(value):
- """Wrapper for inserting float features into Example proto."""
- if not isinstance(value, list):
- value = [value]
- return tf.train.Feature(float_list=tf.train.FloatList(value=value))
-
-
-def _bytes_feature(value):
- """Wrapper for inserting bytes features into Example proto."""
- if six.PY3 and isinstance(value, six.text_type):
- value = six.binary_type(value, encoding='utf-8')
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
-
-def _convert_to_example(filename, image_buffer, label, synset, human, bbox,
- height, width):
- """Build an Example proto for an example.
-
- Args:
- filename: string, path to an image file, e.g., '/path/to/example.JPG'
- image_buffer: string, JPEG encoding of RGB image
- label: integer, identifier for the ground truth for the network
- synset: string, unique WordNet ID specifying the label, e.g., 'n02323233'
- human: string, human-readable label, e.g., 'red fox, Vulpes vulpes'
- bbox: list of bounding boxes; each box is a list of integers
- specifying [xmin, ymin, xmax, ymax]. All boxes are assumed to belong to
- the same label as the image label.
- height: integer, image height in pixels
- width: integer, image width in pixels
- Returns:
- Example proto
- """
- xmin = []
- ymin = []
- xmax = []
- ymax = []
- for b in bbox:
- assert len(b) == 4
- # pylint: disable=expression-not-assigned
- [l.append(point) for l, point in zip([xmin, ymin, xmax, ymax], b)]
- # pylint: enable=expression-not-assigned
-
- colorspace = 'RGB'
- channels = 3
- image_format = 'JPEG'
-
- example = tf.train.Example(features=tf.train.Features(feature={
- 'image/height': _int64_feature(height),
- 'image/width': _int64_feature(width),
- 'image/colorspace': _bytes_feature(colorspace),
- 'image/channels': _int64_feature(channels),
- 'image/class/label': _int64_feature(label),
- 'image/class/synset': _bytes_feature(synset),
- 'image/class/text': _bytes_feature(human),
- 'image/object/bbox/xmin': _float_feature(xmin),
- 'image/object/bbox/xmax': _float_feature(xmax),
- 'image/object/bbox/ymin': _float_feature(ymin),
- 'image/object/bbox/ymax': _float_feature(ymax),
- 'image/object/bbox/label': _int64_feature([label] * len(xmin)),
- 'image/format': _bytes_feature(image_format),
- 'image/filename': _bytes_feature(os.path.basename(filename)),
- 'image/encoded': _bytes_feature(image_buffer)}))
- return example
-
-
-class ImageCoder(object):
- """Helper class that provides TensorFlow image coding utilities."""
-
- def __init__(self):
- # Create a single Session to run all image coding calls.
- self._sess = tf.Session()
-
- # Initializes function that converts PNG to JPEG data.
- self._png_data = tf.placeholder(dtype=tf.string)
- image = tf.image.decode_png(self._png_data, channels=3)
- self._png_to_jpeg = tf.image.encode_jpeg(image, format='rgb', quality=100)
-
- # Initializes function that converts CMYK JPEG data to RGB JPEG data.
- self._cmyk_data = tf.placeholder(dtype=tf.string)
- image = tf.image.decode_jpeg(self._cmyk_data, channels=0)
- self._cmyk_to_rgb = tf.image.encode_jpeg(image, format='rgb', quality=100)
-
- # Initializes function that decodes RGB JPEG data.
- self._decode_jpeg_data = tf.placeholder(dtype=tf.string)
- self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3)
-
- def png_to_jpeg(self, image_data):
- return self._sess.run(self._png_to_jpeg,
- feed_dict={self._png_data: image_data})
-
- def cmyk_to_rgb(self, image_data):
- return self._sess.run(self._cmyk_to_rgb,
- feed_dict={self._cmyk_data: image_data})
-
- def decode_jpeg(self, image_data):
- image = self._sess.run(self._decode_jpeg,
- feed_dict={self._decode_jpeg_data: image_data})
- assert len(image.shape) == 3
- assert image.shape[2] == 3
- return image
-
-
-def _is_png(filename):
- """Determine if a file contains a PNG format image.
-
- Args:
- filename: string, path of the image file.
-
- Returns:
- boolean indicating if the image is a PNG.
- """
- # File list from:
- # https://groups.google.com/forum/embed/?place=forum/torch7#!topic/torch7/fOSTXHIESSU
- return 'n02105855_2933.JPEG' in filename
-
-
-def _is_cmyk(filename):
- """Determine if file contains a CMYK JPEG format image.
-
- Args:
- filename: string, path of the image file.
-
- Returns:
- boolean indicating if the image is a JPEG encoded with CMYK color space.
- """
- # File list from:
- # https://github.com/cytsai/ilsvrc-cmyk-image-list
- blacklist = ['n01739381_1309.JPEG', 'n02077923_14822.JPEG',
- 'n02447366_23489.JPEG', 'n02492035_15739.JPEG',
- 'n02747177_10752.JPEG', 'n03018349_4028.JPEG',
- 'n03062245_4620.JPEG', 'n03347037_9675.JPEG',
- 'n03467068_12171.JPEG', 'n03529860_11437.JPEG',
- 'n03544143_17228.JPEG', 'n03633091_5218.JPEG',
- 'n03710637_5125.JPEG', 'n03961711_5286.JPEG',
- 'n04033995_2932.JPEG', 'n04258138_17003.JPEG',
- 'n04264628_27969.JPEG', 'n04336792_7448.JPEG',
- 'n04371774_5854.JPEG', 'n04596742_4225.JPEG',
- 'n07583066_647.JPEG', 'n13037406_4650.JPEG']
- return filename.split('/')[-1] in blacklist
-
-
-def _process_image(filename, coder):
- """Process a single image file.
-
- Args:
- filename: string, path to an image file e.g., '/path/to/example.JPG'.
- coder: instance of ImageCoder to provide TensorFlow image coding utils.
- Returns:
- image_buffer: string, JPEG encoding of RGB image.
- height: integer, image height in pixels.
- width: integer, image width in pixels.
- """
- # Read the image file.
- with tf.gfile.FastGFile(filename, 'rb') as f:
- image_data = f.read()
-
- # Clean the dirty data.
- if _is_png(filename):
- # 1 image is a PNG.
- print('Converting PNG to JPEG for %s' % filename)
- image_data = coder.png_to_jpeg(image_data)
- elif _is_cmyk(filename):
- # 22 JPEG images are in CMYK colorspace.
- print('Converting CMYK to RGB for %s' % filename)
- image_data = coder.cmyk_to_rgb(image_data)
-
- # Decode the RGB JPEG.
- image = coder.decode_jpeg(image_data)
-
- # Check that image converted to RGB
- assert len(image.shape) == 3
- height = image.shape[0]
- width = image.shape[1]
- assert image.shape[2] == 3
-
- return image_data, height, width
-
-
-def _process_image_files_batch(coder, thread_index, ranges, name, filenames,
- synsets, labels, humans, bboxes, num_shards):
- """Processes and saves list of images as TFRecord in 1 thread.
-
- Args:
- coder: instance of ImageCoder to provide TensorFlow image coding utils.
- thread_index: integer, unique batch to run index is within [0, len(ranges)).
- ranges: list of pairs of integers specifying ranges of each batches to
- analyze in parallel.
- name: string, unique identifier specifying the data set
- filenames: list of strings; each string is a path to an image file
- synsets: list of strings; each string is a unique WordNet ID
- labels: list of integer; each integer identifies the ground truth
- humans: list of strings; each string is a human-readable label
- bboxes: list of bounding boxes for each image. Note that each entry in this
- list might contain from 0+ entries corresponding to the number of bounding
- box annotations for the image.
- num_shards: integer number of shards for this data set.
- """
- # Each thread produces N shards where N = int(num_shards / num_threads).
- # For instance, if num_shards = 128, and the num_threads = 2, then the first
- # thread would produce shards [0, 64).
- num_threads = len(ranges)
- assert not num_shards % num_threads
- num_shards_per_batch = int(num_shards / num_threads)
-
- shard_ranges = np.linspace(ranges[thread_index][0],
- ranges[thread_index][1],
- num_shards_per_batch + 1).astype(int)
- num_files_in_thread = ranges[thread_index][1] - ranges[thread_index][0]
-
- counter = 0
- for s in range(num_shards_per_batch):
- # Generate a sharded version of the file name, e.g. 'train-00002-of-00010'
- shard = thread_index * num_shards_per_batch + s
- output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards)
- output_file = os.path.join(FLAGS.output_directory, output_filename)
- writer = tf.python_io.TFRecordWriter(output_file)
-
- shard_counter = 0
- files_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int)
- for i in files_in_shard:
- filename = filenames[i]
- label = labels[i]
- synset = synsets[i]
- human = humans[i]
- bbox = bboxes[i]
-
- image_buffer, height, width = _process_image(filename, coder)
-
- example = _convert_to_example(filename, image_buffer, label,
- synset, human, bbox,
- height, width)
- writer.write(example.SerializeToString())
- shard_counter += 1
- counter += 1
-
- if not counter % 1000:
- print('%s [thread %d]: Processed %d of %d images in thread batch.' %
- (datetime.now(), thread_index, counter, num_files_in_thread))
- sys.stdout.flush()
-
- writer.close()
- print('%s [thread %d]: Wrote %d images to %s' %
- (datetime.now(), thread_index, shard_counter, output_file))
- sys.stdout.flush()
- shard_counter = 0
- print('%s [thread %d]: Wrote %d images to %d shards.' %
- (datetime.now(), thread_index, counter, num_files_in_thread))
- sys.stdout.flush()
-
-
-def _process_image_files(name, filenames, synsets, labels, humans,
- bboxes, num_shards):
- """Process and save list of images as TFRecord of Example protos.
-
- Args:
- name: string, unique identifier specifying the data set
- filenames: list of strings; each string is a path to an image file
- synsets: list of strings; each string is a unique WordNet ID
- labels: list of integer; each integer identifies the ground truth
- humans: list of strings; each string is a human-readable label
- bboxes: list of bounding boxes for each image. Note that each entry in this
- list might contain from 0+ entries corresponding to the number of bounding
- box annotations for the image.
- num_shards: integer number of shards for this data set.
- """
- assert len(filenames) == len(synsets)
- assert len(filenames) == len(labels)
- assert len(filenames) == len(humans)
- assert len(filenames) == len(bboxes)
-
- # Break all images into batches with a [ranges[i][0], ranges[i][1]].
- spacing = np.linspace(0, len(filenames), FLAGS.num_threads + 1).astype(np.int)
- ranges = []
- threads = []
- for i in range(len(spacing) - 1):
- ranges.append([spacing[i], spacing[i + 1]])
-
- # Launch a thread for each batch.
- print('Launching %d threads for spacings: %s' % (FLAGS.num_threads, ranges))
- sys.stdout.flush()
-
- # Create a mechanism for monitoring when all threads are finished.
- coord = tf.train.Coordinator()
-
- # Create a generic TensorFlow-based utility for converting all image codings.
- coder = ImageCoder()
-
- threads = []
- for thread_index in range(len(ranges)):
- args = (coder, thread_index, ranges, name, filenames,
- synsets, labels, humans, bboxes, num_shards)
- t = threading.Thread(target=_process_image_files_batch, args=args)
- t.start()
- threads.append(t)
-
- # Wait for all the threads to terminate.
- coord.join(threads)
- print('%s: Finished writing all %d images in data set.' %
- (datetime.now(), len(filenames)))
- sys.stdout.flush()
-
-
-def _find_image_files(data_dir, labels_file):
- """Build a list of all images files and labels in the data set.
-
- Args:
- data_dir: string, path to the root directory of images.
-
- Assumes that the ImageNet data set resides in JPEG files located in
- the following directory structure.
-
- data_dir/n01440764/ILSVRC2012_val_00000293.JPEG
- data_dir/n01440764/ILSVRC2012_val_00000543.JPEG
-
- where 'n01440764' is the unique synset label associated with these images.
-
- labels_file: string, path to the labels file.
-
- The list of valid labels are held in this file. Assumes that the file
- contains entries as such:
- n01440764
- n01443537
- n01484850
- where each line corresponds to a label expressed as a synset. We map
- each synset contained in the file to an integer (based on the alphabetical
- ordering) starting with the integer 1 corresponding to the synset
- contained in the first line.
-
- The reason we start the integer labels at 1 is to reserve label 0 as an
- unused background class.
-
- Returns:
- filenames: list of strings; each string is a path to an image file.
- synsets: list of strings; each string is a unique WordNet ID.
- labels: list of integer; each integer identifies the ground truth.
- """
- print('Determining list of input files and labels from %s.' % data_dir)
- challenge_synsets = [l.strip() for l in
- tf.gfile.FastGFile(labels_file, 'r').readlines()]
-
- labels = []
- filenames = []
- synsets = []
-
- # Leave label index 0 empty as a background class.
- label_index = 1
-
- # Construct the list of JPEG files and labels.
- for synset in challenge_synsets:
- jpeg_file_path = '%s/%s/*.JPEG' % (data_dir, synset)
- matching_files = tf.gfile.Glob(jpeg_file_path)
-
- labels.extend([label_index] * len(matching_files))
- synsets.extend([synset] * len(matching_files))
- filenames.extend(matching_files)
-
- if not label_index % 100:
- print('Finished finding files in %d of %d classes.' % (
- label_index, len(challenge_synsets)))
- label_index += 1
-
- # Shuffle the ordering of all image files in order to guarantee
- # random ordering of the images with respect to label in the
- # saved TFRecord files. Make the randomization repeatable.
- shuffled_index = list(range(len(filenames)))
- random.seed(12345)
- random.shuffle(shuffled_index)
-
- filenames = [filenames[i] for i in shuffled_index]
- synsets = [synsets[i] for i in shuffled_index]
- labels = [labels[i] for i in shuffled_index]
-
- print('Found %d JPEG files across %d labels inside %s.' %
- (len(filenames), len(challenge_synsets), data_dir))
- return filenames, synsets, labels
-
-
-def _find_human_readable_labels(synsets, synset_to_human):
- """Build a list of human-readable labels.
-
- Args:
- synsets: list of strings; each string is a unique WordNet ID.
- synset_to_human: dict of synset to human labels, e.g.,
- 'n02119022' --> 'red fox, Vulpes vulpes'
-
- Returns:
- List of human-readable strings corresponding to each synset.
- """
- humans = []
- for s in synsets:
- assert s in synset_to_human, ('Failed to find: %s' % s)
- humans.append(synset_to_human[s])
- return humans
-
-
-def _find_image_bounding_boxes(filenames, image_to_bboxes):
- """Find the bounding boxes for a given image file.
-
- Args:
- filenames: list of strings; each string is a path to an image file.
- image_to_bboxes: dictionary mapping image file names to a list of
- bounding boxes. This list contains 0+ bounding boxes.
- Returns:
- List of bounding boxes for each image. Note that each entry in this
- list might contain from 0+ entries corresponding to the number of bounding
- box annotations for the image.
- """
- num_image_bbox = 0
- bboxes = []
- for f in filenames:
- basename = os.path.basename(f)
- if basename in image_to_bboxes:
- bboxes.append(image_to_bboxes[basename])
- num_image_bbox += 1
- else:
- bboxes.append([])
- print('Found %d images with bboxes out of %d images' % (
- num_image_bbox, len(filenames)))
- return bboxes
-
-
-def _process_dataset(name, directory, num_shards, synset_to_human,
- image_to_bboxes):
- """Process a complete data set and save it as a TFRecord.
-
- Args:
- name: string, unique identifier specifying the data set.
- directory: string, root path to the data set.
- num_shards: integer number of shards for this data set.
- synset_to_human: dict of synset to human labels, e.g.,
- 'n02119022' --> 'red fox, Vulpes vulpes'
- image_to_bboxes: dictionary mapping image file names to a list of
- bounding boxes. This list contains 0+ bounding boxes.
- """
- filenames, synsets, labels = _find_image_files(directory, FLAGS.labels_file)
- humans = _find_human_readable_labels(synsets, synset_to_human)
- bboxes = _find_image_bounding_boxes(filenames, image_to_bboxes)
- _process_image_files(name, filenames, synsets, labels,
- humans, bboxes, num_shards)
-
-
-def _build_synset_lookup(imagenet_metadata_file):
- """Build lookup for synset to human-readable label.
-
- Args:
- imagenet_metadata_file: string, path to file containing mapping from
- synset to human-readable label.
-
- Assumes each line of the file looks like:
-
- n02119247 black fox
- n02119359 silver fox
- n02119477 red fox, Vulpes fulva
-
- where each line corresponds to a unique mapping. Note that each line is
- formatted as \t.
-
- Returns:
- Dictionary of synset to human labels, such as:
- 'n02119022' --> 'red fox, Vulpes vulpes'
- """
- lines = tf.gfile.FastGFile(imagenet_metadata_file, 'r').readlines()
- synset_to_human = {}
- for l in lines:
- if l:
- parts = l.strip().split('\t')
- assert len(parts) == 2
- synset = parts[0]
- human = parts[1]
- synset_to_human[synset] = human
- return synset_to_human
-
-
-def _build_bounding_box_lookup(bounding_box_file):
- """Build a lookup from image file to bounding boxes.
-
- Args:
- bounding_box_file: string, path to file with bounding boxes annotations.
-
- Assumes each line of the file looks like:
-
- n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940
-
- where each line corresponds to one bounding box annotation associated
- with an image. Each line can be parsed as:
-
- , , , ,
-
- Note that there might exist mulitple bounding box annotations associated
- with an image file. This file is the output of process_bounding_boxes.py.
-
- Returns:
- Dictionary mapping image file names to a list of bounding boxes. This list
- contains 0+ bounding boxes.
- """
- lines = tf.gfile.FastGFile(bounding_box_file, 'r').readlines()
- images_to_bboxes = {}
- num_bbox = 0
- num_image = 0
- for l in lines:
- if l:
- parts = l.split(',')
- assert len(parts) == 5, ('Failed to parse: %s' % l)
- filename = parts[0]
- xmin = float(parts[1])
- ymin = float(parts[2])
- xmax = float(parts[3])
- ymax = float(parts[4])
- box = [xmin, ymin, xmax, ymax]
-
- if filename not in images_to_bboxes:
- images_to_bboxes[filename] = []
- num_image += 1
- images_to_bboxes[filename].append(box)
- num_bbox += 1
-
- print('Successfully read %d bounding boxes '
- 'across %d images.' % (num_bbox, num_image))
- return images_to_bboxes
-
-
-def main(unused_argv):
- assert not FLAGS.train_shards % FLAGS.num_threads, (
- 'Please make the FLAGS.num_threads commensurate with FLAGS.train_shards')
- assert not FLAGS.validation_shards % FLAGS.num_threads, (
- 'Please make the FLAGS.num_threads commensurate with '
- 'FLAGS.validation_shards')
- print('Saving results to %s' % FLAGS.output_directory)
-
- # Build a map from synset to human-readable label.
- synset_to_human = _build_synset_lookup(FLAGS.imagenet_metadata_file)
- image_to_bboxes = _build_bounding_box_lookup(FLAGS.bounding_box_file)
-
- # Run it!
- _process_dataset('validation', FLAGS.validation_directory,
- FLAGS.validation_shards, synset_to_human, image_to_bboxes)
- _process_dataset('train', FLAGS.train_directory, FLAGS.train_shards,
- synset_to_human, image_to_bboxes)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/inception/inception/data/download_and_preprocess_flowers.sh b/research/inception/inception/data/download_and_preprocess_flowers.sh
deleted file mode 100755
index ee045c164e803ab38be69fb1933134e7f37f1793..0000000000000000000000000000000000000000
--- a/research/inception/inception/data/download_and_preprocess_flowers.sh
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Script to download and preprocess the flowers data set. This data set
-# provides a demonstration for how to perform fine-tuning (i.e. tranfer
-# learning) from one model to a new data set.
-#
-# This script provides a demonstration for how to prepare an arbitrary
-# data set for training an Inception v3 model.
-#
-# We demonstrate this with the flowers data set which consists of images
-# of labeled flower images from 5 classes:
-#
-# daisy, dandelion, roses, sunflowers, tulips
-#
-# The final output of this script are sharded TFRecord files containing
-# serialized Example protocol buffers. See build_image_data.py for
-# details of how the Example protocol buffer contains image data.
-#
-# usage:
-# ./download_and_preprocess_flowers.sh [data-dir]
-set -e
-
-if [ -z "$1" ]; then
- echo "Usage: download_and_preprocess_flowers.sh [data dir]"
- exit
-fi
-
-# Create the output and temporary directories.
-DATA_DIR="${1%/}"
-SCRATCH_DIR="${DATA_DIR}/raw-data"
-WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-mkdir -p "${DATA_DIR}"
-mkdir -p "${SCRATCH_DIR}"
-
-# Download the flowers data.
-DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz"
-CURRENT_DIR=$(pwd)
-cd "${DATA_DIR}"
-TARBALL="flower_photos.tgz"
-if [ ! -f ${TARBALL} ]; then
- echo "Downloading flower data set."
- curl -o ${TARBALL} "${DATA_URL}"
-else
- echo "Skipping download of flower data."
-fi
-
-# Note the locations of the train and validation data.
-TRAIN_DIRECTORY="${SCRATCH_DIR}/train"
-VALIDATION_DIRECTORY="${SCRATCH_DIR}/validation"
-
-# Expands the data into the flower_photos/ directory and rename it as the
-# train directory.
-tar xf flower_photos.tgz
-rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}"
-mv flower_photos "${TRAIN_DIRECTORY}"
-
-# Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips
-LABELS_FILE="${SCRATCH_DIR}/labels.txt"
-ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}"
-
-# Generate the validation data set.
-while read LABEL; do
- VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}/${LABEL}"
- TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}/${LABEL}"
-
- # Move the first randomly selected 100 images to the validation set.
- mkdir -p "${VALIDATION_DIR_FOR_LABEL}"
- VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | shuf | head -100)
- for IMAGE in ${VALIDATION_IMAGES}; do
- mv -f "${TRAIN_DIRECTORY}/${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}"
- done
-done < "${LABELS_FILE}"
-
-# Build the TFRecords version of the image data.
-cd "${CURRENT_DIR}"
-BUILD_SCRIPT="${WORK_DIR}/build_image_data.py"
-OUTPUT_DIRECTORY="${DATA_DIR}"
-"${BUILD_SCRIPT}" \
- --train_directory="${TRAIN_DIRECTORY}" \
- --validation_directory="${VALIDATION_DIRECTORY}" \
- --output_directory="${OUTPUT_DIRECTORY}" \
- --labels_file="${LABELS_FILE}"
diff --git a/research/inception/inception/data/download_and_preprocess_flowers_mac.sh b/research/inception/inception/data/download_and_preprocess_flowers_mac.sh
deleted file mode 100644
index 154905635b19aeaaea087a8e76afda9b8c624d59..0000000000000000000000000000000000000000
--- a/research/inception/inception/data/download_and_preprocess_flowers_mac.sh
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Script to download and preprocess the flowers data set. This data set
-# provides a demonstration for how to perform fine-tuning (i.e. tranfer
-# learning) from one model to a new data set.
-#
-# This script provides a demonstration for how to prepare an arbitrary
-# data set for training an Inception v3 model.
-#
-# We demonstrate this with the flowers data set which consists of images
-# of labeled flower images from 5 classes:
-#
-# daisy, dandelion, roses, sunflowers, tulips
-#
-# The final output of this script are sharded TFRecord files containing
-# serialized Example protocol buffers. See build_image_data.py for
-# details of how the Example protocol buffer contains image data.
-#
-# usage:
-# ./download_and_preprocess_flowers.sh [data-dir]
-set -e
-
-if [ -z "$1" ]; then
- echo "Usage: download_and_preprocess_flowers.sh [data dir]"
- exit
-fi
-
-# Create the output and temporary directories.
-DATA_DIR="${1%/}"
-SCRATCH_DIR="${DATA_DIR}/raw-data/"
-mkdir -p "${DATA_DIR}"
-mkdir -p "${SCRATCH_DIR}"
-WORK_DIR="$0.runfiles/inception/inception"
-
-# Download the flowers data.
-DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz"
-CURRENT_DIR=$(pwd)
-cd "${DATA_DIR}"
-TARBALL="flower_photos.tgz"
-if [ ! -f ${TARBALL} ]; then
- echo "Downloading flower data set."
- curl -o ${TARBALL} "${DATA_URL}"
-else
- echo "Skipping download of flower data."
-fi
-
-# Note the locations of the train and validation data.
-TRAIN_DIRECTORY="${SCRATCH_DIR}train/"
-VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/"
-
-# Expands the data into the flower_photos/ directory and rename it as the
-# train directory.
-tar xf flower_photos.tgz
-rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}"
-mv flower_photos "${TRAIN_DIRECTORY}"
-
-# Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips
-LABELS_FILE="${SCRATCH_DIR}/labels.txt"
-ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}"
-
-# Generate the validation data set.
-while read LABEL; do
- VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}"
- TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}"
-
- # Move the first randomly selected 100 images to the validation set.
- mkdir -p "${VALIDATION_DIR_FOR_LABEL}"
- VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | gshuf | head -100)
- for IMAGE in ${VALIDATION_IMAGES}; do
- mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}"
- done
-done < "${LABELS_FILE}"
-
-# Build the TFRecords version of the image data.
-cd "${CURRENT_DIR}"
-BUILD_SCRIPT="${WORK_DIR}/build_image_data"
-OUTPUT_DIRECTORY="${DATA_DIR}"
-"${BUILD_SCRIPT}" \
- --train_directory="${TRAIN_DIRECTORY}" \
- --validation_directory="${VALIDATION_DIRECTORY}" \
- --output_directory="${OUTPUT_DIRECTORY}" \
- --labels_file="${LABELS_FILE}"
diff --git a/research/inception/inception/data/download_and_preprocess_imagenet.sh b/research/inception/inception/data/download_and_preprocess_imagenet.sh
deleted file mode 100755
index 6faae831075d4f6bfdc8bf8797219f7a0e4c1797..0000000000000000000000000000000000000000
--- a/research/inception/inception/data/download_and_preprocess_imagenet.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Script to download and preprocess ImageNet Challenge 2012
-# training and validation data set.
-#
-# The final output of this script are sharded TFRecord files containing
-# serialized Example protocol buffers. See build_imagenet_data.py for
-# details of how the Example protocol buffers contain the ImageNet data.
-#
-# The final output of this script appears as such:
-#
-# data_dir/train-00000-of-01024
-# data_dir/train-00001-of-01024
-# ...
-# data_dir/train-01023-of-01024
-#
-# and
-#
-# data_dir/validation-00000-of-00128
-# data_dir/validation-00001-of-00128
-# ...
-# data_dir/validation-00127-of-00128
-#
-# Note that this script may take several hours to run to completion. The
-# conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending
-# on the speed of your machine. Please be patient.
-#
-# **IMPORTANT**
-# To download the raw images, the user must create an account with image-net.org
-# and generate a username and access_key. The latter two are required for
-# downloading the raw images.
-#
-# usage:
-# ./download_and_preprocess_imagenet.sh [data-dir]
-set -e
-
-if [ -z "$1" ]; then
- echo "Usage: download_and_preprocess_imagenet.sh [data dir]"
- exit
-fi
-
-# Create the output and temporary directories.
-DATA_DIR="${1%/}"
-SCRATCH_DIR="${DATA_DIR}/raw-data/"
-mkdir -p "${DATA_DIR}"
-mkdir -p "${SCRATCH_DIR}"
-WORK_DIR="$0.runfiles/inception/inception"
-
-# Download the ImageNet data.
-LABELS_FILE="${WORK_DIR}/data/imagenet_lsvrc_2015_synsets.txt"
-DOWNLOAD_SCRIPT="${WORK_DIR}/data/download_imagenet.sh"
-"${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}"
-
-# Note the locations of the train and validation data.
-TRAIN_DIRECTORY="${SCRATCH_DIR}train/"
-VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/"
-
-# Preprocess the validation data by moving the images into the appropriate
-# sub-directory based on the label (synset) of the image.
-echo "Organizing the validation data into sub-directories."
-PREPROCESS_VAL_SCRIPT="${WORK_DIR}/data/preprocess_imagenet_validation_data.py"
-VAL_LABELS_FILE="${WORK_DIR}/data/imagenet_2012_validation_synset_labels.txt"
-
-"${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}"
-
-# Convert the XML files for bounding box annotations into a single CSV.
-echo "Extracting bounding box information from XML."
-BOUNDING_BOX_SCRIPT="${WORK_DIR}/data/process_bounding_boxes.py"
-BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv"
-BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/"
-
-"${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \
- | sort > "${BOUNDING_BOX_FILE}"
-echo "Finished downloading and preprocessing the ImageNet data."
-
-# Build the TFRecords version of the ImageNet data.
-BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data"
-OUTPUT_DIRECTORY="${DATA_DIR}"
-IMAGENET_METADATA_FILE="${WORK_DIR}/data/imagenet_metadata.txt"
-
-"${BUILD_SCRIPT}" \
- --train_directory="${TRAIN_DIRECTORY}" \
- --validation_directory="${VALIDATION_DIRECTORY}" \
- --output_directory="${OUTPUT_DIRECTORY}" \
- --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \
- --labels_file="${LABELS_FILE}" \
- --bounding_box_file="${BOUNDING_BOX_FILE}"
diff --git a/research/inception/inception/data/download_imagenet.sh b/research/inception/inception/data/download_imagenet.sh
deleted file mode 100755
index f6c77781c0bcaad642ec7a38a7ba00693ef8ef83..0000000000000000000000000000000000000000
--- a/research/inception/inception/data/download_imagenet.sh
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/bin/bash
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Script to download ImageNet Challenge 2012 training and validation data set.
-#
-# Downloads and decompresses raw images and bounding boxes.
-#
-# **IMPORTANT**
-# To download the raw images, the user must create an account with image-net.org
-# and generate a username and access_key. The latter two are required for
-# downloading the raw images.
-#
-# usage:
-# ./download_imagenet.sh [dir name] [synsets file]
-set -e
-
-if [ "x$IMAGENET_ACCESS_KEY" == x -o "x$IMAGENET_USERNAME" == x ]; then
- cat <')
- sys.exit(-1)
- data_dir = sys.argv[1]
- validation_labels_file = sys.argv[2]
-
- # Read in the 50000 synsets associated with the validation data set.
- labels = [l.strip() for l in open(validation_labels_file).readlines()]
- unique_labels = set(labels)
-
- # Make all sub-directories in the validation data dir.
- for label in unique_labels:
- labeled_data_dir = os.path.join(data_dir, label)
- # Catch error if sub-directory exists
- try:
- os.makedirs(labeled_data_dir)
- except OSError as e:
- # Raise all errors but 'EEXIST'
- if e.errno != errno.EEXIST:
- raise
-
- # Move all of the image to the appropriate sub-directory.
- for i in range(len(labels)):
- basename = 'ILSVRC2012_val_000%.5d.JPEG' % (i + 1)
- original_filename = os.path.join(data_dir, basename)
- if not os.path.exists(original_filename):
- print('Failed to find: %s' % original_filename)
- sys.exit(-1)
- new_filename = os.path.join(data_dir, labels[i], basename)
- os.rename(original_filename, new_filename)
diff --git a/research/inception/inception/data/process_bounding_boxes.py b/research/inception/inception/data/process_bounding_boxes.py
deleted file mode 100755
index 5e9fd786e40b6d95b89fcc9f9774aa7f132c1a6f..0000000000000000000000000000000000000000
--- a/research/inception/inception/data/process_bounding_boxes.py
+++ /dev/null
@@ -1,254 +0,0 @@
-#!/usr/bin/python
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Process the ImageNet Challenge bounding boxes for TensorFlow model training.
-
-This script is called as
-
-process_bounding_boxes.py [synsets-file]
-
-Where is a directory containing the downloaded and unpacked bounding box
-data. If [synsets-file] is supplied, then only the bounding boxes whose
-synstes are contained within this file are returned. Note that the
-[synsets-file] file contains synset ids, one per line.
-
-The script dumps out a CSV text file in which each line contains an entry.
- n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940
-
-The entry can be read as:
- , , , ,
-
-The bounding box for contains two points (xmin, ymin) and
-(xmax, ymax) specifying the lower-left corner and upper-right corner of a
-bounding box in *relative* coordinates.
-
-The user supplies a directory where the XML files reside. The directory
-structure in the directory is assumed to look like this:
-
-/nXXXXXXXX/nXXXXXXXX_YYYY.xml
-
-Each XML file contains a bounding box annotation. The script:
-
- (1) Parses the XML file and extracts the filename, label and bounding box info.
-
- (2) The bounding box is specified in the XML files as integer (xmin, ymin) and
- (xmax, ymax) *relative* to image size displayed to the human annotator. The
- size of the image displayed to the human annotator is stored in the XML file
- as integer (height, width).
-
- Note that the displayed size will differ from the actual size of the image
- downloaded from image-net.org. To make the bounding box annotation useable,
- we convert bounding box to floating point numbers relative to displayed
- height and width of the image.
-
- Note that each XML file might contain N bounding box annotations.
-
- Note that the points are all clamped at a range of [0.0, 1.0] because some
- human annotations extend outside the range of the supplied image.
-
- See details here: http://image-net.org/download-bboxes
-
-(3) By default, the script outputs all valid bounding boxes. If a
- [synsets-file] is supplied, only the subset of bounding boxes associated
- with those synsets are outputted. Importantly, one can supply a list of
- synsets in the ImageNet Challenge and output the list of bounding boxes
- associated with the training images of the ILSVRC.
-
- We use these bounding boxes to inform the random distortion of images
- supplied to the network.
-
-If you run this script successfully, you will see the following output
-to stderr:
-> Finished processing 544546 XML files.
-> Skipped 0 XML files not in ImageNet Challenge.
-> Skipped 0 bounding boxes not in ImageNet Challenge.
-> Wrote 615299 bounding boxes from 544546 annotated images.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import glob
-import os.path
-import sys
-import xml.etree.ElementTree as ET
-
-
-class BoundingBox(object):
- pass
-
-
-def GetItem(name, root, index=0):
- count = 0
- for item in root.iter(name):
- if count == index:
- return item.text
- count += 1
- # Failed to find "index" occurrence of item.
- return -1
-
-
-def GetInt(name, root, index=0):
- # In some XML annotation files, the point values are not integers, but floats.
- # So we add a float function to avoid ValueError.
- return int(float(GetItem(name, root, index)))
-
-
-def FindNumberBoundingBoxes(root):
- index = 0
- while True:
- if GetInt('xmin', root, index) == -1:
- break
- index += 1
- return index
-
-
-def ProcessXMLAnnotation(xml_file):
- """Process a single XML file containing a bounding box."""
- # pylint: disable=broad-except
- try:
- tree = ET.parse(xml_file)
- except Exception:
- print('Failed to parse: ' + xml_file, file=sys.stderr)
- return None
- # pylint: enable=broad-except
- root = tree.getroot()
-
- num_boxes = FindNumberBoundingBoxes(root)
- boxes = []
-
- for index in range(num_boxes):
- box = BoundingBox()
- # Grab the 'index' annotation.
- box.xmin = GetInt('xmin', root, index)
- box.ymin = GetInt('ymin', root, index)
- box.xmax = GetInt('xmax', root, index)
- box.ymax = GetInt('ymax', root, index)
-
- box.width = GetInt('width', root)
- box.height = GetInt('height', root)
- box.filename = GetItem('filename', root) + '.JPEG'
- box.label = GetItem('name', root)
-
- xmin = float(box.xmin) / float(box.width)
- xmax = float(box.xmax) / float(box.width)
- ymin = float(box.ymin) / float(box.height)
- ymax = float(box.ymax) / float(box.height)
-
- # Some images contain bounding box annotations that
- # extend outside of the supplied image. See, e.g.
- # n03127925/n03127925_147.xml
- # Additionally, for some bounding boxes, the min > max
- # or the box is entirely outside of the image.
- min_x = min(xmin, xmax)
- max_x = max(xmin, xmax)
- box.xmin_scaled = min(max(min_x, 0.0), 1.0)
- box.xmax_scaled = min(max(max_x, 0.0), 1.0)
-
- min_y = min(ymin, ymax)
- max_y = max(ymin, ymax)
- box.ymin_scaled = min(max(min_y, 0.0), 1.0)
- box.ymax_scaled = min(max(max_y, 0.0), 1.0)
-
- boxes.append(box)
-
- return boxes
-
-if __name__ == '__main__':
- if len(sys.argv) < 2 or len(sys.argv) > 3:
- print('Invalid usage\n'
- 'usage: process_bounding_boxes.py [synsets-file]',
- file=sys.stderr)
- sys.exit(-1)
-
- xml_files = glob.glob(sys.argv[1] + '/*/*.xml')
- print('Identified %d XML files in %s' % (len(xml_files), sys.argv[1]),
- file=sys.stderr)
-
- if len(sys.argv) == 3:
- labels = set([l.strip() for l in open(sys.argv[2]).readlines()])
- print('Identified %d synset IDs in %s' % (len(labels), sys.argv[2]),
- file=sys.stderr)
- else:
- labels = None
-
- skipped_boxes = 0
- skipped_files = 0
- saved_boxes = 0
- saved_files = 0
- for file_index, one_file in enumerate(xml_files):
- # Example: <...>/n06470073/n00141669_6790.xml
- label = os.path.basename(os.path.dirname(one_file))
-
- # Determine if the annotation is from an ImageNet Challenge label.
- if labels is not None and label not in labels:
- skipped_files += 1
- continue
-
- bboxes = ProcessXMLAnnotation(one_file)
- assert bboxes is not None, 'No bounding boxes found in ' + one_file
-
- found_box = False
- for bbox in bboxes:
- if labels is not None:
- if bbox.label != label:
- # Note: There is a slight bug in the bounding box annotation data.
- # Many of the dog labels have the human label 'Scottish_deerhound'
- # instead of the synset ID 'n02092002' in the bbox.label field. As a
- # simple hack to overcome this issue, we only exclude bbox labels
- # *which are synset ID's* that do not match original synset label for
- # the XML file.
- if bbox.label in labels:
- skipped_boxes += 1
- continue
-
- # Guard against improperly specified boxes.
- if (bbox.xmin_scaled >= bbox.xmax_scaled or
- bbox.ymin_scaled >= bbox.ymax_scaled):
- skipped_boxes += 1
- continue
-
- # Note bbox.filename occasionally contains '%s' in the name. This is
- # data set noise that is fixed by just using the basename of the XML file.
- image_filename = os.path.splitext(os.path.basename(one_file))[0]
- print('%s.JPEG,%.4f,%.4f,%.4f,%.4f' %
- (image_filename,
- bbox.xmin_scaled, bbox.ymin_scaled,
- bbox.xmax_scaled, bbox.ymax_scaled))
-
- saved_boxes += 1
- found_box = True
- if found_box:
- saved_files += 1
- else:
- skipped_files += 1
-
- if not file_index % 5000:
- print('--> processed %d of %d XML files.' %
- (file_index + 1, len(xml_files)),
- file=sys.stderr)
- print('--> skipped %d boxes and %d XML files.' %
- (skipped_boxes, skipped_files), file=sys.stderr)
-
- print('Finished processing %d XML files.' % len(xml_files), file=sys.stderr)
- print('Skipped %d XML files not in ImageNet Challenge.' % skipped_files,
- file=sys.stderr)
- print('Skipped %d bounding boxes not in ImageNet Challenge.' % skipped_boxes,
- file=sys.stderr)
- print('Wrote %d bounding boxes from %d annotated images.' %
- (saved_boxes, saved_files),
- file=sys.stderr)
- print('Finished.', file=sys.stderr)
diff --git a/research/inception/inception/dataset.py b/research/inception/inception/dataset.py
deleted file mode 100644
index 752c97e03b0361975d64b72892cc94333e353dfb..0000000000000000000000000000000000000000
--- a/research/inception/inception/dataset.py
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Small library that points to a data set.
-
-Methods of Data class:
- data_files: Returns a python list of all (sharded) data set files.
- num_examples_per_epoch: Returns the number of examples in the data set.
- num_classes: Returns the number of classes in the data set.
- reader: Return a reader for a single entry from the data set.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from abc import ABCMeta
-from abc import abstractmethod
-import os
-
-
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-# Basic model parameters.
-tf.app.flags.DEFINE_string('data_dir', '/tmp/mydata',
- """Path to the processed data, i.e. """
- """TFRecord of Example protos.""")
-
-
-class Dataset(object):
- """A simple class for handling data sets."""
- __metaclass__ = ABCMeta
-
- def __init__(self, name, subset):
- """Initialize dataset using a subset and the path to the data."""
- assert subset in self.available_subsets(), self.available_subsets()
- self.name = name
- self.subset = subset
-
- @abstractmethod
- def num_classes(self):
- """Returns the number of classes in the data set."""
- pass
- # return 10
-
- @abstractmethod
- def num_examples_per_epoch(self):
- """Returns the number of examples in the data subset."""
- pass
- # if self.subset == 'train':
- # return 10000
- # if self.subset == 'validation':
- # return 1000
-
- @abstractmethod
- def download_message(self):
- """Prints a download message for the Dataset."""
- pass
-
- def available_subsets(self):
- """Returns the list of available subsets."""
- return ['train', 'validation']
-
- def data_files(self):
- """Returns a python list of all (sharded) data subset files.
-
- Returns:
- python list of all (sharded) data set files.
- Raises:
- ValueError: if there are not data_files matching the subset.
- """
- tf_record_pattern = os.path.join(FLAGS.data_dir, '%s-*' % self.subset)
- data_files = tf.gfile.Glob(tf_record_pattern)
- if not data_files:
- print('No files found for dataset %s/%s at %s' % (self.name,
- self.subset,
- FLAGS.data_dir))
-
- self.download_message()
- exit(-1)
- return data_files
-
- def reader(self):
- """Return a reader for a single entry from the data set.
-
- See io_ops.py for details of Reader class.
-
- Returns:
- Reader object that reads the data set.
- """
- return tf.TFRecordReader()
diff --git a/research/inception/inception/flowers_data.py b/research/inception/inception/flowers_data.py
deleted file mode 100644
index 022b5234deef035a6150a54ed74445b510f1b148..0000000000000000000000000000000000000000
--- a/research/inception/inception/flowers_data.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Small library that points to the flowers data set.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-
-from inception.dataset import Dataset
-
-
-class FlowersData(Dataset):
- """Flowers data set."""
-
- def __init__(self, subset):
- super(FlowersData, self).__init__('Flowers', subset)
-
- def num_classes(self):
- """Returns the number of classes in the data set."""
- return 5
-
- def num_examples_per_epoch(self):
- """Returns the number of examples in the data subset."""
- if self.subset == 'train':
- return 3170
- if self.subset == 'validation':
- return 500
-
- def download_message(self):
- """Instruction to download and extract the tarball from Flowers website."""
-
- print('Failed to find any Flowers %s files'% self.subset)
- print('')
- print('If you have already downloaded and processed the data, then make '
- 'sure to set --data_dir to point to the directory containing the '
- 'location of the sharded TFRecords.\n')
- print('Please see README.md for instructions on how to build '
- 'the flowers dataset using download_and_preprocess_flowers.\n')
diff --git a/research/inception/inception/flowers_eval.py b/research/inception/inception/flowers_eval.py
deleted file mode 100644
index ae3e9dc14c8dc83368aa83f523ade92e12113554..0000000000000000000000000000000000000000
--- a/research/inception/inception/flowers_eval.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A binary to evaluate Inception on the flowers data set.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-from inception import inception_eval
-from inception.flowers_data import FlowersData
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def main(unused_argv=None):
- dataset = FlowersData(subset=FLAGS.subset)
- assert dataset.data_files()
- if tf.gfile.Exists(FLAGS.eval_dir):
- tf.gfile.DeleteRecursively(FLAGS.eval_dir)
- tf.gfile.MakeDirs(FLAGS.eval_dir)
- inception_eval.evaluate(dataset)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/inception/inception/flowers_train.py b/research/inception/inception/flowers_train.py
deleted file mode 100644
index 1f044a539d48ef6ce011831210b4bc31eba278f3..0000000000000000000000000000000000000000
--- a/research/inception/inception/flowers_train.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A binary to train Inception on the flowers data set.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-
-import tensorflow as tf
-
-from inception import inception_train
-from inception.flowers_data import FlowersData
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def main(_):
- dataset = FlowersData(subset=FLAGS.subset)
- assert dataset.data_files()
- if tf.gfile.Exists(FLAGS.train_dir):
- tf.gfile.DeleteRecursively(FLAGS.train_dir)
- tf.gfile.MakeDirs(FLAGS.train_dir)
- inception_train.train(dataset)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/inception/inception/image_processing.py b/research/inception/inception/image_processing.py
deleted file mode 100644
index fe74f1b3c9958060b15f52df80b11606c7ccf343..0000000000000000000000000000000000000000
--- a/research/inception/inception/image_processing.py
+++ /dev/null
@@ -1,513 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Read and preprocess image data.
-
- Image processing occurs on a single image at a time. Image are read and
- preprocessed in parallel across multiple threads. The resulting images
- are concatenated together to form a single batch for training or evaluation.
-
- -- Provide processed image data for a network:
- inputs: Construct batches of evaluation examples of images.
- distorted_inputs: Construct batches of training examples of images.
- batch_inputs: Construct batches of training or evaluation examples of images.
-
- -- Data processing:
- parse_example_proto: Parses an Example proto containing a training example
- of an image.
-
- -- Image decoding:
- decode_jpeg: Decode a JPEG encoded string into a 3-D float32 Tensor.
-
- -- Image preprocessing:
- image_preprocessing: Decode and preprocess one image for evaluation or training
- distort_image: Distort one image for training a network.
- eval_image: Prepare one image for evaluation.
- distort_color: Distort the color in one image for training.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_integer('batch_size', 32,
- """Number of images to process in a batch.""")
-tf.app.flags.DEFINE_integer('image_size', 299,
- """Provide square images of this size.""")
-tf.app.flags.DEFINE_integer('num_preprocess_threads', 4,
- """Number of preprocessing threads per tower. """
- """Please make this a multiple of 4.""")
-tf.app.flags.DEFINE_integer('num_readers', 4,
- """Number of parallel readers during train.""")
-
-# Images are preprocessed asynchronously using multiple threads specified by
-# --num_preprocss_threads and the resulting processed images are stored in a
-# random shuffling queue. The shuffling queue dequeues --batch_size images
-# for processing on a given Inception tower. A larger shuffling queue guarantees
-# better mixing across examples within a batch and results in slightly higher
-# predictive performance in a trained model. Empirically,
-# --input_queue_memory_factor=16 works well. A value of 16 implies a queue size
-# of 1024*16 images. Assuming RGB 299x299 images, this implies a queue size of
-# 16GB. If the machine is memory limited, then decrease this factor to
-# decrease the CPU memory footprint, accordingly.
-tf.app.flags.DEFINE_integer('input_queue_memory_factor', 16,
- """Size of the queue of preprocessed images. """
- """Default is ideal but try smaller values, e.g. """
- """4, 2 or 1, if host memory is constrained. See """
- """comments in code for more details.""")
-
-
-def inputs(dataset, batch_size=None, num_preprocess_threads=None):
- """Generate batches of ImageNet images for evaluation.
-
- Use this function as the inputs for evaluating a network.
-
- Note that some (minimal) image preprocessing occurs during evaluation
- including central cropping and resizing of the image to fit the network.
-
- Args:
- dataset: instance of Dataset class specifying the dataset.
- batch_size: integer, number of examples in batch
- num_preprocess_threads: integer, total number of preprocessing threads but
- None defaults to FLAGS.num_preprocess_threads.
-
- Returns:
- images: Images. 4D tensor of size [batch_size, FLAGS.image_size,
- image_size, 3].
- labels: 1-D integer Tensor of [FLAGS.batch_size].
- """
- if not batch_size:
- batch_size = FLAGS.batch_size
-
- # Force all input processing onto CPU in order to reserve the GPU for
- # the forward inference and back-propagation.
- with tf.device('/cpu:0'):
- images, labels = batch_inputs(
- dataset, batch_size, train=False,
- num_preprocess_threads=num_preprocess_threads,
- num_readers=1)
-
- return images, labels
-
-
-def distorted_inputs(dataset, batch_size=None, num_preprocess_threads=None):
- """Generate batches of distorted versions of ImageNet images.
-
- Use this function as the inputs for training a network.
-
- Distorting images provides a useful technique for augmenting the data
- set during training in order to make the network invariant to aspects
- of the image that do not effect the label.
-
- Args:
- dataset: instance of Dataset class specifying the dataset.
- batch_size: integer, number of examples in batch
- num_preprocess_threads: integer, total number of preprocessing threads but
- None defaults to FLAGS.num_preprocess_threads.
-
- Returns:
- images: Images. 4D tensor of size [batch_size, FLAGS.image_size,
- FLAGS.image_size, 3].
- labels: 1-D integer Tensor of [batch_size].
- """
- if not batch_size:
- batch_size = FLAGS.batch_size
-
- # Force all input processing onto CPU in order to reserve the GPU for
- # the forward inference and back-propagation.
- with tf.device('/cpu:0'):
- images, labels = batch_inputs(
- dataset, batch_size, train=True,
- num_preprocess_threads=num_preprocess_threads,
- num_readers=FLAGS.num_readers)
- return images, labels
-
-
-def decode_jpeg(image_buffer, scope=None):
- """Decode a JPEG string into one 3-D float image Tensor.
-
- Args:
- image_buffer: scalar string Tensor.
- scope: Optional scope for name_scope.
- Returns:
- 3-D float Tensor with values ranging from [0, 1).
- """
- with tf.name_scope(values=[image_buffer], name=scope,
- default_name='decode_jpeg'):
- # Decode the string as an RGB JPEG.
- # Note that the resulting image contains an unknown height and width
- # that is set dynamically by decode_jpeg. In other words, the height
- # and width of image is unknown at compile-time.
- image = tf.image.decode_jpeg(image_buffer, channels=3)
-
- # After this point, all image pixels reside in [0,1)
- # until the very end, when they're rescaled to (-1, 1). The various
- # adjust_* ops all require this range for dtype float.
- image = tf.image.convert_image_dtype(image, dtype=tf.float32)
- return image
-
-
-def distort_color(image, thread_id=0, scope=None):
- """Distort the color of the image.
-
- Each color distortion is non-commutative and thus ordering of the color ops
- matters. Ideally we would randomly permute the ordering of the color ops.
- Rather than adding that level of complication, we select a distinct ordering
- of color ops for each preprocessing thread.
-
- Args:
- image: Tensor containing single image.
- thread_id: preprocessing thread ID.
- scope: Optional scope for name_scope.
- Returns:
- color-distorted image
- """
- with tf.name_scope(values=[image], name=scope, default_name='distort_color'):
- color_ordering = thread_id % 2
-
- if color_ordering == 0:
- image = tf.image.random_brightness(image, max_delta=32. / 255.)
- image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
- image = tf.image.random_hue(image, max_delta=0.2)
- image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
- elif color_ordering == 1:
- image = tf.image.random_brightness(image, max_delta=32. / 255.)
- image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
- image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
- image = tf.image.random_hue(image, max_delta=0.2)
-
- # The random_* ops do not necessarily clamp.
- image = tf.clip_by_value(image, 0.0, 1.0)
- return image
-
-
-def distort_image(image, height, width, bbox, thread_id=0, scope=None):
- """Distort one image for training a network.
-
- Distorting images provides a useful technique for augmenting the data
- set during training in order to make the network invariant to aspects
- of the image that do not effect the label.
-
- Args:
- image: 3-D float Tensor of image
- height: integer
- width: integer
- bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
- where each coordinate is [0, 1) and the coordinates are arranged
- as [ymin, xmin, ymax, xmax].
- thread_id: integer indicating the preprocessing thread.
- scope: Optional scope for name_scope.
- Returns:
- 3-D float Tensor of distorted image used for training.
- """
- with tf.name_scope(values=[image, height, width, bbox], name=scope,
- default_name='distort_image'):
- # Each bounding box has shape [1, num_boxes, box coords] and
- # the coordinates are ordered [ymin, xmin, ymax, xmax].
-
- # Display the bounding box in the first thread only.
- if not thread_id:
- image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
- bbox)
- tf.summary.image('image_with_bounding_boxes', image_with_box)
-
- # A large fraction of image datasets contain a human-annotated bounding
- # box delineating the region of the image containing the object of interest.
- # We choose to create a new bounding box for the object which is a randomly
- # distorted version of the human-annotated bounding box that obeys an allowed
- # range of aspect ratios, sizes and overlap with the human-annotated
- # bounding box. If no box is supplied, then we assume the bounding box is
- # the entire image.
- sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
- tf.shape(image),
- bounding_boxes=bbox,
- min_object_covered=0.1,
- aspect_ratio_range=[0.75, 1.33],
- area_range=[0.05, 1.0],
- max_attempts=100,
- use_image_if_no_bounding_boxes=True)
- bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box
- if not thread_id:
- image_with_distorted_box = tf.image.draw_bounding_boxes(
- tf.expand_dims(image, 0), distort_bbox)
- tf.summary.image('images_with_distorted_bounding_box',
- image_with_distorted_box)
-
- # Crop the image to the specified bounding box.
- distorted_image = tf.slice(image, bbox_begin, bbox_size)
-
- # This resizing operation may distort the images because the aspect
- # ratio is not respected. We select a resize method in a round robin
- # fashion based on the thread number.
- # Note that ResizeMethod contains 4 enumerated resizing methods.
- resize_method = thread_id % 4
- distorted_image = tf.image.resize_images(distorted_image, [height, width],
- method=resize_method)
- # Restore the shape since the dynamic slice based upon the bbox_size loses
- # the third dimension.
- distorted_image.set_shape([height, width, 3])
- if not thread_id:
- tf.summary.image('cropped_resized_image',
- tf.expand_dims(distorted_image, 0))
-
- # Randomly flip the image horizontally.
- distorted_image = tf.image.random_flip_left_right(distorted_image)
-
- # Randomly distort the colors.
- distorted_image = distort_color(distorted_image, thread_id)
-
- if not thread_id:
- tf.summary.image('final_distorted_image',
- tf.expand_dims(distorted_image, 0))
- return distorted_image
-
-
-def eval_image(image, height, width, scope=None):
- """Prepare one image for evaluation.
-
- Args:
- image: 3-D float Tensor
- height: integer
- width: integer
- scope: Optional scope for name_scope.
- Returns:
- 3-D float Tensor of prepared image.
- """
- with tf.name_scope(values=[image, height, width], name=scope,
- default_name='eval_image'):
- # Crop the central region of the image with an area containing 87.5% of
- # the original image.
- image = tf.image.central_crop(image, central_fraction=0.875)
-
- # Resize the image to the original height and width.
- image = tf.expand_dims(image, 0)
- image = tf.image.resize_bilinear(image, [height, width],
- align_corners=False)
- image = tf.squeeze(image, [0])
- return image
-
-
-def image_preprocessing(image_buffer, bbox, train, thread_id=0):
- """Decode and preprocess one image for evaluation or training.
-
- Args:
- image_buffer: JPEG encoded string Tensor
- bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
- where each coordinate is [0, 1) and the coordinates are arranged as
- [ymin, xmin, ymax, xmax].
- train: boolean
- thread_id: integer indicating preprocessing thread
-
- Returns:
- 3-D float Tensor containing an appropriately scaled image
-
- Raises:
- ValueError: if user does not provide bounding box
- """
- if bbox is None:
- raise ValueError('Please supply a bounding box.')
-
- image = decode_jpeg(image_buffer)
- height = FLAGS.image_size
- width = FLAGS.image_size
-
- if train:
- image = distort_image(image, height, width, bbox, thread_id)
- else:
- image = eval_image(image, height, width)
-
- # Finally, rescale to [-1,1] instead of [0, 1)
- image = tf.subtract(image, 0.5)
- image = tf.multiply(image, 2.0)
- return image
-
-
-def parse_example_proto(example_serialized):
- """Parses an Example proto containing a training example of an image.
-
- The output of the build_image_data.py image preprocessing script is a dataset
- containing serialized Example protocol buffers. Each Example proto contains
- the following fields:
-
- image/height: 462
- image/width: 581
- image/colorspace: 'RGB'
- image/channels: 3
- image/class/label: 615
- image/class/synset: 'n03623198'
- image/class/text: 'knee pad'
- image/object/bbox/xmin: 0.1
- image/object/bbox/xmax: 0.9
- image/object/bbox/ymin: 0.2
- image/object/bbox/ymax: 0.6
- image/object/bbox/label: 615
- image/format: 'JPEG'
- image/filename: 'ILSVRC2012_val_00041207.JPEG'
- image/encoded:
-
- Args:
- example_serialized: scalar Tensor tf.string containing a serialized
- Example protocol buffer.
-
- Returns:
- image_buffer: Tensor tf.string containing the contents of a JPEG file.
- label: Tensor tf.int32 containing the label.
- bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]
- where each coordinate is [0, 1) and the coordinates are arranged as
- [ymin, xmin, ymax, xmax].
- text: Tensor tf.string containing the human-readable label.
- """
- # Dense features in Example proto.
- feature_map = {
- 'image/encoded': tf.FixedLenFeature([], dtype=tf.string,
- default_value=''),
- 'image/class/label': tf.FixedLenFeature([1], dtype=tf.int64,
- default_value=-1),
- 'image/class/text': tf.FixedLenFeature([], dtype=tf.string,
- default_value=''),
- }
- sparse_float32 = tf.VarLenFeature(dtype=tf.float32)
- # Sparse features in Example proto.
- feature_map.update(
- {k: sparse_float32 for k in ['image/object/bbox/xmin',
- 'image/object/bbox/ymin',
- 'image/object/bbox/xmax',
- 'image/object/bbox/ymax']})
-
- features = tf.parse_single_example(example_serialized, feature_map)
- label = tf.cast(features['image/class/label'], dtype=tf.int32)
-
- xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0)
- ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0)
- xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0)
- ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0)
-
- # Note that we impose an ordering of (y, x) just to make life difficult.
- bbox = tf.concat(axis=0, values=[ymin, xmin, ymax, xmax])
-
- # Force the variable number of bounding boxes into the shape
- # [1, num_boxes, coords].
- bbox = tf.expand_dims(bbox, 0)
- bbox = tf.transpose(bbox, [0, 2, 1])
-
- return features['image/encoded'], label, bbox, features['image/class/text']
-
-
-def batch_inputs(dataset, batch_size, train, num_preprocess_threads=None,
- num_readers=1):
- """Contruct batches of training or evaluation examples from the image dataset.
-
- Args:
- dataset: instance of Dataset class specifying the dataset.
- See dataset.py for details.
- batch_size: integer
- train: boolean
- num_preprocess_threads: integer, total number of preprocessing threads
- num_readers: integer, number of parallel readers
-
- Returns:
- images: 4-D float Tensor of a batch of images
- labels: 1-D integer Tensor of [batch_size].
-
- Raises:
- ValueError: if data is not found
- """
- with tf.name_scope('batch_processing'):
- data_files = dataset.data_files()
- if data_files is None:
- raise ValueError('No data files found for this dataset')
-
- # Create filename_queue
- if train:
- filename_queue = tf.train.string_input_producer(data_files,
- shuffle=True,
- capacity=16)
- else:
- filename_queue = tf.train.string_input_producer(data_files,
- shuffle=False,
- capacity=1)
- if num_preprocess_threads is None:
- num_preprocess_threads = FLAGS.num_preprocess_threads
-
- if num_preprocess_threads % 4:
- raise ValueError('Please make num_preprocess_threads a multiple '
- 'of 4 (%d % 4 != 0).', num_preprocess_threads)
-
- if num_readers is None:
- num_readers = FLAGS.num_readers
-
- if num_readers < 1:
- raise ValueError('Please make num_readers at least 1')
-
- # Approximate number of examples per shard.
- examples_per_shard = 1024
- # Size the random shuffle queue to balance between good global
- # mixing (more examples) and memory use (fewer examples).
- # 1 image uses 299*299*3*4 bytes = 1MB
- # The default input_queue_memory_factor is 16 implying a shuffling queue
- # size: examples_per_shard * 16 * 1MB = 17.6GB
- min_queue_examples = examples_per_shard * FLAGS.input_queue_memory_factor
- if train:
- examples_queue = tf.RandomShuffleQueue(
- capacity=min_queue_examples + 3 * batch_size,
- min_after_dequeue=min_queue_examples,
- dtypes=[tf.string])
- else:
- examples_queue = tf.FIFOQueue(
- capacity=examples_per_shard + 3 * batch_size,
- dtypes=[tf.string])
-
- # Create multiple readers to populate the queue of examples.
- if num_readers > 1:
- enqueue_ops = []
- for _ in range(num_readers):
- reader = dataset.reader()
- _, value = reader.read(filename_queue)
- enqueue_ops.append(examples_queue.enqueue([value]))
-
- tf.train.queue_runner.add_queue_runner(
- tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops))
- example_serialized = examples_queue.dequeue()
- else:
- reader = dataset.reader()
- _, example_serialized = reader.read(filename_queue)
-
- images_and_labels = []
- for thread_id in range(num_preprocess_threads):
- # Parse a serialized Example proto to extract the image and metadata.
- image_buffer, label_index, bbox, _ = parse_example_proto(
- example_serialized)
- image = image_preprocessing(image_buffer, bbox, train, thread_id)
- images_and_labels.append([image, label_index])
-
- images, label_index_batch = tf.train.batch_join(
- images_and_labels,
- batch_size=batch_size,
- capacity=2 * num_preprocess_threads * batch_size)
-
- # Reshape images into these desired dimensions.
- height = FLAGS.image_size
- width = FLAGS.image_size
- depth = 3
-
- images = tf.cast(images, tf.float32)
- images = tf.reshape(images, shape=[batch_size, height, width, depth])
-
- # Display the training images in the visualizer.
- tf.summary.image('images', images)
-
- return images, tf.reshape(label_index_batch, [batch_size])
diff --git a/research/inception/inception/imagenet_data.py b/research/inception/inception/imagenet_data.py
deleted file mode 100644
index 0a6d22e1292632f0899355d5aa7183c3f5f33b2c..0000000000000000000000000000000000000000
--- a/research/inception/inception/imagenet_data.py
+++ /dev/null
@@ -1,59 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Small library that points to the ImageNet data set.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-
-from inception.dataset import Dataset
-
-
-class ImagenetData(Dataset):
- """ImageNet data set."""
-
- def __init__(self, subset):
- super(ImagenetData, self).__init__('ImageNet', subset)
-
- def num_classes(self):
- """Returns the number of classes in the data set."""
- return 1000
-
- def num_examples_per_epoch(self):
- """Returns the number of examples in the data set."""
- # Bounding box data consists of 615299 bounding boxes for 544546 images.
- if self.subset == 'train':
- return 1281167
- if self.subset == 'validation':
- return 50000
-
- def download_message(self):
- """Instruction to download and extract the tarball from Flowers website."""
-
- print('Failed to find any ImageNet %s files'% self.subset)
- print('')
- print('If you have already downloaded and processed the data, then make '
- 'sure to set --data_dir to point to the directory containing the '
- 'location of the sharded TFRecords.\n')
- print('If you have not downloaded and prepared the ImageNet data in the '
- 'TFRecord format, you will need to do this at least once. This '
- 'process could take several hours depending on the speed of your '
- 'computer and network connection\n')
- print('Please see README.md for instructions on how to build '
- 'the ImageNet dataset using download_and_preprocess_imagenet.\n')
- print('Note that the raw data size is 300 GB and the processed data size '
- 'is 150 GB. Please ensure you have at least 500GB disk space.')
diff --git a/research/inception/inception/imagenet_distributed_train.py b/research/inception/inception/imagenet_distributed_train.py
deleted file mode 100644
index f3615e012f042649b52e37aeaeeb2c3efc07f92c..0000000000000000000000000000000000000000
--- a/research/inception/inception/imagenet_distributed_train.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-# pylint: disable=line-too-long
-"""A binary to train Inception in a distributed manner using multiple systems.
-
-Please see accompanying README.md for details and instructions.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from inception import inception_distributed_train
-from inception.imagenet_data import ImagenetData
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def main(unused_args):
- assert FLAGS.job_name in ['ps', 'worker'], 'job_name must be ps or worker'
-
- # Extract all the hostnames for the ps and worker jobs to construct the
- # cluster spec.
- ps_hosts = FLAGS.ps_hosts.split(',')
- worker_hosts = FLAGS.worker_hosts.split(',')
- tf.logging.info('PS hosts are: %s' % ps_hosts)
- tf.logging.info('Worker hosts are: %s' % worker_hosts)
-
- cluster_spec = tf.train.ClusterSpec({'ps': ps_hosts,
- 'worker': worker_hosts})
- server = tf.train.Server(
- {'ps': ps_hosts,
- 'worker': worker_hosts},
- job_name=FLAGS.job_name,
- task_index=FLAGS.task_id,
- protocol=FLAGS.protocol)
-
- if FLAGS.job_name == 'ps':
- # `ps` jobs wait for incoming connections from the workers.
- server.join()
- else:
- # `worker` jobs will actually do the work.
- dataset = ImagenetData(subset=FLAGS.subset)
- assert dataset.data_files()
- # Only the chief checks for or creates train_dir.
- if FLAGS.task_id == 0:
- if not tf.gfile.Exists(FLAGS.train_dir):
- tf.gfile.MakeDirs(FLAGS.train_dir)
- inception_distributed_train.train(server.target, dataset, cluster_spec)
-
-if __name__ == '__main__':
- tf.logging.set_verbosity(tf.logging.INFO)
- tf.app.run()
diff --git a/research/inception/inception/imagenet_eval.py b/research/inception/inception/imagenet_eval.py
deleted file mode 100644
index e6f8bac2ee71021914715172296d63dd56b5a6f9..0000000000000000000000000000000000000000
--- a/research/inception/inception/imagenet_eval.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A binary to evaluate Inception on the ImageNet data set.
-
-Note that using the supplied pre-trained inception checkpoint, the eval should
-achieve:
- precision @ 1 = 0.7874 recall @ 5 = 0.9436 [50000 examples]
-
-See the README.md for more details.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-from inception import inception_eval
-from inception.imagenet_data import ImagenetData
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def main(unused_argv=None):
- dataset = ImagenetData(subset=FLAGS.subset)
- assert dataset.data_files()
- if tf.gfile.Exists(FLAGS.eval_dir):
- tf.gfile.DeleteRecursively(FLAGS.eval_dir)
- tf.gfile.MakeDirs(FLAGS.eval_dir)
- inception_eval.evaluate(dataset)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/inception/inception/imagenet_train.py b/research/inception/inception/imagenet_train.py
deleted file mode 100644
index 3ffb55ee963e5b9f8e31915a78eef518324642aa..0000000000000000000000000000000000000000
--- a/research/inception/inception/imagenet_train.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A binary to train Inception on the ImageNet data set.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-
-import tensorflow as tf
-
-from inception import inception_train
-from inception.imagenet_data import ImagenetData
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def main(_):
- dataset = ImagenetData(subset=FLAGS.subset)
- assert dataset.data_files()
- if tf.gfile.Exists(FLAGS.train_dir):
- tf.gfile.DeleteRecursively(FLAGS.train_dir)
- tf.gfile.MakeDirs(FLAGS.train_dir)
- inception_train.train(dataset)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/inception/inception/inception_distributed_train.py b/research/inception/inception/inception_distributed_train.py
deleted file mode 100644
index c1a589acb5fe386fd648ae3fae926ee927c0ca79..0000000000000000000000000000000000000000
--- a/research/inception/inception/inception_distributed_train.py
+++ /dev/null
@@ -1,314 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A library to train Inception using multiple replicas with synchronous update.
-
-Please see accompanying README.md for details and instructions.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from datetime import datetime
-import os.path
-import time
-
-import numpy as np
-import tensorflow as tf
-
-from inception import image_processing
-from inception import inception_model as inception
-from inception.slim import slim
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_string('job_name', '', 'One of "ps", "worker"')
-tf.app.flags.DEFINE_string('ps_hosts', '',
- """Comma-separated list of hostname:port for the """
- """parameter server jobs. e.g. """
- """'machine1:2222,machine2:1111,machine2:2222'""")
-tf.app.flags.DEFINE_string('worker_hosts', '',
- """Comma-separated list of hostname:port for the """
- """worker jobs. e.g. """
- """'machine1:2222,machine2:1111,machine2:2222'""")
-tf.app.flags.DEFINE_string('protocol', 'grpc',
- """Communication protocol to use in distributed """
- """execution (default grpc) """)
-
-tf.app.flags.DEFINE_string('train_dir', '/tmp/imagenet_train',
- """Directory where to write event logs """
- """and checkpoint.""")
-tf.app.flags.DEFINE_integer('max_steps', 1000000, 'Number of batches to run.')
-tf.app.flags.DEFINE_string('subset', 'train', 'Either "train" or "validation".')
-tf.app.flags.DEFINE_boolean('log_device_placement', False,
- 'Whether to log device placement.')
-
-# Task ID is used to select the chief and also to access the local_step for
-# each replica to check staleness of the gradients in SyncReplicasOptimizer.
-tf.app.flags.DEFINE_integer(
- 'task_id', 0, 'Task ID of the worker/replica running the training.')
-
-# More details can be found in the SyncReplicasOptimizer class:
-# tensorflow/python/training/sync_replicas_optimizer.py
-tf.app.flags.DEFINE_integer('num_replicas_to_aggregate', -1,
- """Number of gradients to collect before """
- """updating the parameters.""")
-tf.app.flags.DEFINE_integer('save_interval_secs', 10 * 60,
- 'Save interval seconds.')
-tf.app.flags.DEFINE_integer('save_summaries_secs', 180,
- 'Save summaries interval seconds.')
-
-# **IMPORTANT**
-# Please note that this learning rate schedule is heavily dependent on the
-# hardware architecture, batch size and any changes to the model architecture
-# specification. Selecting a finely tuned learning rate schedule is an
-# empirical process that requires some experimentation. Please see README.md
-# more guidance and discussion.
-#
-# Learning rate decay factor selected from https://arxiv.org/abs/1604.00981
-tf.app.flags.DEFINE_float('initial_learning_rate', 0.045,
- 'Initial learning rate.')
-tf.app.flags.DEFINE_float('num_epochs_per_decay', 2.0,
- 'Epochs after which learning rate decays.')
-tf.app.flags.DEFINE_float('learning_rate_decay_factor', 0.94,
- 'Learning rate decay factor.')
-
-# Constants dictating the learning rate schedule.
-RMSPROP_DECAY = 0.9 # Decay term for RMSProp.
-RMSPROP_MOMENTUM = 0.9 # Momentum in RMSProp.
-RMSPROP_EPSILON = 1.0 # Epsilon term for RMSProp.
-
-
-def train(target, dataset, cluster_spec):
- """Train Inception on a dataset for a number of steps."""
- # Number of workers and parameter servers are inferred from the workers and ps
- # hosts string.
- num_workers = len(cluster_spec.as_dict()['worker'])
- num_parameter_servers = len(cluster_spec.as_dict()['ps'])
- # If no value is given, num_replicas_to_aggregate defaults to be the number of
- # workers.
- if FLAGS.num_replicas_to_aggregate == -1:
- num_replicas_to_aggregate = num_workers
- else:
- num_replicas_to_aggregate = FLAGS.num_replicas_to_aggregate
-
- # Both should be greater than 0 in a distributed training.
- assert num_workers > 0 and num_parameter_servers > 0, (' num_workers and '
- 'num_parameter_servers'
- ' must be > 0.')
-
- # Choose worker 0 as the chief. Note that any worker could be the chief
- # but there should be only one chief.
- is_chief = (FLAGS.task_id == 0)
-
- # Ops are assigned to worker by default.
- with tf.device('/job:worker/task:%d' % FLAGS.task_id):
- # Variables and its related init/assign ops are assigned to ps.
- with slim.scopes.arg_scope(
- [slim.variables.variable, slim.variables.global_step],
- device=slim.variables.VariableDeviceChooser(num_parameter_servers)):
- # Create a variable to count the number of train() calls. This equals the
- # number of updates applied to the variables.
- global_step = slim.variables.global_step()
-
- # Calculate the learning rate schedule.
- num_batches_per_epoch = (dataset.num_examples_per_epoch() /
- FLAGS.batch_size)
- # Decay steps need to be divided by the number of replicas to aggregate.
- decay_steps = int(num_batches_per_epoch * FLAGS.num_epochs_per_decay /
- num_replicas_to_aggregate)
-
- # Decay the learning rate exponentially based on the number of steps.
- lr = tf.train.exponential_decay(FLAGS.initial_learning_rate,
- global_step,
- decay_steps,
- FLAGS.learning_rate_decay_factor,
- staircase=True)
- # Add a summary to track the learning rate.
- tf.summary.scalar('learning_rate', lr)
-
- # Create an optimizer that performs gradient descent.
- opt = tf.train.RMSPropOptimizer(lr,
- RMSPROP_DECAY,
- momentum=RMSPROP_MOMENTUM,
- epsilon=RMSPROP_EPSILON)
-
- images, labels = image_processing.distorted_inputs(
- dataset,
- batch_size=FLAGS.batch_size,
- num_preprocess_threads=FLAGS.num_preprocess_threads)
-
- # Number of classes in the Dataset label set plus 1.
- # Label 0 is reserved for an (unused) background class.
- num_classes = dataset.num_classes() + 1
- logits = inception.inference(images, num_classes, for_training=True)
- # Add classification loss.
- inception.loss(logits, labels)
-
- # Gather all of the losses including regularization losses.
- losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
- losses += tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
-
- total_loss = tf.add_n(losses, name='total_loss')
-
- if is_chief:
- # Compute the moving average of all individual losses and the
- # total loss.
- loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
- loss_averages_op = loss_averages.apply(losses + [total_loss])
-
- # Attach a scalar summmary to all individual losses and the total loss;
- # do the same for the averaged version of the losses.
- for l in losses + [total_loss]:
- loss_name = l.op.name
- # Name each loss as '(raw)' and name the moving average version of the
- # loss as the original loss name.
- tf.summary.scalar(loss_name + ' (raw)', l)
- tf.summary.scalar(loss_name, loss_averages.average(l))
-
- # Add dependency to compute loss_averages.
- with tf.control_dependencies([loss_averages_op]):
- total_loss = tf.identity(total_loss)
-
- # Track the moving averages of all trainable variables.
- # Note that we maintain a 'double-average' of the BatchNormalization
- # global statistics.
- # This is not needed when the number of replicas are small but important
- # for synchronous distributed training with tens of workers/replicas.
- exp_moving_averager = tf.train.ExponentialMovingAverage(
- inception.MOVING_AVERAGE_DECAY, global_step)
-
- variables_to_average = (
- tf.trainable_variables() + tf.moving_average_variables())
-
- # Add histograms for model variables.
- for var in variables_to_average:
- tf.summary.histogram(var.op.name, var)
-
- # Create synchronous replica optimizer.
- opt = tf.train.SyncReplicasOptimizer(
- opt,
- replicas_to_aggregate=num_replicas_to_aggregate,
- total_num_replicas=num_workers,
- variable_averages=exp_moving_averager,
- variables_to_average=variables_to_average)
-
- batchnorm_updates = tf.get_collection(slim.ops.UPDATE_OPS_COLLECTION)
- assert batchnorm_updates, 'Batchnorm updates are missing'
- batchnorm_updates_op = tf.group(*batchnorm_updates)
- # Add dependency to compute batchnorm_updates.
- with tf.control_dependencies([batchnorm_updates_op]):
- total_loss = tf.identity(total_loss)
-
- # Compute gradients with respect to the loss.
- grads = opt.compute_gradients(total_loss)
-
- # Add histograms for gradients.
- for grad, var in grads:
- if grad is not None:
- tf.summary.histogram(var.op.name + '/gradients', grad)
-
- apply_gradients_op = opt.apply_gradients(grads, global_step=global_step)
-
- with tf.control_dependencies([apply_gradients_op]):
- train_op = tf.identity(total_loss, name='train_op')
-
- # Get chief queue_runners and init_tokens, which is used to synchronize
- # replicas. More details can be found in SyncReplicasOptimizer.
- chief_queue_runners = [opt.get_chief_queue_runner()]
- init_tokens_op = opt.get_init_tokens_op()
-
- # Create a saver.
- saver = tf.train.Saver()
-
- # Build the summary operation based on the TF collection of Summaries.
- summary_op = tf.summary.merge_all()
-
- # Build an initialization operation to run below.
- init_op = tf.global_variables_initializer()
-
- # We run the summaries in the same thread as the training operations by
- # passing in None for summary_op to avoid a summary_thread being started.
- # Running summaries and training operations in parallel could run out of
- # GPU memory.
- sv = tf.train.Supervisor(is_chief=is_chief,
- logdir=FLAGS.train_dir,
- init_op=init_op,
- summary_op=None,
- global_step=global_step,
- saver=saver,
- save_model_secs=FLAGS.save_interval_secs)
-
- tf.logging.info('%s Supervisor' % datetime.now())
-
- sess_config = tf.ConfigProto(
- allow_soft_placement=True,
- log_device_placement=FLAGS.log_device_placement)
-
- # Get a session.
- sess = sv.prepare_or_wait_for_session(target, config=sess_config)
-
- # Start the queue runners.
- queue_runners = tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS)
- sv.start_queue_runners(sess, queue_runners)
- tf.logging.info('Started %d queues for processing input data.',
- len(queue_runners))
-
- if is_chief:
- sv.start_queue_runners(sess, chief_queue_runners)
- sess.run(init_tokens_op)
-
- # Train, checking for Nans. Concurrently run the summary operation at a
- # specified interval. Note that the summary_op and train_op never run
- # simultaneously in order to prevent running out of GPU memory.
- next_summary_time = time.time() + FLAGS.save_summaries_secs
- while not sv.should_stop():
- try:
- start_time = time.time()
- loss_value, step = sess.run([train_op, global_step])
- assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
- if step > FLAGS.max_steps:
- break
- duration = time.time() - start_time
-
- if step % 30 == 0:
- examples_per_sec = FLAGS.batch_size / float(duration)
- format_str = ('Worker %d: %s: step %d, loss = %.2f'
- '(%.1f examples/sec; %.3f sec/batch)')
- tf.logging.info(format_str %
- (FLAGS.task_id, datetime.now(), step, loss_value,
- examples_per_sec, duration))
-
- # Determine if the summary_op should be run on the chief worker.
- if is_chief and next_summary_time < time.time():
- tf.logging.info('Running Summary operation on the chief.')
- summary_str = sess.run(summary_op)
- sv.summary_computed(sess, summary_str)
- tf.logging.info('Finished running Summary operation.')
-
- # Determine the next time for running the summary.
- next_summary_time += FLAGS.save_summaries_secs
- except:
- if is_chief:
- tf.logging.info('Chief got exception while running!')
- raise
-
- # Stop the supervisor. This also waits for service threads to finish.
- sv.stop()
-
- # Save after the training ends.
- if is_chief:
- saver.save(sess,
- os.path.join(FLAGS.train_dir, 'model.ckpt'),
- global_step=global_step)
diff --git a/research/inception/inception/inception_eval.py b/research/inception/inception/inception_eval.py
deleted file mode 100644
index e7cfc3c399dd82a915b3a49c7ddd4a8565292f69..0000000000000000000000000000000000000000
--- a/research/inception/inception/inception_eval.py
+++ /dev/null
@@ -1,171 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A library to evaluate Inception on a single GPU.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from datetime import datetime
-import math
-import os.path
-import time
-
-
-import numpy as np
-import tensorflow as tf
-
-from inception import image_processing
-from inception import inception_model as inception
-
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_string('eval_dir', '/tmp/imagenet_eval',
- """Directory where to write event logs.""")
-tf.app.flags.DEFINE_string('checkpoint_dir', '/tmp/imagenet_train',
- """Directory where to read model checkpoints.""")
-
-# Flags governing the frequency of the eval.
-tf.app.flags.DEFINE_integer('eval_interval_secs', 60 * 5,
- """How often to run the eval.""")
-tf.app.flags.DEFINE_boolean('run_once', False,
- """Whether to run eval only once.""")
-
-# Flags governing the data used for the eval.
-tf.app.flags.DEFINE_integer('num_examples', 50000,
- """Number of examples to run. Note that the eval """
- """ImageNet dataset contains 50000 examples.""")
-tf.app.flags.DEFINE_string('subset', 'validation',
- """Either 'validation' or 'train'.""")
-
-
-def _eval_once(saver, summary_writer, top_1_op, top_5_op, summary_op):
- """Runs Eval once.
-
- Args:
- saver: Saver.
- summary_writer: Summary writer.
- top_1_op: Top 1 op.
- top_5_op: Top 5 op.
- summary_op: Summary op.
- """
- with tf.Session() as sess:
- ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
- if ckpt and ckpt.model_checkpoint_path:
- if os.path.isabs(ckpt.model_checkpoint_path):
- # Restores from checkpoint with absolute path.
- saver.restore(sess, ckpt.model_checkpoint_path)
- else:
- # Restores from checkpoint with relative path.
- saver.restore(sess, os.path.join(FLAGS.checkpoint_dir,
- ckpt.model_checkpoint_path))
-
- # Assuming model_checkpoint_path looks something like:
- # /my-favorite-path/imagenet_train/model.ckpt-0,
- # extract global_step from it.
- global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
- print('Successfully loaded model from %s at step=%s.' %
- (ckpt.model_checkpoint_path, global_step))
- else:
- print('No checkpoint file found')
- return
-
- # Start the queue runners.
- coord = tf.train.Coordinator()
- try:
- threads = []
- for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
- threads.extend(qr.create_threads(sess, coord=coord, daemon=True,
- start=True))
-
- num_iter = int(math.ceil(FLAGS.num_examples / FLAGS.batch_size))
- # Counts the number of correct predictions.
- count_top_1 = 0.0
- count_top_5 = 0.0
- total_sample_count = num_iter * FLAGS.batch_size
- step = 0
-
- print('%s: starting evaluation on (%s).' % (datetime.now(), FLAGS.subset))
- start_time = time.time()
- while step < num_iter and not coord.should_stop():
- top_1, top_5 = sess.run([top_1_op, top_5_op])
- count_top_1 += np.sum(top_1)
- count_top_5 += np.sum(top_5)
- step += 1
- if step % 20 == 0:
- duration = time.time() - start_time
- sec_per_batch = duration / 20.0
- examples_per_sec = FLAGS.batch_size / sec_per_batch
- print('%s: [%d batches out of %d] (%.1f examples/sec; %.3f'
- 'sec/batch)' % (datetime.now(), step, num_iter,
- examples_per_sec, sec_per_batch))
- start_time = time.time()
-
- # Compute precision @ 1.
- precision_at_1 = count_top_1 / total_sample_count
- recall_at_5 = count_top_5 / total_sample_count
- print('%s: precision @ 1 = %.4f recall @ 5 = %.4f [%d examples]' %
- (datetime.now(), precision_at_1, recall_at_5, total_sample_count))
-
- summary = tf.Summary()
- summary.ParseFromString(sess.run(summary_op))
- summary.value.add(tag='Precision @ 1', simple_value=precision_at_1)
- summary.value.add(tag='Recall @ 5', simple_value=recall_at_5)
- summary_writer.add_summary(summary, global_step)
-
- except Exception as e: # pylint: disable=broad-except
- coord.request_stop(e)
-
- coord.request_stop()
- coord.join(threads, stop_grace_period_secs=10)
-
-
-def evaluate(dataset):
- """Evaluate model on Dataset for a number of steps."""
- with tf.Graph().as_default():
- # Get images and labels from the dataset.
- images, labels = image_processing.inputs(dataset)
-
- # Number of classes in the Dataset label set plus 1.
- # Label 0 is reserved for an (unused) background class.
- num_classes = dataset.num_classes() + 1
-
- # Build a Graph that computes the logits predictions from the
- # inference model.
- logits, _ = inception.inference(images, num_classes)
-
- # Calculate predictions.
- top_1_op = tf.nn.in_top_k(logits, labels, 1)
- top_5_op = tf.nn.in_top_k(logits, labels, 5)
-
- # Restore the moving average version of the learned variables for eval.
- variable_averages = tf.train.ExponentialMovingAverage(
- inception.MOVING_AVERAGE_DECAY)
- variables_to_restore = variable_averages.variables_to_restore()
- saver = tf.train.Saver(variables_to_restore)
-
- # Build the summary operation based on the TF collection of Summaries.
- summary_op = tf.summary.merge_all()
-
- graph_def = tf.get_default_graph().as_graph_def()
- summary_writer = tf.summary.FileWriter(FLAGS.eval_dir,
- graph_def=graph_def)
-
- while True:
- _eval_once(saver, summary_writer, top_1_op, top_5_op, summary_op)
- if FLAGS.run_once:
- break
- time.sleep(FLAGS.eval_interval_secs)
diff --git a/research/inception/inception/inception_model.py b/research/inception/inception/inception_model.py
deleted file mode 100644
index fedae13ae712f09d23ff020b161d86e87ee46e95..0000000000000000000000000000000000000000
--- a/research/inception/inception/inception_model.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Build the Inception v3 network on ImageNet data set.
-
-The Inception v3 architecture is described in http://arxiv.org/abs/1512.00567
-
-Summary of available functions:
- inference: Compute inference on the model inputs to make a prediction
- loss: Compute the loss of the prediction with respect to the labels
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import re
-
-import tensorflow as tf
-
-from inception.slim import slim
-
-FLAGS = tf.app.flags.FLAGS
-
-# If a model is trained using multiple GPUs, prefix all Op names with tower_name
-# to differentiate the operations. Note that this prefix is removed from the
-# names of the summaries when visualizing a model.
-TOWER_NAME = 'tower'
-
-# Batch normalization. Constant governing the exponential moving average of
-# the 'global' mean and variance for all activations.
-BATCHNORM_MOVING_AVERAGE_DECAY = 0.9997
-
-# The decay to use for the moving average.
-MOVING_AVERAGE_DECAY = 0.9999
-
-
-def inference(images, num_classes, for_training=False, restore_logits=True,
- scope=None):
- """Build Inception v3 model architecture.
-
- See here for reference: http://arxiv.org/abs/1512.00567
-
- Args:
- images: Images returned from inputs() or distorted_inputs().
- num_classes: number of classes
- for_training: If set to `True`, build the inference model for training.
- Kernels that operate differently for inference during training
- e.g. dropout, are appropriately configured.
- restore_logits: whether or not the logits layers should be restored.
- Useful for fine-tuning a model with different num_classes.
- scope: optional prefix string identifying the ImageNet tower.
-
- Returns:
- Logits. 2-D float Tensor.
- Auxiliary Logits. 2-D float Tensor of side-head. Used for training only.
- """
- # Parameters for BatchNorm.
- batch_norm_params = {
- # Decay for the moving averages.
- 'decay': BATCHNORM_MOVING_AVERAGE_DECAY,
- # epsilon to prevent 0s in variance.
- 'epsilon': 0.001,
- }
- # Set weight_decay for weights in Conv and FC layers.
- with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0.00004):
- with slim.arg_scope([slim.ops.conv2d],
- stddev=0.1,
- activation=tf.nn.relu,
- batch_norm_params=batch_norm_params):
- logits, endpoints = slim.inception.inception_v3(
- images,
- dropout_keep_prob=0.8,
- num_classes=num_classes,
- is_training=for_training,
- restore_logits=restore_logits,
- scope=scope)
-
- # Add summaries for viewing model statistics on TensorBoard.
- _activation_summaries(endpoints)
-
- # Grab the logits associated with the side head. Employed during training.
- auxiliary_logits = endpoints['aux_logits']
-
- return logits, auxiliary_logits
-
-
-def loss(logits, labels, batch_size=None):
- """Adds all losses for the model.
-
- Note the final loss is not returned. Instead, the list of losses are collected
- by slim.losses. The losses are accumulated in tower_loss() and summed to
- calculate the total loss.
-
- Args:
- logits: List of logits from inference(). Each entry is a 2-D float Tensor.
- labels: Labels from distorted_inputs or inputs(). 1-D tensor
- of shape [batch_size]
- batch_size: integer
- """
- if not batch_size:
- batch_size = FLAGS.batch_size
-
- # Reshape the labels into a dense Tensor of
- # shape [FLAGS.batch_size, num_classes].
- sparse_labels = tf.reshape(labels, [batch_size, 1])
- indices = tf.reshape(tf.range(batch_size), [batch_size, 1])
- concated = tf.concat(axis=1, values=[indices, sparse_labels])
- num_classes = logits[0].get_shape()[-1].value
- dense_labels = tf.sparse_to_dense(concated,
- [batch_size, num_classes],
- 1.0, 0.0)
-
- # Cross entropy loss for the main softmax prediction.
- slim.losses.cross_entropy_loss(logits[0],
- dense_labels,
- label_smoothing=0.1,
- weight=1.0)
-
- # Cross entropy loss for the auxiliary softmax head.
- slim.losses.cross_entropy_loss(logits[1],
- dense_labels,
- label_smoothing=0.1,
- weight=0.4,
- scope='aux_loss')
-
-
-def _activation_summary(x):
- """Helper to create summaries for activations.
-
- Creates a summary that provides a histogram of activations.
- Creates a summary that measure the sparsity of activations.
-
- Args:
- x: Tensor
- """
- # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
- # session. This helps the clarity of presentation on tensorboard.
- tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name)
- tf.summary.histogram(tensor_name + '/activations', x)
- tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x))
-
-
-def _activation_summaries(endpoints):
- with tf.name_scope('summaries'):
- for act in endpoints.values():
- _activation_summary(act)
diff --git a/research/inception/inception/inception_train.py b/research/inception/inception/inception_train.py
deleted file mode 100644
index e1c32713b2012aec8a18637ec5dd79a1cc84d90f..0000000000000000000000000000000000000000
--- a/research/inception/inception/inception_train.py
+++ /dev/null
@@ -1,357 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A library to train Inception using multiple GPUs with synchronous updates.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import copy
-from datetime import datetime
-import os.path
-import re
-import time
-
-import numpy as np
-import tensorflow as tf
-
-from inception import image_processing
-from inception import inception_model as inception
-from inception.slim import slim
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_string('train_dir', '/tmp/imagenet_train',
- """Directory where to write event logs """
- """and checkpoint.""")
-tf.app.flags.DEFINE_integer('max_steps', 10000000,
- """Number of batches to run.""")
-tf.app.flags.DEFINE_string('subset', 'train',
- """Either 'train' or 'validation'.""")
-
-# Flags governing the hardware employed for running TensorFlow.
-tf.app.flags.DEFINE_integer('num_gpus', 1,
- """How many GPUs to use.""")
-tf.app.flags.DEFINE_boolean('log_device_placement', False,
- """Whether to log device placement.""")
-
-# Flags governing the type of training.
-tf.app.flags.DEFINE_boolean('fine_tune', False,
- """If set, randomly initialize the final layer """
- """of weights in order to train the network on a """
- """new task.""")
-tf.app.flags.DEFINE_string('pretrained_model_checkpoint_path', '',
- """If specified, restore this pretrained model """
- """before beginning any training.""")
-
-# **IMPORTANT**
-# Please note that this learning rate schedule is heavily dependent on the
-# hardware architecture, batch size and any changes to the model architecture
-# specification. Selecting a finely tuned learning rate schedule is an
-# empirical process that requires some experimentation. Please see README.md
-# more guidance and discussion.
-#
-# With 8 Tesla K40's and a batch size = 256, the following setup achieves
-# precision@1 = 73.5% after 100 hours and 100K steps (20 epochs).
-# Learning rate decay factor selected from http://arxiv.org/abs/1404.5997.
-tf.app.flags.DEFINE_float('initial_learning_rate', 0.1,
- """Initial learning rate.""")
-tf.app.flags.DEFINE_float('num_epochs_per_decay', 30.0,
- """Epochs after which learning rate decays.""")
-tf.app.flags.DEFINE_float('learning_rate_decay_factor', 0.16,
- """Learning rate decay factor.""")
-
-# Constants dictating the learning rate schedule.
-RMSPROP_DECAY = 0.9 # Decay term for RMSProp.
-RMSPROP_MOMENTUM = 0.9 # Momentum in RMSProp.
-RMSPROP_EPSILON = 1.0 # Epsilon term for RMSProp.
-
-
-def _tower_loss(images, labels, num_classes, scope, reuse_variables=None):
- """Calculate the total loss on a single tower running the ImageNet model.
-
- We perform 'batch splitting'. This means that we cut up a batch across
- multiple GPUs. For instance, if the batch size = 32 and num_gpus = 2,
- then each tower will operate on an batch of 16 images.
-
- Args:
- images: Images. 4D tensor of size [batch_size, FLAGS.image_size,
- FLAGS.image_size, 3].
- labels: 1-D integer Tensor of [batch_size].
- num_classes: number of classes
- scope: unique prefix string identifying the ImageNet tower, e.g.
- 'tower_0'.
-
- Returns:
- Tensor of shape [] containing the total loss for a batch of data
- """
- # When fine-tuning a model, we do not restore the logits but instead we
- # randomly initialize the logits. The number of classes in the output of the
- # logit is the number of classes in specified Dataset.
- restore_logits = not FLAGS.fine_tune
-
- # Build inference Graph.
- with tf.variable_scope(tf.get_variable_scope(), reuse=reuse_variables):
- logits = inception.inference(images, num_classes, for_training=True,
- restore_logits=restore_logits,
- scope=scope)
-
- # Build the portion of the Graph calculating the losses. Note that we will
- # assemble the total_loss using a custom function below.
- split_batch_size = images.get_shape().as_list()[0]
- inception.loss(logits, labels, batch_size=split_batch_size)
-
- # Assemble all of the losses for the current tower only.
- losses = tf.get_collection(slim.losses.LOSSES_COLLECTION, scope)
-
- # Calculate the total loss for the current tower.
- regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
- total_loss = tf.add_n(losses + regularization_losses, name='total_loss')
-
- # Compute the moving average of all individual losses and the total loss.
- loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
- loss_averages_op = loss_averages.apply(losses + [total_loss])
-
- # Attach a scalar summmary to all individual losses and the total loss; do the
- # same for the averaged version of the losses.
- for l in losses + [total_loss]:
- # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
- # session. This helps the clarity of presentation on TensorBoard.
- loss_name = re.sub('%s_[0-9]*/' % inception.TOWER_NAME, '', l.op.name)
- # Name each loss as '(raw)' and name the moving average version of the loss
- # as the original loss name.
- tf.summary.scalar(loss_name +' (raw)', l)
- tf.summary.scalar(loss_name, loss_averages.average(l))
-
- with tf.control_dependencies([loss_averages_op]):
- total_loss = tf.identity(total_loss)
- return total_loss
-
-
-def _average_gradients(tower_grads):
- """Calculate the average gradient for each shared variable across all towers.
-
- Note that this function provides a synchronization point across all towers.
-
- Args:
- tower_grads: List of lists of (gradient, variable) tuples. The outer list
- is over individual gradients. The inner list is over the gradient
- calculation for each tower.
- Returns:
- List of pairs of (gradient, variable) where the gradient has been averaged
- across all towers.
- """
- average_grads = []
- for grad_and_vars in zip(*tower_grads):
- # Note that each grad_and_vars looks like the following:
- # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
- grads = []
- for g, _ in grad_and_vars:
- # Add 0 dimension to the gradients to represent the tower.
- expanded_g = tf.expand_dims(g, 0)
-
- # Append on a 'tower' dimension which we will average over below.
- grads.append(expanded_g)
-
- # Average over the 'tower' dimension.
- grad = tf.concat(axis=0, values=grads)
- grad = tf.reduce_mean(grad, 0)
-
- # Keep in mind that the Variables are redundant because they are shared
- # across towers. So .. we will just return the first tower's pointer to
- # the Variable.
- v = grad_and_vars[0][1]
- grad_and_var = (grad, v)
- average_grads.append(grad_and_var)
- return average_grads
-
-
-def train(dataset):
- """Train on dataset for a number of steps."""
- with tf.Graph().as_default(), tf.device('/cpu:0'):
- # Create a variable to count the number of train() calls. This equals the
- # number of batches processed * FLAGS.num_gpus.
- global_step = tf.get_variable(
- 'global_step', [],
- initializer=tf.constant_initializer(0), trainable=False)
-
- # Calculate the learning rate schedule.
- num_batches_per_epoch = (dataset.num_examples_per_epoch() /
- FLAGS.batch_size)
- decay_steps = int(num_batches_per_epoch * FLAGS.num_epochs_per_decay)
-
- # Decay the learning rate exponentially based on the number of steps.
- lr = tf.train.exponential_decay(FLAGS.initial_learning_rate,
- global_step,
- decay_steps,
- FLAGS.learning_rate_decay_factor,
- staircase=True)
-
- # Create an optimizer that performs gradient descent.
- opt = tf.train.RMSPropOptimizer(lr, RMSPROP_DECAY,
- momentum=RMSPROP_MOMENTUM,
- epsilon=RMSPROP_EPSILON)
-
- # Get images and labels for ImageNet and split the batch across GPUs.
- assert FLAGS.batch_size % FLAGS.num_gpus == 0, (
- 'Batch size must be divisible by number of GPUs')
- split_batch_size = int(FLAGS.batch_size / FLAGS.num_gpus)
-
- # Override the number of preprocessing threads to account for the increased
- # number of GPU towers.
- num_preprocess_threads = FLAGS.num_preprocess_threads * FLAGS.num_gpus
- images, labels = image_processing.distorted_inputs(
- dataset,
- num_preprocess_threads=num_preprocess_threads)
-
- input_summaries = copy.copy(tf.get_collection(tf.GraphKeys.SUMMARIES))
-
- # Number of classes in the Dataset label set plus 1.
- # Label 0 is reserved for an (unused) background class.
- num_classes = dataset.num_classes() + 1
-
- # Split the batch of images and labels for towers.
- images_splits = tf.split(axis=0, num_or_size_splits=FLAGS.num_gpus, value=images)
- labels_splits = tf.split(axis=0, num_or_size_splits=FLAGS.num_gpus, value=labels)
-
- # Calculate the gradients for each model tower.
- tower_grads = []
- reuse_variables = None
- for i in range(FLAGS.num_gpus):
- with tf.device('/gpu:%d' % i):
- with tf.name_scope('%s_%d' % (inception.TOWER_NAME, i)) as scope:
- # Force all Variables to reside on the CPU.
- with slim.arg_scope([slim.variables.variable], device='/cpu:0'):
- # Calculate the loss for one tower of the ImageNet model. This
- # function constructs the entire ImageNet model but shares the
- # variables across all towers.
- loss = _tower_loss(images_splits[i], labels_splits[i], num_classes,
- scope, reuse_variables)
-
- # Reuse variables for the next tower.
- reuse_variables = True
-
- # Retain the summaries from the final tower.
- summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)
-
- # Retain the Batch Normalization updates operations only from the
- # final tower. Ideally, we should grab the updates from all towers
- # but these stats accumulate extremely fast so we can ignore the
- # other stats from the other towers without significant detriment.
- batchnorm_updates = tf.get_collection(slim.ops.UPDATE_OPS_COLLECTION,
- scope)
-
- # Calculate the gradients for the batch of data on this ImageNet
- # tower.
- grads = opt.compute_gradients(loss)
-
- # Keep track of the gradients across all towers.
- tower_grads.append(grads)
-
- # We must calculate the mean of each gradient. Note that this is the
- # synchronization point across all towers.
- grads = _average_gradients(tower_grads)
-
- # Add a summaries for the input processing and global_step.
- summaries.extend(input_summaries)
-
- # Add a summary to track the learning rate.
- summaries.append(tf.summary.scalar('learning_rate', lr))
-
- # Add histograms for gradients.
- for grad, var in grads:
- if grad is not None:
- summaries.append(
- tf.summary.histogram(var.op.name + '/gradients', grad))
-
- # Apply the gradients to adjust the shared variables.
- apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
-
- # Add histograms for trainable variables.
- for var in tf.trainable_variables():
- summaries.append(tf.summary.histogram(var.op.name, var))
-
- # Track the moving averages of all trainable variables.
- # Note that we maintain a "double-average" of the BatchNormalization
- # global statistics. This is more complicated then need be but we employ
- # this for backward-compatibility with our previous models.
- variable_averages = tf.train.ExponentialMovingAverage(
- inception.MOVING_AVERAGE_DECAY, global_step)
-
- # Another possibility is to use tf.slim.get_variables().
- variables_to_average = (tf.trainable_variables() +
- tf.moving_average_variables())
- variables_averages_op = variable_averages.apply(variables_to_average)
-
- # Group all updates to into a single train op.
- batchnorm_updates_op = tf.group(*batchnorm_updates)
- train_op = tf.group(apply_gradient_op, variables_averages_op,
- batchnorm_updates_op)
-
- # Create a saver.
- saver = tf.train.Saver(tf.global_variables())
-
- # Build the summary operation from the last tower summaries.
- summary_op = tf.summary.merge(summaries)
-
- # Build an initialization operation to run below.
- init = tf.global_variables_initializer()
-
- # Start running operations on the Graph. allow_soft_placement must be set to
- # True to build towers on GPU, as some of the ops do not have GPU
- # implementations.
- sess = tf.Session(config=tf.ConfigProto(
- allow_soft_placement=True,
- log_device_placement=FLAGS.log_device_placement))
- sess.run(init)
-
- if FLAGS.pretrained_model_checkpoint_path:
- assert tf.gfile.Exists(FLAGS.pretrained_model_checkpoint_path)
- variables_to_restore = tf.get_collection(
- slim.variables.VARIABLES_TO_RESTORE)
- restorer = tf.train.Saver(variables_to_restore)
- restorer.restore(sess, FLAGS.pretrained_model_checkpoint_path)
- print('%s: Pre-trained model restored from %s' %
- (datetime.now(), FLAGS.pretrained_model_checkpoint_path))
-
- # Start the queue runners.
- tf.train.start_queue_runners(sess=sess)
-
- summary_writer = tf.summary.FileWriter(
- FLAGS.train_dir,
- graph=sess.graph)
-
- for step in range(FLAGS.max_steps):
- start_time = time.time()
- _, loss_value = sess.run([train_op, loss])
- duration = time.time() - start_time
-
- assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
-
- if step % 10 == 0:
- examples_per_sec = FLAGS.batch_size / float(duration)
- format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
- 'sec/batch)')
- print(format_str % (datetime.now(), step, loss_value,
- examples_per_sec, duration))
-
- if step % 100 == 0:
- summary_str = sess.run(summary_op)
- summary_writer.add_summary(summary_str, step)
-
- # Save the model checkpoint periodically.
- if step % 5000 == 0 or (step + 1) == FLAGS.max_steps:
- checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
- saver.save(sess, checkpoint_path, global_step=step)
diff --git a/research/inception/inception/slim/BUILD b/research/inception/inception/slim/BUILD
deleted file mode 100644
index 174e77d5c2654380232174a2bb8b29c6b9affc5d..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/BUILD
+++ /dev/null
@@ -1,112 +0,0 @@
-# Description:
-# Contains the operations and nets for building TensorFlow-Slim models.
-
-package(default_visibility = ["//inception:internal"])
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-py_library(
- name = "scopes",
- srcs = ["scopes.py"],
-)
-
-py_test(
- name = "scopes_test",
- size = "small",
- srcs = ["scopes_test.py"],
- deps = [
- ":scopes",
- ],
-)
-
-py_library(
- name = "variables",
- srcs = ["variables.py"],
- deps = [
- ":scopes",
- ],
-)
-
-py_test(
- name = "variables_test",
- size = "small",
- srcs = ["variables_test.py"],
- deps = [
- ":variables",
- ],
-)
-
-py_library(
- name = "losses",
- srcs = ["losses.py"],
-)
-
-py_test(
- name = "losses_test",
- size = "small",
- srcs = ["losses_test.py"],
- deps = [
- ":losses",
- ],
-)
-
-py_library(
- name = "ops",
- srcs = ["ops.py"],
- deps = [
- ":losses",
- ":scopes",
- ":variables",
- ],
-)
-
-py_test(
- name = "ops_test",
- size = "small",
- srcs = ["ops_test.py"],
- deps = [
- ":ops",
- ":variables",
- ],
-)
-
-py_library(
- name = "inception",
- srcs = ["inception_model.py"],
- deps = [
- ":ops",
- ":scopes",
- ],
-)
-
-py_test(
- name = "inception_test",
- size = "medium",
- srcs = ["inception_test.py"],
- deps = [
- ":inception",
- ],
-)
-
-py_library(
- name = "slim",
- srcs = ["slim.py"],
- deps = [
- ":inception",
- ":losses",
- ":ops",
- ":scopes",
- ":variables",
- ],
-)
-
-py_test(
- name = "collections_test",
- size = "small",
- srcs = ["collections_test.py"],
- deps = [
- ":slim",
- ],
-)
diff --git a/research/inception/inception/slim/README.md b/research/inception/inception/slim/README.md
deleted file mode 100644
index 36d8b7eb19ae47d8810ed97abe203aa34be50a75..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/README.md
+++ /dev/null
@@ -1,621 +0,0 @@
-# TensorFlow-Slim
-
-TF-Slim is a lightweight library for defining, training and evaluating models in
-TensorFlow. It enables defining complex networks quickly and concisely while
-keeping a model's architecture transparent and its hyperparameters explicit.
-
-[TOC]
-
-## Teaser
-
-As a demonstration of the simplicity of using TF-Slim, compare the simplicity of
-the code necessary for defining the entire [VGG](http://www.robots.ox.ac.uk/~vgg/research/very_deep/) network using TF-Slim to
-the lengthy and verbose nature of defining just the first three layers (out of
-16) using native tensorflow:
-
-```python{.good}
-# VGG16 in TF-Slim.
-def vgg16(inputs):
- with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], stddev=0.01, weight_decay=0.0005):
- net = slim.ops.repeat_op(2, inputs, slim.ops.conv2d, 64, [3, 3], scope='conv1')
- net = slim.ops.max_pool(net, [2, 2], scope='pool1')
- net = slim.ops.repeat_op(2, net, slim.ops.conv2d, 128, [3, 3], scope='conv2')
- net = slim.ops.max_pool(net, [2, 2], scope='pool2')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 256, [3, 3], scope='conv3')
- net = slim.ops.max_pool(net, [2, 2], scope='pool3')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv4')
- net = slim.ops.max_pool(net, [2, 2], scope='pool4')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv5')
- net = slim.ops.max_pool(net, [2, 2], scope='pool5')
- net = slim.ops.flatten(net, scope='flatten5')
- net = slim.ops.fc(net, 4096, scope='fc6')
- net = slim.ops.dropout(net, 0.5, scope='dropout6')
- net = slim.ops.fc(net, 4096, scope='fc7')
- net = slim.ops.dropout(net, 0.5, scope='dropout7')
- net = slim.ops.fc(net, 1000, activation=None, scope='fc8')
- return net
-```
-
-```python{.bad}
-# Layers 1-3 (out of 16) of VGG16 in native tensorflow.
-def vgg16(inputs):
- with tf.name_scope('conv1_1') as scope:
- kernel = tf.Variable(tf.truncated_normal([3, 3, 3, 64], dtype=tf.float32, stddev=1e-1), name='weights')
- conv = tf.nn.conv2d(inputs, kernel, [1, 1, 1, 1], padding='SAME')
- biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=True, name='biases')
- bias = tf.nn.bias_add(conv, biases)
- conv1 = tf.nn.relu(bias, name=scope)
- with tf.name_scope('conv1_2') as scope:
- kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 64], dtype=tf.float32, stddev=1e-1), name='weights')
- conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
- biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=True, name='biases')
- bias = tf.nn.bias_add(conv, biases)
- conv1 = tf.nn.relu(bias, name=scope)
- with tf.name_scope('pool1')
- pool1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID', name='pool1')
-```
-
-## Why TF-Slim?
-
-TF-Slim offers several advantages over just the built-in tensorflow libraries:
-
-* Allows one to define models much more compactly by eliminating boilerplate
- code. This is accomplished through the use of [argument scoping](./scopes.py)
- and numerous high level [operations](./ops.py). These tools increase
- readability and maintainability, reduce the likelihood of an error from
- copy-and-pasting hyperparameter values and simplifies hyperparameter tuning.
-* Makes developing models simple by providing commonly used [loss functions](./losses.py)
-* Provides a concise [definition](./inception_model.py) of [Inception v3](http://arxiv.org/abs/1512.00567) network architecture ready to be used
- out-of-the-box or subsumed into new models.
-
-Additionally TF-Slim was designed with several principles in mind:
-
-* The various modules of TF-Slim (scopes, variables, ops, losses) are
- independent. This flexibility allows users to pick and choose components of
- TF-Slim completely à la carte.
-* TF-Slim is written using a Functional Programming style. That means it's
- super-lightweight and can be used right alongside any of TensorFlow's native
- operations.
-* Makes re-using network architectures easy. This allows users to build new
- networks on top of existing ones as well as fine-tuning pre-trained models
- on new tasks.
-
-## What are the various components of TF-Slim?
-
-TF-Slim is composed of several parts which were designed to exist independently.
-These include:
-
-* [scopes.py](./scopes.py): provides a new scope named `arg_scope` that allows
- a user to define default arguments for specific operations within that
- scope.
-* [variables.py](./variables.py): provides convenience wrappers for variable
- creation and manipulation.
-* [ops.py](./ops.py): provides high level operations for building models using
- tensorflow.
-* [losses.py](./losses.py): contains commonly used loss functions.
-
-## Defining Models
-
-Models can be succinctly defined using TF-Slim by combining its variables,
-operations and scopes. Each of these elements are defined below.
-
-### Variables
-
-Creating [`Variables`](https://www.tensorflow.org/how_tos/variables/index.html)
-in native tensorflow requires either a predefined value or an initialization
-mechanism (random, normally distributed). Furthermore, if a variable needs to be
-created on a specific device, such as a GPU, the specification must be [made
-explicit](https://www.tensorflow.org/how_tos/using_gpu/index.html). To alleviate
-the code required for variable creation, TF-Slim provides a set of thin wrapper
-functions in [variables.py](./variables.py) which allow callers to easily define
-variables.
-
-For example, to create a `weight` variable, initialize it using a truncated
-normal distribution, regularize it with an `l2_loss` and place it on the `CPU`,
-one need only declare the following:
-
-```python
-weights = variables.variable('weights',
- shape=[10, 10, 3 , 3],
- initializer=tf.truncated_normal_initializer(stddev=0.1),
- regularizer=lambda t: losses.l2_loss(t, weight=0.05),
- device='/cpu:0')
-```
-
-In addition to the functionality provided by `tf.Variable`, `slim.variables`
-keeps track of the variables created by `slim.ops` to define a model, which
-allows one to distinguish variables that belong to the model versus other
-variables.
-
-```python
-# Get all the variables defined by the model.
-model_variables = slim.variables.get_variables()
-
-# Get all the variables with the same given name, i.e. 'weights', 'biases'.
-weights = slim.variables.get_variables_by_name('weights')
-biases = slim.variables.get_variables_by_name('biases')
-
-# Get all the variables in VARIABLES_TO_RESTORE collection.
-variables_to_restore = tf.get_collection(slim.variables.VARIABLES_TO_RESTORE)
-
-
-weights = variables.variable('weights',
- shape=[10, 10, 3 , 3],
- initializer=tf.truncated_normal_initializer(stddev=0.1),
- regularizer=lambda t: losses.l2_loss(t, weight=0.05),
- device='/cpu:0')
-```
-
-### Operations (Layers)
-
-While the set of TensorFlow operations is quite extensive, builders of neural
-networks typically think of models in terms of "layers". A layer, such as a
-Convolutional Layer, a Fully Connected Layer or a BatchNorm Layer are more
-abstract than a single TensorFlow operation and typically involve many such
-operations. For example, a Convolutional Layer in a neural network is built
-using several steps:
-
-1. Creating the weight variables
-2. Creating the bias variables
-3. Convolving the weights with the input from the previous layer
-4. Adding the biases to the result of the convolution.
-
-In python code this can be rather laborious:
-
-```python
-input = ...
-with tf.name_scope('conv1_1') as scope:
- kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32,
- stddev=1e-1), name='weights')
- conv = tf.nn.conv2d(input, kernel, [1, 1, 1, 1], padding='SAME')
- biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32),
- trainable=True, name='biases')
- bias = tf.nn.bias_add(conv, biases)
- conv1 = tf.nn.relu(bias, name=scope)
-```
-
-To alleviate the need to duplicate this code repeatedly, TF-Slim provides a
-number of convenient operations defined at the (more abstract) level of neural
-network layers. For example, compare the code above to an invocation of the
-TF-Slim code:
-
-```python
-input = ...
-net = slim.ops.conv2d(input, [3, 3], 128, scope='conv1_1')
-```
-
-TF-Slim provides numerous operations used in building neural networks which
-roughly correspond to such layers. These include:
-
-Layer | TF-Slim Op
---------------------- | ------------------------
-Convolutional Layer | [ops.conv2d](./ops.py)
-Fully Connected Layer | [ops.fc](./ops.py)
-BatchNorm layer | [ops.batch_norm](./ops.py)
-Max Pooling Layer | [ops.max_pool](./ops.py)
-Avg Pooling Layer | [ops.avg_pool](./ops.py)
-Dropout Layer | [ops.dropout](./ops.py)
-
-[ops.py](./ops.py) also includes operations that are not really "layers" per se,
-but are often used to manipulate hidden unit representations during inference:
-
-Operation | TF-Slim Op
---------- | ---------------------
-Flatten | [ops.flatten](./ops.py)
-
-TF-Slim also provides a meta-operation called `repeat_op` that allows one to
-repeatedly perform the same operation. Consider the following snippet from the
-[VGG](https://www.robots.ox.ac.uk/~vgg/research/very_deep/) network whose layers
-perform several convolutions in a row between pooling layers:
-
-```python
-net = ...
-net = slim.ops.conv2d(net, 256, [3, 3], scope='conv3_1')
-net = slim.ops.conv2d(net, 256, [3, 3], scope='conv3_2')
-net = slim.ops.conv2d(net, 256, [3, 3], scope='conv3_3')
-net = slim.ops.max_pool(net, [2, 2], scope='pool3')
-```
-
-This clear duplication of code can be removed via a standard loop:
-
-```python
-net = ...
-for i in range(3):
- net = slim.ops.conv2d(net, 256, [3, 3], scope='conv3_' % (i+1))
-net = slim.ops.max_pool(net, [2, 2], scope='pool3')
-```
-
-While this does reduce the amount of duplication, it can be made even cleaner by
-using the `RepeatOp`:
-
-```python
-net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 256, [3, 3], scope='conv3')
-net = slim.ops.max_pool(net, [2, 2], scope='pool2')
-```
-
-Notice that the RepeatOp not only applies the same argument in-line, it also is
-smart enough to unroll the scopes such that the scopes assigned to each
-subsequent call of `ops.conv2d` is appended with an underscore and iteration
-number. More concretely, the scopes in the example above would be 'conv3_1',
-'conv3_2' and 'conv3_3'.
-
-### Scopes
-
-In addition to the types of scope mechanisms in TensorFlow ([name_scope](https://www.tensorflow.org/api_docs/python/framework.html#name_scope),
-[variable_scope](https://www.tensorflow.org/api_docs/python/state_ops.html#variable_scope),
-TF-Slim adds a new scoping mechanism called "argument scope" or [arg_scope](./scopes.py). This new scope allows a user to specify one or more operations and
-a set of arguments which will be passed to each of the operations defined in the
-`arg_scope`. This functionality is best illustrated by example. Consider the
-following code snippet:
-
-```python
-net = slim.ops.conv2d(inputs, 64, [11, 11], 4, padding='SAME', stddev=0.01, weight_decay=0.0005, scope='conv1')
-net = slim.ops.conv2d(net, 128, [11, 11], padding='VALID', stddev=0.01, weight_decay=0.0005, scope='conv2')
-net = slim.ops.conv2d(net, 256, [11, 11], padding='SAME', stddev=0.01, weight_decay=0.0005, scope='conv3')
-```
-
-It should be clear that these three Convolution layers share many of the same
-hyperparameters. Two have the same padding, all three have the same weight_decay
-and standard deviation of its weights. Not only do the duplicated values make
-the code more difficult to read, it also adds the addition burder to the writer
-of needing to doublecheck that all of the values are identical in each step. One
-solution would be to specify default values using variables:
-
-```python
-padding='SAME'
-stddev=0.01
-weight_decay=0.0005
-net = slim.ops.conv2d(inputs, 64, [11, 11], 4, padding=padding, stddev=stddev, weight_decay=weight_decay, scope='conv1')
-net = slim.ops.conv2d(net, 128, [11, 11], padding='VALID', stddev=stddev, weight_decay=weight_decay, scope='conv2')
-net = slim.ops.conv2d(net, 256, [11, 11], padding=padding, stddev=stddev, weight_decay=weight_decay, scope='conv3')
-
-```
-
-This solution ensures that all three convolutions share the exact same variable
-values but doesn't reduce the code clutter. By using an `arg_scope`, we can both
-ensure that each layer uses the same values and simplify the code:
-
-```python
- with slim.arg_scope([slim.ops.conv2d], padding='SAME', stddev=0.01, weight_decay=0.0005):
- net = slim.ops.conv2d(inputs, 64, [11, 11], scope='conv1')
- net = slim.ops.conv2d(net, 128, [11, 11], padding='VALID', scope='conv2')
- net = slim.ops.conv2d(net, 256, [11, 11], scope='conv3')
-```
-
-As the example illustrates, the use of arg_scope makes the code cleaner, simpler
-and easier to maintain. Notice that while argument values are specifed in the
-arg_scope, they can be overwritten locally. In particular, while the padding
-argument has been set to 'SAME', the second convolution overrides it with the
-value of 'VALID'.
-
-One can also nest `arg_scope`s and use multiple operations in the same scope.
-For example:
-
-```python
-with arg_scope([slim.ops.conv2d, slim.ops.fc], stddev=0.01, weight_decay=0.0005):
- with arg_scope([slim.ops.conv2d], padding='SAME'), slim.arg_scope([slim.ops.fc], bias=1.0):
- net = slim.ops.conv2d(inputs, 64, [11, 11], 4, padding='VALID', scope='conv1')
- net = slim.ops.conv2d(net, 256, [5, 5], stddev=0.03, scope='conv2')
- net = slim.ops.flatten(net)
- net = slim.ops.fc(net, 1000, activation=None, scope='fc')
-```
-
-In this example, the first `arg_scope` applies the same `stddev` and
-`weight_decay` arguments to the `conv2d` and `fc` ops in its scope. In the
-second `arg_scope`, additional default arguments to `conv2d` only are specified.
-
-In addition to `arg_scope`, TF-Slim provides several decorators that wrap the
-use of tensorflow arg scopes. These include `@AddArgScope`, `@AddNameScope`,
-`@AddVariableScope`, `@AddOpScope` and `@AddVariableOpScope`. To illustrate
-their use, consider the following example.
-
-```python
-def MyNewOp(inputs):
- varA = ...
- varB = ...
- outputs = tf.multiply(varA, inputs) + varB
- return outputs
-
-```
-
-In this example, the user has created a new op which creates two variables. To
-ensure that these variables exist within a certain variable scope (to avoid
-collisions with variables with the same name), in standard TF, the op must be
-called within a variable scope:
-
-```python
-inputs = ...
-with tf.variable_scope('layer1'):
- outputs = MyNewOp(inputs)
-```
-
-As an alternative, one can use TF-Slim's decorators to decorate the function and
-simplify the call:
-
-```python
-@AddVariableScope
-def MyNewOp(inputs):
- ...
- return outputs
-
-
-inputs = ...
-outputs = MyNewOp('layer1')
-```
-
-The `@AddVariableScope` decorater simply applies the `tf.variable_scope` scoping
-to the called function taking "layer1" as its argument. This allows the code to
-be written more concisely.
-
-### Losses
-
-The loss function defines a quantity that we want to minimize. For
-classification problems, this is typically the cross entropy between the true
-(one-hot) distribution and the predicted probability distribution across
-classes. For regression problems, this is often the sum-of-squares differences
-between the predicted and true values.
-
-Certain models, such as multi-task learning models, require the use of multiple
-loss functions simultaneously. In other words, the loss function ultimatey being
-minimized is the sum of various other loss functions. For example, consider a
-model that predicts both the type of scene in an image as well as the depth from
-the camera of each pixel. This model's loss function would be the sum of the
-classification loss and depth prediction loss.
-
-TF-Slim provides an easy-to-use mechanism for defining and keeping track of loss
-functions via the [losses.py](./losses.py) module. Consider the simple case
-where we want to train the VGG network:
-
-```python
-# Load the images and labels.
-images, labels = ...
-
-# Create the model.
-predictions = ...
-
-# Define the loss functions and get the total loss.
-loss = losses.cross_entropy_loss(predictions, labels)
-```
-
-In this example, we start by creating the model (using TF-Slim's VGG
-implementation), and add the standard classification loss. Now, lets turn to the
-case where we have a multi-task model that produces multiple outputs:
-
-```python
-# Load the images and labels.
-images, scene_labels, depth_labels = ...
-
-# Create the model.
-scene_predictions, depth_predictions = CreateMultiTaskModel(images)
-
-# Define the loss functions and get the total loss.
-classification_loss = slim.losses.cross_entropy_loss(scene_predictions, scene_labels)
-sum_of_squares_loss = slim.losses.l2loss(depth_predictions - depth_labels)
-
-# The following two lines have the same effect:
-total_loss1 = classification_loss + sum_of_squares_loss
-total_loss2 = tf.get_collection(slim.losses.LOSSES_COLLECTION)
-```
-
-In this example, we have two losses which we add by calling
-`losses.cross_entropy_loss` and `losses.l2loss`. We can obtain the
-total loss by adding them together (`total_loss1`) or by calling
-`losses.GetTotalLoss()`. How did this work? When you create a loss function via
-TF-Slim, TF-Slim adds the loss to a special TensorFlow collection of loss
-functions. This enables you to either manage the total loss manually, or allow
-TF-Slim to manage them for you.
-
-What if you want to let TF-Slim manage the losses for you but have a custom loss
-function? [losses.py](./losses.py) also has a function that adds this loss to
-TF-Slims collection. For example:
-
-```python
-# Load the images and labels.
-images, scene_labels, depth_labels, pose_labels = ...
-
-# Create the model.
-scene_predictions, depth_predictions, pose_predictions = CreateMultiTaskModel(images)
-
-# Define the loss functions and get the total loss.
-classification_loss = slim.losses.cross_entropy_loss(scene_predictions, scene_labels)
-sum_of_squares_loss = slim.losses.l2loss(depth_predictions - depth_labels)
-pose_loss = MyCustomLossFunction(pose_predictions, pose_labels)
-tf.add_to_collection(slim.losses.LOSSES_COLLECTION, pose_loss) # Letting TF-Slim know about the additional loss.
-
-# The following two lines have the same effect:
-total_loss1 = classification_loss + sum_of_squares_loss + pose_loss
-total_loss2 = losses.GetTotalLoss()
-```
-
-In this example, we can again either produce the total loss function manually or
-let TF-Slim know about the additional loss and let TF-Slim handle the losses.
-
-## Putting the Pieces Together
-
-By combining TF-Slim Variables, Operations and scopes, we can write a normally
-very complex network with very few lines of code. For example, the entire [VGG](https://www.robots.ox.ac.uk/~vgg/research/very_deep/) architecture can be
-defined with just the following snippet:
-
-```python
-with arg_scope([slim.ops.conv2d, slim.ops.fc], stddev=0.01, weight_decay=0.0005):
- net = slim.ops.repeat_op(2, inputs, slim.ops.conv2d, 64, [3, 3], scope='conv1')
- net = slim.ops.max_pool(net, [2, 2], scope='pool1')
- net = slim.ops.repeat_op(2, net, slim.ops.conv2d, 128, [3, 3], scope='conv2')
- net = slim.ops.max_pool(net, [2, 2], scope='pool2')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 256, [3, 3], scope='conv3')
- net = slim.ops.max_pool(net, [2, 2], scope='pool3')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv4')
- net = slim.ops.max_pool(net, [2, 2], scope='pool4')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv5')
- net = slim.ops.max_pool(net, [2, 2], scope='pool5')
- net = slim.ops.flatten(net, scope='flatten5')
- net = slim.ops.fc(net, 4096, scope='fc6')
- net = slim.ops.dropout(net, 0.5, scope='dropout6')
- net = slim.ops.fc(net, 4096, scope='fc7')
- net = slim.ops.dropout(net, 0.5, scope='dropout7')
- net = slim.ops.fc(net, 1000, activation=None, scope='fc8')
-return net
-```
-
-## Re-using previously defined network architectures and pre-trained models.
-
-### Brief Recap on Restoring Variables from a Checkpoint
-
-After a model has been trained, it can be restored using `tf.train.Saver()`
-which restores `Variables` from a given checkpoint. For many cases,
-`tf.train.Saver()` provides a simple mechanism to restore all or just a few
-variables.
-
-```python
-# Create some variables.
-v1 = tf.Variable(..., name="v1")
-v2 = tf.Variable(..., name="v2")
-...
-# Add ops to restore all the variables.
-restorer = tf.train.Saver()
-
-# Add ops to restore some variables.
-restorer = tf.train.Saver([v1, v2])
-
-# Later, launch the model, use the saver to restore variables from disk, and
-# do some work with the model.
-with tf.Session() as sess:
- # Restore variables from disk.
- restorer.restore(sess, "/tmp/model.ckpt")
- print("Model restored.")
- # Do some work with the model
- ...
-```
-
-See [Restoring Variables](https://www.tensorflow.org/versions/r0.7/how_tos/variables/index.html#restoring-variables)
-and [Choosing which Variables to Save and Restore](https://www.tensorflow.org/versions/r0.7/how_tos/variables/index.html#choosing-which-variables-to-save-and-restore)
-sections of the [Variables](https://www.tensorflow.org/versions/r0.7/how_tos/variables/index.html) page for
-more details.
-
-### Using slim.variables to Track which Variables need to be Restored
-
-It is often desirable to fine-tune a pre-trained model on an entirely new
-dataset or even a new task. In these situations, one must specify which layers
-of the model should be reused (and consequently loaded from a checkpoint) and
-which layers are new. Indicating which variables or layers should be restored is
-a process that quickly becomes cumbersome when done manually.
-
-To help keep track of which variables to restore, `slim.variables` provides a
-`restore` argument when creating each Variable. By default, all variables are
-marked as `restore=True`, which results in all variables defined by the model
-being restored.
-
-```python
-# Create some variables.
-v1 = slim.variables.variable(name="v1", ..., restore=False)
-v2 = slim.variables.variable(name="v2", ...) # By default restore=True
-...
-# Get list of variables to restore (which contains only 'v2')
-variables_to_restore = tf.get_collection(slim.variables.VARIABLES_TO_RESTORE)
-restorer = tf.train.Saver(variables_to_restore)
-with tf.Session() as sess:
- # Restore variables from disk.
- restorer.restore(sess, "/tmp/model.ckpt")
- print("Model restored.")
- # Do some work with the model
- ...
-```
-
-Additionally, every layer in `slim.ops` that creates slim.variables (such as
-`slim.ops.conv2d`, `slim.ops.fc`, `slim.ops.batch_norm`) also has a `restore`
-argument which controls whether the variables created by that layer should be
-restored or not.
-
-```python
-# Create a small network.
-net = slim.ops.conv2d(images, 32, [7, 7], stride=2, scope='conv1')
-net = slim.ops.conv2d(net, 64, [3, 3], scope='conv2')
-net = slim.ops.conv2d(net, 128, [3, 3], scope='conv3')
-net = slim.ops.max_pool(net, [3, 3], stride=2, scope='pool3')
-net = slim.ops.flatten(net)
-net = slim.ops.fc(net, 10, scope='logits', restore=False)
-...
-
-# VARIABLES_TO_RESTORE would contain the 'weights' and 'bias' defined by 'conv1'
-# 'conv2' and 'conv3' but not the ones defined by 'logits'
-variables_to_restore = tf.get_collection(slim.variables.VARIABLES_TO_RESTORE)
-
-# Create a restorer that would restore only the needed variables.
-restorer = tf.train.Saver(variables_to_restore)
-
-# Create a saver that would save all the variables (including 'logits').
-saver = tf.train.Saver()
-with tf.Session() as sess:
- # Restore variables from disk.
- restorer.restore(sess, "/tmp/model.ckpt")
- print("Model restored.")
-
- # Do some work with the model
- ...
- saver.save(sess, "/tmp/new_model.ckpt")
-```
-
-Note: When restoring variables from a checkpoint, the `Saver` locates the
-variable names in a checkpoint file and maps them to variables in the current
-graph. Above, we created a saver by passing to it a list of variables. In this
-case, the names of the variables to locate in the checkpoint file were
-implicitly obtained from each provided variable's `var.op.name`.
-
-This works well when the variable names in the checkpoint file match those in
-the graph. However, sometimes, we want to restore a model from a checkpoint
-whose variables have different names those in the current graph. In this case,
-we must provide the `Saver` a dictionary that maps from each checkpoint variable
-name to each graph variable. Consider the following example where the checkpoint
-variables names are obtained via a simple function:
-
-```python
-# Assuming that 'conv1/weights' should be restored from 'vgg16/conv1/weights'
-def name_in_checkpoint(var):
- return 'vgg16/' + var.op.name
-
-# Assuming that 'conv1/weights' and 'conv1/bias' should be restored from 'conv1/params1' and 'conv1/params2'
-def name_in_checkpoint(var):
- if "weights" in var.op.name:
- return var.op.name.replace("weights", "params1")
- if "bias" in var.op.name:
- return var.op.name.replace("bias", "params2")
-
-variables_to_restore = tf.get_collection(slim.variables.VARIABLES_TO_RESTORE)
-variables_to_restore = {name_in_checkpoint(var):var for var in variables_to_restore}
-restorer = tf.train.Saver(variables_to_restore)
-with tf.Session() as sess:
- # Restore variables from disk.
- restorer.restore(sess, "/tmp/model.ckpt")
-```
-
-### Reusing the VGG16 network defined in TF-Slim on a different task, i.e. PASCAL-VOC.
-
-Assuming one have already a pre-trained VGG16 model, one just need to replace
-the last layer `fc8` with a new layer `fc8_pascal` and use `restore=False`.
-
-```python
-def vgg16_pascal(inputs):
- with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], stddev=0.01, weight_decay=0.0005):
- net = slim.ops.repeat_op(2, inputs, slim.ops.conv2d, 64, [3, 3], scope='conv1')
- net = slim.ops.max_pool(net, [2, 2], scope='pool1')
- net = slim.ops.repeat_op(2, net, slim.ops.conv2d, 128, [3, 3], scope='conv2')
- net = slim.ops.max_pool(net, [2, 2], scope='pool2')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 256, [3, 3], scope='conv3')
- net = slim.ops.max_pool(net, [2, 2], scope='pool3')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv4')
- net = slim.ops.max_pool(net, [2, 2], scope='pool4')
- net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv5')
- net = slim.ops.max_pool(net, [2, 2], scope='pool5')
- net = slim.ops.flatten(net, scope='flatten5')
- net = slim.ops.fc(net, 4096, scope='fc6')
- net = slim.ops.dropout(net, 0.5, scope='dropout6')
- net = slim.ops.fc(net, 4096, scope='fc7')
- net = slim.ops.dropout(net, 0.5, scope='dropout7')
- # To reuse vgg16 on PASCAL-VOC, just change the last layer.
- net = slim.ops.fc(net, 21, activation=None, scope='fc8_pascal', restore=False)
- return net
-```
-
-## Authors
-
-Sergio Guadarrama and Nathan Silberman
diff --git a/research/inception/inception/slim/collections_test.py b/research/inception/inception/slim/collections_test.py
deleted file mode 100644
index 2a1f170edaaedae337df8e0b552a03dd82b263d4..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/collections_test.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for inception."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from inception.slim import slim
-
-
-def get_variables(scope=None):
- return slim.variables.get_variables(scope)
-
-
-def get_variables_by_name(name):
- return slim.variables.get_variables_by_name(name)
-
-
-class CollectionsTest(tf.test.TestCase):
-
- def testVariables(self):
- batch_size = 5
- height, width = 299, 299
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- with slim.arg_scope([slim.ops.conv2d],
- batch_norm_params={'decay': 0.9997}):
- slim.inception.inception_v3(inputs)
- self.assertEqual(len(get_variables()), 388)
- self.assertEqual(len(get_variables_by_name('weights')), 98)
- self.assertEqual(len(get_variables_by_name('biases')), 2)
- self.assertEqual(len(get_variables_by_name('beta')), 96)
- self.assertEqual(len(get_variables_by_name('gamma')), 0)
- self.assertEqual(len(get_variables_by_name('moving_mean')), 96)
- self.assertEqual(len(get_variables_by_name('moving_variance')), 96)
-
- def testVariablesWithoutBatchNorm(self):
- batch_size = 5
- height, width = 299, 299
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- with slim.arg_scope([slim.ops.conv2d],
- batch_norm_params=None):
- slim.inception.inception_v3(inputs)
- self.assertEqual(len(get_variables()), 196)
- self.assertEqual(len(get_variables_by_name('weights')), 98)
- self.assertEqual(len(get_variables_by_name('biases')), 98)
- self.assertEqual(len(get_variables_by_name('beta')), 0)
- self.assertEqual(len(get_variables_by_name('gamma')), 0)
- self.assertEqual(len(get_variables_by_name('moving_mean')), 0)
- self.assertEqual(len(get_variables_by_name('moving_variance')), 0)
-
- def testVariablesByLayer(self):
- batch_size = 5
- height, width = 299, 299
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- with slim.arg_scope([slim.ops.conv2d],
- batch_norm_params={'decay': 0.9997}):
- slim.inception.inception_v3(inputs)
- self.assertEqual(len(get_variables()), 388)
- self.assertEqual(len(get_variables('conv0')), 4)
- self.assertEqual(len(get_variables('conv1')), 4)
- self.assertEqual(len(get_variables('conv2')), 4)
- self.assertEqual(len(get_variables('conv3')), 4)
- self.assertEqual(len(get_variables('conv4')), 4)
- self.assertEqual(len(get_variables('mixed_35x35x256a')), 28)
- self.assertEqual(len(get_variables('mixed_35x35x288a')), 28)
- self.assertEqual(len(get_variables('mixed_35x35x288b')), 28)
- self.assertEqual(len(get_variables('mixed_17x17x768a')), 16)
- self.assertEqual(len(get_variables('mixed_17x17x768b')), 40)
- self.assertEqual(len(get_variables('mixed_17x17x768c')), 40)
- self.assertEqual(len(get_variables('mixed_17x17x768d')), 40)
- self.assertEqual(len(get_variables('mixed_17x17x768e')), 40)
- self.assertEqual(len(get_variables('mixed_8x8x2048a')), 36)
- self.assertEqual(len(get_variables('mixed_8x8x2048b')), 36)
- self.assertEqual(len(get_variables('logits')), 2)
- self.assertEqual(len(get_variables('aux_logits')), 10)
-
- def testVariablesToRestore(self):
- batch_size = 5
- height, width = 299, 299
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- with slim.arg_scope([slim.ops.conv2d],
- batch_norm_params={'decay': 0.9997}):
- slim.inception.inception_v3(inputs)
- variables_to_restore = tf.get_collection(
- slim.variables.VARIABLES_TO_RESTORE)
- self.assertEqual(len(variables_to_restore), 388)
- self.assertListEqual(variables_to_restore, get_variables())
-
- def testVariablesToRestoreWithoutLogits(self):
- batch_size = 5
- height, width = 299, 299
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- with slim.arg_scope([slim.ops.conv2d],
- batch_norm_params={'decay': 0.9997}):
- slim.inception.inception_v3(inputs, restore_logits=False)
- variables_to_restore = tf.get_collection(
- slim.variables.VARIABLES_TO_RESTORE)
- self.assertEqual(len(variables_to_restore), 384)
-
- def testRegularizationLosses(self):
- batch_size = 5
- height, width = 299, 299
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0.00004):
- slim.inception.inception_v3(inputs)
- losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
- self.assertEqual(len(losses), len(get_variables_by_name('weights')))
-
- def testTotalLossWithoutRegularization(self):
- batch_size = 5
- height, width = 299, 299
- num_classes = 1001
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- dense_labels = tf.random_uniform((batch_size, num_classes))
- with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0):
- logits, end_points = slim.inception.inception_v3(
- inputs,
- num_classes=num_classes)
- # Cross entropy loss for the main softmax prediction.
- slim.losses.cross_entropy_loss(logits,
- dense_labels,
- label_smoothing=0.1,
- weight=1.0)
- # Cross entropy loss for the auxiliary softmax head.
- slim.losses.cross_entropy_loss(end_points['aux_logits'],
- dense_labels,
- label_smoothing=0.1,
- weight=0.4,
- scope='aux_loss')
- losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
- self.assertEqual(len(losses), 2)
-
- def testTotalLossWithRegularization(self):
- batch_size = 5
- height, width = 299, 299
- num_classes = 1000
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- dense_labels = tf.random_uniform((batch_size, num_classes))
- with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0.00004):
- logits, end_points = slim.inception.inception_v3(inputs, num_classes)
- # Cross entropy loss for the main softmax prediction.
- slim.losses.cross_entropy_loss(logits,
- dense_labels,
- label_smoothing=0.1,
- weight=1.0)
- # Cross entropy loss for the auxiliary softmax head.
- slim.losses.cross_entropy_loss(end_points['aux_logits'],
- dense_labels,
- label_smoothing=0.1,
- weight=0.4,
- scope='aux_loss')
- losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
- self.assertEqual(len(losses), 2)
- reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
- self.assertEqual(len(reg_losses), 98)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/inception/inception/slim/inception_model.py b/research/inception/inception/slim/inception_model.py
deleted file mode 100644
index 6136ab1ba68716f4f135110a4d5c518b732b23df..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/inception_model.py
+++ /dev/null
@@ -1,356 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Inception-v3 expressed in TensorFlow-Slim.
-
- Usage:
-
- # Parameters for BatchNorm.
- batch_norm_params = {
- # Decay for the batch_norm moving averages.
- 'decay': BATCHNORM_MOVING_AVERAGE_DECAY,
- # epsilon to prevent 0s in variance.
- 'epsilon': 0.001,
- }
- # Set weight_decay for weights in Conv and FC layers.
- with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0.00004):
- with slim.arg_scope([slim.ops.conv2d],
- stddev=0.1,
- activation=tf.nn.relu,
- batch_norm_params=batch_norm_params):
- # Force all Variables to reside on the CPU.
- with slim.arg_scope([slim.variables.variable], device='/cpu:0'):
- logits, endpoints = slim.inception.inception_v3(
- images,
- dropout_keep_prob=0.8,
- num_classes=num_classes,
- is_training=for_training,
- restore_logits=restore_logits,
- scope=scope)
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from inception.slim import ops
-from inception.slim import scopes
-
-
-def inception_v3(inputs,
- dropout_keep_prob=0.8,
- num_classes=1000,
- is_training=True,
- restore_logits=True,
- scope=''):
- """Latest Inception from http://arxiv.org/abs/1512.00567.
-
- "Rethinking the Inception Architecture for Computer Vision"
-
- Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens,
- Zbigniew Wojna
-
- Args:
- inputs: a tensor of size [batch_size, height, width, channels].
- dropout_keep_prob: dropout keep_prob.
- num_classes: number of predicted classes.
- is_training: whether is training or not.
- restore_logits: whether or not the logits layers should be restored.
- Useful for fine-tuning a model with different num_classes.
- scope: Optional scope for name_scope.
-
- Returns:
- a list containing 'logits', 'aux_logits' Tensors.
- """
- # end_points will collect relevant activations for external use, for example
- # summaries or losses.
- end_points = {}
- with tf.name_scope(scope, 'inception_v3', [inputs]):
- with scopes.arg_scope([ops.conv2d, ops.fc, ops.batch_norm, ops.dropout],
- is_training=is_training):
- with scopes.arg_scope([ops.conv2d, ops.max_pool, ops.avg_pool],
- stride=1, padding='VALID'):
- # 299 x 299 x 3
- end_points['conv0'] = ops.conv2d(inputs, 32, [3, 3], stride=2,
- scope='conv0')
- # 149 x 149 x 32
- end_points['conv1'] = ops.conv2d(end_points['conv0'], 32, [3, 3],
- scope='conv1')
- # 147 x 147 x 32
- end_points['conv2'] = ops.conv2d(end_points['conv1'], 64, [3, 3],
- padding='SAME', scope='conv2')
- # 147 x 147 x 64
- end_points['pool1'] = ops.max_pool(end_points['conv2'], [3, 3],
- stride=2, scope='pool1')
- # 73 x 73 x 64
- end_points['conv3'] = ops.conv2d(end_points['pool1'], 80, [1, 1],
- scope='conv3')
- # 73 x 73 x 80.
- end_points['conv4'] = ops.conv2d(end_points['conv3'], 192, [3, 3],
- scope='conv4')
- # 71 x 71 x 192.
- end_points['pool2'] = ops.max_pool(end_points['conv4'], [3, 3],
- stride=2, scope='pool2')
- # 35 x 35 x 192.
- net = end_points['pool2']
- # Inception blocks
- with scopes.arg_scope([ops.conv2d, ops.max_pool, ops.avg_pool],
- stride=1, padding='SAME'):
- # mixed: 35 x 35 x 256.
- with tf.variable_scope('mixed_35x35x256a'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 64, [1, 1])
- with tf.variable_scope('branch5x5'):
- branch5x5 = ops.conv2d(net, 48, [1, 1])
- branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
- with tf.variable_scope('branch3x3dbl'):
- branch3x3dbl = ops.conv2d(net, 64, [1, 1])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 32, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch5x5, branch3x3dbl, branch_pool])
- end_points['mixed_35x35x256a'] = net
- # mixed_1: 35 x 35 x 288.
- with tf.variable_scope('mixed_35x35x288a'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 64, [1, 1])
- with tf.variable_scope('branch5x5'):
- branch5x5 = ops.conv2d(net, 48, [1, 1])
- branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
- with tf.variable_scope('branch3x3dbl'):
- branch3x3dbl = ops.conv2d(net, 64, [1, 1])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 64, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch5x5, branch3x3dbl, branch_pool])
- end_points['mixed_35x35x288a'] = net
- # mixed_2: 35 x 35 x 288.
- with tf.variable_scope('mixed_35x35x288b'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 64, [1, 1])
- with tf.variable_scope('branch5x5'):
- branch5x5 = ops.conv2d(net, 48, [1, 1])
- branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
- with tf.variable_scope('branch3x3dbl'):
- branch3x3dbl = ops.conv2d(net, 64, [1, 1])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 64, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch5x5, branch3x3dbl, branch_pool])
- end_points['mixed_35x35x288b'] = net
- # mixed_3: 17 x 17 x 768.
- with tf.variable_scope('mixed_17x17x768a'):
- with tf.variable_scope('branch3x3'):
- branch3x3 = ops.conv2d(net, 384, [3, 3], stride=2, padding='VALID')
- with tf.variable_scope('branch3x3dbl'):
- branch3x3dbl = ops.conv2d(net, 64, [1, 1])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3],
- stride=2, padding='VALID')
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.max_pool(net, [3, 3], stride=2, padding='VALID')
- net = tf.concat(axis=3, values=[branch3x3, branch3x3dbl, branch_pool])
- end_points['mixed_17x17x768a'] = net
- # mixed4: 17 x 17 x 768.
- with tf.variable_scope('mixed_17x17x768b'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 192, [1, 1])
- with tf.variable_scope('branch7x7'):
- branch7x7 = ops.conv2d(net, 128, [1, 1])
- branch7x7 = ops.conv2d(branch7x7, 128, [1, 7])
- branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
- with tf.variable_scope('branch7x7dbl'):
- branch7x7dbl = ops.conv2d(net, 128, [1, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [7, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [1, 7])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [7, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch7x7, branch7x7dbl, branch_pool])
- end_points['mixed_17x17x768b'] = net
- # mixed_5: 17 x 17 x 768.
- with tf.variable_scope('mixed_17x17x768c'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 192, [1, 1])
- with tf.variable_scope('branch7x7'):
- branch7x7 = ops.conv2d(net, 160, [1, 1])
- branch7x7 = ops.conv2d(branch7x7, 160, [1, 7])
- branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
- with tf.variable_scope('branch7x7dbl'):
- branch7x7dbl = ops.conv2d(net, 160, [1, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [1, 7])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch7x7, branch7x7dbl, branch_pool])
- end_points['mixed_17x17x768c'] = net
- # mixed_6: 17 x 17 x 768.
- with tf.variable_scope('mixed_17x17x768d'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 192, [1, 1])
- with tf.variable_scope('branch7x7'):
- branch7x7 = ops.conv2d(net, 160, [1, 1])
- branch7x7 = ops.conv2d(branch7x7, 160, [1, 7])
- branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
- with tf.variable_scope('branch7x7dbl'):
- branch7x7dbl = ops.conv2d(net, 160, [1, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [1, 7])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch7x7, branch7x7dbl, branch_pool])
- end_points['mixed_17x17x768d'] = net
- # mixed_7: 17 x 17 x 768.
- with tf.variable_scope('mixed_17x17x768e'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 192, [1, 1])
- with tf.variable_scope('branch7x7'):
- branch7x7 = ops.conv2d(net, 192, [1, 1])
- branch7x7 = ops.conv2d(branch7x7, 192, [1, 7])
- branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
- with tf.variable_scope('branch7x7dbl'):
- branch7x7dbl = ops.conv2d(net, 192, [1, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [7, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [7, 1])
- branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch7x7, branch7x7dbl, branch_pool])
- end_points['mixed_17x17x768e'] = net
- # Auxiliary Head logits
- aux_logits = tf.identity(end_points['mixed_17x17x768e'])
- with tf.variable_scope('aux_logits'):
- aux_logits = ops.avg_pool(aux_logits, [5, 5], stride=3,
- padding='VALID')
- aux_logits = ops.conv2d(aux_logits, 128, [1, 1], scope='proj')
- # Shape of feature map before the final layer.
- shape = aux_logits.get_shape()
- aux_logits = ops.conv2d(aux_logits, 768, shape[1:3], stddev=0.01,
- padding='VALID')
- aux_logits = ops.flatten(aux_logits)
- aux_logits = ops.fc(aux_logits, num_classes, activation=None,
- stddev=0.001, restore=restore_logits)
- end_points['aux_logits'] = aux_logits
- # mixed_8: 8 x 8 x 1280.
- # Note that the scope below is not changed to not void previous
- # checkpoints.
- # (TODO) Fix the scope when appropriate.
- with tf.variable_scope('mixed_17x17x1280a'):
- with tf.variable_scope('branch3x3'):
- branch3x3 = ops.conv2d(net, 192, [1, 1])
- branch3x3 = ops.conv2d(branch3x3, 320, [3, 3], stride=2,
- padding='VALID')
- with tf.variable_scope('branch7x7x3'):
- branch7x7x3 = ops.conv2d(net, 192, [1, 1])
- branch7x7x3 = ops.conv2d(branch7x7x3, 192, [1, 7])
- branch7x7x3 = ops.conv2d(branch7x7x3, 192, [7, 1])
- branch7x7x3 = ops.conv2d(branch7x7x3, 192, [3, 3],
- stride=2, padding='VALID')
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.max_pool(net, [3, 3], stride=2, padding='VALID')
- net = tf.concat(axis=3, values=[branch3x3, branch7x7x3, branch_pool])
- end_points['mixed_17x17x1280a'] = net
- # mixed_9: 8 x 8 x 2048.
- with tf.variable_scope('mixed_8x8x2048a'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 320, [1, 1])
- with tf.variable_scope('branch3x3'):
- branch3x3 = ops.conv2d(net, 384, [1, 1])
- branch3x3 = tf.concat(axis=3, values=[ops.conv2d(branch3x3, 384, [1, 3]),
- ops.conv2d(branch3x3, 384, [3, 1])])
- with tf.variable_scope('branch3x3dbl'):
- branch3x3dbl = ops.conv2d(net, 448, [1, 1])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 384, [3, 3])
- branch3x3dbl = tf.concat(axis=3, values=[ops.conv2d(branch3x3dbl, 384, [1, 3]),
- ops.conv2d(branch3x3dbl, 384, [3, 1])])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch3x3, branch3x3dbl, branch_pool])
- end_points['mixed_8x8x2048a'] = net
- # mixed_10: 8 x 8 x 2048.
- with tf.variable_scope('mixed_8x8x2048b'):
- with tf.variable_scope('branch1x1'):
- branch1x1 = ops.conv2d(net, 320, [1, 1])
- with tf.variable_scope('branch3x3'):
- branch3x3 = ops.conv2d(net, 384, [1, 1])
- branch3x3 = tf.concat(axis=3, values=[ops.conv2d(branch3x3, 384, [1, 3]),
- ops.conv2d(branch3x3, 384, [3, 1])])
- with tf.variable_scope('branch3x3dbl'):
- branch3x3dbl = ops.conv2d(net, 448, [1, 1])
- branch3x3dbl = ops.conv2d(branch3x3dbl, 384, [3, 3])
- branch3x3dbl = tf.concat(axis=3, values=[ops.conv2d(branch3x3dbl, 384, [1, 3]),
- ops.conv2d(branch3x3dbl, 384, [3, 1])])
- with tf.variable_scope('branch_pool'):
- branch_pool = ops.avg_pool(net, [3, 3])
- branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
- net = tf.concat(axis=3, values=[branch1x1, branch3x3, branch3x3dbl, branch_pool])
- end_points['mixed_8x8x2048b'] = net
- # Final pooling and prediction
- with tf.variable_scope('logits'):
- shape = net.get_shape()
- net = ops.avg_pool(net, shape[1:3], padding='VALID', scope='pool')
- # 1 x 1 x 2048
- net = ops.dropout(net, dropout_keep_prob, scope='dropout')
- net = ops.flatten(net, scope='flatten')
- # 2048
- logits = ops.fc(net, num_classes, activation=None, scope='logits',
- restore=restore_logits)
- # 1000
- end_points['logits'] = logits
- end_points['predictions'] = tf.nn.softmax(logits, name='predictions')
- return logits, end_points
-
-
-def inception_v3_parameters(weight_decay=0.00004, stddev=0.1,
- batch_norm_decay=0.9997, batch_norm_epsilon=0.001):
- """Yields the scope with the default parameters for inception_v3.
-
- Args:
- weight_decay: the weight decay for weights variables.
- stddev: standard deviation of the truncated guassian weight distribution.
- batch_norm_decay: decay for the moving average of batch_norm momentums.
- batch_norm_epsilon: small float added to variance to avoid dividing by zero.
-
- Yields:
- a arg_scope with the parameters needed for inception_v3.
- """
- # Set weight_decay for weights in Conv and FC layers.
- with scopes.arg_scope([ops.conv2d, ops.fc],
- weight_decay=weight_decay):
- # Set stddev, activation and parameters for batch_norm.
- with scopes.arg_scope([ops.conv2d],
- stddev=stddev,
- activation=tf.nn.relu,
- batch_norm_params={
- 'decay': batch_norm_decay,
- 'epsilon': batch_norm_epsilon}) as arg_scope:
- yield arg_scope
diff --git a/research/inception/inception/slim/inception_test.py b/research/inception/inception/slim/inception_test.py
deleted file mode 100644
index 231dea298f4b761aa90224df1c263873bc890ac5..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/inception_test.py
+++ /dev/null
@@ -1,134 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for slim.inception."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from inception.slim import inception_model as inception
-
-
-class InceptionTest(tf.test.TestCase):
-
- def testBuildLogits(self):
- batch_size = 5
- height, width = 299, 299
- num_classes = 1000
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- logits, _ = inception.inception_v3(inputs, num_classes)
- self.assertTrue(logits.op.name.startswith('logits'))
- self.assertListEqual(logits.get_shape().as_list(),
- [batch_size, num_classes])
-
- def testBuildEndPoints(self):
- batch_size = 5
- height, width = 299, 299
- num_classes = 1000
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- _, end_points = inception.inception_v3(inputs, num_classes)
- self.assertTrue('logits' in end_points)
- logits = end_points['logits']
- self.assertListEqual(logits.get_shape().as_list(),
- [batch_size, num_classes])
- self.assertTrue('aux_logits' in end_points)
- aux_logits = end_points['aux_logits']
- self.assertListEqual(aux_logits.get_shape().as_list(),
- [batch_size, num_classes])
- pre_pool = end_points['mixed_8x8x2048b']
- self.assertListEqual(pre_pool.get_shape().as_list(),
- [batch_size, 8, 8, 2048])
-
- def testVariablesSetDevice(self):
- batch_size = 5
- height, width = 299, 299
- num_classes = 1000
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- # Force all Variables to reside on the device.
- with tf.variable_scope('on_cpu'), tf.device('/cpu:0'):
- inception.inception_v3(inputs, num_classes)
- with tf.variable_scope('on_gpu'), tf.device('/gpu:0'):
- inception.inception_v3(inputs, num_classes)
- for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='on_cpu'):
- self.assertDeviceEqual(v.device, '/cpu:0')
- for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='on_gpu'):
- self.assertDeviceEqual(v.device, '/gpu:0')
-
- def testHalfSizeImages(self):
- batch_size = 5
- height, width = 150, 150
- num_classes = 1000
- with self.test_session():
- inputs = tf.random_uniform((batch_size, height, width, 3))
- logits, end_points = inception.inception_v3(inputs, num_classes)
- self.assertTrue(logits.op.name.startswith('logits'))
- self.assertListEqual(logits.get_shape().as_list(),
- [batch_size, num_classes])
- pre_pool = end_points['mixed_8x8x2048b']
- self.assertListEqual(pre_pool.get_shape().as_list(),
- [batch_size, 3, 3, 2048])
-
- def testUnknowBatchSize(self):
- batch_size = 1
- height, width = 299, 299
- num_classes = 1000
- with self.test_session() as sess:
- inputs = tf.placeholder(tf.float32, (None, height, width, 3))
- logits, _ = inception.inception_v3(inputs, num_classes)
- self.assertTrue(logits.op.name.startswith('logits'))
- self.assertListEqual(logits.get_shape().as_list(),
- [None, num_classes])
- images = tf.random_uniform((batch_size, height, width, 3))
- sess.run(tf.global_variables_initializer())
- output = sess.run(logits, {inputs: images.eval()})
- self.assertEquals(output.shape, (batch_size, num_classes))
-
- def testEvaluation(self):
- batch_size = 2
- height, width = 299, 299
- num_classes = 1000
- with self.test_session() as sess:
- eval_inputs = tf.random_uniform((batch_size, height, width, 3))
- logits, _ = inception.inception_v3(eval_inputs, num_classes,
- is_training=False)
- predictions = tf.argmax(logits, 1)
- sess.run(tf.global_variables_initializer())
- output = sess.run(predictions)
- self.assertEquals(output.shape, (batch_size,))
-
- def testTrainEvalWithReuse(self):
- train_batch_size = 5
- eval_batch_size = 2
- height, width = 150, 150
- num_classes = 1000
- with self.test_session() as sess:
- train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
- inception.inception_v3(train_inputs, num_classes)
- tf.get_variable_scope().reuse_variables()
- eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
- logits, _ = inception.inception_v3(eval_inputs, num_classes,
- is_training=False)
- predictions = tf.argmax(logits, 1)
- sess.run(tf.global_variables_initializer())
- output = sess.run(predictions)
- self.assertEquals(output.shape, (eval_batch_size,))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/inception/inception/slim/losses.py b/research/inception/inception/slim/losses.py
deleted file mode 100644
index 78298d092fab3afc264e427fb060602c27ea97b0..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/losses.py
+++ /dev/null
@@ -1,174 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Contains convenience wrappers for various Neural Network TensorFlow losses.
-
- All the losses defined here add themselves to the LOSSES_COLLECTION
- collection.
-
- l1_loss: Define a L1 Loss, useful for regularization, i.e. lasso.
- l2_loss: Define a L2 Loss, useful for regularization, i.e. weight decay.
- cross_entropy_loss: Define a cross entropy loss using
- softmax_cross_entropy_with_logits. Useful for classification.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-# In order to gather all losses in a network, the user should use this
-# key for get_collection, i.e:
-# losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
-LOSSES_COLLECTION = '_losses'
-
-
-def l1_regularizer(weight=1.0, scope=None):
- """Define a L1 regularizer.
-
- Args:
- weight: scale the loss by this factor.
- scope: Optional scope for name_scope.
-
- Returns:
- a regularizer function.
- """
- def regularizer(tensor):
- with tf.name_scope(scope, 'L1Regularizer', [tensor]):
- l1_weight = tf.convert_to_tensor(weight,
- dtype=tensor.dtype.base_dtype,
- name='weight')
- return tf.multiply(l1_weight, tf.reduce_sum(tf.abs(tensor)), name='value')
- return regularizer
-
-
-def l2_regularizer(weight=1.0, scope=None):
- """Define a L2 regularizer.
-
- Args:
- weight: scale the loss by this factor.
- scope: Optional scope for name_scope.
-
- Returns:
- a regularizer function.
- """
- def regularizer(tensor):
- with tf.name_scope(scope, 'L2Regularizer', [tensor]):
- l2_weight = tf.convert_to_tensor(weight,
- dtype=tensor.dtype.base_dtype,
- name='weight')
- return tf.multiply(l2_weight, tf.nn.l2_loss(tensor), name='value')
- return regularizer
-
-
-def l1_l2_regularizer(weight_l1=1.0, weight_l2=1.0, scope=None):
- """Define a L1L2 regularizer.
-
- Args:
- weight_l1: scale the L1 loss by this factor.
- weight_l2: scale the L2 loss by this factor.
- scope: Optional scope for name_scope.
-
- Returns:
- a regularizer function.
- """
- def regularizer(tensor):
- with tf.name_scope(scope, 'L1L2Regularizer', [tensor]):
- weight_l1_t = tf.convert_to_tensor(weight_l1,
- dtype=tensor.dtype.base_dtype,
- name='weight_l1')
- weight_l2_t = tf.convert_to_tensor(weight_l2,
- dtype=tensor.dtype.base_dtype,
- name='weight_l2')
- reg_l1 = tf.multiply(weight_l1_t, tf.reduce_sum(tf.abs(tensor)),
- name='value_l1')
- reg_l2 = tf.multiply(weight_l2_t, tf.nn.l2_loss(tensor),
- name='value_l2')
- return tf.add(reg_l1, reg_l2, name='value')
- return regularizer
-
-
-def l1_loss(tensor, weight=1.0, scope=None):
- """Define a L1Loss, useful for regularize, i.e. lasso.
-
- Args:
- tensor: tensor to regularize.
- weight: scale the loss by this factor.
- scope: Optional scope for name_scope.
-
- Returns:
- the L1 loss op.
- """
- with tf.name_scope(scope, 'L1Loss', [tensor]):
- weight = tf.convert_to_tensor(weight,
- dtype=tensor.dtype.base_dtype,
- name='loss_weight')
- loss = tf.multiply(weight, tf.reduce_sum(tf.abs(tensor)), name='value')
- tf.add_to_collection(LOSSES_COLLECTION, loss)
- return loss
-
-
-def l2_loss(tensor, weight=1.0, scope=None):
- """Define a L2Loss, useful for regularize, i.e. weight decay.
-
- Args:
- tensor: tensor to regularize.
- weight: an optional weight to modulate the loss.
- scope: Optional scope for name_scope.
-
- Returns:
- the L2 loss op.
- """
- with tf.name_scope(scope, 'L2Loss', [tensor]):
- weight = tf.convert_to_tensor(weight,
- dtype=tensor.dtype.base_dtype,
- name='loss_weight')
- loss = tf.multiply(weight, tf.nn.l2_loss(tensor), name='value')
- tf.add_to_collection(LOSSES_COLLECTION, loss)
- return loss
-
-
-def cross_entropy_loss(logits, one_hot_labels, label_smoothing=0,
- weight=1.0, scope=None):
- """Define a Cross Entropy loss using softmax_cross_entropy_with_logits.
-
- It can scale the loss by weight factor, and smooth the labels.
-
- Args:
- logits: [batch_size, num_classes] logits outputs of the network .
- one_hot_labels: [batch_size, num_classes] target one_hot_encoded labels.
- label_smoothing: if greater than 0 then smooth the labels.
- weight: scale the loss by this factor.
- scope: Optional scope for name_scope.
-
- Returns:
- A tensor with the softmax_cross_entropy loss.
- """
- logits.get_shape().assert_is_compatible_with(one_hot_labels.get_shape())
- with tf.name_scope(scope, 'CrossEntropyLoss', [logits, one_hot_labels]):
- num_classes = one_hot_labels.get_shape()[-1].value
- one_hot_labels = tf.cast(one_hot_labels, logits.dtype)
- if label_smoothing > 0:
- smooth_positives = 1.0 - label_smoothing
- smooth_negatives = label_smoothing / num_classes
- one_hot_labels = one_hot_labels * smooth_positives + smooth_negatives
- cross_entropy = tf.contrib.nn.deprecated_flipped_softmax_cross_entropy_with_logits(
- logits, one_hot_labels, name='xentropy')
-
- weight = tf.convert_to_tensor(weight,
- dtype=logits.dtype.base_dtype,
- name='loss_weight')
- loss = tf.multiply(weight, tf.reduce_mean(cross_entropy), name='value')
- tf.add_to_collection(LOSSES_COLLECTION, loss)
- return loss
diff --git a/research/inception/inception/slim/losses_test.py b/research/inception/inception/slim/losses_test.py
deleted file mode 100644
index e267f6520779f63be0becf41ceccc7de494e14f7..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/losses_test.py
+++ /dev/null
@@ -1,177 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for slim.losses."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-from inception.slim import losses
-
-
-class LossesTest(tf.test.TestCase):
-
- def testL1Loss(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- weights = tf.constant(1.0, shape=shape)
- wd = 0.01
- loss = losses.l1_loss(weights, wd)
- self.assertEquals(loss.op.name, 'L1Loss/value')
- self.assertAlmostEqual(loss.eval(), num_elem * wd, 5)
-
- def testL2Loss(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- weights = tf.constant(1.0, shape=shape)
- wd = 0.01
- loss = losses.l2_loss(weights, wd)
- self.assertEquals(loss.op.name, 'L2Loss/value')
- self.assertAlmostEqual(loss.eval(), num_elem * wd / 2, 5)
-
-
-class RegularizersTest(tf.test.TestCase):
-
- def testL1Regularizer(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- loss = losses.l1_regularizer()(tensor)
- self.assertEquals(loss.op.name, 'L1Regularizer/value')
- self.assertAlmostEqual(loss.eval(), num_elem, 5)
-
- def testL1RegularizerWithScope(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- loss = losses.l1_regularizer(scope='L1')(tensor)
- self.assertEquals(loss.op.name, 'L1/value')
- self.assertAlmostEqual(loss.eval(), num_elem, 5)
-
- def testL1RegularizerWithWeight(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- weight = 0.01
- loss = losses.l1_regularizer(weight)(tensor)
- self.assertEquals(loss.op.name, 'L1Regularizer/value')
- self.assertAlmostEqual(loss.eval(), num_elem * weight, 5)
-
- def testL2Regularizer(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- loss = losses.l2_regularizer()(tensor)
- self.assertEquals(loss.op.name, 'L2Regularizer/value')
- self.assertAlmostEqual(loss.eval(), num_elem / 2, 5)
-
- def testL2RegularizerWithScope(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- loss = losses.l2_regularizer(scope='L2')(tensor)
- self.assertEquals(loss.op.name, 'L2/value')
- self.assertAlmostEqual(loss.eval(), num_elem / 2, 5)
-
- def testL2RegularizerWithWeight(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- weight = 0.01
- loss = losses.l2_regularizer(weight)(tensor)
- self.assertEquals(loss.op.name, 'L2Regularizer/value')
- self.assertAlmostEqual(loss.eval(), num_elem * weight / 2, 5)
-
- def testL1L2Regularizer(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- loss = losses.l1_l2_regularizer()(tensor)
- self.assertEquals(loss.op.name, 'L1L2Regularizer/value')
- self.assertAlmostEqual(loss.eval(), num_elem + num_elem / 2, 5)
-
- def testL1L2RegularizerWithScope(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- loss = losses.l1_l2_regularizer(scope='L1L2')(tensor)
- self.assertEquals(loss.op.name, 'L1L2/value')
- self.assertAlmostEqual(loss.eval(), num_elem + num_elem / 2, 5)
-
- def testL1L2RegularizerWithWeights(self):
- with self.test_session():
- shape = [5, 5, 5]
- num_elem = 5 * 5 * 5
- tensor = tf.constant(1.0, shape=shape)
- weight_l1 = 0.01
- weight_l2 = 0.05
- loss = losses.l1_l2_regularizer(weight_l1, weight_l2)(tensor)
- self.assertEquals(loss.op.name, 'L1L2Regularizer/value')
- self.assertAlmostEqual(loss.eval(),
- num_elem * weight_l1 + num_elem * weight_l2 / 2, 5)
-
-
-class CrossEntropyLossTest(tf.test.TestCase):
-
- def testCrossEntropyLossAllCorrect(self):
- with self.test_session():
- logits = tf.constant([[10.0, 0.0, 0.0],
- [0.0, 10.0, 0.0],
- [0.0, 0.0, 10.0]])
- labels = tf.constant([[1, 0, 0],
- [0, 1, 0],
- [0, 0, 1]])
- loss = losses.cross_entropy_loss(logits, labels)
- self.assertEquals(loss.op.name, 'CrossEntropyLoss/value')
- self.assertAlmostEqual(loss.eval(), 0.0, 3)
-
- def testCrossEntropyLossAllWrong(self):
- with self.test_session():
- logits = tf.constant([[10.0, 0.0, 0.0],
- [0.0, 10.0, 0.0],
- [0.0, 0.0, 10.0]])
- labels = tf.constant([[0, 0, 1],
- [1, 0, 0],
- [0, 1, 0]])
- loss = losses.cross_entropy_loss(logits, labels)
- self.assertEquals(loss.op.name, 'CrossEntropyLoss/value')
- self.assertAlmostEqual(loss.eval(), 10.0, 3)
-
- def testCrossEntropyLossAllWrongWithWeight(self):
- with self.test_session():
- logits = tf.constant([[10.0, 0.0, 0.0],
- [0.0, 10.0, 0.0],
- [0.0, 0.0, 10.0]])
- labels = tf.constant([[0, 0, 1],
- [1, 0, 0],
- [0, 1, 0]])
- loss = losses.cross_entropy_loss(logits, labels, weight=0.5)
- self.assertEquals(loss.op.name, 'CrossEntropyLoss/value')
- self.assertAlmostEqual(loss.eval(), 5.0, 3)
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/inception/inception/slim/ops.py b/research/inception/inception/slim/ops.py
deleted file mode 100644
index 54fda4eb81f3a138d9bb2748c21164b88570ede9..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/ops.py
+++ /dev/null
@@ -1,473 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Contains convenience wrappers for typical Neural Network TensorFlow layers.
-
- Additionally it maintains a collection with update_ops that need to be
- updated after the ops have been computed, for example to update moving means
- and moving variances of batch_norm.
-
- Ops that have different behavior during training or eval have an is_training
- parameter. Additionally Ops that contain variables.variable have a trainable
- parameter, which control if the ops variables are trainable or not.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-
-from tensorflow.python.training import moving_averages
-
-from inception.slim import losses
-from inception.slim import scopes
-from inception.slim import variables
-
-# Used to keep the update ops done by batch_norm.
-UPDATE_OPS_COLLECTION = '_update_ops_'
-
-
-@scopes.add_arg_scope
-def batch_norm(inputs,
- decay=0.999,
- center=True,
- scale=False,
- epsilon=0.001,
- moving_vars='moving_vars',
- activation=None,
- is_training=True,
- trainable=True,
- restore=True,
- scope=None,
- reuse=None):
- """Adds a Batch Normalization layer.
-
- Args:
- inputs: a tensor of size [batch_size, height, width, channels]
- or [batch_size, channels].
- decay: decay for the moving average.
- center: If True, subtract beta. If False, beta is not created and ignored.
- scale: If True, multiply by gamma. If False, gamma is
- not used. When the next layer is linear (also e.g. ReLU), this can be
- disabled since the scaling can be done by the next layer.
- epsilon: small float added to variance to avoid dividing by zero.
- moving_vars: collection to store the moving_mean and moving_variance.
- activation: activation function.
- is_training: whether or not the model is in training mode.
- trainable: whether or not the variables should be trainable or not.
- restore: whether or not the variables should be marked for restore.
- scope: Optional scope for variable_scope.
- reuse: whether or not the layer and its variables should be reused. To be
- able to reuse the layer scope must be given.
-
- Returns:
- a tensor representing the output of the operation.
-
- """
- inputs_shape = inputs.get_shape()
- with tf.variable_scope(scope, 'BatchNorm', [inputs], reuse=reuse):
- axis = list(range(len(inputs_shape) - 1))
- params_shape = inputs_shape[-1:]
- # Allocate parameters for the beta and gamma of the normalization.
- beta, gamma = None, None
- if center:
- beta = variables.variable('beta',
- params_shape,
- initializer=tf.zeros_initializer(),
- trainable=trainable,
- restore=restore)
- if scale:
- gamma = variables.variable('gamma',
- params_shape,
- initializer=tf.ones_initializer(),
- trainable=trainable,
- restore=restore)
- # Create moving_mean and moving_variance add them to
- # GraphKeys.MOVING_AVERAGE_VARIABLES collections.
- moving_collections = [moving_vars, tf.GraphKeys.MOVING_AVERAGE_VARIABLES]
- moving_mean = variables.variable('moving_mean',
- params_shape,
- initializer=tf.zeros_initializer(),
- trainable=False,
- restore=restore,
- collections=moving_collections)
- moving_variance = variables.variable('moving_variance',
- params_shape,
- initializer=tf.ones_initializer(),
- trainable=False,
- restore=restore,
- collections=moving_collections)
- if is_training:
- # Calculate the moments based on the individual batch.
- mean, variance = tf.nn.moments(inputs, axis)
-
- update_moving_mean = moving_averages.assign_moving_average(
- moving_mean, mean, decay)
- tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_mean)
- update_moving_variance = moving_averages.assign_moving_average(
- moving_variance, variance, decay)
- tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_variance)
- else:
- # Just use the moving_mean and moving_variance.
- mean = moving_mean
- variance = moving_variance
- # Normalize the activations.
- outputs = tf.nn.batch_normalization(
- inputs, mean, variance, beta, gamma, epsilon)
- outputs.set_shape(inputs.get_shape())
- if activation:
- outputs = activation(outputs)
- return outputs
-
-
-def _two_element_tuple(int_or_tuple):
- """Converts `int_or_tuple` to height, width.
-
- Several of the functions that follow accept arguments as either
- a tuple of 2 integers or a single integer. A single integer
- indicates that the 2 values of the tuple are the same.
-
- This functions normalizes the input value by always returning a tuple.
-
- Args:
- int_or_tuple: A list of 2 ints, a single int or a tf.TensorShape.
-
- Returns:
- A tuple with 2 values.
-
- Raises:
- ValueError: If `int_or_tuple` it not well formed.
- """
- if isinstance(int_or_tuple, (list, tuple)):
- if len(int_or_tuple) != 2:
- raise ValueError('Must be a list with 2 elements: %s' % int_or_tuple)
- return int(int_or_tuple[0]), int(int_or_tuple[1])
- if isinstance(int_or_tuple, int):
- return int(int_or_tuple), int(int_or_tuple)
- if isinstance(int_or_tuple, tf.TensorShape):
- if len(int_or_tuple) == 2:
- return int_or_tuple[0], int_or_tuple[1]
- raise ValueError('Must be an int, a list with 2 elements or a TensorShape of '
- 'length 2')
-
-
-@scopes.add_arg_scope
-def conv2d(inputs,
- num_filters_out,
- kernel_size,
- stride=1,
- padding='SAME',
- activation=tf.nn.relu,
- stddev=0.01,
- bias=0.0,
- weight_decay=0,
- batch_norm_params=None,
- is_training=True,
- trainable=True,
- restore=True,
- scope=None,
- reuse=None):
- """Adds a 2D convolution followed by an optional batch_norm layer.
-
- conv2d creates a variable called 'weights', representing the convolutional
- kernel, that is convolved with the input. If `batch_norm_params` is None, a
- second variable called 'biases' is added to the result of the convolution
- operation.
-
- Args:
- inputs: a tensor of size [batch_size, height, width, channels].
- num_filters_out: the number of output filters.
- kernel_size: a list of length 2: [kernel_height, kernel_width] of
- of the filters. Can be an int if both values are the same.
- stride: a list of length 2: [stride_height, stride_width].
- Can be an int if both strides are the same. Note that presently
- both strides must have the same value.
- padding: one of 'VALID' or 'SAME'.
- activation: activation function.
- stddev: standard deviation of the truncated guassian weight distribution.
- bias: the initial value of the biases.
- weight_decay: the weight decay.
- batch_norm_params: parameters for the batch_norm. If is None don't use it.
- is_training: whether or not the model is in training mode.
- trainable: whether or not the variables should be trainable or not.
- restore: whether or not the variables should be marked for restore.
- scope: Optional scope for variable_scope.
- reuse: whether or not the layer and its variables should be reused. To be
- able to reuse the layer scope must be given.
- Returns:
- a tensor representing the output of the operation.
-
- """
- with tf.variable_scope(scope, 'Conv', [inputs], reuse=reuse):
- kernel_h, kernel_w = _two_element_tuple(kernel_size)
- stride_h, stride_w = _two_element_tuple(stride)
- num_filters_in = inputs.get_shape()[-1]
- weights_shape = [kernel_h, kernel_w,
- num_filters_in, num_filters_out]
- weights_initializer = tf.truncated_normal_initializer(stddev=stddev)
- l2_regularizer = None
- if weight_decay and weight_decay > 0:
- l2_regularizer = losses.l2_regularizer(weight_decay)
- weights = variables.variable('weights',
- shape=weights_shape,
- initializer=weights_initializer,
- regularizer=l2_regularizer,
- trainable=trainable,
- restore=restore)
- conv = tf.nn.conv2d(inputs, weights, [1, stride_h, stride_w, 1],
- padding=padding)
- if batch_norm_params is not None:
- with scopes.arg_scope([batch_norm], is_training=is_training,
- trainable=trainable, restore=restore):
- outputs = batch_norm(conv, **batch_norm_params)
- else:
- bias_shape = [num_filters_out,]
- bias_initializer = tf.constant_initializer(bias)
- biases = variables.variable('biases',
- shape=bias_shape,
- initializer=bias_initializer,
- trainable=trainable,
- restore=restore)
- outputs = tf.nn.bias_add(conv, biases)
- if activation:
- outputs = activation(outputs)
- return outputs
-
-
-@scopes.add_arg_scope
-def fc(inputs,
- num_units_out,
- activation=tf.nn.relu,
- stddev=0.01,
- bias=0.0,
- weight_decay=0,
- batch_norm_params=None,
- is_training=True,
- trainable=True,
- restore=True,
- scope=None,
- reuse=None):
- """Adds a fully connected layer followed by an optional batch_norm layer.
-
- FC creates a variable called 'weights', representing the fully connected
- weight matrix, that is multiplied by the input. If `batch_norm` is None, a
- second variable called 'biases' is added to the result of the initial
- vector-matrix multiplication.
-
- Args:
- inputs: a [B x N] tensor where B is the batch size and N is the number of
- input units in the layer.
- num_units_out: the number of output units in the layer.
- activation: activation function.
- stddev: the standard deviation for the weights.
- bias: the initial value of the biases.
- weight_decay: the weight decay.
- batch_norm_params: parameters for the batch_norm. If is None don't use it.
- is_training: whether or not the model is in training mode.
- trainable: whether or not the variables should be trainable or not.
- restore: whether or not the variables should be marked for restore.
- scope: Optional scope for variable_scope.
- reuse: whether or not the layer and its variables should be reused. To be
- able to reuse the layer scope must be given.
-
- Returns:
- the tensor variable representing the result of the series of operations.
- """
- with tf.variable_scope(scope, 'FC', [inputs], reuse=reuse):
- num_units_in = inputs.get_shape()[1]
- weights_shape = [num_units_in, num_units_out]
- weights_initializer = tf.truncated_normal_initializer(stddev=stddev)
- l2_regularizer = None
- if weight_decay and weight_decay > 0:
- l2_regularizer = losses.l2_regularizer(weight_decay)
- weights = variables.variable('weights',
- shape=weights_shape,
- initializer=weights_initializer,
- regularizer=l2_regularizer,
- trainable=trainable,
- restore=restore)
- if batch_norm_params is not None:
- outputs = tf.matmul(inputs, weights)
- with scopes.arg_scope([batch_norm], is_training=is_training,
- trainable=trainable, restore=restore):
- outputs = batch_norm(outputs, **batch_norm_params)
- else:
- bias_shape = [num_units_out,]
- bias_initializer = tf.constant_initializer(bias)
- biases = variables.variable('biases',
- shape=bias_shape,
- initializer=bias_initializer,
- trainable=trainable,
- restore=restore)
- outputs = tf.nn.xw_plus_b(inputs, weights, biases)
- if activation:
- outputs = activation(outputs)
- return outputs
-
-
-def one_hot_encoding(labels, num_classes, scope=None):
- """Transform numeric labels into onehot_labels.
-
- Args:
- labels: [batch_size] target labels.
- num_classes: total number of classes.
- scope: Optional scope for name_scope.
- Returns:
- one hot encoding of the labels.
- """
- with tf.name_scope(scope, 'OneHotEncoding', [labels]):
- batch_size = labels.get_shape()[0]
- indices = tf.expand_dims(tf.range(0, batch_size), 1)
- labels = tf.cast(tf.expand_dims(labels, 1), indices.dtype)
- concated = tf.concat(axis=1, values=[indices, labels])
- onehot_labels = tf.sparse_to_dense(
- concated, tf.stack([batch_size, num_classes]), 1.0, 0.0)
- onehot_labels.set_shape([batch_size, num_classes])
- return onehot_labels
-
-
-@scopes.add_arg_scope
-def max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None):
- """Adds a Max Pooling layer.
-
- It is assumed by the wrapper that the pooling is only done per image and not
- in depth or batch.
-
- Args:
- inputs: a tensor of size [batch_size, height, width, depth].
- kernel_size: a list of length 2: [kernel_height, kernel_width] of the
- pooling kernel over which the op is computed. Can be an int if both
- values are the same.
- stride: a list of length 2: [stride_height, stride_width].
- Can be an int if both strides are the same. Note that presently
- both strides must have the same value.
- padding: the padding method, either 'VALID' or 'SAME'.
- scope: Optional scope for name_scope.
-
- Returns:
- a tensor representing the results of the pooling operation.
- Raises:
- ValueError: if 'kernel_size' is not a 2-D list
- """
- with tf.name_scope(scope, 'MaxPool', [inputs]):
- kernel_h, kernel_w = _two_element_tuple(kernel_size)
- stride_h, stride_w = _two_element_tuple(stride)
- return tf.nn.max_pool(inputs,
- ksize=[1, kernel_h, kernel_w, 1],
- strides=[1, stride_h, stride_w, 1],
- padding=padding)
-
-
-@scopes.add_arg_scope
-def avg_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None):
- """Adds a Avg Pooling layer.
-
- It is assumed by the wrapper that the pooling is only done per image and not
- in depth or batch.
-
- Args:
- inputs: a tensor of size [batch_size, height, width, depth].
- kernel_size: a list of length 2: [kernel_height, kernel_width] of the
- pooling kernel over which the op is computed. Can be an int if both
- values are the same.
- stride: a list of length 2: [stride_height, stride_width].
- Can be an int if both strides are the same. Note that presently
- both strides must have the same value.
- padding: the padding method, either 'VALID' or 'SAME'.
- scope: Optional scope for name_scope.
-
- Returns:
- a tensor representing the results of the pooling operation.
- """
- with tf.name_scope(scope, 'AvgPool', [inputs]):
- kernel_h, kernel_w = _two_element_tuple(kernel_size)
- stride_h, stride_w = _two_element_tuple(stride)
- return tf.nn.avg_pool(inputs,
- ksize=[1, kernel_h, kernel_w, 1],
- strides=[1, stride_h, stride_w, 1],
- padding=padding)
-
-
-@scopes.add_arg_scope
-def dropout(inputs, keep_prob=0.5, is_training=True, scope=None):
- """Returns a dropout layer applied to the input.
-
- Args:
- inputs: the tensor to pass to the Dropout layer.
- keep_prob: the probability of keeping each input unit.
- is_training: whether or not the model is in training mode. If so, dropout is
- applied and values scaled. Otherwise, inputs is returned.
- scope: Optional scope for name_scope.
-
- Returns:
- a tensor representing the output of the operation.
- """
- if is_training and keep_prob > 0:
- with tf.name_scope(scope, 'Dropout', [inputs]):
- return tf.nn.dropout(inputs, keep_prob)
- else:
- return inputs
-
-
-def flatten(inputs, scope=None):
- """Flattens the input while maintaining the batch_size.
-
- Assumes that the first dimension represents the batch.
-
- Args:
- inputs: a tensor of size [batch_size, ...].
- scope: Optional scope for name_scope.
-
- Returns:
- a flattened tensor with shape [batch_size, k].
- Raises:
- ValueError: if inputs.shape is wrong.
- """
- if len(inputs.get_shape()) < 2:
- raise ValueError('Inputs must be have a least 2 dimensions')
- dims = inputs.get_shape()[1:]
- k = dims.num_elements()
- with tf.name_scope(scope, 'Flatten', [inputs]):
- return tf.reshape(inputs, [-1, k])
-
-
-def repeat_op(repetitions, inputs, op, *args, **kwargs):
- """Build a sequential Tower starting from inputs by using an op repeatedly.
-
- It creates new scopes for each operation by increasing the counter.
- Example: given repeat_op(3, _, ops.conv2d, 64, [3, 3], scope='conv1')
- it will repeat the given op under the following variable_scopes:
- conv1/Conv
- conv1/Conv_1
- conv1/Conv_2
-
- Args:
- repetitions: number or repetitions.
- inputs: a tensor of size [batch_size, height, width, channels].
- op: an operation.
- *args: args for the op.
- **kwargs: kwargs for the op.
-
- Returns:
- a tensor result of applying the operation op, num times.
- Raises:
- ValueError: if the op is unknown or wrong.
- """
- scope = kwargs.pop('scope', None)
- with tf.variable_scope(scope, 'RepeatOp', [inputs]):
- tower = inputs
- for _ in range(repetitions):
- tower = op(tower, *args, **kwargs)
- return tower
diff --git a/research/inception/inception/slim/ops_test.py b/research/inception/inception/slim/ops_test.py
deleted file mode 100644
index 13dc5d9aacf6e283540a406d419a67d2d7215161..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/ops_test.py
+++ /dev/null
@@ -1,687 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for slim.ops."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import numpy as np
-import tensorflow as tf
-
-from inception.slim import ops
-from inception.slim import scopes
-from inception.slim import variables
-
-
-class ConvTest(tf.test.TestCase):
-
- def testCreateConv(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, [3, 3])
- self.assertEquals(output.op.name, 'Conv/Relu')
- self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32])
-
- def testCreateSquareConv(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, 3)
- self.assertEquals(output.op.name, 'Conv/Relu')
- self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32])
-
- def testCreateConvWithTensorShape(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, images.get_shape()[1:3])
- self.assertEquals(output.op.name, 'Conv/Relu')
- self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32])
-
- def testCreateFullyConv(self):
- height, width = 6, 6
- with self.test_session():
- images = tf.random_uniform((5, height, width, 32), seed=1)
- output = ops.conv2d(images, 64, images.get_shape()[1:3], padding='VALID')
- self.assertEquals(output.op.name, 'Conv/Relu')
- self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 64])
-
- def testCreateVerticalConv(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, [3, 1])
- self.assertEquals(output.op.name, 'Conv/Relu')
- self.assertListEqual(output.get_shape().as_list(),
- [5, height, width, 32])
-
- def testCreateHorizontalConv(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, [1, 3])
- self.assertEquals(output.op.name, 'Conv/Relu')
- self.assertListEqual(output.get_shape().as_list(),
- [5, height, width, 32])
-
- def testCreateConvWithStride(self):
- height, width = 6, 6
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, [3, 3], stride=2)
- self.assertEquals(output.op.name, 'Conv/Relu')
- self.assertListEqual(output.get_shape().as_list(),
- [5, height/2, width/2, 32])
-
- def testCreateConvCreatesWeightsAndBiasesVars(self):
- height, width = 3, 3
- images = tf.random_uniform((5, height, width, 3), seed=1)
- with self.test_session():
- self.assertFalse(variables.get_variables('conv1/weights'))
- self.assertFalse(variables.get_variables('conv1/biases'))
- ops.conv2d(images, 32, [3, 3], scope='conv1')
- self.assertTrue(variables.get_variables('conv1/weights'))
- self.assertTrue(variables.get_variables('conv1/biases'))
-
- def testCreateConvWithScope(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, [3, 3], scope='conv1')
- self.assertEquals(output.op.name, 'conv1/Relu')
-
- def testCreateConvWithoutActivation(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, [3, 3], activation=None)
- self.assertEquals(output.op.name, 'Conv/BiasAdd')
-
- def testCreateConvValid(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.conv2d(images, 32, [3, 3], padding='VALID')
- self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 32])
-
- def testCreateConvWithWD(self):
- height, width = 3, 3
- with self.test_session() as sess:
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.conv2d(images, 32, [3, 3], weight_decay=0.01)
- wd = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)[0]
- self.assertEquals(wd.op.name,
- 'Conv/weights/Regularizer/L2Regularizer/value')
- sess.run(tf.global_variables_initializer())
- self.assertTrue(sess.run(wd) <= 0.01)
-
- def testCreateConvWithoutWD(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.conv2d(images, 32, [3, 3], weight_decay=0)
- self.assertEquals(
- tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES), [])
-
- def testReuseVars(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.conv2d(images, 32, [3, 3], scope='conv1')
- self.assertEquals(len(variables.get_variables()), 2)
- ops.conv2d(images, 32, [3, 3], scope='conv1', reuse=True)
- self.assertEquals(len(variables.get_variables()), 2)
-
- def testNonReuseVars(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.conv2d(images, 32, [3, 3])
- self.assertEquals(len(variables.get_variables()), 2)
- ops.conv2d(images, 32, [3, 3])
- self.assertEquals(len(variables.get_variables()), 4)
-
- def testReuseConvWithWD(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.conv2d(images, 32, [3, 3], weight_decay=0.01, scope='conv1')
- self.assertEquals(len(variables.get_variables()), 2)
- self.assertEquals(
- len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
- ops.conv2d(images, 32, [3, 3], weight_decay=0.01, scope='conv1',
- reuse=True)
- self.assertEquals(len(variables.get_variables()), 2)
- self.assertEquals(
- len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
-
- def testConvWithBatchNorm(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 32), seed=1)
- with scopes.arg_scope([ops.conv2d], batch_norm_params={'decay': 0.9}):
- net = ops.conv2d(images, 32, [3, 3])
- net = ops.conv2d(net, 32, [3, 3])
- self.assertEquals(len(variables.get_variables()), 8)
- self.assertEquals(len(variables.get_variables('Conv/BatchNorm')), 3)
- self.assertEquals(len(variables.get_variables('Conv_1/BatchNorm')), 3)
-
- def testReuseConvWithBatchNorm(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 32), seed=1)
- with scopes.arg_scope([ops.conv2d], batch_norm_params={'decay': 0.9}):
- net = ops.conv2d(images, 32, [3, 3], scope='Conv')
- net = ops.conv2d(net, 32, [3, 3], scope='Conv', reuse=True)
- self.assertEquals(len(variables.get_variables()), 4)
- self.assertEquals(len(variables.get_variables('Conv/BatchNorm')), 3)
- self.assertEquals(len(variables.get_variables('Conv_1/BatchNorm')), 0)
-
-
-class FCTest(tf.test.TestCase):
-
- def testCreateFC(self):
- height, width = 3, 3
- with self.test_session():
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- output = ops.fc(inputs, 32)
- self.assertEquals(output.op.name, 'FC/Relu')
- self.assertListEqual(output.get_shape().as_list(), [5, 32])
-
- def testCreateFCWithScope(self):
- height, width = 3, 3
- with self.test_session():
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- output = ops.fc(inputs, 32, scope='fc1')
- self.assertEquals(output.op.name, 'fc1/Relu')
-
- def testCreateFcCreatesWeightsAndBiasesVars(self):
- height, width = 3, 3
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- with self.test_session():
- self.assertFalse(variables.get_variables('fc1/weights'))
- self.assertFalse(variables.get_variables('fc1/biases'))
- ops.fc(inputs, 32, scope='fc1')
- self.assertTrue(variables.get_variables('fc1/weights'))
- self.assertTrue(variables.get_variables('fc1/biases'))
-
- def testReuseVars(self):
- height, width = 3, 3
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- with self.test_session():
- ops.fc(inputs, 32, scope='fc1')
- self.assertEquals(len(variables.get_variables('fc1')), 2)
- ops.fc(inputs, 32, scope='fc1', reuse=True)
- self.assertEquals(len(variables.get_variables('fc1')), 2)
-
- def testNonReuseVars(self):
- height, width = 3, 3
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- with self.test_session():
- ops.fc(inputs, 32)
- self.assertEquals(len(variables.get_variables('FC')), 2)
- ops.fc(inputs, 32)
- self.assertEquals(len(variables.get_variables('FC')), 4)
-
- def testCreateFCWithoutActivation(self):
- height, width = 3, 3
- with self.test_session():
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- output = ops.fc(inputs, 32, activation=None)
- self.assertEquals(output.op.name, 'FC/xw_plus_b')
-
- def testCreateFCWithWD(self):
- height, width = 3, 3
- with self.test_session() as sess:
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- ops.fc(inputs, 32, weight_decay=0.01)
- wd = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)[0]
- self.assertEquals(wd.op.name,
- 'FC/weights/Regularizer/L2Regularizer/value')
- sess.run(tf.global_variables_initializer())
- self.assertTrue(sess.run(wd) <= 0.01)
-
- def testCreateFCWithoutWD(self):
- height, width = 3, 3
- with self.test_session():
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- ops.fc(inputs, 32, weight_decay=0)
- self.assertEquals(
- tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES), [])
-
- def testReuseFCWithWD(self):
- height, width = 3, 3
- with self.test_session():
- inputs = tf.random_uniform((5, height * width * 3), seed=1)
- ops.fc(inputs, 32, weight_decay=0.01, scope='fc')
- self.assertEquals(len(variables.get_variables()), 2)
- self.assertEquals(
- len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
- ops.fc(inputs, 32, weight_decay=0.01, scope='fc', reuse=True)
- self.assertEquals(len(variables.get_variables()), 2)
- self.assertEquals(
- len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
-
- def testFCWithBatchNorm(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height * width * 3), seed=1)
- with scopes.arg_scope([ops.fc], batch_norm_params={}):
- net = ops.fc(images, 27)
- net = ops.fc(net, 27)
- self.assertEquals(len(variables.get_variables()), 8)
- self.assertEquals(len(variables.get_variables('FC/BatchNorm')), 3)
- self.assertEquals(len(variables.get_variables('FC_1/BatchNorm')), 3)
-
- def testReuseFCWithBatchNorm(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height * width * 3), seed=1)
- with scopes.arg_scope([ops.fc], batch_norm_params={'decay': 0.9}):
- net = ops.fc(images, 27, scope='fc1')
- net = ops.fc(net, 27, scope='fc1', reuse=True)
- self.assertEquals(len(variables.get_variables()), 4)
- self.assertEquals(len(variables.get_variables('fc1/BatchNorm')), 3)
-
-
-class MaxPoolTest(tf.test.TestCase):
-
- def testCreateMaxPool(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.max_pool(images, [3, 3])
- self.assertEquals(output.op.name, 'MaxPool/MaxPool')
- self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
-
- def testCreateSquareMaxPool(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.max_pool(images, 3)
- self.assertEquals(output.op.name, 'MaxPool/MaxPool')
- self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
-
- def testCreateMaxPoolWithScope(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.max_pool(images, [3, 3], scope='pool1')
- self.assertEquals(output.op.name, 'pool1/MaxPool')
-
- def testCreateMaxPoolSAME(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.max_pool(images, [3, 3], padding='SAME')
- self.assertListEqual(output.get_shape().as_list(), [5, 2, 2, 3])
-
- def testCreateMaxPoolStrideSAME(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.max_pool(images, [3, 3], stride=1, padding='SAME')
- self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3])
-
- def testGlobalMaxPool(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.max_pool(images, images.get_shape()[1:3], stride=1)
- self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
-
-
-class AvgPoolTest(tf.test.TestCase):
-
- def testCreateAvgPool(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.avg_pool(images, [3, 3])
- self.assertEquals(output.op.name, 'AvgPool/AvgPool')
- self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
-
- def testCreateSquareAvgPool(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.avg_pool(images, 3)
- self.assertEquals(output.op.name, 'AvgPool/AvgPool')
- self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
-
- def testCreateAvgPoolWithScope(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.avg_pool(images, [3, 3], scope='pool1')
- self.assertEquals(output.op.name, 'pool1/AvgPool')
-
- def testCreateAvgPoolSAME(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.avg_pool(images, [3, 3], padding='SAME')
- self.assertListEqual(output.get_shape().as_list(), [5, 2, 2, 3])
-
- def testCreateAvgPoolStrideSAME(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.avg_pool(images, [3, 3], stride=1, padding='SAME')
- self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3])
-
- def testGlobalAvgPool(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.avg_pool(images, images.get_shape()[1:3], stride=1)
- self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
-
-
-class OneHotEncodingTest(tf.test.TestCase):
-
- def testOneHotEncodingCreate(self):
- with self.test_session():
- labels = tf.constant([0, 1, 2])
- output = ops.one_hot_encoding(labels, num_classes=3)
- self.assertEquals(output.op.name, 'OneHotEncoding/SparseToDense')
- self.assertListEqual(output.get_shape().as_list(), [3, 3])
-
- def testOneHotEncoding(self):
- with self.test_session():
- labels = tf.constant([0, 1, 2])
- one_hot_labels = tf.constant([[1, 0, 0],
- [0, 1, 0],
- [0, 0, 1]])
- output = ops.one_hot_encoding(labels, num_classes=3)
- self.assertAllClose(output.eval(), one_hot_labels.eval())
-
-
-class DropoutTest(tf.test.TestCase):
-
- def testCreateDropout(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.dropout(images)
- self.assertEquals(output.op.name, 'Dropout/dropout/mul')
- output.get_shape().assert_is_compatible_with(images.get_shape())
-
- def testCreateDropoutNoTraining(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1, name='images')
- output = ops.dropout(images, is_training=False)
- self.assertEquals(output, images)
-
-
-class FlattenTest(tf.test.TestCase):
-
- def testFlatten4D(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1, name='images')
- output = ops.flatten(images)
- self.assertEquals(output.get_shape().num_elements(),
- images.get_shape().num_elements())
- self.assertEqual(output.get_shape()[0], images.get_shape()[0])
-
- def testFlatten3D(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width), seed=1, name='images')
- output = ops.flatten(images)
- self.assertEquals(output.get_shape().num_elements(),
- images.get_shape().num_elements())
- self.assertEqual(output.get_shape()[0], images.get_shape()[0])
-
- def testFlattenBatchSize(self):
- height, width = 3, 3
- with self.test_session() as sess:
- images = tf.random_uniform((5, height, width, 3), seed=1, name='images')
- inputs = tf.placeholder(tf.int32, (None, height, width, 3))
- output = ops.flatten(inputs)
- self.assertEquals(output.get_shape().as_list(),
- [None, height * width * 3])
- output = sess.run(output, {inputs: images.eval()})
- self.assertEquals(output.size,
- images.get_shape().num_elements())
- self.assertEqual(output.shape[0], images.get_shape()[0])
-
-
-class BatchNormTest(tf.test.TestCase):
-
- def testCreateOp(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- output = ops.batch_norm(images)
- self.assertTrue(output.op.name.startswith('BatchNorm/batchnorm'))
- self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3])
-
- def testCreateVariables(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.batch_norm(images)
- beta = variables.get_variables_by_name('beta')[0]
- self.assertEquals(beta.op.name, 'BatchNorm/beta')
- gamma = variables.get_variables_by_name('gamma')
- self.assertEquals(gamma, [])
- moving_mean = tf.moving_average_variables()[0]
- moving_variance = tf.moving_average_variables()[1]
- self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
- self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
-
- def testCreateVariablesWithScale(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.batch_norm(images, scale=True)
- beta = variables.get_variables_by_name('beta')[0]
- gamma = variables.get_variables_by_name('gamma')[0]
- self.assertEquals(beta.op.name, 'BatchNorm/beta')
- self.assertEquals(gamma.op.name, 'BatchNorm/gamma')
- moving_mean = tf.moving_average_variables()[0]
- moving_variance = tf.moving_average_variables()[1]
- self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
- self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
-
- def testCreateVariablesWithoutCenterWithScale(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.batch_norm(images, center=False, scale=True)
- beta = variables.get_variables_by_name('beta')
- self.assertEquals(beta, [])
- gamma = variables.get_variables_by_name('gamma')[0]
- self.assertEquals(gamma.op.name, 'BatchNorm/gamma')
- moving_mean = tf.moving_average_variables()[0]
- moving_variance = tf.moving_average_variables()[1]
- self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
- self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
-
- def testCreateVariablesWithoutCenterWithoutScale(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.batch_norm(images, center=False, scale=False)
- beta = variables.get_variables_by_name('beta')
- self.assertEquals(beta, [])
- gamma = variables.get_variables_by_name('gamma')
- self.assertEquals(gamma, [])
- moving_mean = tf.moving_average_variables()[0]
- moving_variance = tf.moving_average_variables()[1]
- self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
- self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
-
- def testMovingAverageVariables(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.batch_norm(images, scale=True)
- moving_mean = tf.moving_average_variables()[0]
- moving_variance = tf.moving_average_variables()[1]
- self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
- self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
-
- def testUpdateOps(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.batch_norm(images)
- update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
- update_moving_mean = update_ops[0]
- update_moving_variance = update_ops[1]
- self.assertEquals(update_moving_mean.op.name,
- 'BatchNorm/AssignMovingAvg')
- self.assertEquals(update_moving_variance.op.name,
- 'BatchNorm/AssignMovingAvg_1')
-
- def testReuseVariables(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.batch_norm(images, scale=True, scope='bn')
- ops.batch_norm(images, scale=True, scope='bn', reuse=True)
- beta = variables.get_variables_by_name('beta')
- gamma = variables.get_variables_by_name('gamma')
- self.assertEquals(len(beta), 1)
- self.assertEquals(len(gamma), 1)
- moving_vars = tf.get_collection('moving_vars')
- self.assertEquals(len(moving_vars), 2)
-
- def testReuseUpdateOps(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- ops.batch_norm(images, scope='bn')
- self.assertEquals(len(tf.get_collection(ops.UPDATE_OPS_COLLECTION)), 2)
- ops.batch_norm(images, scope='bn', reuse=True)
- self.assertEquals(len(tf.get_collection(ops.UPDATE_OPS_COLLECTION)), 4)
-
- def testCreateMovingVars(self):
- height, width = 3, 3
- with self.test_session():
- images = tf.random_uniform((5, height, width, 3), seed=1)
- _ = ops.batch_norm(images, moving_vars='moving_vars')
- moving_mean = tf.get_collection('moving_vars',
- 'BatchNorm/moving_mean')
- self.assertEquals(len(moving_mean), 1)
- self.assertEquals(moving_mean[0].op.name, 'BatchNorm/moving_mean')
- moving_variance = tf.get_collection('moving_vars',
- 'BatchNorm/moving_variance')
- self.assertEquals(len(moving_variance), 1)
- self.assertEquals(moving_variance[0].op.name, 'BatchNorm/moving_variance')
-
- def testComputeMovingVars(self):
- height, width = 3, 3
- with self.test_session() as sess:
- image_shape = (10, height, width, 3)
- image_values = np.random.rand(*image_shape)
- expected_mean = np.mean(image_values, axis=(0, 1, 2))
- expected_var = np.var(image_values, axis=(0, 1, 2))
- images = tf.constant(image_values, shape=image_shape, dtype=tf.float32)
- output = ops.batch_norm(images, decay=0.1)
- update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
- with tf.control_dependencies(update_ops):
- output = tf.identity(output)
- # Initialize all variables
- sess.run(tf.global_variables_initializer())
- moving_mean = variables.get_variables('BatchNorm/moving_mean')[0]
- moving_variance = variables.get_variables('BatchNorm/moving_variance')[0]
- mean, variance = sess.run([moving_mean, moving_variance])
- # After initialization moving_mean == 0 and moving_variance == 1.
- self.assertAllClose(mean, [0] * 3)
- self.assertAllClose(variance, [1] * 3)
- for _ in range(10):
- sess.run([output])
- mean = moving_mean.eval()
- variance = moving_variance.eval()
- # After 10 updates with decay 0.1 moving_mean == expected_mean and
- # moving_variance == expected_var.
- self.assertAllClose(mean, expected_mean)
- self.assertAllClose(variance, expected_var)
-
- def testEvalMovingVars(self):
- height, width = 3, 3
- with self.test_session() as sess:
- image_shape = (10, height, width, 3)
- image_values = np.random.rand(*image_shape)
- expected_mean = np.mean(image_values, axis=(0, 1, 2))
- expected_var = np.var(image_values, axis=(0, 1, 2))
- images = tf.constant(image_values, shape=image_shape, dtype=tf.float32)
- output = ops.batch_norm(images, decay=0.1, is_training=False)
- update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
- with tf.control_dependencies(update_ops):
- output = tf.identity(output)
- # Initialize all variables
- sess.run(tf.global_variables_initializer())
- moving_mean = variables.get_variables('BatchNorm/moving_mean')[0]
- moving_variance = variables.get_variables('BatchNorm/moving_variance')[0]
- mean, variance = sess.run([moving_mean, moving_variance])
- # After initialization moving_mean == 0 and moving_variance == 1.
- self.assertAllClose(mean, [0] * 3)
- self.assertAllClose(variance, [1] * 3)
- # Simulate assigment from saver restore.
- init_assigns = [tf.assign(moving_mean, expected_mean),
- tf.assign(moving_variance, expected_var)]
- sess.run(init_assigns)
- for _ in range(10):
- sess.run([output], {images: np.random.rand(*image_shape)})
- mean = moving_mean.eval()
- variance = moving_variance.eval()
- # Although we feed different images, the moving_mean and moving_variance
- # shouldn't change.
- self.assertAllClose(mean, expected_mean)
- self.assertAllClose(variance, expected_var)
-
- def testReuseVars(self):
- height, width = 3, 3
- with self.test_session() as sess:
- image_shape = (10, height, width, 3)
- image_values = np.random.rand(*image_shape)
- expected_mean = np.mean(image_values, axis=(0, 1, 2))
- expected_var = np.var(image_values, axis=(0, 1, 2))
- images = tf.constant(image_values, shape=image_shape, dtype=tf.float32)
- output = ops.batch_norm(images, decay=0.1, is_training=False)
- update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
- with tf.control_dependencies(update_ops):
- output = tf.identity(output)
- # Initialize all variables
- sess.run(tf.global_variables_initializer())
- moving_mean = variables.get_variables('BatchNorm/moving_mean')[0]
- moving_variance = variables.get_variables('BatchNorm/moving_variance')[0]
- mean, variance = sess.run([moving_mean, moving_variance])
- # After initialization moving_mean == 0 and moving_variance == 1.
- self.assertAllClose(mean, [0] * 3)
- self.assertAllClose(variance, [1] * 3)
- # Simulate assigment from saver restore.
- init_assigns = [tf.assign(moving_mean, expected_mean),
- tf.assign(moving_variance, expected_var)]
- sess.run(init_assigns)
- for _ in range(10):
- sess.run([output], {images: np.random.rand(*image_shape)})
- mean = moving_mean.eval()
- variance = moving_variance.eval()
- # Although we feed different images, the moving_mean and moving_variance
- # shouldn't change.
- self.assertAllClose(mean, expected_mean)
- self.assertAllClose(variance, expected_var)
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/inception/inception/slim/scopes.py b/research/inception/inception/slim/scopes.py
deleted file mode 100644
index 2c2fb0a2efa7d30eaddb36fc30265f30cbaeb9ef..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/scopes.py
+++ /dev/null
@@ -1,170 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Contains the new arg_scope used for TF-Slim ops.
-
- Allows one to define models much more compactly by eliminating boilerplate
- code. This is accomplished through the use of argument scoping (arg_scope).
-
- Example of how to use scopes.arg_scope:
-
- with scopes.arg_scope(ops.conv2d, padding='SAME',
- stddev=0.01, weight_decay=0.0005):
- net = ops.conv2d(inputs, 64, [11, 11], 4, padding='VALID', scope='conv1')
- net = ops.conv2d(net, 256, [5, 5], scope='conv2')
-
- The first call to conv2d will overwrite padding:
- ops.conv2d(inputs, 64, [11, 11], 4, padding='VALID',
- stddev=0.01, weight_decay=0.0005, scope='conv1')
-
- The second call to Conv will use predefined args:
- ops.conv2d(inputs, 256, [5, 5], padding='SAME',
- stddev=0.01, weight_decay=0.0005, scope='conv2')
-
- Example of how to reuse an arg_scope:
- with scopes.arg_scope(ops.conv2d, padding='SAME',
- stddev=0.01, weight_decay=0.0005) as conv2d_arg_scope:
- net = ops.conv2d(net, 256, [5, 5], scope='conv1')
- ....
-
- with scopes.arg_scope(conv2d_arg_scope):
- net = ops.conv2d(net, 256, [5, 5], scope='conv2')
-
- Example of how to use scopes.add_arg_scope:
-
- @scopes.add_arg_scope
- def conv2d(*args, **kwargs)
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import contextlib
-import functools
-
-from tensorflow.python.framework import ops
-
-_ARGSTACK_KEY = ("__arg_stack",)
-
-_DECORATED_OPS = set()
-
-
-def _get_arg_stack():
- stack = ops.get_collection(_ARGSTACK_KEY)
- if stack:
- return stack[0]
- else:
- stack = [{}]
- ops.add_to_collection(_ARGSTACK_KEY, stack)
- return stack
-
-
-def _current_arg_scope():
- stack = _get_arg_stack()
- return stack[-1]
-
-
-def _add_op(op):
- key_op = (op.__module__, op.__name__)
- if key_op not in _DECORATED_OPS:
- _DECORATED_OPS.add(key_op)
-
-
-@contextlib.contextmanager
-def arg_scope(list_ops_or_scope, **kwargs):
- """Stores the default arguments for the given set of list_ops.
-
- For usage, please see examples at top of the file.
-
- Args:
- list_ops_or_scope: List or tuple of operations to set argument scope for or
- a dictionary containg the current scope. When list_ops_or_scope is a dict,
- kwargs must be empty. When list_ops_or_scope is a list or tuple, then
- every op in it need to be decorated with @add_arg_scope to work.
- **kwargs: keyword=value that will define the defaults for each op in
- list_ops. All the ops need to accept the given set of arguments.
-
- Yields:
- the current_scope, which is a dictionary of {op: {arg: value}}
- Raises:
- TypeError: if list_ops is not a list or a tuple.
- ValueError: if any op in list_ops has not be decorated with @add_arg_scope.
- """
- if isinstance(list_ops_or_scope, dict):
- # Assumes that list_ops_or_scope is a scope that is being reused.
- if kwargs:
- raise ValueError("When attempting to re-use a scope by suppling a"
- "dictionary, kwargs must be empty.")
- current_scope = list_ops_or_scope.copy()
- try:
- _get_arg_stack().append(current_scope)
- yield current_scope
- finally:
- _get_arg_stack().pop()
- else:
- # Assumes that list_ops_or_scope is a list/tuple of ops with kwargs.
- if not isinstance(list_ops_or_scope, (list, tuple)):
- raise TypeError("list_ops_or_scope must either be a list/tuple or reused"
- "scope (i.e. dict)")
- try:
- current_scope = _current_arg_scope().copy()
- for op in list_ops_or_scope:
- key_op = (op.__module__, op.__name__)
- if not has_arg_scope(op):
- raise ValueError("%s is not decorated with @add_arg_scope", key_op)
- if key_op in current_scope:
- current_kwargs = current_scope[key_op].copy()
- current_kwargs.update(kwargs)
- current_scope[key_op] = current_kwargs
- else:
- current_scope[key_op] = kwargs.copy()
- _get_arg_stack().append(current_scope)
- yield current_scope
- finally:
- _get_arg_stack().pop()
-
-
-def add_arg_scope(func):
- """Decorates a function with args so it can be used within an arg_scope.
-
- Args:
- func: function to decorate.
-
- Returns:
- A tuple with the decorated function func_with_args().
- """
- @functools.wraps(func)
- def func_with_args(*args, **kwargs):
- current_scope = _current_arg_scope()
- current_args = kwargs
- key_func = (func.__module__, func.__name__)
- if key_func in current_scope:
- current_args = current_scope[key_func].copy()
- current_args.update(kwargs)
- return func(*args, **current_args)
- _add_op(func)
- return func_with_args
-
-
-def has_arg_scope(func):
- """Checks whether a func has been decorated with @add_arg_scope or not.
-
- Args:
- func: function to check.
-
- Returns:
- a boolean.
- """
- key_op = (func.__module__, func.__name__)
- return key_op in _DECORATED_OPS
diff --git a/research/inception/inception/slim/scopes_test.py b/research/inception/inception/slim/scopes_test.py
deleted file mode 100644
index cd349399ed7300dde38ac9bcb9818abc9d0680b4..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/scopes_test.py
+++ /dev/null
@@ -1,162 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests slim.scopes."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import tensorflow as tf
-from inception.slim import scopes
-
-
-@scopes.add_arg_scope
-def func1(*args, **kwargs):
- return (args, kwargs)
-
-
-@scopes.add_arg_scope
-def func2(*args, **kwargs):
- return (args, kwargs)
-
-
-class ArgScopeTest(tf.test.TestCase):
-
- def testEmptyArgScope(self):
- with self.test_session():
- self.assertEqual(scopes._current_arg_scope(), {})
-
- def testCurrentArgScope(self):
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- key_op = (func1.__module__, func1.__name__)
- current_scope = {key_op: func1_kwargs.copy()}
- with self.test_session():
- with scopes.arg_scope([func1], a=1, b=None, c=[1]) as scope:
- self.assertDictEqual(scope, current_scope)
-
- def testCurrentArgScopeNested(self):
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- func2_kwargs = {'b': 2, 'd': [2]}
- key = lambda f: (f.__module__, f.__name__)
- current_scope = {key(func1): func1_kwargs.copy(),
- key(func2): func2_kwargs.copy()}
- with self.test_session():
- with scopes.arg_scope([func1], a=1, b=None, c=[1]):
- with scopes.arg_scope([func2], b=2, d=[2]) as scope:
- self.assertDictEqual(scope, current_scope)
-
- def testReuseArgScope(self):
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- key_op = (func1.__module__, func1.__name__)
- current_scope = {key_op: func1_kwargs.copy()}
- with self.test_session():
- with scopes.arg_scope([func1], a=1, b=None, c=[1]) as scope1:
- pass
- with scopes.arg_scope(scope1) as scope:
- self.assertDictEqual(scope, current_scope)
-
- def testReuseArgScopeNested(self):
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- func2_kwargs = {'b': 2, 'd': [2]}
- key = lambda f: (f.__module__, f.__name__)
- current_scope1 = {key(func1): func1_kwargs.copy()}
- current_scope2 = {key(func1): func1_kwargs.copy(),
- key(func2): func2_kwargs.copy()}
- with self.test_session():
- with scopes.arg_scope([func1], a=1, b=None, c=[1]) as scope1:
- with scopes.arg_scope([func2], b=2, d=[2]) as scope2:
- pass
- with scopes.arg_scope(scope1):
- self.assertDictEqual(scopes._current_arg_scope(), current_scope1)
- with scopes.arg_scope(scope2):
- self.assertDictEqual(scopes._current_arg_scope(), current_scope2)
-
- def testSimpleArgScope(self):
- func1_args = (0,)
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- with self.test_session():
- with scopes.arg_scope([func1], a=1, b=None, c=[1]):
- args, kwargs = func1(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
-
- def testSimpleArgScopeWithTuple(self):
- func1_args = (0,)
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- with self.test_session():
- with scopes.arg_scope((func1,), a=1, b=None, c=[1]):
- args, kwargs = func1(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
-
- def testOverwriteArgScope(self):
- func1_args = (0,)
- func1_kwargs = {'a': 1, 'b': 2, 'c': [1]}
- with scopes.arg_scope([func1], a=1, b=None, c=[1]):
- args, kwargs = func1(0, b=2)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
-
- def testNestedArgScope(self):
- func1_args = (0,)
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- with scopes.arg_scope([func1], a=1, b=None, c=[1]):
- args, kwargs = func1(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
- func1_kwargs['b'] = 2
- with scopes.arg_scope([func1], b=2):
- args, kwargs = func1(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
-
- def testSharedArgScope(self):
- func1_args = (0,)
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- with scopes.arg_scope([func1, func2], a=1, b=None, c=[1]):
- args, kwargs = func1(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
- args, kwargs = func2(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
-
- def testSharedArgScopeTuple(self):
- func1_args = (0,)
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- with scopes.arg_scope((func1, func2), a=1, b=None, c=[1]):
- args, kwargs = func1(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
- args, kwargs = func2(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
-
- def testPartiallySharedArgScope(self):
- func1_args = (0,)
- func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
- func2_args = (1,)
- func2_kwargs = {'a': 1, 'b': None, 'd': [2]}
- with scopes.arg_scope([func1, func2], a=1, b=None):
- with scopes.arg_scope([func1], c=[1]), scopes.arg_scope([func2], d=[2]):
- args, kwargs = func1(0)
- self.assertTupleEqual(args, func1_args)
- self.assertDictEqual(kwargs, func1_kwargs)
- args, kwargs = func2(1)
- self.assertTupleEqual(args, func2_args)
- self.assertDictEqual(kwargs, func2_kwargs)
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/inception/inception/slim/slim.py b/research/inception/inception/slim/slim.py
deleted file mode 100644
index b7a5c0f8c52b66db899835480c331ffafdc386e2..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/slim.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""TF-Slim grouped API. Please see README.md for details and usage."""
-# pylint: disable=unused-import
-
-# Collapse tf-slim into a single namespace.
-from inception.slim import inception_model as inception
-from inception.slim import losses
-from inception.slim import ops
-from inception.slim import scopes
-from inception.slim import variables
-from inception.slim.scopes import arg_scope
diff --git a/research/inception/inception/slim/variables.py b/research/inception/inception/slim/variables.py
deleted file mode 100644
index 1d967b79e9563724b1114995a732cfd4dd486afd..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/variables.py
+++ /dev/null
@@ -1,289 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Contains convenience wrappers for creating variables in TF-Slim.
-
-The variables module is typically used for defining model variables from the
-ops routines (see slim.ops). Such variables are used for training, evaluation
-and inference of models.
-
-All the variables created through this module would be added to the
-MODEL_VARIABLES collection, if you create a model variable outside slim, it can
-be added with slim.variables.add_variable(external_variable, reuse).
-
-Usage:
- weights_initializer = tf.truncated_normal_initializer(stddev=0.01)
- l2_regularizer = lambda t: losses.l2_loss(t, weight=0.0005)
- weights = variables.variable('weights',
- shape=[100, 100],
- initializer=weights_initializer,
- regularizer=l2_regularizer,
- device='/cpu:0')
-
- biases = variables.variable('biases',
- shape=[100],
- initializer=tf.zeros_initializer(),
- device='/cpu:0')
-
- # More complex example.
-
- net = slim.ops.conv2d(input, 32, [3, 3], scope='conv1')
- net = slim.ops.conv2d(net, 64, [3, 3], scope='conv2')
- with slim.arg_scope([variables.variable], restore=False):
- net = slim.ops.conv2d(net, 64, [3, 3], scope='conv3')
-
- # Get all model variables from all the layers.
- model_variables = slim.variables.get_variables()
-
- # Get all model variables from a specific the layer, i.e 'conv1'.
- conv1_variables = slim.variables.get_variables('conv1')
-
- # Get all weights from all the layers.
- weights = slim.variables.get_variables_by_name('weights')
-
- # Get all bias from all the layers.
- biases = slim.variables.get_variables_by_name('biases')
-
- # Get all variables to restore.
- # (i.e. only those created by 'conv1' and 'conv2')
- variables_to_restore = slim.variables.get_variables_to_restore()
-
-************************************************
-* Initializing model variables from a checkpoint
-************************************************
-
-# Create some variables.
-v1 = slim.variables.variable(name="v1", ..., restore=False)
-v2 = slim.variables.variable(name="v2", ...) # By default restore=True
-...
-# The list of variables to restore should only contain 'v2'.
-variables_to_restore = slim.variables.get_variables_to_restore()
-restorer = tf.train.Saver(variables_to_restore)
-with tf.Session() as sess:
- # Restore variables from disk.
- restorer.restore(sess, "/tmp/model.ckpt")
- print("Model restored.")
- # Do some work with the model
- ...
-
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from inception.slim import scopes
-
-# Collection containing all the variables created using slim.variables
-MODEL_VARIABLES = '_model_variables_'
-
-# Collection containing the slim.variables that are created with restore=True.
-VARIABLES_TO_RESTORE = '_variables_to_restore_'
-
-
-def add_variable(var, restore=True):
- """Adds a variable to the MODEL_VARIABLES collection.
-
- Optionally it will add the variable to the VARIABLES_TO_RESTORE collection.
- Args:
- var: a variable.
- restore: whether the variable should be added to the
- VARIABLES_TO_RESTORE collection.
-
- """
- collections = [MODEL_VARIABLES]
- if restore:
- collections.append(VARIABLES_TO_RESTORE)
- for collection in collections:
- if var not in tf.get_collection(collection):
- tf.add_to_collection(collection, var)
-
-
-def get_variables(scope=None, suffix=None):
- """Gets the list of variables, filtered by scope and/or suffix.
-
- Args:
- scope: an optional scope for filtering the variables to return.
- suffix: an optional suffix for filtering the variables to return.
-
- Returns:
- a copied list of variables with scope and suffix.
- """
- candidates = tf.get_collection(MODEL_VARIABLES, scope)[:]
- if suffix is not None:
- candidates = [var for var in candidates if var.op.name.endswith(suffix)]
- return candidates
-
-
-def get_variables_to_restore():
- """Gets the list of variables to restore.
-
- Returns:
- a copied list of variables.
- """
- return tf.get_collection(VARIABLES_TO_RESTORE)[:]
-
-
-def get_variables_by_name(given_name, scope=None):
- """Gets the list of variables that were given that name.
-
- Args:
- given_name: name given to the variable without scope.
- scope: an optional scope for filtering the variables to return.
-
- Returns:
- a copied list of variables with the given name and prefix.
- """
- return get_variables(scope=scope, suffix=given_name)
-
-
-def get_unique_variable(name):
- """Gets the variable uniquely identified by that name.
-
- Args:
- name: a name that uniquely identifies the variable.
-
- Returns:
- a tensorflow variable.
-
- Raises:
- ValueError: if no variable uniquely identified by the name exists.
- """
- candidates = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, name)
- if not candidates:
- raise ValueError('Couldnt find variable %s' % name)
-
- for candidate in candidates:
- if candidate.op.name == name:
- return candidate
- raise ValueError('Variable %s does not uniquely identify a variable', name)
-
-
-class VariableDeviceChooser(object):
- """Slim device chooser for variables.
-
- When using a parameter server it will assign them in a round-robin fashion.
- When not using a parameter server it allows GPU:0 placement otherwise CPU:0.
- """
-
- def __init__(self,
- num_parameter_servers=0,
- ps_device='/job:ps',
- placement='CPU:0'):
- """Initialize VariableDeviceChooser.
-
- Args:
- num_parameter_servers: number of parameter servers.
- ps_device: string representing the parameter server device.
- placement: string representing the placement of the variable either CPU:0
- or GPU:0. When using parameter servers forced to CPU:0.
- """
- self._num_ps = num_parameter_servers
- self._ps_device = ps_device
- self._placement = placement if num_parameter_servers == 0 else 'CPU:0'
- self._next_task_id = 0
-
- def __call__(self, op):
- device_string = ''
- if self._num_ps > 0:
- task_id = self._next_task_id
- self._next_task_id = (self._next_task_id + 1) % self._num_ps
- device_string = '%s/task:%d' % (self._ps_device, task_id)
- device_string += '/%s' % self._placement
- return device_string
-
-
-# TODO(sguada) Remove once get_variable is able to colocate op.devices.
-def variable_device(device, name):
- """Fix the variable device to colocate its ops."""
- if callable(device):
- var_name = tf.get_variable_scope().name + '/' + name
- var_def = tf.NodeDef(name=var_name, op='Variable')
- device = device(var_def)
- if device is None:
- device = ''
- return device
-
-
-@scopes.add_arg_scope
-def global_step(device=''):
- """Returns the global step variable.
-
- Args:
- device: Optional device to place the variable. It can be an string or a
- function that is called to get the device for the variable.
-
- Returns:
- the tensor representing the global step variable.
- """
- global_step_ref = tf.get_collection(tf.GraphKeys.GLOBAL_STEP)
- if global_step_ref:
- return global_step_ref[0]
- else:
- collections = [
- VARIABLES_TO_RESTORE,
- tf.GraphKeys.GLOBAL_VARIABLES,
- tf.GraphKeys.GLOBAL_STEP,
- ]
- # Get the device for the variable.
- with tf.device(variable_device(device, 'global_step')):
- return tf.get_variable('global_step', shape=[], dtype=tf.int64,
- initializer=tf.zeros_initializer(),
- trainable=False, collections=collections)
-
-
-@scopes.add_arg_scope
-def variable(name, shape=None, dtype=tf.float32, initializer=None,
- regularizer=None, trainable=True, collections=None, device='',
- restore=True):
- """Gets an existing variable with these parameters or creates a new one.
-
- It also add itself to a group with its name.
-
- Args:
- name: the name of the new or existing variable.
- shape: shape of the new or existing variable.
- dtype: type of the new or existing variable (defaults to `DT_FLOAT`).
- initializer: initializer for the variable if one is created.
- regularizer: a (Tensor -> Tensor or None) function; the result of
- applying it on a newly created variable will be added to the collection
- GraphKeys.REGULARIZATION_LOSSES and can be used for regularization.
- trainable: If `True` also add the variable to the graph collection
- `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
- collections: A list of collection names to which the Variable will be added.
- Note that the variable is always also added to the tf.GraphKeys.GLOBAL_VARIABLES
- and MODEL_VARIABLES collections.
- device: Optional device to place the variable. It can be an string or a
- function that is called to get the device for the variable.
- restore: whether the variable should be added to the
- VARIABLES_TO_RESTORE collection.
-
- Returns:
- The created or existing variable.
- """
- collections = list(collections or [])
-
- # Make sure variables are added to tf.GraphKeys.GLOBAL_VARIABLES and MODEL_VARIABLES
- collections += [tf.GraphKeys.GLOBAL_VARIABLES, MODEL_VARIABLES]
- # Add to VARIABLES_TO_RESTORE if necessary
- if restore:
- collections.append(VARIABLES_TO_RESTORE)
- # Remove duplicates
- collections = set(collections)
- # Get the device for the variable.
- with tf.device(variable_device(device, name)):
- return tf.get_variable(name, shape=shape, dtype=dtype,
- initializer=initializer, regularizer=regularizer,
- trainable=trainable, collections=collections)
diff --git a/research/inception/inception/slim/variables_test.py b/research/inception/inception/slim/variables_test.py
deleted file mode 100644
index b8c1944dfeb0fba7ad99f104b0c366c41d737c63..0000000000000000000000000000000000000000
--- a/research/inception/inception/slim/variables_test.py
+++ /dev/null
@@ -1,392 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for slim.variables."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from inception.slim import scopes
-from inception.slim import variables
-
-
-class VariablesTest(tf.test.TestCase):
-
- def testCreateVariable(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [5])
- self.assertEquals(a.op.name, 'A/a')
- self.assertListEqual(a.get_shape().as_list(), [5])
-
- def testGetVariables(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [5])
- with tf.variable_scope('B'):
- b = variables.variable('a', [5])
- self.assertEquals([a, b], variables.get_variables())
- self.assertEquals([a], variables.get_variables('A'))
- self.assertEquals([b], variables.get_variables('B'))
-
- def testGetVariablesSuffix(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [5])
- with tf.variable_scope('A'):
- b = variables.variable('b', [5])
- self.assertEquals([a], variables.get_variables(suffix='a'))
- self.assertEquals([b], variables.get_variables(suffix='b'))
-
- def testGetVariableWithSingleVar(self):
- with self.test_session():
- with tf.variable_scope('parent'):
- a = variables.variable('child', [5])
- self.assertEquals(a, variables.get_unique_variable('parent/child'))
-
- def testGetVariableWithDistractors(self):
- with self.test_session():
- with tf.variable_scope('parent'):
- a = variables.variable('child', [5])
- with tf.variable_scope('child'):
- variables.variable('grandchild1', [7])
- variables.variable('grandchild2', [9])
- self.assertEquals(a, variables.get_unique_variable('parent/child'))
-
- def testGetVariableThrowsExceptionWithNoMatch(self):
- var_name = 'cant_find_me'
- with self.test_session():
- with self.assertRaises(ValueError):
- variables.get_unique_variable(var_name)
-
- def testGetThrowsExceptionWithChildrenButNoMatch(self):
- var_name = 'parent/child'
- with self.test_session():
- with tf.variable_scope(var_name):
- variables.variable('grandchild1', [7])
- variables.variable('grandchild2', [9])
- with self.assertRaises(ValueError):
- variables.get_unique_variable(var_name)
-
- def testGetVariablesToRestore(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [5])
- with tf.variable_scope('B'):
- b = variables.variable('a', [5])
- self.assertEquals([a, b], variables.get_variables_to_restore())
-
- def testNoneGetVariablesToRestore(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [5], restore=False)
- with tf.variable_scope('B'):
- b = variables.variable('a', [5], restore=False)
- self.assertEquals([], variables.get_variables_to_restore())
- self.assertEquals([a, b], variables.get_variables())
-
- def testGetMixedVariablesToRestore(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [5])
- b = variables.variable('b', [5], restore=False)
- with tf.variable_scope('B'):
- c = variables.variable('c', [5])
- d = variables.variable('d', [5], restore=False)
- self.assertEquals([a, b, c, d], variables.get_variables())
- self.assertEquals([a, c], variables.get_variables_to_restore())
-
- def testReuseVariable(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [])
- with tf.variable_scope('A', reuse=True):
- b = variables.variable('a', [])
- self.assertEquals(a, b)
- self.assertListEqual([a], variables.get_variables())
-
- def testVariableWithDevice(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [], device='cpu:0')
- b = variables.variable('b', [], device='cpu:1')
- self.assertDeviceEqual(a.device, 'cpu:0')
- self.assertDeviceEqual(b.device, 'cpu:1')
-
- def testVariableWithDeviceFromScope(self):
- with self.test_session():
- with tf.device('/cpu:0'):
- a = variables.variable('a', [])
- b = variables.variable('b', [], device='cpu:1')
- self.assertDeviceEqual(a.device, 'cpu:0')
- self.assertDeviceEqual(b.device, 'cpu:1')
-
- def testVariableWithDeviceFunction(self):
- class DevFn(object):
-
- def __init__(self):
- self.counter = -1
-
- def __call__(self, op):
- self.counter += 1
- return 'cpu:%d' % self.counter
-
- with self.test_session():
- with scopes.arg_scope([variables.variable], device=DevFn()):
- a = variables.variable('a', [])
- b = variables.variable('b', [])
- c = variables.variable('c', [], device='cpu:12')
- d = variables.variable('d', [])
- with tf.device('cpu:99'):
- e_init = tf.constant(12)
- e = variables.variable('e', initializer=e_init)
- self.assertDeviceEqual(a.device, 'cpu:0')
- self.assertDeviceEqual(a.initial_value.device, 'cpu:0')
- self.assertDeviceEqual(b.device, 'cpu:1')
- self.assertDeviceEqual(b.initial_value.device, 'cpu:1')
- self.assertDeviceEqual(c.device, 'cpu:12')
- self.assertDeviceEqual(c.initial_value.device, 'cpu:12')
- self.assertDeviceEqual(d.device, 'cpu:2')
- self.assertDeviceEqual(d.initial_value.device, 'cpu:2')
- self.assertDeviceEqual(e.device, 'cpu:3')
- self.assertDeviceEqual(e.initial_value.device, 'cpu:99')
-
- def testVariableWithReplicaDeviceSetter(self):
- with self.test_session():
- with tf.device(tf.train.replica_device_setter(ps_tasks=2)):
- a = variables.variable('a', [])
- b = variables.variable('b', [])
- c = variables.variable('c', [], device='cpu:12')
- d = variables.variable('d', [])
- with tf.device('cpu:99'):
- e_init = tf.constant(12)
- e = variables.variable('e', initializer=e_init)
- # The values below highlight how the replica_device_setter puts initial
- # values on the worker job, and how it merges explicit devices.
- self.assertDeviceEqual(a.device, '/job:ps/task:0/cpu:0')
- self.assertDeviceEqual(a.initial_value.device, '/job:worker/cpu:0')
- self.assertDeviceEqual(b.device, '/job:ps/task:1/cpu:0')
- self.assertDeviceEqual(b.initial_value.device, '/job:worker/cpu:0')
- self.assertDeviceEqual(c.device, '/job:ps/task:0/cpu:12')
- self.assertDeviceEqual(c.initial_value.device, '/job:worker/cpu:12')
- self.assertDeviceEqual(d.device, '/job:ps/task:1/cpu:0')
- self.assertDeviceEqual(d.initial_value.device, '/job:worker/cpu:0')
- self.assertDeviceEqual(e.device, '/job:ps/task:0/cpu:0')
- self.assertDeviceEqual(e.initial_value.device, '/job:worker/cpu:99')
-
- def testVariableWithVariableDeviceChooser(self):
-
- with tf.Graph().as_default():
- device_fn = variables.VariableDeviceChooser(num_parameter_servers=2)
- with scopes.arg_scope([variables.variable], device=device_fn):
- a = variables.variable('a', [])
- b = variables.variable('b', [])
- c = variables.variable('c', [], device='cpu:12')
- d = variables.variable('d', [])
- with tf.device('cpu:99'):
- e_init = tf.constant(12)
- e = variables.variable('e', initializer=e_init)
- # The values below highlight how the VariableDeviceChooser puts initial
- # values on the same device as the variable job.
- self.assertDeviceEqual(a.device, '/job:ps/task:0/cpu:0')
- self.assertDeviceEqual(a.initial_value.device, a.device)
- self.assertDeviceEqual(b.device, '/job:ps/task:1/cpu:0')
- self.assertDeviceEqual(b.initial_value.device, b.device)
- self.assertDeviceEqual(c.device, '/cpu:12')
- self.assertDeviceEqual(c.initial_value.device, c.device)
- self.assertDeviceEqual(d.device, '/job:ps/task:0/cpu:0')
- self.assertDeviceEqual(d.initial_value.device, d.device)
- self.assertDeviceEqual(e.device, '/job:ps/task:1/cpu:0')
- self.assertDeviceEqual(e.initial_value.device, '/cpu:99')
-
- def testVariableGPUPlacement(self):
-
- with tf.Graph().as_default():
- device_fn = variables.VariableDeviceChooser(placement='gpu:0')
- with scopes.arg_scope([variables.variable], device=device_fn):
- a = variables.variable('a', [])
- b = variables.variable('b', [])
- c = variables.variable('c', [], device='cpu:12')
- d = variables.variable('d', [])
- with tf.device('cpu:99'):
- e_init = tf.constant(12)
- e = variables.variable('e', initializer=e_init)
- # The values below highlight how the VariableDeviceChooser puts initial
- # values on the same device as the variable job.
- self.assertDeviceEqual(a.device, '/gpu:0')
- self.assertDeviceEqual(a.initial_value.device, a.device)
- self.assertDeviceEqual(b.device, '/gpu:0')
- self.assertDeviceEqual(b.initial_value.device, b.device)
- self.assertDeviceEqual(c.device, '/cpu:12')
- self.assertDeviceEqual(c.initial_value.device, c.device)
- self.assertDeviceEqual(d.device, '/gpu:0')
- self.assertDeviceEqual(d.initial_value.device, d.device)
- self.assertDeviceEqual(e.device, '/gpu:0')
- self.assertDeviceEqual(e.initial_value.device, '/cpu:99')
-
- def testVariableCollection(self):
- with self.test_session():
- a = variables.variable('a', [], collections='A')
- b = variables.variable('b', [], collections='B')
- self.assertEquals(a, tf.get_collection('A')[0])
- self.assertEquals(b, tf.get_collection('B')[0])
-
- def testVariableCollections(self):
- with self.test_session():
- a = variables.variable('a', [], collections=['A', 'C'])
- b = variables.variable('b', [], collections=['B', 'C'])
- self.assertEquals(a, tf.get_collection('A')[0])
- self.assertEquals(b, tf.get_collection('B')[0])
-
- def testVariableCollectionsWithArgScope(self):
- with self.test_session():
- with scopes.arg_scope([variables.variable], collections='A'):
- a = variables.variable('a', [])
- b = variables.variable('b', [])
- self.assertListEqual([a, b], tf.get_collection('A'))
-
- def testVariableCollectionsWithArgScopeNested(self):
- with self.test_session():
- with scopes.arg_scope([variables.variable], collections='A'):
- a = variables.variable('a', [])
- with scopes.arg_scope([variables.variable], collections='B'):
- b = variables.variable('b', [])
- self.assertEquals(a, tf.get_collection('A')[0])
- self.assertEquals(b, tf.get_collection('B')[0])
-
- def testVariableCollectionsWithArgScopeNonNested(self):
- with self.test_session():
- with scopes.arg_scope([variables.variable], collections='A'):
- a = variables.variable('a', [])
- with scopes.arg_scope([variables.variable], collections='B'):
- b = variables.variable('b', [])
- variables.variable('c', [])
- self.assertListEqual([a], tf.get_collection('A'))
- self.assertListEqual([b], tf.get_collection('B'))
-
- def testVariableRestoreWithArgScopeNested(self):
- with self.test_session():
- with scopes.arg_scope([variables.variable], restore=True):
- a = variables.variable('a', [])
- with scopes.arg_scope([variables.variable],
- trainable=False,
- collections=['A', 'B']):
- b = variables.variable('b', [])
- c = variables.variable('c', [])
- self.assertListEqual([a, b, c], variables.get_variables_to_restore())
- self.assertListEqual([a, c], tf.trainable_variables())
- self.assertListEqual([b], tf.get_collection('A'))
- self.assertListEqual([b], tf.get_collection('B'))
-
-
-class GetVariablesByNameTest(tf.test.TestCase):
-
- def testGetVariableGivenNameScoped(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [5])
- b = variables.variable('b', [5])
- self.assertEquals([a], variables.get_variables_by_name('a'))
- self.assertEquals([b], variables.get_variables_by_name('b'))
-
- def testGetVariablesByNameReturnsByValueWithScope(self):
- with self.test_session():
- with tf.variable_scope('A'):
- a = variables.variable('a', [5])
- matched_variables = variables.get_variables_by_name('a')
-
- # If variables.get_variables_by_name returns the list by reference, the
- # following append should persist, and be returned, in subsequent calls
- # to variables.get_variables_by_name('a').
- matched_variables.append(4)
-
- matched_variables = variables.get_variables_by_name('a')
- self.assertEquals([a], matched_variables)
-
- def testGetVariablesByNameReturnsByValueWithoutScope(self):
- with self.test_session():
- a = variables.variable('a', [5])
- matched_variables = variables.get_variables_by_name('a')
-
- # If variables.get_variables_by_name returns the list by reference, the
- # following append should persist, and be returned, in subsequent calls
- # to variables.get_variables_by_name('a').
- matched_variables.append(4)
-
- matched_variables = variables.get_variables_by_name('a')
- self.assertEquals([a], matched_variables)
-
-
-class GlobalStepTest(tf.test.TestCase):
-
- def testStable(self):
- with tf.Graph().as_default():
- gs = variables.global_step()
- gs2 = variables.global_step()
- self.assertTrue(gs is gs2)
-
- def testDevice(self):
- with tf.Graph().as_default():
- with scopes.arg_scope([variables.global_step], device='/gpu:0'):
- gs = variables.global_step()
- self.assertDeviceEqual(gs.device, '/gpu:0')
-
- def testDeviceFn(self):
- class DevFn(object):
-
- def __init__(self):
- self.counter = -1
-
- def __call__(self, op):
- self.counter += 1
- return '/cpu:%d' % self.counter
-
- with tf.Graph().as_default():
- with scopes.arg_scope([variables.global_step], device=DevFn()):
- gs = variables.global_step()
- gs2 = variables.global_step()
- self.assertDeviceEqual(gs.device, '/cpu:0')
- self.assertEquals(gs, gs2)
- self.assertDeviceEqual(gs2.device, '/cpu:0')
-
- def testReplicaDeviceSetter(self):
- device_fn = tf.train.replica_device_setter(2)
- with tf.Graph().as_default():
- with scopes.arg_scope([variables.global_step], device=device_fn):
- gs = variables.global_step()
- gs2 = variables.global_step()
- self.assertEquals(gs, gs2)
- self.assertDeviceEqual(gs.device, '/job:ps/task:0')
- self.assertDeviceEqual(gs.initial_value.device, '/job:ps/task:0')
- self.assertDeviceEqual(gs2.device, '/job:ps/task:0')
- self.assertDeviceEqual(gs2.initial_value.device, '/job:ps/task:0')
-
- def testVariableWithVariableDeviceChooser(self):
-
- with tf.Graph().as_default():
- device_fn = variables.VariableDeviceChooser()
- with scopes.arg_scope([variables.global_step], device=device_fn):
- gs = variables.global_step()
- gs2 = variables.global_step()
- self.assertEquals(gs, gs2)
- self.assertDeviceEqual(gs.device, 'cpu:0')
- self.assertDeviceEqual(gs.initial_value.device, gs.device)
- self.assertDeviceEqual(gs2.device, 'cpu:0')
- self.assertDeviceEqual(gs2.initial_value.device, gs2.device)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/keypointnet/CONTRIBUTING.md b/research/keypointnet/CONTRIBUTING.md
deleted file mode 100644
index 939e5341e74dc2371c8b47f0e27b50581bed5f63..0000000000000000000000000000000000000000
--- a/research/keypointnet/CONTRIBUTING.md
+++ /dev/null
@@ -1,28 +0,0 @@
-# How to Contribute
-
-We'd love to accept your patches and contributions to this project. There are
-just a few small guidelines you need to follow.
-
-## Contributor License Agreement
-
-Contributions to this project must be accompanied by a Contributor License
-Agreement. You (or your employer) retain the copyright to your contribution;
-this simply gives us permission to use and redistribute your contributions as
-part of the project. Head over to to see
-your current agreements on file or to sign a new one.
-
-You generally only need to submit a CLA once, so if you've already submitted one
-(even if it was for a different project), you probably don't need to do it
-again.
-
-## Code reviews
-
-All submissions, including submissions by project members, require review. We
-use GitHub pull requests for this purpose. Consult
-[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
-information on using pull requests.
-
-## Community Guidelines
-
-This project follows [Google's Open Source Community
-Guidelines](https://opensource.google.com/conduct/).
diff --git a/research/keypointnet/LICENSE b/research/keypointnet/LICENSE
deleted file mode 100644
index d645695673349e3947e8e5ae42332d0ac3164cd7..0000000000000000000000000000000000000000
--- a/research/keypointnet/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/research/keypointnet/README.md b/research/keypointnet/README.md
deleted file mode 100644
index 8de88ca5a18816984302a9c20639364a7c8cde53..0000000000000000000000000000000000000000
--- a/research/keypointnet/README.md
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-
-
-# KeypointNet
-This is an implementation of the keypoint network proposed in "Discovery of
-Latent 3D Keypoints via End-to-end Geometric Reasoning
-[[pdf](https://arxiv.org/pdf/1807.03146.pdf)]". Given a single 2D image of a
-known class, this network can predict a set of 3D keypoints that are consistent
-across viewing angles of the same object and across object instances. These
-keypoints and their detectors are discovered and learned automatically without
-keypoint location supervision [[demo](https://keypointnet.github.io)].
-
-## Datasets:
- ShapeNet's rendering for
- [Cars](https://storage.googleapis.com/discovery-3dkeypoints-data/cars_with_keypoints.zip),
- [Planes](https://storage.googleapis.com/discovery-3dkeypoints-data/planes_with_keypoints.zip),
- [Chairs](https://storage.googleapis.com/discovery-3dkeypoints-data/chairs_with_keypoints.zip).
-
- Each set contains:
-1. tfrecords
-2. train.txt, a list of tfrecords used for training.
-2. dev.txt, a list of tfrecords used for validation.
-3. test.txt, a list of tfrecords used for testing.
-4. projection.txt, storing the global 4x4 camera projection matrix.
-5. job.txt, storing ShapeNet's object IDs in each tfrecord.
-
-## Training:
- Run `main.py --model_dir=MODEL_DIR --dset=DSET`
-
- where MODEL_DIR is a folder for storing model checkpoints: (see [tf.estimator](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator)), and DSET should point to the folder containing tfrecords (download above).
-
-## Inference:
- Run `main.py --model_dir=MODEL_DIR --input=INPUT --predict`
-
- where MODEL_DIR is the model checkpoint folder, and INPUT is a folder containing png or jpeg test images.
- We trained the network using the total batch size of 256 (8 x 32 replicas). You may have to tune the learning rate if your batch size is different.
-
-## Code credit:
- Supasorn Suwajanakorn
-
-## Contact:
- supasorn@gmail.com, [snavely,tompson,mnorouzi]@google.com
-
-
-(This is not an officially supported Google product)
diff --git a/research/keypointnet/main.py b/research/keypointnet/main.py
deleted file mode 100644
index 04b30159404e01529c898ee75fb1ed78f705f539..0000000000000000000000000000000000000000
--- a/research/keypointnet/main.py
+++ /dev/null
@@ -1,697 +0,0 @@
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# =============================================================================
-"""KeypointNet!!
-
-A reimplementation of 'Discovery of Latent 3D Keypoints via End-to-end
-Geometric Reasoning' keypoint network. Given a single 2D image of a known class,
-this network can predict a set of 3D keypoints that are consistent across
-viewing angles of the same object and across object instances. These keypoints
-and their detectors are discovered and learned automatically without
-keypoint location supervision.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-import matplotlib.pyplot as plt
-import numpy as np
-import os
-from scipy import misc
-import sys
-import tensorflow as tf
-import tensorflow.contrib.slim as slim
-import utils
-
-FLAGS = tf.app.flags.FLAGS
-
-tf.app.flags.DEFINE_boolean("predict", False, "Running inference if true")
-tf.app.flags.DEFINE_string(
- "input",
- "",
- "Input folder containing images")
-tf.app.flags.DEFINE_string("model_dir", None, "Estimator model_dir")
-tf.app.flags.DEFINE_string(
- "dset",
- "",
- "Path to the directory containing the dataset.")
-tf.app.flags.DEFINE_integer("steps", 200000, "Training steps")
-tf.app.flags.DEFINE_integer("batch_size", 8, "Size of mini-batch.")
-tf.app.flags.DEFINE_string(
- "hparams", "",
- "A comma-separated list of `name=value` hyperparameter values. This flag "
- "is used to override hyperparameter settings either when manually "
- "selecting hyperparameters or when using Vizier.")
-tf.app.flags.DEFINE_integer(
- "sync_replicas", -1,
- "If > 0, use SyncReplicasOptimizer and use this many replicas per sync.")
-
-# Fixed input size 128 x 128.
-vw = vh = 128
-
-
-def create_input_fn(split, batch_size):
- """Returns input_fn for tf.estimator.Estimator.
-
- Reads tfrecords and construts input_fn for either training or eval. All
- tfrecords not in test.txt or dev.txt will be assigned to training set.
-
- Args:
- split: A string indicating the split. Can be either 'train' or 'validation'.
- batch_size: The batch size!
-
- Returns:
- input_fn for tf.estimator.Estimator.
-
- Raises:
- IOError: If test.txt or dev.txt are not found.
- """
-
- if (not os.path.exists(os.path.join(FLAGS.dset, "test.txt")) or
- not os.path.exists(os.path.join(FLAGS.dset, "dev.txt"))):
- raise IOError("test.txt or dev.txt not found")
-
- with open(os.path.join(FLAGS.dset, "test.txt"), "r") as f:
- testset = [x.strip() for x in f.readlines()]
-
- with open(os.path.join(FLAGS.dset, "dev.txt"), "r") as f:
- validset = [x.strip() for x in f.readlines()]
-
- files = os.listdir(FLAGS.dset)
- filenames = []
- for f in files:
- sp = os.path.splitext(f)
- if sp[1] != ".tfrecord" or sp[0] in testset:
- continue
-
- if ((split == "validation" and sp[0] in validset) or
- (split == "train" and sp[0] not in validset)):
- filenames.append(os.path.join(FLAGS.dset, f))
-
- def input_fn():
- """input_fn for tf.estimator.Estimator."""
-
- def parser(serialized_example):
- """Parses a single tf.Example into image and label tensors."""
- fs = tf.parse_single_example(
- serialized_example,
- features={
- "img0": tf.FixedLenFeature([], tf.string),
- "img1": tf.FixedLenFeature([], tf.string),
- "mv0": tf.FixedLenFeature([16], tf.float32),
- "mvi0": tf.FixedLenFeature([16], tf.float32),
- "mv1": tf.FixedLenFeature([16], tf.float32),
- "mvi1": tf.FixedLenFeature([16], tf.float32),
- })
-
- fs["img0"] = tf.div(tf.to_float(tf.image.decode_png(fs["img0"], 4)), 255)
- fs["img1"] = tf.div(tf.to_float(tf.image.decode_png(fs["img1"], 4)), 255)
-
- fs["img0"].set_shape([vh, vw, 4])
- fs["img1"].set_shape([vh, vw, 4])
-
- # fs["lr0"] = [fs["mv0"][0]]
- # fs["lr1"] = [fs["mv1"][0]]
-
- fs["lr0"] = tf.convert_to_tensor([fs["mv0"][0]])
- fs["lr1"] = tf.convert_to_tensor([fs["mv1"][0]])
-
- return fs
-
- np.random.shuffle(filenames)
- dataset = tf.data.TFRecordDataset(filenames)
- dataset = dataset.map(parser, num_parallel_calls=4)
- dataset = dataset.shuffle(400).repeat().batch(batch_size)
- dataset = dataset.prefetch(buffer_size=256)
-
- return dataset.make_one_shot_iterator().get_next(), None
-
- return input_fn
-
-
-class Transformer(object):
- """A utility for projecting 3D points to 2D coordinates and vice versa.
-
- 3D points are represented in 4D-homogeneous world coordinates. The pixel
- coordinates are represented in normalized device coordinates [-1, 1].
- See https://learnopengl.com/Getting-started/Coordinate-Systems.
- """
-
- def __get_matrix(self, lines):
- return np.array([[float(y) for y in x.strip().split(" ")] for x in lines])
-
- def __read_projection_matrix(self, filename):
- if not os.path.exists(filename):
- filename = "/cns/vz-d/home/supasorn/datasets/cars/projection.txt"
- with open(filename, "r") as f:
- lines = f.readlines()
- return self.__get_matrix(lines)
-
- def __init__(self, w, h, dataset_dir):
- self.w = w
- self.h = h
- p = self.__read_projection_matrix(dataset_dir + "projection.txt")
-
- # transposed of inversed projection matrix.
- self.pinv_t = tf.constant([[1.0 / p[0, 0], 0, 0,
- 0], [0, 1.0 / p[1, 1], 0, 0], [0, 0, 1, 0],
- [0, 0, 0, 1]])
- self.f = p[0, 0]
-
- def project(self, xyzw):
- """Projects homogeneous 3D coordinates to normalized device coordinates."""
-
- z = xyzw[:, :, 2:3] + 1e-8
- return tf.concat([-self.f * xyzw[:, :, :2] / z, z], axis=2)
-
- def unproject(self, xyz):
- """Unprojects normalized device coordinates with depth to 3D coordinates."""
-
- z = xyz[:, :, 2:]
- xy = -xyz * z
-
- def batch_matmul(a, b):
- return tf.reshape(
- tf.matmul(tf.reshape(a, [-1, a.shape[2].value]), b),
- [-1, a.shape[1].value, a.shape[2].value])
-
- return batch_matmul(
- tf.concat([xy[:, :, :2], z, tf.ones_like(z)], axis=2), self.pinv_t)
-
-
-def meshgrid(h):
- """Returns a meshgrid ranging from [-1, 1] in x, y axes."""
-
- r = np.arange(0.5, h, 1) / (h / 2) - 1
- ranx, rany = tf.meshgrid(r, -r)
- return tf.to_float(ranx), tf.to_float(rany)
-
-
-def estimate_rotation(xyz0, xyz1, pconf, noise):
- """Estimates the rotation between two sets of keypoints.
-
- The rotation is estimated by first subtracting mean from each set of keypoints
- and computing SVD of the covariance matrix.
-
- Args:
- xyz0: [batch, num_kp, 3] The first set of keypoints.
- xyz1: [batch, num_kp, 3] The second set of keypoints.
- pconf: [batch, num_kp] The weights used to compute the rotation estimate.
- noise: A number indicating the noise added to the keypoints.
-
- Returns:
- [batch, 3, 3] A batch of transposed 3 x 3 rotation matrices.
- """
-
- xyz0 += tf.random_normal(tf.shape(xyz0), mean=0, stddev=noise)
- xyz1 += tf.random_normal(tf.shape(xyz1), mean=0, stddev=noise)
-
- pconf2 = tf.expand_dims(pconf, 2)
- cen0 = tf.reduce_sum(xyz0 * pconf2, 1, keepdims=True)
- cen1 = tf.reduce_sum(xyz1 * pconf2, 1, keepdims=True)
-
- x = xyz0 - cen0
- y = xyz1 - cen1
-
- cov = tf.matmul(tf.matmul(x, tf.matrix_diag(pconf), transpose_a=True), y)
- _, u, v = tf.svd(cov, full_matrices=True)
-
- d = tf.matrix_determinant(tf.matmul(v, u, transpose_b=True))
- ud = tf.concat(
- [u[:, :, :-1], u[:, :, -1:] * tf.expand_dims(tf.expand_dims(d, 1), 1)],
- axis=2)
- return tf.matmul(ud, v, transpose_b=True)
-
-
-def relative_pose_loss(xyz0, xyz1, rot, pconf, noise):
- """Computes the relative pose loss (chordal, angular).
-
- Args:
- xyz0: [batch, num_kp, 3] The first set of keypoints.
- xyz1: [batch, num_kp, 3] The second set of keypoints.
- rot: [batch, 4, 4] The ground-truth rotation matrices.
- pconf: [batch, num_kp] The weights used to compute the rotation estimate.
- noise: A number indicating the noise added to the keypoints.
-
- Returns:
- A tuple (chordal loss, angular loss).
- """
-
- r_transposed = estimate_rotation(xyz0, xyz1, pconf, noise)
- rotation = rot[:, :3, :3]
- frob_sqr = tf.reduce_sum(tf.square(r_transposed - rotation), axis=[1, 2])
- frob = tf.sqrt(frob_sqr)
-
- return tf.reduce_mean(frob_sqr), \
- 2.0 * tf.reduce_mean(tf.asin(tf.minimum(1.0, frob / (2 * math.sqrt(2)))))
-
-
-def separation_loss(xyz, delta):
- """Computes the separation loss.
-
- Args:
- xyz: [batch, num_kp, 3] Input keypoints.
- delta: A separation threshold. Incur 0 cost if the distance >= delta.
-
- Returns:
- The seperation loss.
- """
-
- num_kp = tf.shape(xyz)[1]
- t1 = tf.tile(xyz, [1, num_kp, 1])
-
- t2 = tf.reshape(tf.tile(xyz, [1, 1, num_kp]), tf.shape(t1))
- diffsq = tf.square(t1 - t2)
-
- # -> [batch, num_kp ^ 2]
- lensqr = tf.reduce_sum(diffsq, axis=2)
-
- return (tf.reduce_sum(tf.maximum(-lensqr + delta, 0.0)) / tf.to_float(
- num_kp * FLAGS.batch_size * 2))
-
-
-def consistency_loss(uv0, uv1, pconf):
- """Computes multi-view consistency loss between two sets of keypoints.
-
- Args:
- uv0: [batch, num_kp, 2] The first set of keypoint 2D coordinates.
- uv1: [batch, num_kp, 2] The second set of keypoint 2D coordinates.
- pconf: [batch, num_kp] The weights used to compute the rotation estimate.
-
- Returns:
- The consistency loss.
- """
-
- # [batch, num_kp, 2]
- wd = tf.square(uv0 - uv1) * tf.expand_dims(pconf, 2)
- wd = tf.reduce_sum(wd, axis=[1, 2])
- return tf.reduce_mean(wd)
-
-
-def variance_loss(probmap, ranx, rany, uv):
- """Computes the variance loss as part of Sillhouette consistency.
-
- Args:
- probmap: [batch, num_kp, h, w] The distribution map of keypoint locations.
- ranx: X-axis meshgrid.
- rany: Y-axis meshgrid.
- uv: [batch, num_kp, 2] Keypoint locations (in NDC).
-
- Returns:
- The variance loss.
- """
-
- ran = tf.stack([ranx, rany], axis=2)
-
- sh = tf.shape(ran)
- # [batch, num_kp, vh, vw, 2]
- ran = tf.reshape(ran, [1, 1, sh[0], sh[1], 2])
-
- sh = tf.shape(uv)
- uv = tf.reshape(uv, [sh[0], sh[1], 1, 1, 2])
-
- diff = tf.reduce_sum(tf.square(uv - ran), axis=4)
- diff *= probmap
-
- return tf.reduce_mean(tf.reduce_sum(diff, axis=[2, 3]))
-
-
-def dilated_cnn(images, num_filters, is_training):
- """Constructs a base dilated convolutional network.
-
- Args:
- images: [batch, h, w, 3] Input RGB images.
- num_filters: The number of filters for all layers.
- is_training: True if this function is called during training.
-
- Returns:
- Output of this dilated CNN.
- """
-
- net = images
-
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- normalizer_fn=slim.batch_norm,
- activation_fn=lambda x: tf.nn.leaky_relu(x, alpha=0.1),
- normalizer_params={"is_training": is_training}):
- for i, r in enumerate([1, 1, 2, 4, 8, 16, 1, 2, 4, 8, 16, 1]):
- net = slim.conv2d(net, num_filters, [3, 3], rate=r, scope="dconv%d" % i)
-
- return net
-
-
-def orientation_network(images, num_filters, is_training):
- """Constructs a network that infers the orientation of an object.
-
- Args:
- images: [batch, h, w, 3] Input RGB images.
- num_filters: The number of filters for all layers.
- is_training: True if this function is called during training.
-
- Returns:
- Output of the orientation network.
- """
-
- with tf.variable_scope("OrientationNetwork"):
- net = dilated_cnn(images, num_filters, is_training)
-
- modules = 2
- prob = slim.conv2d(net, 2, [3, 3], rate=1, activation_fn=None)
- prob = tf.transpose(prob, [0, 3, 1, 2])
-
- prob = tf.reshape(prob, [-1, modules, vh * vw])
- prob = tf.nn.softmax(prob)
- ranx, rany = meshgrid(vh)
-
- prob = tf.reshape(prob, [-1, 2, vh, vw])
-
- sx = tf.reduce_sum(prob * ranx, axis=[2, 3])
- sy = tf.reduce_sum(prob * rany, axis=[2, 3]) # -> batch x modules
-
- out_xy = tf.reshape(tf.stack([sx, sy], -1), [-1, modules, 2])
-
- return out_xy
-
-
-def keypoint_network(rgba,
- num_filters,
- num_kp,
- is_training,
- lr_gt=None,
- anneal=1):
- """Constructs our main keypoint network that predicts 3D keypoints.
-
- Args:
- rgba: [batch, h, w, 4] Input RGB images with alpha channel.
- num_filters: The number of filters for all layers.
- num_kp: The number of keypoints.
- is_training: True if this function is called during training.
- lr_gt: The groundtruth orientation flag used at the beginning of training.
- Then we linearly anneal in the prediction.
- anneal: A number between [0, 1] where 1 means using the ground-truth
- orientation and 0 means using our estimate.
-
- Returns:
- uv: [batch, num_kp, 2] 2D locations of keypoints.
- z: [batch, num_kp] The depth of keypoints.
- orient: [batch, 2, 2] Two 2D coordinates that correspond to [1, 0, 0] and
- [-1, 0, 0] in object space.
- sill: The Sillhouette loss.
- variance: The variance loss.
- prob_viz: A visualization of all predicted keypoints.
- prob_vizs: A list of visualizations of each keypoint.
-
- """
-
- images = rgba[:, :, :, :3]
-
- # [batch, 1]
- orient = orientation_network(images, num_filters * 0.5, is_training)
-
- # [batch, 1]
- lr_estimated = tf.maximum(0.0, tf.sign(orient[:, 0, :1] - orient[:, 1, :1]))
-
- if lr_gt is None:
- lr = lr_estimated
- else:
- lr_gt = tf.maximum(0.0, tf.sign(lr_gt[:, :1]))
- lr = tf.round(lr_gt * anneal + lr_estimated * (1 - anneal))
-
- lrtiled = tf.tile(
- tf.expand_dims(tf.expand_dims(lr, 1), 1),
- [1, images.shape[1], images.shape[2], 1])
-
- images = tf.concat([images, lrtiled], axis=3)
-
- mask = rgba[:, :, :, 3]
- mask = tf.cast(tf.greater(mask, tf.zeros_like(mask)), dtype=tf.float32)
-
- net = dilated_cnn(images, num_filters, is_training)
-
- # The probability distribution map.
- prob = slim.conv2d(
- net, num_kp, [3, 3], rate=1, scope="conv_xy", activation_fn=None)
-
- # We added the fixed camera distance as a bias.
- z = -30 + slim.conv2d(
- net, num_kp, [3, 3], rate=1, scope="conv_z", activation_fn=None)
-
- prob = tf.transpose(prob, [0, 3, 1, 2])
- z = tf.transpose(z, [0, 3, 1, 2])
-
- prob = tf.reshape(prob, [-1, num_kp, vh * vw])
- prob = tf.nn.softmax(prob, name="softmax")
-
- ranx, rany = meshgrid(vh)
- prob = tf.reshape(prob, [-1, num_kp, vh, vw])
-
- # These are for visualizing the distribution maps.
- prob_viz = tf.expand_dims(tf.reduce_sum(prob, 1), 3)
- prob_vizs = [tf.expand_dims(prob[:, i, :, :], 3) for i in range(num_kp)]
-
- sx = tf.reduce_sum(prob * ranx, axis=[2, 3])
- sy = tf.reduce_sum(prob * rany, axis=[2, 3]) # -> batch x num_kp
-
- # [batch, num_kp]
- sill = tf.reduce_sum(prob * tf.expand_dims(mask, 1), axis=[2, 3])
- sill = tf.reduce_mean(-tf.log(sill + 1e-12))
-
- z = tf.reduce_sum(prob * z, axis=[2, 3])
- uv = tf.reshape(tf.stack([sx, sy], -1), [-1, num_kp, 2])
-
- variance = variance_loss(prob, ranx, rany, uv)
-
- return uv, z, orient, sill, variance, prob_viz, prob_vizs
-
-
-def model_fn(features, labels, mode, hparams):
- """Returns model_fn for tf.estimator.Estimator."""
-
- del labels
-
- is_training = (mode == tf.estimator.ModeKeys.TRAIN)
- t = Transformer(vw, vh, FLAGS.dset)
-
- def func1(x):
- return tf.transpose(tf.reshape(features[x], [-1, 4, 4]), [0, 2, 1])
-
- mv = [func1("mv%d" % i) for i in range(2)]
- mvi = [func1("mvi%d" % i) for i in range(2)]
-
- uvz = [None] * 2
- uvz_proj = [None] * 2 # uvz coordinates projected on to the other view.
- viz = [None] * 2
- vizs = [None] * 2
-
- loss_sill = 0
- loss_variance = 0
- loss_con = 0
- loss_sep = 0
- loss_lr = 0
-
- for i in range(2):
- with tf.variable_scope("KeypointNetwork", reuse=i > 0):
- # anneal: 1 = using ground-truth, 0 = using our estimate orientation.
- anneal = tf.to_float(hparams.lr_anneal_end - tf.train.get_global_step())
- anneal = tf.clip_by_value(
- anneal / (hparams.lr_anneal_end - hparams.lr_anneal_start), 0.0, 1.0)
-
- uv, z, orient, sill, variance, viz[i], vizs[i] = keypoint_network(
- features["img%d" % i],
- hparams.num_filters,
- hparams.num_kp,
- is_training,
- lr_gt=features["lr%d" % i],
- anneal=anneal)
-
- # x-positive/negative axes (dominant direction).
- xp_axis = tf.tile(
- tf.constant([[[1.0, 0, 0, 1], [-1.0, 0, 0, 1]]]),
- [tf.shape(orient)[0], 1, 1])
-
- # [batch, 2, 4] = [batch, 2, 4] x [batch, 4, 4]
- xp = tf.matmul(xp_axis, mv[i])
-
- # [batch, 2, 3]
- xp = t.project(xp)
-
- loss_lr += tf.losses.mean_squared_error(orient[:, :, :2], xp[:, :, :2])
- loss_variance += variance
- loss_sill += sill
-
- uv = tf.reshape(uv, [-1, hparams.num_kp, 2])
- z = tf.reshape(z, [-1, hparams.num_kp, 1])
-
- # [batch, num_kp, 3]
- uvz[i] = tf.concat([uv, z], axis=2)
-
- world_coords = tf.matmul(t.unproject(uvz[i]), mvi[i])
-
- # [batch, num_kp, 3]
- uvz_proj[i] = t.project(tf.matmul(world_coords, mv[1 - i]))
-
- pconf = tf.ones(
- [tf.shape(uv)[0], tf.shape(uv)[1]], dtype=tf.float32) / hparams.num_kp
-
- for i in range(2):
- loss_con += consistency_loss(uvz_proj[i][:, :, :2], uvz[1 - i][:, :, :2],
- pconf)
- loss_sep += separation_loss(
- t.unproject(uvz[i])[:, :, :3], hparams.sep_delta)
-
- chordal, angular = relative_pose_loss(
- t.unproject(uvz[0])[:, :, :3],
- t.unproject(uvz[1])[:, :, :3], tf.matmul(mvi[0], mv[1]), pconf,
- hparams.noise)
-
- loss = (
- hparams.loss_pose * angular +
- hparams.loss_con * loss_con +
- hparams.loss_sep * loss_sep +
- hparams.loss_sill * loss_sill +
- hparams.loss_lr * loss_lr +
- hparams.loss_variance * loss_variance
- )
-
- def touint8(img):
- return tf.cast(img * 255.0, tf.uint8)
-
- with tf.variable_scope("output"):
- tf.summary.image("0_img0", touint8(features["img0"][:, :, :, :3]))
- tf.summary.image("1_combined", viz[0])
- for i in range(hparams.num_kp):
- tf.summary.image("2_f%02d" % i, vizs[0][i])
-
- with tf.variable_scope("stats"):
- tf.summary.scalar("anneal", anneal)
- tf.summary.scalar("closs", loss_con)
- tf.summary.scalar("seploss", loss_sep)
- tf.summary.scalar("angular", angular)
- tf.summary.scalar("chordal", chordal)
- tf.summary.scalar("lrloss", loss_lr)
- tf.summary.scalar("sill", loss_sill)
- tf.summary.scalar("vloss", loss_variance)
-
- return {
- "loss": loss,
- "predictions": {
- "img0": features["img0"],
- "img1": features["img1"],
- "uvz0": uvz[0],
- "uvz1": uvz[1]
- },
- "eval_metric_ops": {
- "closs": tf.metrics.mean(loss_con),
- "angular_loss": tf.metrics.mean(angular),
- "chordal_loss": tf.metrics.mean(chordal),
- }
- }
-
-
-def predict(input_folder, hparams):
- """Predicts keypoints on all images in input_folder."""
-
- cols = plt.cm.get_cmap("rainbow")(
- np.linspace(0, 1.0, hparams.num_kp))[:, :4]
-
- img = tf.placeholder(tf.float32, shape=(1, 128, 128, 4))
-
- with tf.variable_scope("KeypointNetwork"):
- ret = keypoint_network(
- img, hparams.num_filters, hparams.num_kp, False)
-
- uv = tf.reshape(ret[0], [-1, hparams.num_kp, 2])
- z = tf.reshape(ret[1], [-1, hparams.num_kp, 1])
- uvz = tf.concat([uv, z], axis=2)
-
- sess = tf.Session()
- saver = tf.train.Saver()
- ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir)
-
- print("loading model: ", ckpt.model_checkpoint_path)
- saver.restore(sess, ckpt.model_checkpoint_path)
-
- files = [x for x in os.listdir(input_folder)
- if x[-3:] in ["jpg", "png"]]
-
- output_folder = os.path.join(input_folder, "output")
- if not os.path.exists(output_folder):
- os.mkdir(output_folder)
-
- for f in files:
- orig = misc.imread(os.path.join(input_folder, f)).astype(float) / 255
- if orig.shape[2] == 3:
- orig = np.concatenate((orig, np.ones_like(orig[:, :, :1])), axis=2)
-
- uv_ret = sess.run(uvz, feed_dict={img: np.expand_dims(orig, 0)})
-
- utils.draw_ndc_points(orig, uv_ret.reshape(hparams.num_kp, 3), cols)
- misc.imsave(os.path.join(output_folder, f), orig)
-
-
-def _default_hparams():
- """Returns default or overridden user-specified hyperparameters."""
-
- hparams = tf.contrib.training.HParams(
- num_filters=64, # Number of filters.
- num_kp=10, # Numer of keypoints.
-
- loss_pose=0.2, # Pose Loss.
- loss_con=1.0, # Multiview consistency Loss.
- loss_sep=1.0, # Seperation Loss.
- loss_sill=1.0, # Sillhouette Loss.
- loss_lr=1.0, # Orientation Loss.
- loss_variance=0.5, # Variance Loss (part of Sillhouette loss).
-
- sep_delta=0.05, # Seperation threshold.
- noise=0.1, # Noise added during estimating rotation.
-
- learning_rate=1.0e-3,
- lr_anneal_start=30000, # When to anneal in the orientation prediction.
- lr_anneal_end=60000, # When to use the prediction completely.
- )
- if FLAGS.hparams:
- hparams = hparams.parse(FLAGS.hparams)
- return hparams
-
-
-def main(argv):
- del argv
-
- hparams = _default_hparams()
-
- if FLAGS.predict:
- predict(FLAGS.input, hparams)
- else:
- utils.train_and_eval(
- model_dir=FLAGS.model_dir,
- model_fn=model_fn,
- input_fn=create_input_fn,
- hparams=hparams,
- steps=FLAGS.steps,
- batch_size=FLAGS.batch_size,
- save_checkpoints_secs=600,
- eval_throttle_secs=1800,
- eval_steps=5,
- sync_replicas=FLAGS.sync_replicas,
- )
-
-
-if __name__ == "__main__":
- sys.excepthook = utils.colored_hook(
- os.path.dirname(os.path.realpath(__file__)))
- tf.app.run()
diff --git a/research/keypointnet/tools/gen_tfrecords.py b/research/keypointnet/tools/gen_tfrecords.py
deleted file mode 100644
index 2f973b7fe5f16951dbfa01edd2a759b96b4f79db..0000000000000000000000000000000000000000
--- a/research/keypointnet/tools/gen_tfrecords.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# =============================================================================
-"""An example script to generate a tfrecord file from a folder containing the
-renderings.
-
-Example usage:
- python gen_tfrecords.py --input=FOLDER --output=output.tfrecord
-
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import os
-from scipy import misc
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-tf.app.flags.DEFINE_string("input", "", "Input folder containing images")
-tf.app.flags.DEFINE_string("output", "", "Output tfrecord.")
-
-
-def get_matrix(lines):
- return np.array([[float(y) for y in x.strip().split(" ")] for x in lines])
-
-
-def read_model_view_matrices(filename):
- with open(filename, "r") as f:
- lines = f.readlines()
- return get_matrix(lines[:4]), get_matrix(lines[4:])
-
-
-def bytes_feature(values):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[values]))
-
-
-def generate():
- with tf.python_io.TFRecordWriter(FLAGS.output) as tfrecord_writer:
- with tf.Graph().as_default():
- im0 = tf.placeholder(dtype=tf.uint8)
- im1 = tf.placeholder(dtype=tf.uint8)
- encoded0 = tf.image.encode_png(im0)
- encoded1 = tf.image.encode_png(im1)
-
- with tf.Session() as sess:
- count = 0
- indir = FLAGS.input + "/"
- while tf.gfile.Exists(indir + "%06d.txt" % count):
- print("saving %06d" % count)
- image0 = misc.imread(indir + "%06d.png" % (count * 2))
- image1 = misc.imread(indir + "%06d.png" % (count * 2 + 1))
-
- mat0, mat1 = read_model_view_matrices(indir + "%06d.txt" % count)
-
- mati0 = np.linalg.inv(mat0).flatten()
- mati1 = np.linalg.inv(mat1).flatten()
- mat0 = mat0.flatten()
- mat1 = mat1.flatten()
-
- st0, st1 = sess.run([encoded0, encoded1],
- feed_dict={im0: image0, im1: image1})
-
- example = tf.train.Example(features=tf.train.Features(feature={
- 'img0': bytes_feature(st0),
- 'img1': bytes_feature(st1),
- 'mv0': tf.train.Feature(
- float_list=tf.train.FloatList(value=mat0)),
- 'mvi0': tf.train.Feature(
- float_list=tf.train.FloatList(value=mati0)),
- 'mv1': tf.train.Feature(
- float_list=tf.train.FloatList(value=mat1)),
- 'mvi1': tf.train.Feature(
- float_list=tf.train.FloatList(value=mati1)),
- }))
-
- tfrecord_writer.write(example.SerializeToString())
- count += 1
-
-
-def main(argv):
- del argv
- generate()
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/keypointnet/tools/render.py b/research/keypointnet/tools/render.py
deleted file mode 100644
index 3a8872675d83cc414d6348dbc7a56e924541b8d7..0000000000000000000000000000000000000000
--- a/research/keypointnet/tools/render.py
+++ /dev/null
@@ -1,310 +0,0 @@
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# =============================================================================
-"""Script to render object views from ShapeNet obj models.
-
-Example usage:
- blender -b --python render.py -- -m model.obj -o output/ -s 128 -n 120 -fov 5
-
-"""
-from __future__ import print_function
-
-import argparse
-import itertools
-import json
-from math import pi
-import os
-import random
-import sys
-from mathutils import Vector
-import math
-import mathutils
-import time
-import copy
-
-import bpy
-
-sys.path.append(os.path.dirname(__file__))
-
-BG_LUMINANCE = 0
-
-
-def look_at(obj_camera, point):
- loc_camera = obj_camera.location
- direction = point - loc_camera
- # point the cameras '-Z' and use its 'Y' as up
- rot_quat = direction.to_track_quat('-Z', 'Y')
-
- obj_camera.rotation_euler = rot_quat.to_euler()
-
-
-def roll_camera(obj_camera):
- roll_rotate = mathutils.Euler(
- (0, 0, random.random() * math.pi - math.pi * 0.5), 'XYZ')
- obj_camera.rotation_euler = (obj_camera.rotation_euler.to_matrix() *
- roll_rotate.to_matrix()).to_euler()
-
-
-def norm(x):
- return math.sqrt(x[0] * x[0] + x[1] * x[1] + x[2] * x[2])
-
-
-def normalize(x):
- n = norm(x)
- x[0] /= n
- x[1] /= n
- x[2] /= n
-
-
-def random_top_sphere():
- xyz = [random.normalvariate(0, 1) for x in range(3)]
- normalize(xyz)
-
- if xyz[2] < 0:
- xyz[2] *= -1
- return xyz
-
-
-def perturb_sphere(loc, size):
- while True:
- xyz = [random.normalvariate(0, 1) for x in range(3)]
- normalize(xyz)
-
- nloc = [loc[i] + xyz[i] * random.random() * size for i in range(3)]
- normalize(nloc)
-
- if nloc[2] >= 0:
- return nloc
-
-
-def perturb(loc, size):
- while True:
- nloc = [loc[i] + random.random() * size * 2 - size for i in range(3)]
- if nloc[2] >= 0:
- return nloc
-
- bpy.ops.object.mode_set()
-
-
-def delete_all_objects():
- bpy.ops.object.select_by_type(type="MESH")
- bpy.ops.object.delete(use_global=False)
-
-
-def set_scene(render_size, fov, alpha=False):
- """Set up default scene properties."""
- delete_all_objects()
-
- cam = bpy.data.cameras["Camera"]
- cam.angle = fov * pi / 180
-
- light = bpy.data.objects["Lamp"]
- light.location = (0, 0, 1)
- look_at(light, Vector((0.0, 0, 0)))
- bpy.data.lamps['Lamp'].type = "HEMI"
- bpy.data.lamps['Lamp'].energy = 1
- bpy.data.lamps['Lamp'].use_specular = False
- bpy.data.lamps['Lamp'].use_diffuse = True
-
- bpy.context.scene.world.horizon_color = (
- BG_LUMINANCE, BG_LUMINANCE, BG_LUMINANCE)
-
- bpy.context.scene.render.resolution_x = render_size
- bpy.context.scene.render.resolution_y = render_size
- bpy.context.scene.render.resolution_percentage = 100
-
- bpy.context.scene.render.use_antialiasing = True
- bpy.context.scene.render.antialiasing_samples = '5'
-
-
-def get_modelview_matrix():
- cam = bpy.data.objects["Camera"]
- bpy.context.scene.update()
-
- # when apply to object with CV coordinate i.e. to_blender * obj
- # this gives object in blender coordinate
- to_blender = mathutils.Matrix(
- ((1., 0., 0., 0.),
- (0., 0., -1., 0.),
- (0., 1., 0., 0.),
- (0., 0., 0., 1.)))
- return cam.matrix_world.inverted() * to_blender
-
-
-def print_matrix(f, mat):
- for i in range(4):
- for j in range(4):
- f.write("%lf " % mat[i][j])
- f.write("\n")
-
-
-def mul(loc, v):
- return [loc[i] * v for i in range(3)]
-
-
-def merge_all():
- bpy.ops.object.select_by_type(type="MESH")
- bpy.context.scene.objects.active = bpy.context.selected_objects[0]
- bpy.ops.object.join()
- obj = bpy.context.scene.objects.active
- bpy.ops.object.origin_set(type="ORIGIN_CENTER_OF_MASS")
- return obj
-
-
-def insert_frame(obj, frame_number):
- obj.keyframe_insert(data_path="location", frame=frame_number)
- obj.keyframe_insert(data_path="rotation_euler", frame=frame_number)
- obj.keyframe_insert(data_path="scale", frame=frame_number)
-
-
-def render(output_prefix):
- bpy.context.scene.render.filepath = output_prefix
- bpy.context.scene.render.image_settings.file_format = "PNG"
- bpy.context.scene.render.alpha_mode = "TRANSPARENT"
- bpy.context.scene.render.image_settings.color_mode = "RGBA"
- bpy.ops.render.render(write_still=True, animation=True)
-
-
-def render_obj(
- obj_fn, save_dir, n, perturb_size, rotate=False, roll=False, scale=1.0):
-
- # Load object.
- bpy.ops.import_scene.obj(filepath=obj_fn)
- cur_obj = merge_all()
-
- scale = 2.0 / max(cur_obj.dimensions) * scale
- cur_obj.scale = (scale, scale, scale)
- # Using the center of mass as the origin doesn't really work, because Blender
- # assumes the object is a solid shell. This seems to generate better-looking
- # rotations.
-
- bpy.ops.object.origin_set(type='ORIGIN_GEOMETRY', center='BOUNDS')
-
- # bpy.ops.mesh.primitive_cube_add(location=(0, 0, 1))
- # cube = bpy.data.objects["Cube"]
- # cube.scale = (0.2, 0.2, 0.2)
-
- for polygon in cur_obj.data.polygons:
- polygon.use_smooth = True
-
- bpy.ops.object.select_all(action="DESELECT")
-
- camera = bpy.data.objects["Camera"]
-
- # os.system("mkdir " + save_dir)
- for i in range(n):
- fo = open(save_dir + "/%06d.txt" % i, "w")
- d = 30
- shift = 0.2
- if rotate:
- t = 1.0 * i / (n-1) * 2 * math.pi
- loc = [math.sin(t), math.cos(t), 1]
-
- normalize(loc)
- camera.location = mul(loc, d)
- look_at(camera, Vector((0.0, 0, 0)))
-
- print_matrix(fo, get_modelview_matrix())
- print_matrix(fo, get_modelview_matrix())
-
- insert_frame(camera, 2 * i)
- insert_frame(camera, 2 * i + 1)
-
- else:
- loc = random_top_sphere()
-
- camera.location = mul(loc, d)
- look_at(camera, Vector((0.0, 0, 0)))
-
- if roll:
- roll_camera(camera)
- camera.location = perturb(mul(loc, d), shift)
-
- print_matrix(fo, get_modelview_matrix())
- insert_frame(camera, 2 * i)
-
- if perturb_size > 0:
- loc = perturb_sphere(loc, perturb_size)
- else:
- loc = random_top_sphere()
-
- camera.location = mul(loc, d)
- look_at(camera, Vector((0.0, 0, 0)))
- if roll:
- roll_camera(camera)
- camera.location = perturb(mul(loc, d), shift)
-
- print_matrix(fo, get_modelview_matrix())
- insert_frame(camera, 2 * i + 1)
-
- fo.close()
-
- # Create a bunch of views of the object
- bpy.context.scene.frame_start = 0
- bpy.context.scene.frame_end = 2 * n - 1
-
- stem = os.path.join(save_dir, '######')
- render(stem)
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('-m', '--model', dest='model',
- required=True,
- help='Path to model obj file.')
- parser.add_argument('-o', '--output_dir', dest='output_dir',
- required=True,
- help='Where to output files.')
- parser.add_argument('-s', '--output_size', dest='output_size',
- required=True,
- help='Width and height of output in pixels, e.g. 32x32.')
- parser.add_argument('-n', '--num_frames', dest='n', type=int,
- required=True,
- help='Number of frames to generate per clip.')
-
- parser.add_argument('-scale', '--scale', dest='scale', type=float,
- help='object scaling', default=1)
-
- parser.add_argument('-perturb', '--perturb', dest='perturb', type=float,
- help='sphere perturbation', default=0)
-
- parser.add_argument('-rotate', '--rotate', dest='rotate', action='store_true',
- help='render rotating test set')
-
- parser.add_argument('-roll', '--roll', dest='roll', action='store_true',
- help='add roll')
-
- parser.add_argument(
- '-fov', '--fov', dest='fov', type=float, required=True,
- help='field of view')
-
- if '--' not in sys.argv:
- parser.print_help()
- exit(1)
-
- argv = sys.argv[sys.argv.index('--') + 1:]
- args, _ = parser.parse_known_args(argv)
-
- random.seed(args.model + str(time.time()) + str(os.getpid()))
- # random.seed(0)
-
- set_scene(int(args.output_size), args.fov)
- render_obj(
- args.model, args.output_dir, args.n, args.perturb, args.rotate,
- args.roll, args.scale)
- exit()
-
-
-if __name__ == '__main__':
- main()
diff --git a/research/keypointnet/utils.py b/research/keypointnet/utils.py
deleted file mode 100644
index 148b7a3ed843638cff597be0c462b7e335df9857..0000000000000000000000000000000000000000
--- a/research/keypointnet/utils.py
+++ /dev/null
@@ -1,307 +0,0 @@
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# =============================================================================
-"""Utility functions for KeypointNet.
-
-These are helper / tensorflow related functions. The actual implementation and
-algorithm is in main.py.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-import numpy as np
-import os
-import re
-import tensorflow as tf
-import tensorflow.contrib.slim as slim
-import time
-import traceback
-
-
-class TrainingHook(tf.train.SessionRunHook):
- """A utility for displaying training information such as the loss, percent
- completed, estimated finish date and time."""
-
- def __init__(self, steps):
- self.steps = steps
-
- self.last_time = time.time()
- self.last_est = self.last_time
-
- self.eta_interval = int(math.ceil(0.1 * self.steps))
- self.current_interval = 0
-
- def before_run(self, run_context):
- graph = tf.get_default_graph()
- return tf.train.SessionRunArgs(
- {"loss": graph.get_collection("total_loss")[0]})
-
- def after_run(self, run_context, run_values):
- step = run_context.session.run(tf.train.get_global_step())
- now = time.time()
-
- if self.current_interval < self.eta_interval:
- self.duration = now - self.last_est
- self.current_interval += 1
- if step % self.eta_interval == 0:
- self.duration = now - self.last_est
- self.last_est = now
-
- eta_time = float(self.steps - step) / self.current_interval * \
- self.duration
- m, s = divmod(eta_time, 60)
- h, m = divmod(m, 60)
- eta = "%d:%02d:%02d" % (h, m, s)
-
- print("%.2f%% (%d/%d): %.3e t %.3f @ %s (%s)" % (
- step * 100.0 / self.steps,
- step,
- self.steps,
- run_values.results["loss"],
- now - self.last_time,
- time.strftime("%a %d %H:%M:%S", time.localtime(time.time() + eta_time)),
- eta))
-
- self.last_time = now
-
-
-def standard_model_fn(
- func, steps, run_config=None, sync_replicas=0, optimizer_fn=None):
- """Creates model_fn for tf.Estimator.
-
- Args:
- func: A model_fn with prototype model_fn(features, labels, mode, hparams).
- steps: Training steps.
- run_config: tf.estimatorRunConfig (usually passed in from TF_CONFIG).
- sync_replicas: The number of replicas used to compute gradient for
- synchronous training.
- optimizer_fn: The type of the optimizer. Default to Adam.
-
- Returns:
- model_fn for tf.estimator.Estimator.
- """
-
- def fn(features, labels, mode, params):
- """Returns model_fn for tf.estimator.Estimator."""
-
- is_training = (mode == tf.estimator.ModeKeys.TRAIN)
- ret = func(features, labels, mode, params)
-
- tf.add_to_collection("total_loss", ret["loss"])
- train_op = None
-
- training_hooks = []
- if is_training:
- training_hooks.append(TrainingHook(steps))
-
- if optimizer_fn is None:
- optimizer = tf.train.AdamOptimizer(params.learning_rate)
- else:
- optimizer = optimizer_fn
-
- if run_config is not None and run_config.num_worker_replicas > 1:
- sr = sync_replicas
- if sr <= 0:
- sr = run_config.num_worker_replicas
-
- optimizer = tf.train.SyncReplicasOptimizer(
- optimizer,
- replicas_to_aggregate=sr,
- total_num_replicas=run_config.num_worker_replicas)
-
- training_hooks.append(
- optimizer.make_session_run_hook(
- run_config.is_chief, num_tokens=run_config.num_worker_replicas))
-
- optimizer = tf.contrib.estimator.clip_gradients_by_norm(optimizer, 5)
- train_op = slim.learning.create_train_op(ret["loss"], optimizer)
-
- if "eval_metric_ops" not in ret:
- ret["eval_metric_ops"] = {}
-
- return tf.estimator.EstimatorSpec(
- mode=mode,
- predictions=ret["predictions"],
- loss=ret["loss"],
- train_op=train_op,
- eval_metric_ops=ret["eval_metric_ops"],
- training_hooks=training_hooks)
- return fn
-
-
-def train_and_eval(
- model_dir,
- steps,
- batch_size,
- model_fn,
- input_fn,
- hparams,
- keep_checkpoint_every_n_hours=0.5,
- save_checkpoints_secs=180,
- save_summary_steps=50,
- eval_steps=20,
- eval_start_delay_secs=10,
- eval_throttle_secs=300,
- sync_replicas=0):
- """Trains and evaluates our model. Supports local and distributed training.
-
- Args:
- model_dir: The output directory for trained parameters, checkpoints, etc.
- steps: Training steps.
- batch_size: Batch size.
- model_fn: A func with prototype model_fn(features, labels, mode, hparams).
- input_fn: A input function for the tf.estimator.Estimator.
- hparams: tf.HParams containing a set of hyperparameters.
- keep_checkpoint_every_n_hours: Number of hours between each checkpoint
- to be saved.
- save_checkpoints_secs: Save checkpoints every this many seconds.
- save_summary_steps: Save summaries every this many steps.
- eval_steps: Number of steps to evaluate model.
- eval_start_delay_secs: Start evaluating after waiting for this many seconds.
- eval_throttle_secs: Do not re-evaluate unless the last evaluation was
- started at least this many seconds ago
- sync_replicas: Number of synchronous replicas for distributed training.
-
- Returns:
- None
- """
-
- run_config = tf.estimator.RunConfig(
- keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
- save_checkpoints_secs=save_checkpoints_secs,
- save_summary_steps=save_summary_steps)
-
- estimator = tf.estimator.Estimator(
- model_dir=model_dir,
- model_fn=standard_model_fn(
- model_fn,
- steps,
- run_config,
- sync_replicas=sync_replicas),
- params=hparams, config=run_config)
-
- train_spec = tf.estimator.TrainSpec(
- input_fn=input_fn(split="train", batch_size=batch_size),
- max_steps=steps)
-
- eval_spec = tf.estimator.EvalSpec(
- input_fn=input_fn(split="validation", batch_size=batch_size),
- steps=eval_steps,
- start_delay_secs=eval_start_delay_secs,
- throttle_secs=eval_throttle_secs)
-
- tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
-
-
-def draw_circle(rgb, u, v, col, r):
- """Draws a simple anti-aliasing circle in-place.
-
- Args:
- rgb: Input image to be modified.
- u: Horizontal coordinate.
- v: Vertical coordinate.
- col: Color.
- r: Radius.
- """
-
- ir = int(math.ceil(r))
- for i in range(-ir-1, ir+2):
- for j in range(-ir-1, ir+2):
- nu = int(round(u + i))
- nv = int(round(v + j))
- if nu < 0 or nu >= rgb.shape[1] or nv < 0 or nv >= rgb.shape[0]:
- continue
-
- du = abs(nu - u)
- dv = abs(nv - v)
-
- # need sqrt to keep scale
- t = math.sqrt(du * du + dv * dv) - math.sqrt(r * r)
- if t < 0:
- rgb[nv, nu, :] = col
- else:
- t = 1 - t
- if t > 0:
- # t = t ** 0.3
- rgb[nv, nu, :] = col * t + rgb[nv, nu, :] * (1-t)
-
-
-def draw_ndc_points(rgb, xy, cols):
- """Draws keypoints onto an input image.
-
- Args:
- rgb: Input image to be modified.
- xy: [n x 2] matrix of 2D locations.
- cols: A list of colors for the keypoints.
- """
-
- vh, vw = rgb.shape[0], rgb.shape[1]
-
- for j in range(len(cols)):
- x, y = xy[j, :2]
- x = (min(max(x, -1), 1) * vw / 2 + vw / 2) - 0.5
- y = vh - 0.5 - (min(max(y, -1), 1) * vh / 2 + vh / 2)
-
- x = int(round(x))
- y = int(round(y))
- if x < 0 or y < 0 or x >= vw or y >= vh:
- continue
-
- rad = 1.5
- rad *= rgb.shape[0] / 128.0
- draw_circle(rgb, x, y, np.array([0.0, 0.0, 0.0, 1.0]), rad * 1.5)
- draw_circle(rgb, x, y, cols[j], rad)
-
-
-def colored_hook(home_dir):
- """Colorizes python's error message.
-
- Args:
- home_dir: directory where code resides (to highlight your own files).
- Returns:
- The traceback hook.
- """
-
- def hook(type_, value, tb):
- def colorize(text, color, own=0):
- """Returns colorized text."""
- endcolor = "\x1b[0m"
- codes = {
- "green": "\x1b[0;32m",
- "green_own": "\x1b[1;32;40m",
- "red": "\x1b[0;31m",
- "red_own": "\x1b[1;31m",
- "yellow": "\x1b[0;33m",
- "yellow_own": "\x1b[1;33m",
- "black": "\x1b[0;90m",
- "black_own": "\x1b[1;90m",
- "cyan": "\033[1;36m",
- }
- return codes[color + ("_own" if own else "")] + text + endcolor
-
- for filename, line_num, func, text in traceback.extract_tb(tb):
- basename = os.path.basename(filename)
- own = (home_dir in filename) or ("/" not in filename)
-
- print(colorize("\"" + basename + '"', "green", own) + " in " + func)
- print("%s: %s" % (
- colorize("%5d" % line_num, "red", own),
- colorize(text, "yellow", own)))
- print(" %s" % colorize(filename, "black", own))
-
- print(colorize("%s: %s" % (type_.__name__, value), "cyan"))
- return hook
diff --git a/research/learned_optimizer/.gitignore b/research/learned_optimizer/.gitignore
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/learned_optimizer/BUILD b/research/learned_optimizer/BUILD
deleted file mode 100644
index 629c9a06b51d10eb7cab69ed0d9dd0bfa52fd2f0..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/BUILD
+++ /dev/null
@@ -1,33 +0,0 @@
-# Learning to Optimize Learning (LOL)
-
-package(default_visibility = ["//visibility:public"])
-
-# Libraries
-# =========
-
-py_library(
- name = "metaopt",
- srcs = ["metaopt.py"],
- deps = [
- "//learned_optimizer/problems:datasets",
- "//learned_optimizer/problems:problem_generator",
- ],
-)
-
-# Binaries
-# ========
-py_binary(
- name = "metarun",
- srcs = ["metarun.py"],
- deps = [
- ":metaopt",
- "//learned_optimizer/optimizer:coordinatewise_rnn",
- "//learned_optimizer/optimizer:global_learning_rate",
- "//learned_optimizer/optimizer:hierarchical_rnn",
- "//learned_optimizer/optimizer:learning_rate_schedule",
- "//learned_optimizer/optimizer:trainable_adam",
- "//learned_optimizer/problems:problem_sets",
- "//learned_optimizer/problems:problem_spec",
- ],
-)
-
diff --git a/research/learned_optimizer/README.md b/research/learned_optimizer/README.md
deleted file mode 100644
index 6a32514f053f97bc64dc87c4ec972c8223a83fe2..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/README.md
+++ /dev/null
@@ -1,47 +0,0 @@
-
-
-
-
-# Learned Optimizer
-
-Code for [Learned Optimizers that Scale and Generalize](https://arxiv.org/abs/1703.04813).
-
-## Requirements
-
-* Bazel ([install](https://bazel.build/versions/master/docs/install.html))
-* TensorFlow >= v1.3
-* Python 2.7.x
-
-## Training a Learned Optimizer
-
-## Code Overview
-In the top-level directory, ```metaopt.py``` contains the code to train and test a learned optimizer. ```metarun.py``` packages the actual training procedure into a
-single file, defining and exposing many flags to tune the procedure, from selecting the optimizer type and problem set to more fine-grained hyperparameter settings.
-There is no testing binary; testing can be done ad-hoc via ```metaopt.test_optimizer``` by passing an optimizer object and a directory with a checkpoint.
-
-The ```optimizer``` directory contains a base ```trainable_optimizer.py``` class and a number of extensions, including the ```hierarchical_rnn``` optimizer used in
-the paper, a ```coordinatewise_rnn``` optimizer that more closely matches previous work, and a number of simpler optimizers to demonstrate the basic mechanics of
-a learnable optimizer.
-
-The ```problems``` directory contains the code to build the problems that were used in the meta-training set.
-
-### Binaries
-```metarun.py```: meta-training of a learned optimizer
-
-### Command-Line Flags
-The flags most relevant to meta-training are defined in ```metarun.py```. The default values will meta-train a HierarchicalRNN optimizer with the hyperparameter
-settings used in the paper.
-
-### Using a Learned Optimizer as a Black Box
-The ```trainable_optimizer``` inherits from ```tf.train.Optimizer```, so a properly instantiated version can be used to train any model in any APIs that accept
-this class. There are just 2 caveats:
-
-1. If using the Hierarchical RNN optimizer, the apply_gradients return type must be changed (see comments inline for what exactly must be removed)
-
-2. Care must be taken to restore the variables from the optimizer without overriding them. Optimizer variables should be loaded manually using a pretrained checkpoint
-and a ```tf.train.Saver``` with only the optimizer variables. Then, when constructing the session, ensure that any automatic variable initialization does not
-re-initialize the loaded optimizer variables.
-
-## Contact for Issues
-
-* Olga Wichrowska (@olganw), Niru Maheswaranathan (@nirum)
diff --git a/research/learned_optimizer/metaopt.py b/research/learned_optimizer/metaopt.py
deleted file mode 100644
index 62c06272d3096ed63296744792c8742826380536..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/metaopt.py
+++ /dev/null
@@ -1,639 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Helper utilities for training and testing optimizers."""
-
-from collections import defaultdict
-import random
-import sys
-import time
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from learned_optimizer.optimizer import trainable_optimizer
-from learned_optimizer.optimizer import utils
-from learned_optimizer.problems import datasets
-from learned_optimizer.problems import problem_generator
-
-tf.app.flags.DEFINE_integer("ps_tasks", 0,
- """Number of tasks in the ps job.
- If 0 no ps job is used.""")
-tf.app.flags.DEFINE_float("nan_l2_reg", 1e-2,
- """Strength of l2-reg when NaNs are encountered.""")
-tf.app.flags.DEFINE_float("l2_reg", 0.,
- """Lambda value for parameter regularization.""")
-# Default is 0.9
-tf.app.flags.DEFINE_float("rms_decay", 0.9,
- """Decay value for the RMSProp metaoptimizer.""")
-# Default is 1e-10
-tf.app.flags.DEFINE_float("rms_epsilon", 1e-20,
- """Epsilon value for the RMSProp metaoptimizer.""")
-tf.app.flags.DEFINE_boolean("set_profiling", False,
- """Enable memory usage and computation time """
- """tracing for tensorflow nodes (available in """
- """TensorBoard).""")
-tf.app.flags.DEFINE_boolean("reset_rnn_params", True,
- """Reset the parameters of the optimizer
- from one meta-iteration to the next.""")
-
-FLAGS = tf.app.flags.FLAGS
-OPTIMIZER_SCOPE = "LOL"
-OPT_SUM_COLLECTION = "LOL_summaries"
-
-
-def sigmoid_weights(n, slope=0.1, offset=5):
- """Generates a sigmoid, scaled to sum to 1.
-
- This function is used to generate weights that serve to mask out
- the early objective values of an optimization problem such that
- initial variation in the objective is phased out (hence the sigmoid
- starts at zero and ramps up to the maximum value, and the total
- weight is normalized to sum to one)
-
- Args:
- n: the number of samples
- slope: slope of the sigmoid (Default: 0.1)
- offset: threshold of the sigmoid (Default: 5)
-
- Returns:
- No
- """
- x = np.arange(n)
- y = 1. / (1. + np.exp(-slope * (x-offset)))
- y_normalized = y / np.sum(y)
- return y_normalized
-
-
-def sample_numiter(scale, min_steps=50):
- """Samples a number of iterations from an exponential distribution.
-
- Args:
- scale: parameter for the exponential distribution
- min_steps: minimum number of steps to run (additive)
-
- Returns:
- num_steps: An integer equal to a rounded sample from the exponential
- distribution + the value of min_steps.
- """
- return int(np.round(np.random.exponential(scale=scale)) + min_steps)
-
-
-def train_optimizer(logdir,
- optimizer_spec,
- problems_and_data,
- num_problems,
- num_meta_iterations,
- num_unroll_func,
- num_partial_unroll_itrs_func,
- learning_rate=1e-4,
- gradient_clip=5.,
- is_chief=False,
- select_random_problems=True,
- callbacks=None,
- obj_train_max_multiplier=-1,
- out=sys.stdout):
- """Trains the meta-parameters of this optimizer.
-
- Args:
- logdir: a directory filepath for storing model checkpoints (must exist)
- optimizer_spec: specification for an Optimizer (see utils.Spec)
- problems_and_data: a list of tuples containing three elements: a problem
- specification (see utils.Spec), a dataset (see datasets.Dataset), and
- a batch_size (int) for generating a problem and corresponding dataset. If
- the problem doesn't have data, set dataset to None.
- num_problems: the number of problems to sample during meta-training
- num_meta_iterations: the number of iterations (steps) to run the
- meta-optimizer for on each subproblem.
- num_unroll_func: called once per meta iteration and returns the number of
- unrolls to do for that meta iteration.
- num_partial_unroll_itrs_func: called once per unroll and returns the number
- of iterations to do for that unroll.
- learning_rate: learning rate of the RMSProp meta-optimizer (Default: 1e-4)
- gradient_clip: value to clip gradients at (Default: 5.0)
- is_chief: whether this is the chief task (Default: False)
- select_random_problems: whether to select training problems randomly
- (Default: True)
- callbacks: a list of callback functions that is run after every random
- problem draw
- obj_train_max_multiplier: the maximum increase in the objective value over
- a single training run. Ignored if < 0.
- out: where to write output to, e.g. a file handle (Default: sys.stdout)
-
- Raises:
- ValueError: If one of the subproblems has a negative objective value.
- """
-
- if select_random_problems:
- # iterate over random draws of problem / dataset pairs
- sampler = (random.choice(problems_and_data) for _ in range(num_problems))
- else:
- # iterate over a random shuffle of problems, looping if necessary
- num_repeats = (num_problems / len(problems_and_data)) + 1
- random.shuffle(problems_and_data)
- sampler = (problems_and_data * num_repeats)[:num_problems]
-
- for problem_itr, (problem_spec, dataset, batch_size) in enumerate(sampler):
-
- # timer used to time how long it takes to initialize a problem
- problem_start_time = time.time()
-
- # if dataset is None, use the EMPTY_DATASET
- if dataset is None:
- dataset = datasets.EMPTY_DATASET
- batch_size = dataset.size
-
- # build a new graph for this problem
- graph = tf.Graph()
- real_device_setter = tf.train.replica_device_setter(FLAGS.ps_tasks)
-
- def custom_device_setter(op):
- # Places the local variables onto the workers.
- if trainable_optimizer.is_local_state_variable(op):
- return "/job:worker"
- else:
- return real_device_setter(op)
-
- if real_device_setter:
- device_setter = custom_device_setter
- else:
- device_setter = None
-
- with graph.as_default(), graph.device(device_setter):
-
- # initialize a problem
- problem = problem_spec.build()
-
- # build the optimizer
- opt = optimizer_spec.build()
-
- # get the meta-objective for training the optimizer
- train_output = opt.train(problem, dataset)
-
- state_keys = opt.state_keys
- for key, val in zip(state_keys, train_output.output_state[0]):
- finite_val = utils.make_finite(val, replacement=tf.zeros_like(val))
- tf.summary.histogram("State/{}".format(key), finite_val,
- collections=[OPT_SUM_COLLECTION])
-
- tf.summary.scalar("MetaObjective", train_output.metaobj,
- collections=[OPT_SUM_COLLECTION])
-
- # Per-problem meta-objective
- tf.summary.scalar(problem_spec.callable.__name__ + "_MetaObjective",
- train_output.metaobj,
- collections=[OPT_SUM_COLLECTION])
-
- # create the meta-train_op
- global_step = tf.Variable(0, name="global_step", trainable=False)
- meta_parameters = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
- scope=OPTIMIZER_SCOPE)
- # parameter regularization
- reg_l2 = FLAGS.l2_reg * sum([tf.reduce_sum(param ** 2)
- for param in meta_parameters])
-
- # compute the meta-gradients
- meta_opt = tf.train.RMSPropOptimizer(learning_rate, decay=FLAGS.rms_decay,
- use_locking=True,
- epsilon=FLAGS.rms_epsilon)
- grads_and_vars = meta_opt.compute_gradients(train_output.metaobj + reg_l2,
- meta_parameters)
-
- # clip the gradients
- clipped_grads_and_vars = []
- for grad, var in grads_and_vars:
- clipped_grad = tf.clip_by_value(
- utils.make_finite(grad, replacement=tf.zeros_like(var)),
- -gradient_clip, gradient_clip)
- clipped_grads_and_vars.append((clipped_grad, var))
-
- # histogram summary of grads and vars
- for grad, var in grads_and_vars:
- tf.summary.histogram(
- var.name + "_rawgrad",
- utils.make_finite(
- grad, replacement=tf.zeros_like(grad)),
- collections=[OPT_SUM_COLLECTION])
- for grad, var in clipped_grads_and_vars:
- tf.summary.histogram(var.name + "_var", var,
- collections=[OPT_SUM_COLLECTION])
- tf.summary.histogram(var.name + "_grad", grad,
- collections=[OPT_SUM_COLLECTION])
-
- # builds the train and summary operations
- train_op = meta_opt.apply_gradients(clipped_grads_and_vars,
- global_step=global_step)
-
- # only grab summaries defined for LOL, not inside the problem
- summary_op = tf.summary.merge_all(key=OPT_SUM_COLLECTION)
-
- # make sure the state gets propagated after the gradients and summaries
- # were computed.
- with tf.control_dependencies([train_op, summary_op]):
- propagate_loop_state_ops = []
- for dest, src in zip(
- train_output.init_loop_vars, train_output.output_loop_vars):
- propagate_loop_state_ops.append(dest.assign(src))
- propagate_loop_state_op = tf.group(*propagate_loop_state_ops)
-
- # create the supervisor
- sv = tf.train.Supervisor(
- graph=graph,
- is_chief=is_chief,
- logdir=logdir,
- summary_op=None,
- save_model_secs=0, # we save checkpoints manually
- global_step=global_step,
- )
-
- with sv.managed_session() as sess:
-
- init_time = time.time() - problem_start_time
- out.write("--------- Problem #{} ---------\n".format(problem_itr))
- out.write("{callable.__name__}{args}{kwargs}\n".format(
- **problem_spec.__dict__))
- out.write("Took {} seconds to initialize.\n".format(init_time))
- out.flush()
-
- # For profiling summaries
- if FLAGS.set_profiling:
- summary_writer = tf.summary.FileWriter(logdir, graph=sess.graph)
-
- # used to store information during training
- metadata = defaultdict(list)
-
- for k in range(num_meta_iterations):
-
- if sv.should_stop():
- break
-
- problem.init_fn(sess)
-
- # set run options (for profiling)
- full_trace_opt = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
- run_options = full_trace_opt if FLAGS.set_profiling else None
- run_metadata = tf.RunMetadata() if FLAGS.set_profiling else None
-
- num_unrolls = num_unroll_func()
- partial_unroll_iters = [
- num_partial_unroll_itrs_func() for _ in xrange(num_unrolls)
- ]
- total_num_iter = sum(partial_unroll_iters)
-
- objective_weights = [np.ones(num) / float(num)
- for num in partial_unroll_iters]
- db = dataset.batch_indices(total_num_iter, batch_size)
- dataset_batches = []
- last_index = 0
- for num in partial_unroll_iters:
- dataset_batches.append(db[last_index:last_index + num])
- last_index += num
-
- train_start_time = time.time()
-
- unroll_itr = 0
- additional_log_info = ""
-
- for unroll_itr in range(num_unrolls):
- first_unroll = unroll_itr == 0
- if FLAGS.reset_rnn_params:
- reset_state = first_unroll and k == 0
- else:
- reset_state = first_unroll
-
- feed = {
- train_output.obj_weights: objective_weights[unroll_itr],
- train_output.batches: dataset_batches[unroll_itr],
- train_output.first_unroll: first_unroll,
- train_output.reset_state: reset_state,
- }
-
- # run the train and summary ops
- # when a "save_diagnostics" flag is turned on
- fetches_list = [
- train_output.metaobj,
- train_output.problem_objectives,
- train_output.initial_obj,
- summary_op,
- clipped_grads_and_vars,
- train_op
- ]
- if unroll_itr + 1 < num_unrolls:
- fetches_list += [propagate_loop_state_op]
-
- fetched = sess.run(fetches_list, feed_dict=feed,
- options=run_options, run_metadata=run_metadata)
- meta_obj = fetched[0]
- sub_obj = fetched[1]
- init_obj = fetched[2]
- summ = fetched[3]
- meta_grads_and_params = fetched[4]
-
- # assert that the subproblem objectives are non-negative
- # (this is so that we can rescale the objective by the initial value
- # and not worry about rescaling by a negative value)
- if np.any(sub_obj < 0):
- raise ValueError(
- "Training problem objectives must be nonnegative.")
- # If the objective has increased more than we want, exit this
- # training run and start over on another meta iteration.
- if obj_train_max_multiplier > 0 and (
- sub_obj[-1] > (init_obj +
- abs(init_obj) * (obj_train_max_multiplier - 1))):
- msg = " Broke early at {} out of {} unrolls. ".format(
- unroll_itr + 1, num_unrolls)
- additional_log_info += msg
- break
-
- # only the chief task is allowed to write the summary
- if is_chief:
- sv.summary_computed(sess, summ)
-
- metadata["subproblem_objs"].append(sub_obj)
- # store training metadata to pass to the callback
- metadata["meta_objs"].append(meta_obj)
- metadata["meta_grads_and_params"].append(meta_grads_and_params)
-
- optimization_time = time.time() - train_start_time
-
- if FLAGS.set_profiling:
- summary_name = "%02d_iter%04d_%02d" % (FLAGS.task, problem_itr, k)
- summary_writer.add_run_metadata(run_metadata, summary_name)
-
- metadata["global_step"].append(sess.run(global_step))
- metadata["runtimes"].append(optimization_time)
-
- # write a diagnostic message to the output
- args = (k, meta_obj, optimization_time,
- sum(partial_unroll_iters[:unroll_itr+1]))
- out.write(" [{:02}] {}, {} seconds, {} iters ".format(*args))
- out.write("(unrolled {} steps)".format(
- ", ".join([str(s) for s in partial_unroll_iters[:unroll_itr+1]])))
- out.write("{}\n".format(additional_log_info))
- out.flush()
-
- if FLAGS.set_profiling:
- summary_writer.close()
-
- # force a checkpoint save before we load a new problem
- # only the chief task has the save_path and can write the checkpoint
- if is_chief:
- sv.saver.save(sess, sv.save_path, global_step=global_step)
-
- # run the callbacks on the chief
- if is_chief and callbacks is not None:
- for callback in callbacks:
- if hasattr(callback, "__call__"):
- problem_name = problem_spec.callable.__name__
- callback(problem_name, problem_itr, logdir, metadata)
-
-
-def test_optimizer(optimizer,
- problem,
- num_iter,
- dataset=datasets.EMPTY_DATASET,
- batch_size=None,
- seed=None,
- graph=None,
- logdir=None,
- record_every=None):
- """Tests an optimization algorithm on a given problem.
-
- Args:
- optimizer: Either a tf.train.Optimizer instance, or an Optimizer instance
- inheriting from trainable_optimizer.py
- problem: A Problem instance that defines an optimization problem to solve
- num_iter: The number of iterations of the optimizer to run
- dataset: The dataset to train the problem against
- batch_size: The number of samples per batch. If None (default), the
- batch size is set to the full batch (dataset.size)
- seed: A random seed used for drawing the initial parameters, or a list of
- numpy arrays used to explicitly initialize the parameters.
- graph: The tensorflow graph to execute (if None, uses the default graph)
- logdir: A directory containing model checkpoints. If given, then the
- parameters of the optimizer are loaded from the latest checkpoint
- in this folder.
- record_every: if an integer, stores the parameters, objective, and gradient
- every recored_every iterations. If None, nothing is stored
-
- Returns:
- objective_values: A list of the objective values during optimization
- parameters: The parameters obtained after training
- records: A dictionary containing lists of the parameters and gradients
- during optimization saved every record_every iterations (empty if
- record_every is set to None)
- """
-
- if dataset is None:
- dataset = datasets.EMPTY_DATASET
- batch_size = dataset.size
- else:
- # default batch size is the entire dataset
- batch_size = dataset.size if batch_size is None else batch_size
-
- graph = tf.get_default_graph() if graph is None else graph
- with graph.as_default():
-
- # define the parameters of the optimization problem
- if isinstance(seed, (list, tuple)):
- # seed is a list of arrays
- params = problem_generator.init_fixed_variables(seed)
- else:
- # seed is an int or None
- params = problem.init_variables(seed)
-
- data_placeholder = tf.placeholder(tf.float32)
- labels_placeholder = tf.placeholder(tf.int32)
-
- # get the problem objective and gradient(s)
- obj = problem.objective(params, data_placeholder, labels_placeholder)
- gradients = problem.gradients(obj, params)
-
- vars_to_preinitialize = params
-
- with tf.Session(graph=graph) as sess:
- # initialize the parameter scope variables; necessary for apply_gradients
- sess.run(tf.variables_initializer(vars_to_preinitialize))
- coord = tf.train.Coordinator()
- threads = tf.train.start_queue_runners(sess=sess, coord=coord)
-
- # create the train operation and training variables
- try:
- train_op, real_params = optimizer.apply_gradients(zip(gradients, params))
- obj = problem.objective(real_params, data_placeholder, labels_placeholder)
- except TypeError:
- # If all goes well, this exception should only be thrown when we are using
- # a non-hrnn optimizer.
- train_op = optimizer.apply_gradients(zip(gradients, params))
-
- vars_to_restore = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
- scope=OPTIMIZER_SCOPE)
- vars_to_initialize = list(
- set(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)) -
- set(vars_to_restore) - set(vars_to_preinitialize))
- # load or initialize optimizer variables
- if logdir is not None:
- restorer = tf.Saver(var_list=vars_to_restore)
- ckpt = tf.train.latest_checkpoint(logdir)
- restorer.restore(sess, ckpt)
- else:
- sess.run(tf.variables_initializer(vars_to_restore))
- # initialize all the other variables
- sess.run(tf.variables_initializer(vars_to_initialize))
-
- problem.init_fn(sess)
-
- # generate the minibatch indices
- batch_inds = dataset.batch_indices(num_iter, batch_size)
-
- # run the train operation for n iterations and save the objectives
- records = defaultdict(list)
- objective_values = []
- for itr, batch in enumerate(batch_inds):
-
- # data to feed in
- feed = {data_placeholder: dataset.data[batch],
- labels_placeholder: dataset.labels[batch]}
- full_feed = {data_placeholder: dataset.data,
- labels_placeholder: dataset.labels}
-
- # record stuff
- if record_every is not None and (itr % record_every) == 0:
- def grad_value(g):
- if isinstance(g, tf.IndexedSlices):
- return g.values
- else:
- return g
-
- records_fetch = {}
- for p in params:
- for key in optimizer.get_slot_names():
- v = optimizer.get_slot(p, key)
- records_fetch[p.name + "_" + key] = v
- gav_fetch = [(grad_value(g), v) for g, v in zip(gradients, params)]
-
- _, gav_eval, records_eval = sess.run(
- (obj, gav_fetch, records_fetch), feed_dict=feed)
- full_obj_eval = sess.run([obj], feed_dict=full_feed)
-
- records["objective"].append(full_obj_eval)
- records["grad_norm"].append([np.linalg.norm(g.ravel())
- for g, _ in gav_eval])
- records["param_norm"].append([np.linalg.norm(v.ravel())
- for _, v in gav_eval])
- records["grad"].append([g for g, _ in gav_eval])
- records["param"].append([v for _, v in gav_eval])
- records["iter"].append(itr)
-
- for k, v in records_eval.iteritems():
- records[k].append(v)
-
- # run the optimization train operation
- objective_values.append(sess.run([train_op, obj], feed_dict=feed)[1])
-
- # final parameters
- parameters = [sess.run(p) for p in params]
- coord.request_stop()
- coord.join(threads)
-
- return objective_values, parameters, records
-
-
-def run_wall_clock_test(optimizer,
- problem,
- num_steps,
- dataset=datasets.EMPTY_DATASET,
- seed=None,
- logdir=None,
- batch_size=None):
- """Runs optimization with the given parameters and return average iter time.
-
- Args:
- optimizer: The tf.train.Optimizer instance
- problem: The problem to optimize (a problem_generator.Problem)
- num_steps: The number of steps to run optimization for
- dataset: The dataset to train the problem against
- seed: The seed used for drawing the initial parameters, or a list of
- numpy arrays used to explicitly initialize the parameters
- logdir: A directory containing model checkpoints. If given, then the
- parameters of the optimizer are loaded from the latest checkpoint
- in this folder.
- batch_size: The number of samples per batch.
-
- Returns:
- The average time in seconds for a single optimization iteration.
- """
- if dataset is None:
- dataset = datasets.EMPTY_DATASET
- batch_size = dataset.size
- else:
- # default batch size is the entire dataset
- batch_size = dataset.size if batch_size is None else batch_size
-
- # define the parameters of the optimization problem
- if isinstance(seed, (list, tuple)):
- # seed is a list of arrays
- params = problem_generator.init_fixed_variables(seed)
- else:
- # seed is an int or None
- params = problem.init_variables(seed)
-
- data_placeholder = tf.placeholder(tf.float32)
- labels_placeholder = tf.placeholder(tf.int32)
-
- obj = problem.objective(params, data_placeholder, labels_placeholder)
- gradients = problem.gradients(obj, params)
- vars_to_preinitialize = params
-
- with tf.Session(graph=tf.get_default_graph()) as sess:
- # initialize the parameter scope variables; necessary for apply_gradients
- sess.run(tf.variables_initializer(vars_to_preinitialize))
- train_op = optimizer.apply_gradients(zip(gradients, params))
- if isinstance(train_op, tuple) or isinstance(train_op, list):
- # LOL apply_gradients returns a tuple. Regular optimizers do not.
- train_op = train_op[0]
- vars_to_restore = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
- scope=OPTIMIZER_SCOPE)
- vars_to_initialize = list(
- set(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)) -
- set(vars_to_restore) - set(vars_to_preinitialize))
- # load or initialize optimizer variables
- if logdir is not None:
- restorer = tf.Saver(var_list=vars_to_restore)
- ckpt = tf.train.latest_checkpoint(logdir)
- restorer.restore(sess, ckpt)
- else:
- sess.run(tf.variables_initializer(vars_to_restore))
- # initialize all the other variables
- sess.run(tf.variables_initializer(vars_to_initialize))
-
- problem.init_fn(sess)
-
- # generate the minibatch indices
- batch_inds = dataset.batch_indices(num_steps, batch_size)
-
- avg_iter_time = []
- for batch in batch_inds:
- # data to feed in
- feed = {data_placeholder: dataset.data[batch],
- labels_placeholder: dataset.labels[batch]}
-
- # run the optimization train operation
- start = time.time()
- sess.run([train_op], feed_dict=feed)
- avg_iter_time.append(time.time() - start)
-
- return np.median(np.array(avg_iter_time))
diff --git a/research/learned_optimizer/metarun.py b/research/learned_optimizer/metarun.py
deleted file mode 100644
index 45a29623c7fd1381cef590c4e8440d8749585b72..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/metarun.py
+++ /dev/null
@@ -1,394 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Scripts for meta-optimization."""
-
-from __future__ import print_function
-
-import os
-
-import tensorflow as tf
-
-import metaopt
-from learned_optimizer.optimizer import coordinatewise_rnn
-from learned_optimizer.optimizer import global_learning_rate
-from learned_optimizer.optimizer import hierarchical_rnn
-from learned_optimizer.optimizer import learning_rate_schedule
-from learned_optimizer.optimizer import trainable_adam
-from learned_optimizer.problems import problem_sets as ps
-from learned_optimizer.problems import problem_spec
-
-tf.app.flags.DEFINE_string("train_dir", "/tmp/lol/",
- """Directory to store parameters and results.""")
-
-tf.app.flags.DEFINE_integer("task", 0,
- """Task id of the replica running the training.""")
-tf.app.flags.DEFINE_integer("worker_tasks", 1,
- """Number of tasks in the worker job.""")
-
-tf.app.flags.DEFINE_integer("num_problems", 1000,
- """Number of sub-problems to run.""")
-tf.app.flags.DEFINE_integer("num_meta_iterations", 5,
- """Number of meta-iterations to optimize.""")
-tf.app.flags.DEFINE_integer("num_unroll_scale", 40,
- """The scale parameter of the exponential
- distribution from which the number of partial
- unrolls is drawn""")
-tf.app.flags.DEFINE_integer("min_num_unrolls", 1,
- """The minimum number of unrolls per problem.""")
-tf.app.flags.DEFINE_integer("num_partial_unroll_itr_scale", 200,
- """The scale parameter of the exponential
- distribution from which the number of iterations
- per unroll is drawn.""")
-tf.app.flags.DEFINE_integer("min_num_itr_partial_unroll", 50,
- """The minimum number of iterations for one
- unroll.""")
-
-tf.app.flags.DEFINE_string("optimizer", "HierarchicalRNN",
- """Which meta-optimizer to train.""")
-
-# CoordinatewiseRNN-specific flags
-tf.app.flags.DEFINE_integer("cell_size", 20,
- """Size of the RNN hidden state in each layer.""")
-tf.app.flags.DEFINE_integer("num_cells", 2,
- """Number of RNN layers.""")
-tf.app.flags.DEFINE_string("cell_cls", "GRUCell",
- """Type of RNN cell to use.""")
-
-# Metaoptimization parameters
-tf.app.flags.DEFINE_float("meta_learning_rate", 1e-6,
- """The learning rate for the meta-optimizer.""")
-tf.app.flags.DEFINE_float("gradient_clip_level", 1e4,
- """The level to clip gradients to.""")
-
-# Training set selection
-tf.app.flags.DEFINE_boolean("include_quadratic_problems", False,
- """Include non-noisy quadratic problems.""")
-tf.app.flags.DEFINE_boolean("include_noisy_quadratic_problems", True,
- """Include noisy quadratic problems.""")
-tf.app.flags.DEFINE_boolean("include_large_quadratic_problems", True,
- """Include very large quadratic problems.""")
-tf.app.flags.DEFINE_boolean("include_bowl_problems", True,
- """Include 2D bowl problems.""")
-tf.app.flags.DEFINE_boolean("include_softmax_2_class_problems", True,
- """Include 2-class logistic regression problems.""")
-tf.app.flags.DEFINE_boolean("include_noisy_softmax_2_class_problems", True,
- """Include noisy 2-class logistic regression
- problems.""")
-tf.app.flags.DEFINE_boolean("include_optimization_test_problems", True,
- """Include non-noisy versions of classic
- optimization test problems, e.g. Rosenbrock.""")
-tf.app.flags.DEFINE_boolean("include_noisy_optimization_test_problems", True,
- """Include gradient-noise versions of classic
- optimization test problems, e.g. Rosenbrock""")
-tf.app.flags.DEFINE_boolean("include_fully_connected_random_2_class_problems",
- True, """Include MLP problems for 2 classes.""")
-tf.app.flags.DEFINE_boolean("include_matmul_problems", True,
- """Include matrix multiplication problems.""")
-tf.app.flags.DEFINE_boolean("include_log_objective_problems", True,
- """Include problems where the objective is the log
- objective of another problem, e.g. Bowl.""")
-tf.app.flags.DEFINE_boolean("include_rescale_problems", True,
- """Include problems where the parameters are scaled
- version of the original parameters.""")
-tf.app.flags.DEFINE_boolean("include_norm_problems", True,
- """Include problems where the objective is the
- N-norm of another problem, e.g. Quadratic.""")
-tf.app.flags.DEFINE_boolean("include_sum_problems", True,
- """Include problems where the objective is the sum
- of the objectives of the subproblems that make
- up the problem parameters. Per-problem tensors
- are still independent of each other.""")
-tf.app.flags.DEFINE_boolean("include_sparse_gradient_problems", True,
- """Include problems where the gradient is set to 0
- with some high probability.""")
-tf.app.flags.DEFINE_boolean("include_sparse_softmax_problems", False,
- """Include sparse softmax problems.""")
-tf.app.flags.DEFINE_boolean("include_one_hot_sparse_softmax_problems", False,
- """Include one-hot sparse softmax problems.""")
-tf.app.flags.DEFINE_boolean("include_noisy_bowl_problems", True,
- """Include noisy bowl problems.""")
-tf.app.flags.DEFINE_boolean("include_noisy_norm_problems", True,
- """Include noisy norm problems.""")
-tf.app.flags.DEFINE_boolean("include_noisy_sum_problems", True,
- """Include noisy sum problems.""")
-tf.app.flags.DEFINE_boolean("include_sum_of_quadratics_problems", False,
- """Include sum of quadratics problems.""")
-tf.app.flags.DEFINE_boolean("include_projection_quadratic_problems", False,
- """Include projection quadratic problems.""")
-tf.app.flags.DEFINE_boolean("include_outward_snake_problems", False,
- """Include outward snake problems.""")
-tf.app.flags.DEFINE_boolean("include_dependency_chain_problems", False,
- """Include dependency chain problems.""")
-tf.app.flags.DEFINE_boolean("include_min_max_well_problems", False,
- """Include min-max well problems.""")
-
-# Optimizer parameters: initialization and scale values
-tf.app.flags.DEFINE_float("min_lr", 1e-6,
- """The minimum initial learning rate.""")
-tf.app.flags.DEFINE_float("max_lr", 1e-2,
- """The maximum initial learning rate.""")
-
-# Optimizer parameters: small features.
-tf.app.flags.DEFINE_boolean("zero_init_lr_weights", True,
- """Whether to initialize the learning rate weights
- to 0 rather than the scaled random initialization
- used for other RNN variables.""")
-tf.app.flags.DEFINE_boolean("use_relative_lr", True,
- """Whether to use the relative learning rate as an
- input during training. Can only be used if
- learnable_decay is also True.""")
-tf.app.flags.DEFINE_boolean("use_extreme_indicator", False,
- """Whether to use the extreme indicator for learning
- rates as an input during training. Can only be
- used if learnable_decay is also True.""")
-tf.app.flags.DEFINE_boolean("use_log_means_squared", True,
- """Whether to track the log of the mean squared
- grads instead of the means squared grads.""")
-tf.app.flags.DEFINE_boolean("use_problem_lr_mean", True,
- """Whether to use the mean over all learning rates
- in the problem when calculating the relative
- learning rate.""")
-
-# Optimizer parameters: major features
-tf.app.flags.DEFINE_boolean("learnable_decay", True,
- """Whether to learn weights that dynamically
- modulate the input scale via RMS decay.""")
-tf.app.flags.DEFINE_boolean("dynamic_output_scale", True,
- """Whether to learn weights that dynamically
- modulate the output scale.""")
-tf.app.flags.DEFINE_boolean("use_log_objective", True,
- """Whether to use the log of the scaled objective
- rather than just the scaled obj for training.""")
-tf.app.flags.DEFINE_boolean("use_attention", False,
- """Whether to learn where to attend.""")
-tf.app.flags.DEFINE_boolean("use_second_derivatives", True,
- """Whether to use second derivatives.""")
-tf.app.flags.DEFINE_integer("num_gradient_scales", 4,
- """How many different timescales to keep for
- gradient history. If > 1, also learns a scale
- factor for gradient history.""")
-tf.app.flags.DEFINE_float("max_log_lr", 33,
- """The maximum log learning rate allowed.""")
-tf.app.flags.DEFINE_float("objective_training_max_multiplier", -1,
- """How much the objective can grow before training on
- this problem / param pair is terminated. Sets a max
- on the objective value when multiplied by the
- initial objective. If <= 0, not used.""")
-tf.app.flags.DEFINE_boolean("use_gradient_shortcut", True,
- """Whether to add a learned affine projection of the
- gradient to the update delta in addition to the
- gradient function computed by the RNN.""")
-tf.app.flags.DEFINE_boolean("use_lr_shortcut", False,
- """Whether to add the difference between the current
- learning rate and the desired learning rate to
- the RNN input.""")
-tf.app.flags.DEFINE_boolean("use_grad_products", True,
- """Whether to use gradient products in the input to
- the RNN. Only applicable when num_gradient_scales
- > 1.""")
-tf.app.flags.DEFINE_boolean("use_multiple_scale_decays", False,
- """Whether to use many-timescale scale decays.""")
-tf.app.flags.DEFINE_boolean("use_numerator_epsilon", False,
- """Whether to use epsilon in the numerator of the
- log objective.""")
-tf.app.flags.DEFINE_boolean("learnable_inp_decay", True,
- """Whether to learn input decay weight and bias.""")
-tf.app.flags.DEFINE_boolean("learnable_rnn_init", True,
- """Whether to learn RNN state initialization.""")
-
-FLAGS = tf.app.flags.FLAGS
-
-# The Size of the RNN hidden state in each layer:
-# [PerParam, PerTensor, Global]. The length of this list must be 1, 2, or 3.
-# If less than 3, the Global and/or PerTensor RNNs will not be created.
-
-HRNN_CELL_SIZES = [10, 20, 20]
-
-
-
-def register_optimizers():
- opts = {}
- opts["CoordinatewiseRNN"] = coordinatewise_rnn.CoordinatewiseRNN
- opts["GlobalLearningRate"] = global_learning_rate.GlobalLearningRate
- opts["HierarchicalRNN"] = hierarchical_rnn.HierarchicalRNN
- opts["LearningRateSchedule"] = learning_rate_schedule.LearningRateSchedule
- opts["TrainableAdam"] = trainable_adam.TrainableAdam
- return opts
-
-
-def main(unused_argv):
- """Runs the main script."""
-
- opts = register_optimizers()
-
- # Choose a set of problems to optimize. By default this includes quadratics,
- # 2-dimensional bowls, 2-class softmax problems, and non-noisy optimization
- # test problems (e.g. Rosenbrock, Beale)
- problems_and_data = []
-
- if FLAGS.include_sparse_softmax_problems:
- problems_and_data.extend(ps.sparse_softmax_2_class_sparse_problems())
-
- if FLAGS.include_one_hot_sparse_softmax_problems:
- problems_and_data.extend(
- ps.one_hot_sparse_softmax_2_class_sparse_problems())
-
- if FLAGS.include_quadratic_problems:
- problems_and_data.extend(ps.quadratic_problems())
-
- if FLAGS.include_noisy_quadratic_problems:
- problems_and_data.extend(ps.quadratic_problems_noisy())
-
- if FLAGS.include_large_quadratic_problems:
- problems_and_data.extend(ps.quadratic_problems_large())
-
- if FLAGS.include_bowl_problems:
- problems_and_data.extend(ps.bowl_problems())
-
- if FLAGS.include_noisy_bowl_problems:
- problems_and_data.extend(ps.bowl_problems_noisy())
-
- if FLAGS.include_softmax_2_class_problems:
- problems_and_data.extend(ps.softmax_2_class_problems())
-
- if FLAGS.include_noisy_softmax_2_class_problems:
- problems_and_data.extend(ps.softmax_2_class_problems_noisy())
-
- if FLAGS.include_optimization_test_problems:
- problems_and_data.extend(ps.optimization_test_problems())
-
- if FLAGS.include_noisy_optimization_test_problems:
- problems_and_data.extend(ps.optimization_test_problems_noisy())
-
- if FLAGS.include_fully_connected_random_2_class_problems:
- problems_and_data.extend(ps.fully_connected_random_2_class_problems())
-
- if FLAGS.include_matmul_problems:
- problems_and_data.extend(ps.matmul_problems())
-
- if FLAGS.include_log_objective_problems:
- problems_and_data.extend(ps.log_objective_problems())
-
- if FLAGS.include_rescale_problems:
- problems_and_data.extend(ps.rescale_problems())
-
- if FLAGS.include_norm_problems:
- problems_and_data.extend(ps.norm_problems())
-
- if FLAGS.include_noisy_norm_problems:
- problems_and_data.extend(ps.norm_problems_noisy())
-
- if FLAGS.include_sum_problems:
- problems_and_data.extend(ps.sum_problems())
-
- if FLAGS.include_noisy_sum_problems:
- problems_and_data.extend(ps.sum_problems_noisy())
-
- if FLAGS.include_sparse_gradient_problems:
- problems_and_data.extend(ps.sparse_gradient_problems())
- if FLAGS.include_fully_connected_random_2_class_problems:
- problems_and_data.extend(ps.sparse_gradient_problems_mlp())
-
- if FLAGS.include_min_max_well_problems:
- problems_and_data.extend(ps.min_max_well_problems())
-
- if FLAGS.include_sum_of_quadratics_problems:
- problems_and_data.extend(ps.sum_of_quadratics_problems())
-
- if FLAGS.include_projection_quadratic_problems:
- problems_and_data.extend(ps.projection_quadratic_problems())
-
- if FLAGS.include_outward_snake_problems:
- problems_and_data.extend(ps.outward_snake_problems())
-
- if FLAGS.include_dependency_chain_problems:
- problems_and_data.extend(ps.dependency_chain_problems())
-
- # log directory
- logdir = os.path.join(FLAGS.train_dir,
- "{}_{}_{}_{}".format(FLAGS.optimizer,
- FLAGS.cell_cls,
- FLAGS.cell_size,
- FLAGS.num_cells))
-
- # get the optimizer class and arguments
- optimizer_cls = opts[FLAGS.optimizer]
-
- assert len(HRNN_CELL_SIZES) in [1, 2, 3]
- optimizer_args = (HRNN_CELL_SIZES,)
-
- optimizer_kwargs = {
- "init_lr_range": (FLAGS.min_lr, FLAGS.max_lr),
- "learnable_decay": FLAGS.learnable_decay,
- "dynamic_output_scale": FLAGS.dynamic_output_scale,
- "cell_cls": getattr(tf.contrib.rnn, FLAGS.cell_cls),
- "use_attention": FLAGS.use_attention,
- "use_log_objective": FLAGS.use_log_objective,
- "num_gradient_scales": FLAGS.num_gradient_scales,
- "zero_init_lr_weights": FLAGS.zero_init_lr_weights,
- "use_log_means_squared": FLAGS.use_log_means_squared,
- "use_relative_lr": FLAGS.use_relative_lr,
- "use_extreme_indicator": FLAGS.use_extreme_indicator,
- "max_log_lr": FLAGS.max_log_lr,
- "obj_train_max_multiplier": FLAGS.objective_training_max_multiplier,
- "use_problem_lr_mean": FLAGS.use_problem_lr_mean,
- "use_gradient_shortcut": FLAGS.use_gradient_shortcut,
- "use_second_derivatives": FLAGS.use_second_derivatives,
- "use_lr_shortcut": FLAGS.use_lr_shortcut,
- "use_grad_products": FLAGS.use_grad_products,
- "use_multiple_scale_decays": FLAGS.use_multiple_scale_decays,
- "use_numerator_epsilon": FLAGS.use_numerator_epsilon,
- "learnable_inp_decay": FLAGS.learnable_inp_decay,
- "learnable_rnn_init": FLAGS.learnable_rnn_init,
- }
- optimizer_spec = problem_spec.Spec(
- optimizer_cls, optimizer_args, optimizer_kwargs)
-
- # make log directory
- tf.gfile.MakeDirs(logdir)
-
- is_chief = FLAGS.task == 0
- # if this is a distributed run, make the chief run through problems in order
- select_random_problems = FLAGS.worker_tasks == 1 or not is_chief
-
- def num_unrolls():
- return metaopt.sample_numiter(FLAGS.num_unroll_scale, FLAGS.min_num_unrolls)
-
- def num_partial_unroll_itrs():
- return metaopt.sample_numiter(FLAGS.num_partial_unroll_itr_scale,
- FLAGS.min_num_itr_partial_unroll)
-
- # run it
- metaopt.train_optimizer(
- logdir,
- optimizer_spec,
- problems_and_data,
- FLAGS.num_problems,
- FLAGS.num_meta_iterations,
- num_unrolls,
- num_partial_unroll_itrs,
- learning_rate=FLAGS.meta_learning_rate,
- gradient_clip=FLAGS.gradient_clip_level,
- is_chief=is_chief,
- select_random_problems=select_random_problems,
- obj_train_max_multiplier=FLAGS.objective_training_max_multiplier,
- callbacks=[])
-
- return 0
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/learned_optimizer/optimizer/BUILD b/research/learned_optimizer/optimizer/BUILD
deleted file mode 100644
index 8953e7592ace416b786be2a6fa59f4c537c82644..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/BUILD
+++ /dev/null
@@ -1,69 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-# Libraries
-# =========
-py_library(
- name = "coordinatewise_rnn",
- srcs = ["coordinatewise_rnn.py"],
- deps = [
- ":trainable_optimizer",
- ":utils",
- ],
-)
-
-py_library(
- name = "global_learning_rate",
- srcs = ["global_learning_rate.py"],
- deps = [
- ":trainable_optimizer",
- ],
-)
-
-py_library(
- name = "hierarchical_rnn",
- srcs = ["hierarchical_rnn.py"],
- deps = [
- ":rnn_cells",
- ":trainable_optimizer",
- ":utils",
- ],
-)
-
-py_library(
- name = "learning_rate_schedule",
- srcs = ["learning_rate_schedule.py"],
- deps = [
- ":trainable_optimizer",
- ],
-)
-
-py_library(
- name = "rnn_cells",
- srcs = ["rnn_cells.py"],
- deps = [
- ":utils",
- ],
-)
-
-py_library(
- name = "trainable_adam",
- srcs = ["trainable_adam.py"],
- deps = [
- ":trainable_optimizer",
- ":utils",
- ],
-)
-
-py_library(
- name = "trainable_optimizer",
- srcs = ["trainable_optimizer.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "utils",
- srcs = ["utils.py"],
- deps = [
- ],
-)
diff --git a/research/learned_optimizer/optimizer/coordinatewise_rnn.py b/research/learned_optimizer/optimizer/coordinatewise_rnn.py
deleted file mode 100644
index 3d699504b7a3d86643bea6b295d20b2434131a99..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/coordinatewise_rnn.py
+++ /dev/null
@@ -1,316 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Collection of trainable optimizers for meta-optimization."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-
-import numpy as np
-import tensorflow as tf
-
-from learned_optimizer.optimizer import utils
-from learned_optimizer.optimizer import trainable_optimizer as opt
-
-
-# Default was 1e-3
-tf.app.flags.DEFINE_float("crnn_rnn_readout_scale", 0.5,
- """The initialization scale for the RNN readouts.""")
-tf.app.flags.DEFINE_float("crnn_default_decay_var_init", 2.2,
- """The default initializer value for any decay/
- momentum style variables and constants.
- sigmoid(2.2) ~ 0.9, sigmoid(-2.2) ~ 0.01.""")
-
-FLAGS = tf.flags.FLAGS
-
-
-class CoordinatewiseRNN(opt.TrainableOptimizer):
- """RNN that operates on each coordinate of the problem independently."""
-
- def __init__(self,
- cell_sizes,
- cell_cls,
- init_lr_range=(1., 1.),
- dynamic_output_scale=True,
- learnable_decay=True,
- zero_init_lr_weights=False,
- **kwargs):
- """Initializes the RNN per-parameter optimizer.
-
- Args:
- cell_sizes: List of hidden state sizes for each RNN cell in the network
- cell_cls: tf.contrib.rnn class for specifying the RNN cell type
- init_lr_range: the range in which to initialize the learning rates.
- dynamic_output_scale: whether to learn weights that dynamically modulate
- the output scale (default: True)
- learnable_decay: whether to learn weights that dynamically modulate the
- input scale via RMS style decay (default: True)
- zero_init_lr_weights: whether to initialize the lr weights to zero
- **kwargs: args passed to TrainableOptimizer's constructor
-
- Raises:
- ValueError: If the init lr range is not of length 2.
- ValueError: If the init lr range is not a valid range (min > max).
- """
- if len(init_lr_range) != 2:
- raise ValueError(
- "Initial LR range must be len 2, was {}".format(len(init_lr_range)))
- if init_lr_range[0] > init_lr_range[1]:
- raise ValueError("Initial LR range min is greater than max.")
- self.init_lr_range = init_lr_range
-
- self.zero_init_lr_weights = zero_init_lr_weights
- self.reuse_vars = False
-
- # create the RNN cell
- with tf.variable_scope(opt.OPTIMIZER_SCOPE):
- self.component_cells = [cell_cls(sz) for sz in cell_sizes]
- self.cell = tf.contrib.rnn.MultiRNNCell(self.component_cells)
-
- # random normal initialization scaled by the output size
- scale_factor = FLAGS.crnn_rnn_readout_scale / math.sqrt(cell_sizes[-1])
- scaled_init = tf.random_normal_initializer(0., scale_factor)
-
- # weights for projecting the hidden state to a parameter update
- self.update_weights = tf.get_variable("update_weights",
- shape=(cell_sizes[-1], 1),
- initializer=scaled_init)
-
- self._initialize_decay(learnable_decay, (cell_sizes[-1], 1), scaled_init)
-
- self._initialize_lr(dynamic_output_scale, (cell_sizes[-1], 1),
- scaled_init)
-
- state_size = sum([sum(state_size) for state_size in self.cell.state_size])
- self._init_vector = tf.get_variable(
- "init_vector", shape=[1, state_size],
- initializer=tf.random_uniform_initializer(-1., 1.))
-
- state_keys = ["rms", "rnn", "learning_rate", "decay"]
- super(CoordinatewiseRNN, self).__init__("cRNN", state_keys, **kwargs)
-
- def _initialize_decay(
- self, learnable_decay, weights_tensor_shape, scaled_init):
- """Initializes the decay weights and bias variables or tensors.
-
- Args:
- learnable_decay: Whether to use learnable decay.
- weights_tensor_shape: The shape the weight tensor should take.
- scaled_init: The scaled initialization for the weights tensor.
- """
- if learnable_decay:
-
- # weights for projecting the hidden state to the RMS decay term
- self.decay_weights = tf.get_variable("decay_weights",
- shape=weights_tensor_shape,
- initializer=scaled_init)
- self.decay_bias = tf.get_variable(
- "decay_bias", shape=(1,),
- initializer=tf.constant_initializer(
- FLAGS.crnn_default_decay_var_init))
- else:
- self.decay_weights = tf.zeros_like(self.update_weights)
- self.decay_bias = tf.constant(FLAGS.crnn_default_decay_var_init)
-
- def _initialize_lr(
- self, dynamic_output_scale, weights_tensor_shape, scaled_init):
- """Initializes the learning rate weights and bias variables or tensors.
-
- Args:
- dynamic_output_scale: Whether to use a dynamic output scale.
- weights_tensor_shape: The shape the weight tensor should take.
- scaled_init: The scaled initialization for the weights tensor.
- """
- if dynamic_output_scale:
- zero_init = tf.constant_initializer(0.)
- wt_init = zero_init if self.zero_init_lr_weights else scaled_init
- self.lr_weights = tf.get_variable("learning_rate_weights",
- shape=weights_tensor_shape,
- initializer=wt_init)
- self.lr_bias = tf.get_variable("learning_rate_bias", shape=(1,),
- initializer=zero_init)
- else:
- self.lr_weights = tf.zeros_like(self.update_weights)
- self.lr_bias = tf.zeros([1, 1])
-
- def _initialize_state(self, var):
- """Return a dictionary mapping names of state variables to their values."""
- vectorized_shape = [var.get_shape().num_elements(), 1]
-
- min_lr = self.init_lr_range[0]
- max_lr = self.init_lr_range[1]
- if min_lr == max_lr:
- init_lr = tf.constant(min_lr, shape=vectorized_shape)
- else:
- actual_vals = tf.random_uniform(vectorized_shape,
- np.log(min_lr),
- np.log(max_lr))
- init_lr = tf.exp(actual_vals)
-
- ones = tf.ones(vectorized_shape)
- rnn_init = ones * self._init_vector
-
- return {
- "rms": tf.ones(vectorized_shape),
- "learning_rate": init_lr,
- "rnn": rnn_init,
- "decay": tf.ones(vectorized_shape),
- }
-
- def _compute_update(self, param, grad, state):
- """Update parameters given the gradient and state.
-
- Args:
- param: tensor of parameters
- grad: tensor of gradients with the same shape as param
- state: a dictionary containing any state for the optimizer
-
- Returns:
- updated_param: updated parameters
- updated_state: updated state variables in a dictionary
- """
-
- with tf.variable_scope(opt.OPTIMIZER_SCOPE) as scope:
-
- if self.reuse_vars:
- scope.reuse_variables()
- else:
- self.reuse_vars = True
-
- param_shape = tf.shape(param)
-
- (grad_values, decay_state, rms_state, rnn_state, learning_rate_state,
- grad_indices) = self._extract_gradients_and_internal_state(
- grad, state, param_shape)
-
- # Vectorize and scale the gradients.
- grad_scaled, rms = utils.rms_scaling(grad_values, decay_state, rms_state)
-
- # Apply the RNN update.
- rnn_state_tuples = self._unpack_rnn_state_into_tuples(rnn_state)
- rnn_output, rnn_state_tuples = self.cell(grad_scaled, rnn_state_tuples)
- rnn_state = self._pack_tuples_into_rnn_state(rnn_state_tuples)
-
- # Compute the update direction (a linear projection of the RNN output).
- delta = utils.project(rnn_output, self.update_weights)
-
- # The updated decay is an affine projection of the hidden state
- decay = utils.project(rnn_output, self.decay_weights,
- bias=self.decay_bias, activation=tf.nn.sigmoid)
-
- # Compute the change in learning rate (an affine projection of the RNN
- # state, passed through a 2x sigmoid, so the change is bounded).
- learning_rate_change = 2. * utils.project(rnn_output, self.lr_weights,
- bias=self.lr_bias,
- activation=tf.nn.sigmoid)
-
- # Update the learning rate.
- new_learning_rate = learning_rate_change * learning_rate_state
-
- # Apply the update to the parameters.
- update = tf.reshape(new_learning_rate * delta, tf.shape(grad_values))
-
- if isinstance(grad, tf.IndexedSlices):
- update = utils.stack_tensor(update, grad_indices, param,
- param_shape[:1])
- rms = utils.update_slices(rms, grad_indices, state["rms"], param_shape)
- new_learning_rate = utils.update_slices(new_learning_rate, grad_indices,
- state["learning_rate"],
- param_shape)
- rnn_state = utils.update_slices(rnn_state, grad_indices, state["rnn"],
- param_shape)
- decay = utils.update_slices(decay, grad_indices, state["decay"],
- param_shape)
-
- new_param = param - update
-
- # Collect the update and new state.
- new_state = {
- "rms": rms,
- "learning_rate": new_learning_rate,
- "rnn": rnn_state,
- "decay": decay,
- }
-
- return new_param, new_state
-
- def _extract_gradients_and_internal_state(self, grad, state, param_shape):
- """Extracts the gradients and relevant internal state.
-
- If the gradient is sparse, extracts the appropriate slices from the state.
-
- Args:
- grad: The current gradient.
- state: The current state.
- param_shape: The shape of the parameter (used if gradient is sparse).
-
- Returns:
- grad_values: The gradient value tensor.
- decay_state: The current decay state.
- rms_state: The current rms state.
- rnn_state: The current state of the internal rnns.
- learning_rate_state: The current learning rate state.
- grad_indices: The indices for the gradient tensor, if sparse.
- None otherwise.
- """
- if isinstance(grad, tf.IndexedSlices):
- grad_indices, grad_values = utils.accumulate_sparse_gradients(grad)
- decay_state = utils.slice_tensor(state["decay"], grad_indices,
- param_shape)
- rms_state = utils.slice_tensor(state["rms"], grad_indices, param_shape)
- rnn_state = utils.slice_tensor(state["rnn"], grad_indices, param_shape)
- learning_rate_state = utils.slice_tensor(state["learning_rate"],
- grad_indices, param_shape)
- decay_state.set_shape([None, 1])
- rms_state.set_shape([None, 1])
- else:
- grad_values = grad
- grad_indices = None
-
- decay_state = state["decay"]
- rms_state = state["rms"]
- rnn_state = state["rnn"]
- learning_rate_state = state["learning_rate"]
- return (grad_values, decay_state, rms_state, rnn_state, learning_rate_state,
- grad_indices)
-
- def _unpack_rnn_state_into_tuples(self, rnn_state):
- """Creates state tuples from the rnn state vector."""
- rnn_state_tuples = []
- cur_state_pos = 0
- for cell in self.component_cells:
- total_state_size = sum(cell.state_size)
- cur_state = tf.slice(rnn_state, [0, cur_state_pos],
- [-1, total_state_size])
- cur_state_tuple = tf.split(value=cur_state, num_or_size_splits=2,
- axis=1)
- rnn_state_tuples.append(cur_state_tuple)
- cur_state_pos += total_state_size
- return rnn_state_tuples
-
- def _pack_tuples_into_rnn_state(self, rnn_state_tuples):
- """Creates a single state vector concatenated along column axis."""
- rnn_state = None
- for new_state_tuple in rnn_state_tuples:
- new_c, new_h = new_state_tuple
- if rnn_state is None:
- rnn_state = tf.concat([new_c, new_h], axis=1)
- else:
- rnn_state = tf.concat([rnn_state, tf.concat([new_c, new_h], 1)], axis=1)
- return rnn_state
-
diff --git a/research/learned_optimizer/optimizer/global_learning_rate.py b/research/learned_optimizer/optimizer/global_learning_rate.py
deleted file mode 100644
index bcf102fff054e9fe9e92d4379538f6394314fe1c..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/global_learning_rate.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A trainable optimizer that learns a single global learning rate."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from learned_optimizer.optimizer import trainable_optimizer
-
-
-class GlobalLearningRate(trainable_optimizer.TrainableOptimizer):
- """Optimizes for a single global learning rate."""
-
- def __init__(self, initial_rate=1e-3, **kwargs):
- """Initializes the global learning rate."""
- with tf.variable_scope(trainable_optimizer.OPTIMIZER_SCOPE):
- initializer = tf.constant_initializer(initial_rate)
- self.learning_rate = tf.get_variable("global_learning_rate", shape=(),
- initializer=initializer)
- super(GlobalLearningRate, self).__init__("GLR", [], **kwargs)
-
- def _compute_update(self, param, grad, state):
- return param - tf.scalar_mul(self.learning_rate, grad), state
-
diff --git a/research/learned_optimizer/optimizer/hierarchical_rnn.py b/research/learned_optimizer/optimizer/hierarchical_rnn.py
deleted file mode 100644
index 953b72b5d04724a11a0e95385bbe0c6a0d91289d..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/hierarchical_rnn.py
+++ /dev/null
@@ -1,792 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Collection of trainable optimizers for meta-optimization."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import math
-
-import numpy as np
-import tensorflow as tf
-
-from tensorflow.python.ops import state_ops
-from learned_optimizer.optimizer import rnn_cells
-from learned_optimizer.optimizer import trainable_optimizer as opt
-from learned_optimizer.optimizer import utils
-
-# Default was 0.1
-tf.app.flags.DEFINE_float("biasgrucell_scale", 0.5,
- """The scale for the internal BiasGRUCell vars.""")
-# Default was 0
-tf.app.flags.DEFINE_float("biasgrucell_gate_bias_init", 2.2,
- """The bias for the internal BiasGRUCell reset and
- update gate variables.""")
-# Default was 1e-3
-tf.app.flags.DEFINE_float("hrnn_rnn_readout_scale", 0.5,
- """The initialization scale for the RNN readouts.""")
-tf.app.flags.DEFINE_float("hrnn_default_decay_var_init", 2.2,
- """The default initializer value for any decay/
- momentum style variables and constants.
- sigmoid(2.2) ~ 0.9, sigmoid(-2.2) ~ 0.01.""")
-# Default was 2.2
-tf.app.flags.DEFINE_float("scale_decay_bias_init", 3.2,
- """The initialization for the scale decay bias. This
- is the initial bias for the timescale for the
- exponential avg of the mean square gradients.""")
-tf.app.flags.DEFINE_float("learning_rate_momentum_logit_init", 3.2,
- """Initialization for the learning rate momentum.""")
-# Default was 0.1
-tf.app.flags.DEFINE_float("hrnn_affine_scale", 0.5,
- """The initialization scale for the weight matrix of
- the bias variables in layer0 and 1 of the hrnn.""")
-
-FLAGS = tf.flags.FLAGS
-
-
-class HierarchicalRNN(opt.TrainableOptimizer):
- """3 level hierarchical RNN.
-
- Optionally uses second order gradient information and has decoupled evaluation
- and update locations.
- """
-
- def __init__(self, level_sizes, init_lr_range=(1e-6, 1e-2),
- learnable_decay=True, dynamic_output_scale=True,
- use_attention=False, use_log_objective=True,
- num_gradient_scales=4, zero_init_lr_weights=True,
- use_log_means_squared=True, use_relative_lr=True,
- use_extreme_indicator=False, max_log_lr=33,
- obj_train_max_multiplier=-1, use_problem_lr_mean=False,
- use_gradient_shortcut=False, use_lr_shortcut=False,
- use_grad_products=False, use_multiple_scale_decays=False,
- learnable_inp_decay=True, learnable_rnn_init=True,
- random_seed=None, **kwargs):
- """Initializes the RNN per-parameter optimizer.
-
- The hierarchy consists of up to three levels:
- Level 0: per parameter RNN
- Level 1: per tensor RNN
- Level 2: global RNN
-
- Args:
- level_sizes: list or tuple with 1, 2, or 3 integers, the number of units
- in each RNN in the hierarchy (level0, level1, level2).
- length 1: only coordinatewise rnn's will be used
- length 2: coordinatewise and tensor-level rnn's will be used
- length 3: a single global-level rnn will be used in addition to
- coordinatewise and tensor-level
- init_lr_range: the range in which to initialize the learning rates
- learnable_decay: whether to learn weights that dynamically modulate the
- input scale via RMS style decay
- dynamic_output_scale: whether to learn weights that dynamically modulate
- the output scale
- use_attention: whether to use attention to train the optimizer
- use_log_objective: whether to train on the log of the objective
- num_gradient_scales: the number of scales to use for gradient history
- zero_init_lr_weights: whether to initialize the lr weights to zero
- use_log_means_squared: whether to track the log of the means_squared,
- used as a measure of signal vs. noise in gradient.
- use_relative_lr: whether to use the relative learning rate as an
- input during training (requires learnable_decay=True)
- use_extreme_indicator: whether to use the extreme indicator for learning
- rates as an input during training (requires learnable_decay=True)
- max_log_lr: the maximum log learning rate allowed during train or test
- obj_train_max_multiplier: max objective increase during a training run
- use_problem_lr_mean: whether to use the mean over all learning rates in
- the problem when calculating the relative learning rate as opposed to
- the per-tensor mean
- use_gradient_shortcut: Whether to add a learned affine projection of the
- gradient to the update delta in addition to the gradient function
- computed by the RNN
- use_lr_shortcut: Whether to add as input the difference between the log lr
- and the desired log lr (1e-3)
- use_grad_products: Whether to use gradient products in the rnn input.
- Only applicable if num_gradient_scales > 1
- use_multiple_scale_decays: Whether to use multiple scales for the scale
- decay, as with input decay
- learnable_inp_decay: Whether to learn the input decay weights and bias.
- learnable_rnn_init: Whether to learn the RNN state initialization.
- random_seed: Random seed for random variable initializers. (Default: None)
- **kwargs: args passed to TrainableOptimizer's constructor
-
- Raises:
- ValueError: If level_sizes is not a length 1, 2, or 3 list.
- ValueError: If there are any non-integer sizes in level_sizes.
- ValueError: If the init lr range is not of length 2.
- ValueError: If the init lr range is not a valid range (min > max).
- """
- if len(level_sizes) not in [1, 2, 3]:
- raise ValueError("HierarchicalRNN only supports 1, 2, or 3 levels in the "
- "hierarchy, but {} were requested.".format(
- len(level_sizes)))
- if any(not isinstance(level, int) for level in level_sizes):
- raise ValueError("Level sizes must be integer values, were {}".format(
- level_sizes))
- if len(init_lr_range) != 2:
- raise ValueError(
- "Initial LR range must be len 2, was {}".format(len(init_lr_range)))
- if init_lr_range[0] > init_lr_range[1]:
- raise ValueError("Initial LR range min is greater than max.")
-
- self.learnable_decay = learnable_decay
- self.dynamic_output_scale = dynamic_output_scale
- self.use_attention = use_attention
- self.use_log_objective = use_log_objective
- self.num_gradient_scales = num_gradient_scales
- self.zero_init_lr_weights = zero_init_lr_weights
- self.use_log_means_squared = use_log_means_squared
- self.use_relative_lr = use_relative_lr
- self.use_extreme_indicator = use_extreme_indicator
- self.max_log_lr = max_log_lr
- self.use_problem_lr_mean = use_problem_lr_mean
- self.use_gradient_shortcut = use_gradient_shortcut
- self.use_lr_shortcut = use_lr_shortcut
- self.use_grad_products = use_grad_products
- self.use_multiple_scale_decays = use_multiple_scale_decays
- self.learnable_inp_decay = learnable_inp_decay
- self.learnable_rnn_init = learnable_rnn_init
-
- self.random_seed = random_seed
-
- self.num_layers = len(level_sizes)
- self.init_lr_range = init_lr_range
-
- self.reuse_vars = None
- self.reuse_global_state = None
- self.cells = []
- self.init_vectors = []
-
- with tf.variable_scope(opt.OPTIMIZER_SCOPE):
-
- self._initialize_rnn_cells(level_sizes)
-
- # get the cell size for the per-parameter RNN (Level 0)
- cell_size = level_sizes[0]
-
- # Random normal initialization scaled by the output size. This is the
- # scale for the RNN *readouts*. RNN internal weight scale is set in the
- # BiasGRUCell call.
- scale_factor = FLAGS.hrnn_rnn_readout_scale / math.sqrt(cell_size)
- scaled_init = tf.random_normal_initializer(0., scale_factor,
- seed=self.random_seed)
-
- # weights for projecting the hidden state to a parameter update
- self.update_weights = tf.get_variable("update_weights",
- shape=(cell_size, 1),
- initializer=scaled_init)
-
- if self.use_attention:
- # weights for projecting the hidden state to the location at which the
- # gradient is attended
- self.attention_weights = tf.get_variable(
- "attention_weights",
- initializer=self.update_weights.initialized_value())
-
- # weights for projecting the hidden state to the RMS decay term
- self._initialize_scale_decay((cell_size, 1), scaled_init)
- self._initialize_input_decay((cell_size, 1), scaled_init)
-
- self._initialize_lr((cell_size, 1), scaled_init)
-
- state_keys = ["parameter", "layer", "scl_decay", "inp_decay", "true_param"]
-
- if self.dynamic_output_scale:
- state_keys.append("log_learning_rate")
-
- for i in range(self.num_gradient_scales):
- state_keys.append("grad_accum{}".format(i + 1))
- state_keys.append("ms{}".format(i + 1))
-
- super(HierarchicalRNN, self).__init__(
- "hRNN", state_keys, use_attention=use_attention,
- use_log_objective=use_log_objective,
- obj_train_max_multiplier=obj_train_max_multiplier, **kwargs)
-
- def _initialize_rnn_cells(self, level_sizes):
- """Initializes the RNN cells to use in the hierarchical RNN."""
-
- # RNN Cell layers (0 -> lowest, 1 -> middle, 2 -> global)
- for level in range(self.num_layers):
- scope = "Level{}_RNN".format(level)
- with tf.variable_scope(scope):
- hcell = rnn_cells.BiasGRUCell(
- level_sizes[level],
- scale=FLAGS.biasgrucell_scale,
- gate_bias_init=FLAGS.biasgrucell_gate_bias_init,
- random_seed=self.random_seed)
- self.cells.append(hcell)
- if self.learnable_rnn_init:
- self.init_vectors.append(tf.Variable(
- tf.random_uniform([1, hcell.state_size], -1., 1.,
- seed=self.random_seed),
- name="init_vector"))
- else:
- self.init_vectors.append(
- tf.random_uniform([1, hcell.state_size], -1., 1.,
- seed=self.random_seed))
-
- def _initialize_scale_decay(self, weights_tensor_shape, scaled_init):
- """Initializes the scale decay weights and bias variables or tensors.
-
- Args:
- weights_tensor_shape: The shape the weight tensor should take.
- scaled_init: The scaled initialization for the weights tensor.
- """
- if self.learnable_decay:
- self.scl_decay_weights = tf.get_variable("scl_decay_weights",
- shape=weights_tensor_shape,
- initializer=scaled_init)
- scl_decay_bias_init = tf.constant_initializer(
- FLAGS.scale_decay_bias_init)
- self.scl_decay_bias = tf.get_variable("scl_decay_bias",
- shape=(1,),
- initializer=scl_decay_bias_init)
- else:
- self.scl_decay_weights = tf.zeros_like(self.update_weights)
- self.scl_decay_bias = tf.log(0.93 / (1. - 0.93))
-
- def _initialize_input_decay(self, weights_tensor_shape, scaled_init):
- """Initializes the input scale decay weights and bias variables or tensors.
-
- Args:
- weights_tensor_shape: The shape the weight tensor should take.
- scaled_init: The scaled initialization for the weights tensor.
- """
- if (self.learnable_decay and self.num_gradient_scales > 1 and
- self.learnable_inp_decay):
- self.inp_decay_weights = tf.get_variable("inp_decay_weights",
- shape=weights_tensor_shape,
- initializer=scaled_init)
- inp_decay_bias_init = tf.constant_initializer(
- FLAGS.hrnn_default_decay_var_init)
- self.inp_decay_bias = tf.get_variable("inp_decay_bias",
- shape=(1,),
- initializer=inp_decay_bias_init)
- else:
- self.inp_decay_weights = tf.zeros_like(self.update_weights)
- self.inp_decay_bias = tf.log(0.89 / (1. - 0.89))
-
- def _initialize_lr(self, weights_tensor_shape, scaled_init):
- """Initializes the learning rate weights and bias variables or tensors.
-
- Args:
- weights_tensor_shape: The shape the weight tensor should take.
- scaled_init: The scaled initialization for the weights tensor.
- """
- if self.dynamic_output_scale:
- zero_init = tf.constant_initializer(0.)
- wt_init = zero_init if self.zero_init_lr_weights else scaled_init
- self.lr_weights = tf.get_variable("learning_rate_weights",
- shape=weights_tensor_shape,
- initializer=wt_init)
- self.lr_bias = tf.get_variable("learning_rate_bias", shape=(1,),
- initializer=zero_init)
- else:
- self.lr_weights = tf.zeros_like(self.update_weights)
- self.lr_bias = tf.zeros([1, 1])
-
- def _initialize_state(self, var):
- """Return a dictionary mapping names of state variables to their values."""
- var_vectorized = tf.reshape(var, [-1, 1])
- ndim = var_vectorized.get_shape().as_list()[0]
-
- state = {
- # parameter init tensor is [var_ndim x layer0_cell_size]
- "parameter": tf.ones([ndim, 1]) * self.init_vectors[0],
- "scl_decay": tf.zeros_like(var_vectorized),
- "inp_decay": tf.zeros_like(var_vectorized),
- "true_param": var,
- }
-
- if self.num_layers > 1:
- # layer init tensor is [1 x layer1_cell_size]
- state["layer"] = tf.ones([1, 1]) * self.init_vectors[1]
-
- if self.dynamic_output_scale:
- min_lr = self.init_lr_range[0]
- max_lr = self.init_lr_range[1]
- if min_lr == max_lr:
- log_init_lr = tf.log(min_lr * tf.ones_like(var_vectorized))
- else:
- # Use a random offset to increase the likelihood that the average of the
- # LRs for this variable is different from the LRs for other variables.
- actual_vals = tf.random_uniform(var_vectorized.get_shape().as_list(),
- np.log(min_lr) / 2.,
- np.log(max_lr) / 2.,
- seed=self.random_seed)
- offset = tf.random_uniform((), np.log(min_lr) / 2., np.log(max_lr) / 2.,
- seed=self.random_seed)
- log_init_lr = actual_vals + offset
- # Clip the log learning rate to the flag at the top end, and to
- # (log(min int32) - 1) at the bottom
- clipped = tf.clip_by_value(log_init_lr, -33, self.max_log_lr)
- state["log_learning_rate"] = clipped
-
- for i in range(self.num_gradient_scales):
- state["grad_accum{}".format(i + 1)] = tf.zeros_like(var_vectorized)
- state["ms{}".format(i + 1)] = tf.zeros_like(var_vectorized)
-
- return state
-
- def _initialize_global_state(self):
- if self.num_layers < 3:
- return []
- rnn_global_init = tf.ones([1, 1]) * self.init_vectors[2]
- return [rnn_global_init]
-
- def _compute_updates(self, params, grads, states, global_state):
- # Store the updated parameters and states.
- updated_params = []
- updated_attention = []
- updated_states = []
-
- with tf.variable_scope(opt.OPTIMIZER_SCOPE):
-
- mean_log_lr = self._compute_mean_log_lr(states)
-
- # Iterate over the layers.
- for param, grad_unflat, state in zip(params, grads, states):
-
- with tf.variable_scope("PerTensor", reuse=self.reuse_vars):
- self.reuse_vars = True
- grad = tf.reshape(grad_unflat, [-1, 1])
-
- # Create the RNN input. We will optionally extend it with additional
- # features such as curvature and gradient signal vs. noise.
- (grads_scaled, mean_squared_gradients,
- grads_accum) = self._compute_scaled_and_ms_grads(grad, state)
- rnn_input = [g for g in grads_scaled]
-
- self._extend_rnn_input(rnn_input, state, grads_scaled,
- mean_squared_gradients, mean_log_lr)
-
- # Concatenate any features we've collected.
- rnn_input_tensor = tf.concat(rnn_input, 1)
-
- layer_state, new_param_state = self._update_rnn_cells(
- state, global_state, rnn_input_tensor,
- len(rnn_input) != len(grads_scaled))
-
- (scl_decay, inp_decay, new_log_lr, update_step, lr_attend,
- attention_delta) = self._compute_rnn_state_projections(
- state, new_param_state, grads_scaled)
-
- # Apply updates and store state variables.
- if self.use_attention:
- truth = state["true_param"]
- updated_param = truth - update_step
- attention_step = tf.reshape(lr_attend * attention_delta,
- truth.get_shape())
- updated_attention.append(truth - attention_step)
- else:
- updated_param = param - update_step
- updated_attention.append(updated_param)
- updated_params.append(updated_param)
-
- # Collect the new state.
- new_state = {
- "parameter": new_param_state,
- "scl_decay": scl_decay,
- "inp_decay": inp_decay,
- "true_param": updated_param,
- }
- if layer_state is not None:
- new_state["layer"] = layer_state
-
- if self.dynamic_output_scale:
- new_state["log_learning_rate"] = new_log_lr
-
- for i in range(self.num_gradient_scales):
- new_state["grad_accum{}".format(i + 1)] = grads_accum[i]
- new_state["ms{}".format(i + 1)] = mean_squared_gradients[i]
- updated_states.append(new_state)
-
- updated_global_state = self._compute_updated_global_state([layer_state],
- global_state)
-
- return (updated_params, updated_states, [updated_global_state],
- updated_attention)
-
- def _compute_mean_log_lr(self, states):
- """Computes the mean log learning rate across all variables."""
- if self.use_problem_lr_mean and self.use_relative_lr:
-
- sum_log_lr = 0.
- count_log_lr = 0.
- for state in states:
- sum_log_lr += tf.reduce_sum(state["log_learning_rate"])
- # Note: get_shape().num_elements()=num elements in the original tensor.
- count_log_lr += state["log_learning_rate"].get_shape().num_elements()
- return sum_log_lr / count_log_lr
-
- def _compute_scaled_and_ms_grads(self, grad, state):
- """Computes the scaled gradient and the mean squared gradients.
-
- Gradients are also accumulated across different timescales if appropriate.
-
- Args:
- grad: The gradient tensor for this layer.
- state: The optimizer state for this layer.
-
- Returns:
- The scaled gradients, mean squared gradients, and accumulated gradients.
- """
- input_decays = [state["inp_decay"]]
- scale_decays = [state["scl_decay"]]
- if self.use_multiple_scale_decays and self.num_gradient_scales > 1:
- for i in range(self.num_gradient_scales - 1):
- scale_decays.append(tf.sqrt(scale_decays[i]))
-
- for i in range(self.num_gradient_scales - 1):
- # Each accumulator on twice the timescale of the one before.
- input_decays.append(tf.sqrt(input_decays[i]))
- grads_accum = []
- grads_scaled = []
- mean_squared_gradients = []
-
- # populate the scaled gradients and associated mean_squared values
- if self.num_gradient_scales > 0:
- for i, decay in enumerate(input_decays):
- if self.num_gradient_scales == 1:
- # We don't accumulate if no scales, just take the current gradient.
- grad_accum = grad
- else:
- # The state vars are 1-indexed.
- old_accum = state["grad_accum{}".format(i + 1)]
- grad_accum = grad * (1. - decay) + old_accum * decay
-
- grads_accum.append(grad_accum)
-
- sd = scale_decays[i if self.use_multiple_scale_decays else 0]
- grad_scaled, ms = utils.rms_scaling(grad_accum, sd,
- state["ms{}".format(i + 1)],
- update_ms=True)
- grads_scaled.append(grad_scaled)
- mean_squared_gradients.append(ms)
-
- return grads_scaled, mean_squared_gradients, grads_accum
-
- def _extend_rnn_input(self, rnn_input, state, grads_scaled,
- mean_squared_gradients, mean_log_lr):
- """Computes additional rnn inputs and adds them to the rnn_input list."""
- if self.num_gradient_scales > 1 and self.use_grad_products:
- # This gives a measure of curvature relative to input averaging
- # lengthscale and to the learning rate
- grad_products = [a * b for a, b in
- zip(grads_scaled[:-1], grads_scaled[1:])]
- rnn_input.extend([g for g in grad_products])
-
- if self.use_log_means_squared:
- log_means_squared = [tf.log(ms + 1e-16)
- for ms in mean_squared_gradients]
-
- avg = tf.reduce_mean(log_means_squared, axis=0)
- # This gives a measure of the signal vs. noise contribution to the
- # gradient, at the current averaging lengthscale. If all the noise
- # is averaged out, and if updates are small, these will be 0.
- mean_log_means_squared = [m - avg for m in log_means_squared]
-
- rnn_input.extend([m for m in mean_log_means_squared])
-
- if self.use_relative_lr or self.use_extreme_indicator:
- if not self.dynamic_output_scale:
- raise Exception("Relative LR and Extreme Indicator features "
- "require dynamic_output_scale to be set to True.")
- log_lr_vec = tf.reshape(state["log_learning_rate"], [-1, 1])
- if self.use_relative_lr:
- if self.use_problem_lr_mean:
- # Learning rate of this dimension vs. rest of target problem.
- relative_lr = log_lr_vec - mean_log_lr
- else:
- # Learning rate of this dimension vs. rest of tensor.
- relative_lr = log_lr_vec - tf.reduce_mean(log_lr_vec)
- rnn_input.append(relative_lr)
- if self.use_extreme_indicator:
- # Indicator of extremely large or extremely small learning rate.
- extreme_indicator = (tf.nn.relu(log_lr_vec - tf.log(1.)) -
- tf.nn.relu(tf.log(1e-6) - log_lr_vec))
- rnn_input.append(extreme_indicator)
-
- if self.use_lr_shortcut:
- log_lr_vec = tf.reshape(state["log_learning_rate"], [-1, 1])
- rnn_input.append(log_lr_vec - tf.log(1e-3))
-
- def _update_rnn_cells(self, state, global_state, rnn_input_tensor,
- use_additional_features):
- """Updates the component RNN cells with the given state and tensor.
-
- Args:
- state: The current state of the optimizer.
- global_state: The current global RNN state.
- rnn_input_tensor: The input tensor to the RNN.
- use_additional_features: Whether the rnn input tensor contains additional
- features beyond the scaled gradients (affects whether the rnn input
- tensor is used as input to the RNN.)
-
- Returns:
- layer_state: The new state of the per-tensor RNN.
- new_param_state: The new state of the per-parameter RNN.
- """
- # lowest level (per parameter)
- # input -> gradient for this parameter
- # bias -> output from the layer RNN
- with tf.variable_scope("Layer0_RNN"):
- total_bias = None
- if self.num_layers > 1:
- sz = 3 * self.cells[0].state_size # size of the concatenated bias
- param_bias = utils.affine([state["layer"]], sz,
- scope="Param/Affine",
- scale=FLAGS.hrnn_affine_scale,
- random_seed=self.random_seed)
- total_bias = param_bias
- if self.num_layers == 3:
- global_bias = utils.affine(global_state, sz,
- scope="Global/Affine",
- scale=FLAGS.hrnn_affine_scale,
- random_seed=self.random_seed)
- total_bias += global_bias
-
- new_param_state, _ = self.cells[0](
- rnn_input_tensor, state["parameter"], bias=total_bias)
-
- if self.num_layers > 1:
- # middle level (per layer)
- # input -> average hidden state from each parameter in this layer
- # bias -> output from the RNN at the global level
- with tf.variable_scope("Layer1_RNN"):
- if not use_additional_features:
- # Restore old behavior and only add the mean of the new params.
- layer_input = tf.reduce_mean(new_param_state, 0, keep_dims=True)
- else:
- layer_input = tf.reduce_mean(
- tf.concat((new_param_state, rnn_input_tensor), 1), 0,
- keep_dims=True)
- if self.num_layers == 3:
- sz = 3 * self.cells[1].state_size
- layer_bias = utils.affine(global_state, sz,
- scale=FLAGS.hrnn_affine_scale,
- random_seed=self.random_seed)
- layer_state, _ = self.cells[1](
- layer_input, state["layer"], bias=layer_bias)
- else:
- layer_state, _ = self.cells[1](layer_input, state["layer"])
- else:
- layer_state = None
-
- return layer_state, new_param_state
-
- def _compute_rnn_state_projections(self, state, new_param_state,
- grads_scaled):
- """Computes the RNN state-based updates to parameters and update steps."""
- # Compute the update direction (a linear projection of the RNN output).
- update_weights = self.update_weights
-
- update_delta = utils.project(new_param_state, update_weights)
- if self.use_gradient_shortcut:
- # Include an affine projection of just the direction of the gradient
- # so that RNN hidden states are freed up to store more complex
- # functions of the gradient and other parameters.
- grads_scaled_tensor = tf.concat([g for g in grads_scaled], 1)
- update_delta += utils.affine(grads_scaled_tensor, 1,
- scope="GradsToDelta",
- include_bias=False,
- vec_mean=1. / len(grads_scaled),
- random_seed=self.random_seed)
- if self.dynamic_output_scale:
- denom = tf.sqrt(tf.reduce_mean(update_delta ** 2) + 1e-16)
-
- update_delta /= denom
-
- if self.use_attention:
- attention_weights = self.attention_weights
- attention_delta = utils.project(new_param_state,
- attention_weights)
- if self.use_gradient_shortcut:
- attention_delta += utils.affine(grads_scaled_tensor, 1,
- scope="GradsToAttnDelta",
- include_bias=False,
- vec_mean=1. / len(grads_scaled),
- random_seed=self.random_seed)
- if self.dynamic_output_scale:
- attention_delta /= tf.sqrt(
- tf.reduce_mean(attention_delta ** 2) + 1e-16)
- else:
- attention_delta = None
-
- # The updated decay is an affine projection of the hidden state.
- scl_decay = utils.project(new_param_state, self.scl_decay_weights,
- bias=self.scl_decay_bias,
- activation=tf.nn.sigmoid)
- # This is only used if learnable_decay and num_gradient_scales > 1
- inp_decay = utils.project(new_param_state, self.inp_decay_weights,
- bias=self.inp_decay_bias,
- activation=tf.nn.sigmoid)
-
- # Also update the learning rate.
- lr_param, lr_attend, new_log_lr = self._compute_new_learning_rate(
- state, new_param_state)
-
- update_step = tf.reshape(lr_param * update_delta,
- state["true_param"].get_shape())
-
- return (scl_decay, inp_decay, new_log_lr, update_step, lr_attend,
- attention_delta)
-
- def _compute_new_learning_rate(self, state, new_param_state):
- if self.dynamic_output_scale:
- # Compute the change in learning rate (an affine projection of the
- # RNN state, passed through a sigmoid or log depending on flags).
- # Update the learning rate, w/ momentum.
- lr_change = utils.project(new_param_state, self.lr_weights,
- bias=self.lr_bias)
- step_log_lr = state["log_learning_rate"] + lr_change
-
- # Clip the log learning rate to the flag at the top end, and to
- # (log(min int32) - 1) at the bottom
-
- # Check out this hack: we want to be able to compute the gradient
- # of the downstream result w.r.t lr weights and bias, even if the
- # value of step_log_lr is outside the clip range. So we clip,
- # subtract off step_log_lr, and wrap all that in a stop_gradient so
- # TF never tries to take the gradient of the clip... or the
- # subtraction. Then we add BACK step_log_lr so that downstream still
- # receives the clipped value. But the GRADIENT of step_log_lr will
- # be the gradient of the unclipped value, which we added back in
- # after stop_gradients.
- step_log_lr += tf.stop_gradient(
- tf.clip_by_value(step_log_lr, -33, self.max_log_lr)
- - step_log_lr)
-
- lr_momentum_logit = tf.get_variable(
- "learning_rate_momentum_logit",
- initializer=FLAGS.learning_rate_momentum_logit_init)
- lrm = tf.nn.sigmoid(lr_momentum_logit)
- new_log_lr = (lrm * state["log_learning_rate"] +
- (1. - lrm) * step_log_lr)
- param_stepsize_offset = tf.get_variable("param_stepsize_offset",
- initializer=-1.)
- lr_param = tf.exp(step_log_lr + param_stepsize_offset)
- lr_attend = tf.exp(step_log_lr) if self.use_attention else lr_param
- else:
- # Dynamic output scale is off, LR param is always 1.
- lr_param = 2. * utils.project(new_param_state, self.lr_weights,
- bias=self.lr_bias,
- activation=tf.nn.sigmoid)
- new_log_lr = None
- lr_attend = lr_param
-
- return lr_param, lr_attend, new_log_lr
-
- def _compute_updated_global_state(self, layer_states, global_state):
- """Computes the new global state gives the layers states and old state.
-
- Args:
- layer_states: The current layer states.
- global_state: The old global state.
-
- Returns:
- The updated global state.
- """
- updated_global_state = []
- if self.num_layers == 3:
- # highest (global) layer
- # input -> average hidden state from each layer-specific RNN
- # bias -> None
- with tf.variable_scope("Layer2_RNN", reuse=self.reuse_global_state):
- self.reuse_global_state = True
- global_input = tf.reduce_mean(tf.concat(layer_states, 0), 0,
- keep_dims=True)
- updated_global_state, _ = self.cells[2](global_input, global_state[0])
- return updated_global_state
-
- def apply_gradients(self, grads_and_vars, global_step=None, name=None):
- """Overwrites the tf.train.Optimizer interface for applying gradients."""
-
- # Pull out the variables.
- grads_and_vars = tuple(grads_and_vars) # Make sure repeat iteration works.
- for g, v in grads_and_vars:
- if not isinstance(g, (tf.Tensor, tf.IndexedSlices, type(None))):
- raise TypeError(
- "Gradient must be a Tensor, IndexedSlices, or None: %s" % g)
- if not isinstance(v, tf.Variable):
- raise TypeError(
- "Variable must be a tf.Variable: %s" % v)
- if g is not None:
- self._assert_valid_dtypes([g, v])
- var_list = [v for g, v in grads_and_vars if g is not None]
- if not var_list:
- raise ValueError("No gradients provided for any variable: %s" %
- (grads_and_vars,))
-
- # Create slots for the variables.
- with tf.control_dependencies(None):
- self._create_slots(var_list)
-
- # Store update ops in this list.
- with tf.op_scope([], name, self._name) as name:
-
- # Prepare the global state.
- with tf.variable_scope(self._name, reuse=self.reuse_global_state):
- gs = self._initialize_global_state()
- if gs:
- global_state = [tf.get_variable("global_state", initializer=gs[0])]
- else:
- global_state = []
-
- # Get the states for each variable in the list.
- states = [{key: self.get_slot(var, key) for key in self.get_slot_names()}
- for var in var_list]
-
- # Compute updated values.
- grads, params = zip(*grads_and_vars)
- args = (params, grads, states, global_state)
- updates = self._compute_updates(*args)
- new_params, new_states, new_global_state, new_attention = updates
- # Assign op for new global state.
- update_ops = [tf.assign(gs, ngs)
- for gs, ngs in zip(global_state, new_global_state)]
-
- # Create the assign ops for the params and state variables.
- args = (params, states, new_params, new_attention, new_states)
- for var, state, new_var, new_var_attend, new_state in zip(*args):
- # Assign updates to the state variables.
- state_assign_ops = [tf.assign(state_var, new_state[key])
- for key, state_var in state.items()]
-
- # Update the parameter.
- with tf.control_dependencies(state_assign_ops):
- if self.use_attention:
- # Assign to the attended location, rather than the actual location
- # so that the gradients are computed where attention is.
- param_update_op = var.assign(new_var_attend)
- else:
- param_update_op = var.assign(new_var)
-
- with tf.name_scope("update_" + var.op.name): #, tf.colocate_with(var):
- update_ops.append(param_update_op)
-
- real_params = [self.get_slot(var, "true_param") for var in var_list]
-
- if global_step is None:
- # NOTE: if using the optimizer in a non-test-optimizer setting (e.g.
- # on Inception), remove the real_params return value. Otherwise
- # the code will throw an error.
- return self._finish(update_ops, name), real_params
- else:
- with tf.control_dependencies([self._finish(update_ops, "update")]):
- return state_ops.assign_add(global_step, 1, name=name).op, real_params
diff --git a/research/learned_optimizer/optimizer/learning_rate_schedule.py b/research/learned_optimizer/optimizer/learning_rate_schedule.py
deleted file mode 100644
index 53db8addd3d152bfa02630ec6e37f0cc1776abc8..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/learning_rate_schedule.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A trainable optimizer that learns a learning rate schedule."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from learned_optimizer.optimizer import trainable_optimizer
-
-
-class LearningRateSchedule(trainable_optimizer.TrainableOptimizer):
- """Learns a learning rate schedule over a fixed number of iterations."""
-
- def __init__(self, initial_rate=0.0, n_steps=1000, **kwargs):
- """Initializes the learning rates."""
- self.max_index = tf.constant(n_steps-1, dtype=tf.int32)
-
- with tf.variable_scope(trainable_optimizer.OPTIMIZER_SCOPE):
- initializer = tf.constant_initializer(initial_rate)
- self.learning_rates = tf.get_variable("learning_rates",
- shape=([n_steps,]),
- initializer=initializer)
-
- super(LearningRateSchedule, self).__init__("LRS", ["itr"], **kwargs)
-
- def _initialize_state(self, var):
- """Return a dictionary mapping names of state variables to their values."""
- return {
- "itr": tf.constant(0, dtype=tf.int32),
- }
-
- def _compute_update(self, param, grad, state):
- """Compute updates of parameters."""
-
- # get the learning rate at the current index, if the index
- # is greater than the number of available learning rates,
- # use the last one
- index = tf.minimum(state["itr"], self.max_index)
- learning_rate = tf.gather(self.learning_rates, index)
-
- # update the parameters: parameter - learning_rate * gradient
- updated_param = param - tf.scalar_mul(learning_rate, grad)
-
- return updated_param, {"itr": state["itr"] + 1}
diff --git a/research/learned_optimizer/optimizer/rnn_cells.py b/research/learned_optimizer/optimizer/rnn_cells.py
deleted file mode 100644
index 3d68de04ca5318bb0f264d4f4647ddbc6fbe08e0..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/rnn_cells.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Custom RNN cells for hierarchical RNNs."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from learned_optimizer.optimizer import utils
-
-
-class BiasGRUCell(tf.contrib.rnn.RNNCell):
- """GRU cell (cf. http://arxiv.org/abs/1406.1078) with an additional bias."""
-
- def __init__(self, num_units, activation=tf.tanh, scale=0.1,
- gate_bias_init=0., random_seed=None):
- self._num_units = num_units
- self._activation = activation
- self._scale = scale
- self._gate_bias_init = gate_bias_init
- self._random_seed = random_seed
-
- @property
- def state_size(self):
- return self._num_units
-
- @property
- def output_size(self):
- return self._num_units
-
- def __call__(self, inputs, state, bias=None):
- # Split the injected bias vector into a bias for the r, u, and c updates.
- if bias is None:
- bias = tf.zeros((1, 3))
-
- r_bias, u_bias, c_bias = tf.split(bias, 3, 1)
-
- with tf.variable_scope(type(self).__name__): # "BiasGRUCell"
- with tf.variable_scope("gates"): # Reset gate and update gate.
- proj = utils.affine([inputs, state], 2 * self._num_units,
- scale=self._scale, bias_init=self._gate_bias_init,
- random_seed=self._random_seed)
- r_lin, u_lin = tf.split(proj, 2, 1)
- r, u = tf.nn.sigmoid(r_lin + r_bias), tf.nn.sigmoid(u_lin + u_bias)
-
- with tf.variable_scope("candidate"):
- proj = utils.affine([inputs, r * state], self._num_units,
- scale=self._scale, random_seed=self._random_seed)
- c = self._activation(proj + c_bias)
-
- new_h = u * state + (1 - u) * c
-
- return new_h, new_h
diff --git a/research/learned_optimizer/optimizer/trainable_adam.py b/research/learned_optimizer/optimizer/trainable_adam.py
deleted file mode 100644
index 638217f1b723da8633dc7a82623392eaaf190829..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/trainable_adam.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A trainable ADAM optimizer that learns its internal variables."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-from learned_optimizer.optimizer import trainable_optimizer as opt
-from learned_optimizer.optimizer import utils
-
-
-class TrainableAdam(opt.TrainableOptimizer):
- """Adam optimizer with learnable scalar parameters.
-
- See Kingma et. al., 2014 for algorithm (http://arxiv.org/abs/1412.6980).
- """
-
- def __init__(self,
- learning_rate=1e-3,
- beta1=0.9,
- beta2=0.999,
- epsilon=1e-8,
- **kwargs):
- """Initializes the TrainableAdam optimizer with the given initial values.
-
- Args:
- learning_rate: The learning rate (default: 1e-3).
- beta1: The exponential decay rate for the 1st moment estimates.
- beta2: The exponential decay rate for the 2nd moment estimates.
- epsilon: A small constant for numerical stability.
- **kwargs: Any additional keyword arguments for TrainableOptimizer.
-
- Raises:
- ValueError: if the learning rate or epsilon is not positive
- ValueError: if beta1 or beta2 is not in (0, 1).
- """
- if learning_rate <= 0:
- raise ValueError("Learning rate must be positive.")
- if epsilon <= 0:
- raise ValueError("Epsilon must be positive.")
- if not 0 < beta1 < 1 or not 0 < beta2 < 1:
- raise ValueError("Beta values must be between 0 and 1, exclusive.")
-
- self._reuse_vars = False
-
- with tf.variable_scope(opt.OPTIMIZER_SCOPE):
- def inv_sigmoid(x):
- return np.log(x / (1.0 - x))
-
- self.log_learning_rate = tf.get_variable(
- "log_learning_rate",
- shape=[],
- initializer=tf.constant_initializer(np.log(learning_rate)))
- self.beta1_logit = tf.get_variable(
- "beta1_logit",
- shape=[],
- initializer=tf.constant_initializer(inv_sigmoid(beta1)))
- self.beta2_logit = tf.get_variable(
- "beta2_logit",
- shape=[],
- initializer=tf.constant_initializer(inv_sigmoid(beta2)))
- self.log_epsilon = tf.get_variable(
- "log_epsilon",
- shape=[],
- initializer=tf.constant_initializer(np.log(epsilon)))
-
- # Key names are derived from Algorithm 1 described in
- # https://arxiv.org/pdf/1412.6980.pdf
- state_keys = ["m", "v", "t"]
- super(TrainableAdam, self).__init__("Adam", state_keys, **kwargs)
-
- def _initialize_state(self, var):
- """Returns a dictionary mapping names of state variables to their values."""
- vectorized_shape = var.get_shape().num_elements(), 1
-
- return {key: tf.zeros(vectorized_shape) for key in self.state_keys}
-
- def _compute_update(self, param, grad, state):
- """Calculates the new internal state and parameters.
-
- If the gradient is sparse, updates the appropriate slices in the internal
- state and stacks the update tensor.
-
- Args:
- param: A tensor of parameters.
- grad: A tensor of gradients with the same shape as param.
- state: A dictionary containing any state for the optimizer.
-
- Returns:
- updated_param: The updated parameters.
- updated_state: The updated state variables in a dictionary.
- """
-
- with tf.variable_scope(opt.OPTIMIZER_SCOPE) as scope:
-
- if self._reuse_vars:
- scope.reuse_variables()
- else:
- self._reuse_vars = True
-
- (grad_values, first_moment, second_moment, timestep, grad_indices
- ) = self._extract_gradients_and_internal_state(
- grad, state, tf.shape(param))
-
- beta1 = tf.nn.sigmoid(self.beta1_logit)
- beta2 = tf.nn.sigmoid(self.beta2_logit)
- epsilon = tf.exp(self.log_epsilon) + 1e-10
- learning_rate = tf.exp(self.log_learning_rate)
-
- old_grad_shape = tf.shape(grad_values)
- grad_values = tf.reshape(grad_values, [-1, 1])
-
- new_timestep = timestep + 1
- new_first_moment = self._update_adam_estimate(
- first_moment, grad_values, beta1)
- new_second_moment = self._debias_adam_estimate(
- second_moment, tf.square(grad_values), beta2)
-
- debiased_first_moment = self._debias_adam_estimate(
- new_first_moment, beta1, new_timestep)
- debiased_second_moment = self._debias_adam_estimate(
- new_second_moment, beta2, new_timestep)
-
- # Propagating through the square root of 0 is very bad for stability.
- update = (learning_rate * debiased_first_moment /
- (tf.sqrt(debiased_second_moment + 1e-10) + epsilon))
-
- update = tf.reshape(update, old_grad_shape)
-
- if grad_indices is not None:
- param_shape = tf.shape(param)
- update = utils.stack_tensor(
- update, grad_indices, param, param_shape[:1])
- new_first_moment = utils.update_slices(
- new_first_moment, grad_indices, state["m"], param_shape)
- new_second_moment = utils.update_slices(
- new_second_moment, grad_indices, state["v"], param_shape)
- new_timestep = utils.update_slices(
- new_timestep, grad_indices, state["t"], param_shape)
-
- new_param = param - update
-
- # collect the update and new state
- new_state = {
- "m": new_first_moment,
- "v": new_second_moment,
- "t": new_timestep
- }
-
- return new_param, new_state
-
- def _update_adam_estimate(self, estimate, value, beta):
- """Returns a beta-weighted average of estimate and value."""
- return (beta * estimate) + ((1 - beta) * value)
-
- def _debias_adam_estimate(self, estimate, beta, t_step):
- """Returns a debiased estimate based on beta and the timestep."""
- return estimate / (1 - tf.pow(beta, t_step))
-
- def _extract_gradients_and_internal_state(self, grad, state, param_shape):
- """Extracts the gradients and relevant internal state.
-
- If the gradient is sparse, extracts the appropriate slices from the state.
-
- Args:
- grad: The current gradient.
- state: The current state.
- param_shape: The shape of the parameter (used if gradient is sparse).
-
- Returns:
- grad_values: The gradient value tensor.
- first_moment: The first moment tensor (internal state).
- second_moment: The second moment tensor (internal state).
- timestep: The current timestep (internal state).
- grad_indices: The indices for the gradient tensor, if sparse.
- None otherwise.
- """
- grad_values = grad
- grad_indices = None
- first_moment = state["m"]
- second_moment = state["v"]
- timestep = state["t"]
-
- if isinstance(grad, tf.IndexedSlices):
- grad_indices, grad_values = utils.accumulate_sparse_gradients(grad)
- first_moment = utils.slice_tensor(
- first_moment, grad_indices, param_shape)
- second_moment = utils.slice_tensor(
- second_moment, grad_indices, param_shape)
- timestep = utils.slice_tensor(timestep, grad_indices, param_shape)
-
- return grad_values, first_moment, second_moment, timestep, grad_indices
-
diff --git a/research/learned_optimizer/optimizer/trainable_optimizer.py b/research/learned_optimizer/optimizer/trainable_optimizer.py
deleted file mode 100644
index 955112a9dd1d3b0af5ae2f5f0fe8eff65d2dbfc7..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/trainable_optimizer.py
+++ /dev/null
@@ -1,574 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A base class definition for trainable optimizers."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import itertools
-
-import tensorflow as tf
-
-from tensorflow.python.framework import tensor_shape
-
-OPTIMIZER_SCOPE = "LOL"
-_LOCAL_VARIABLE_PREFIX = "local_state_"
-_LOCAL_STATE_VARIABLE_COLLECTION = "local_state_collection"
-EPSILON = 1e-6
-
-
-class TrainableOptimizer(tf.train.Optimizer):
- """Base class for trainable optimizers.
-
- A trainable optimizer is an optimizer that has parameters that can themselves
- be learned (meta-optimized).
-
- Subclasses must implement:
- _compute_update(self, param, grad, state)
- """
-
- def __init__(self, name, state_keys, use_attention=False,
- use_log_objective=False, obj_train_max_multiplier=-1,
- use_second_derivatives=True, use_numerator_epsilon=False,
- **kwargs):
- """Initializes the optimizer with the given name and settings.
-
- Args:
- name: The name string for this optimizer.
- state_keys: The names of any required state variables (list)
- use_attention: Whether this optimizer uses attention (Default: True)
- use_log_objective: Whether this optimizer uses the logarithm of the
- objective when computing the loss (Default: False)
- obj_train_max_multiplier: The maximum multiplier for the increase in the
- objective before meta-training is stopped. If <= 0, meta-training is
- not stopped early. (Default: -1)
- use_second_derivatives: Whether this optimizer uses second derivatives in
- meta-training. This should be set to False if some second derivatives
- in the meta-training problem set are not defined in Tensorflow.
- (Default: True)
- use_numerator_epsilon: Whether to use epsilon in the numerator when
- scaling the problem objective during meta-training. (Default: False)
- **kwargs: Any additional keyword arguments.
- """
- self.use_second_derivatives = use_second_derivatives
- self.state_keys = sorted(state_keys)
- self.use_attention = use_attention
- self.use_log_objective = use_log_objective
- self.obj_train_max_multiplier = obj_train_max_multiplier
- self.use_numerator_epsilon = use_numerator_epsilon
-
- use_locking = False
- super(TrainableOptimizer, self).__init__(use_locking, name)
-
- def _create_slots(self, var_list):
- """Creates all slots needed by the variables.
-
- Args:
- var_list: A list of `Variable` objects.
- """
- for var in var_list:
- init_states = self._initialize_state(var)
- for slot_name in sorted(init_states):
- slot_var_name = "{}_{}".format(self.get_name(), slot_name)
- value = init_states[slot_name]
- self._get_or_make_slot(var, value, slot_name, slot_var_name)
-
- def _initialize_state(self, var):
- """Initializes any state required for this variable.
-
- Args:
- var: a tensor containing parameters to be optimized
-
- Returns:
- state: a dictionary mapping state keys to initial state values (tensors)
- """
- return {}
-
- def _initialize_global_state(self):
- """Initializes any global state values."""
- return []
-
- def _apply_common(self, grad, var):
- """Applies the optimizer updates to the variables.
-
- Note: this should only get called via _apply_dense or _apply_sparse when
- using the optimizer via optimizer.minimize or optimizer.apply_gradients.
- During meta-training, the optimizer.train function should be used to
- construct an optimization path that is differentiable.
-
- Args:
- grad: A tensor representing the gradient.
- var: A tf.Variable with the same shape as grad.
-
- Returns:
- update_op: A tensorflow op that assigns new values to the variable, and
- also defines dependencies that update the state variables for the
- optimizer.
- """
- state = {key: self.get_slot(var, key) for key in self.get_slot_names()}
- new_var, new_state = self._compute_update(var, grad, state)
- state_assign_ops = [tf.assign(state_var, new_state[key])
- for key, state_var in state.items()]
- with tf.control_dependencies(state_assign_ops):
- update_op = var.assign(new_var)
-
- return update_op
-
- def _apply_dense(self, grad, var):
- """Adds ops to apply dense gradients to 'var'."""
- return self._apply_common(grad, var)
-
- def _apply_sparse(self, grad, var):
- """Adds ops to apply sparse gradients to 'var'."""
- return self._apply_common(grad, var)
-
- def _compute_update(self, param, grad, state):
- """Computes the update step for optimization.
-
- Args:
- param: A tensor of parameters to optimize.
- grad: The gradient tensor of the objective with respect to the parameters.
- (It has the same shape as param.)
- state: A dictionary containing any extra state required by the optimizer.
-
- Returns:
- updated_params: The updated parameters.
- updated_state: The dictionary of updated state variable(s).
- """
- raise NotImplementedError
-
- def _compute_updates(self, params, grads, states, global_state):
- """Maps the compute update functions for each parameter.
-
- This function can be overriden by a subclass if the subclass wants to
- combine information across the different parameters in the list.
-
- Args:
- params: A list of parameter tensors.
- grads: A list of gradients corresponding to each parameter.
- states: A list of state variables corresponding to each parameter.
- global_state: A list of global state variables for the problem.
-
- Returns:
- new_params: The updated parameters.
- new_states: The updated states.
- new_global_state: The updated global state.
- attention_params: A list of attention parameters. This is the same as
- new_params if the optimizer does not use attention.
- """
- # Zip up the arguments to _compute_update.
- args = zip(params, grads, states)
-
- # Call compute_update on each set of parameter/gradient/state args.
- new_params, new_states = zip(*list(
- itertools.starmap(self._compute_update, args)))
-
- # Global state is unused in the basic case, just pass it through.
- return list(new_params), list(new_states), global_state, list(new_params)
-
- def train(self, problem, dataset):
- """Creates graph operations to train the optimizer.
-
- Args:
- problem: A problem_generator.Problem instance to train on.
- dataset: A datasets.Dataset tuple to use when training.
-
- Returns:
- meta_objective: A tensorflow operation for computing the meta-objective
- obj_weights: A tensor placeholder for feeding in the objective weights
- obj_values: The subproblem objective values during optimization
- batches: The batch indexes tensor for overriding with feed_dict
- first_unroll: A placeholder signifying if this is a first unroll
- (this will propagate the gradients slightly differently).
- reset_state: A placeholder signifying that the rnn state should be reset.
- output_state: The final state of the optimizer
- init_loop_vars_to_override: Local variables that can be assigned to
- propagate the optimizer and problem state for unrolling
- final_loop_vals: Final values of the loop variables that can be
- assigned to init_loop_vars_to_override.
- """
-
- # Placeholder for the objective weights
- obj_weights = tf.placeholder(tf.float32)
- num_iter = tf.shape(obj_weights)[0]
-
- # Unpack the dataset and generate the minibatches for training
- data, labels = dataset
- # Convert the ndarrays to tensors so we can pass them back in via feed_dict
- data = tf.constant(data)
- labels = tf.constant(labels)
- batches = tf.placeholder(tf.int32)
- first_unroll = tf.placeholder_with_default(False, [])
- reset_state = tf.placeholder_with_default(False, [])
-
- training_output = collections.namedtuple("TrainingOutput",
- ["metaobj",
- "obj_weights",
- "problem_objectives",
- "initial_obj",
- "batches",
- "first_unroll",
- "reset_state",
- "output_state",
- "init_loop_vars",
- "output_loop_vars"])
-
- def loop_body(itr, obj_accum, params, attend_params, flattened_states,
- global_state, all_obj, unused_init_obj, data,
- labels, batches):
- """Body of the meta-training while loop for optimizing a sub-problem.
-
- Args:
- itr: The current meta-training iteration.
- obj_accum: The accumulated objective over all training steps so far.
- params: The parameters of the sub-problem.
- attend_params: The parameters of the sub-problems at the attended
- location.
- flattened_states: The states of the trainable optimizer, sorted and
- flattened into a list (since a while loop can't handle nested lists
- or dictionaries).
- global_state: The global state of the optimizer.
- all_obj: The list of all objective values in the training process.
- unused_init_obj: The initial objective (unused here, but needed in the
- variable list because it's used in a stopping condition in the
- loop_cond.)
- data: The data for this problem.
- labels: The labels corresponding to the data.
- batches: The batch indexes needed for shuffled minibatch creation.
-
- Returns:
- itr: The updated meta-training iteration.
- obj_accum: The updated accumulated objective.
- params: The new parameters of the sub-problem.
- attend_params: The new parameters of the sub-problems at the attended
- location.
- flattened_states: The new states of the trainable optimizer.
- global_state: The updated global state.
- all_obj: The updates list of all objective values.
- unused_init_obj: The initial objective.
- data: The data for this problem.
- labels: The labels corresponding to the data.
- batches: The batch indexes needed for shuffled minibatch creation.
- """
- batch_indices = tf.gather(batches, itr)
- batch_data = tf.gather(data, batch_indices)
- batch_labels = tf.gather(labels, batch_indices)
-
- # Compute the objective over the entire dataset (full batch).
- obj = problem.objective(params, data, labels)
-
- # Compute the gradients on just the current batch
- if self.use_attention:
- current_obj = problem.objective(attend_params, batch_data, batch_labels)
- grads = problem.gradients(current_obj, attend_params)
- else:
- current_obj = problem.objective(params, batch_data, batch_labels)
- grads = problem.gradients(current_obj, params)
-
- if not self.use_second_derivatives:
- new_grads = []
- for grad in grads:
- if isinstance(grad, tf.IndexedSlices):
- new_grads.append(
- tf.IndexedSlices(tf.stop_gradient(grad.values), grad.indices))
- else:
- new_grads.append(tf.stop_gradient(grad))
- grads = new_grads
-
- # store the objective value for the entire problem at each iteration
- all_obj = tf.concat([all_obj, tf.reshape(obj, (1,))], 0)
-
- # accumulate the weighted objective for the entire dataset
- acc = tf.gather(obj_weights, itr) * obj
-
- obj_accum = tf.add(obj_accum, acc)
- # Set the shape to keep the shape invariant for obj_accum. Without this,
- # the graph builder thinks the tensor shape is unknown on the 2nd iter.
- obj_accum.set_shape([])
-
- # convert flattened_states to dictionaries
- dict_states = [dict(zip(self.state_keys, flat_state))
- for flat_state in flattened_states]
-
- # compute the new parameters and states
- args = (params, grads, dict_states, global_state)
- updates = self._compute_updates(*args)
- new_params, new_states, new_global_state, new_attend_params = updates
-
- # flatten the states
- new_flattened_states = map(flatten_and_sort, new_states)
-
- return [itr + 1, obj_accum, new_params, new_attend_params,
- new_flattened_states, new_global_state, all_obj, unused_init_obj,
- data, labels, batches]
-
- def loop_cond(itr, obj_accum, unused_params, unused_attend_params,
- unused_flattened_states, unused_global_state, all_obj,
- init_obj, *args):
- """Termination conditions of the sub-problem optimization loop."""
- del args # unused
-
- cond1 = tf.less(itr, num_iter) # We've run < num_iter times
- cond2 = tf.is_finite(obj_accum) # The objective is still finite
-
- if self.obj_train_max_multiplier > 0:
- current_obj = tf.gather(all_obj, itr)
- # Account for negative init_obj too
- max_diff = (self.obj_train_max_multiplier - 1) * tf.abs(init_obj)
- max_obj = init_obj + max_diff
- # The objective is a reasonable multiplier of the original objective
- cond3 = tf.less(current_obj, max_obj)
-
- return tf.logical_and(tf.logical_and(cond1, cond2), cond3,
- name="training_loop_cond")
- else:
- return tf.logical_and(cond1, cond2, name="training_loop_cond")
-
- init = self._initialize_training_loop_parameters(
- problem, data, labels, batches, first_unroll, reset_state)
- loop_vars, invariants, initial_obj, init_loop_vars_to_override = init
-
- loop_output = tf.while_loop(loop_cond, loop_body, loop_vars,
- swap_memory=True, shape_invariants=invariants)
- meta_obj, problem_objectives = loop_output[1], loop_output[6]
-
- # The meta objective is normalized by the initial objective at the start of
- # the series of partial unrolls.
- scaled_meta_objective = self.scale_objective(
- meta_obj, problem_objectives, initial_obj)
-
- final_loop_vals = (
- [initial_obj] + loop_output[2] + loop_output[3] + loop_output[5])
- final_loop_vals.extend(itertools.chain(*loop_output[4]))
-
- return training_output(scaled_meta_objective,
- obj_weights,
- problem_objectives,
- initial_obj,
- batches,
- first_unroll,
- reset_state,
- loop_output[4],
- init_loop_vars_to_override,
- final_loop_vals)
-
- def _initialize_training_loop_parameters(
- self, problem, data, labels, batches, first_unroll, reset_state):
- """Initializes the vars and params needed for the training process.
-
- Args:
- problem: The problem being optimized.
- data: The data for the problem.
- labels: The corresponding labels for the data.
- batches: The indexes needed to create shuffled batches of the data.
- first_unroll: Whether this is the first unroll in a partial unrolling.
- reset_state: Whether RNN state variables should be reset.
-
- Returns:
- loop_vars: The while loop variables for training.
- invariants: The corresponding variable shapes (required by while loop).
- initial_obj: The initial objective (used later for scaling).
- init_loop_vars_to_override: The loop vars that can be overridden when
- performing training via partial unrolls.
- """
- # Extract these separately so we don't have to make inter-variable
- # dependencies.
- initial_tensors = problem.init_tensors()
-
- return_initial_tensor_values = first_unroll
- initial_params_vars, initial_params = local_state_variables(
- initial_tensors, return_initial_tensor_values)
- initial_attend_params_vars, initial_attend_params = local_state_variables(
- initial_tensors, return_initial_tensor_values)
- # Recalculate the initial objective for the list on each partial unroll with
- # the new initial_params. initial_obj holds the value from the very first
- # unroll.
- initial_obj_init = problem.objective(initial_params, data, labels)
- return_initial_obj_init = first_unroll
- [initial_obj_var], [initial_obj] = local_state_variables(
- [initial_obj_init], return_initial_obj_init)
-
- # Initialize the loop variables.
- initial_itr = tf.constant(0, dtype=tf.int32)
- initial_meta_obj = tf.constant(0, dtype=tf.float32)
- # N.B. the use of initial_obj_init here rather than initial_obj
- initial_problem_objectives = tf.reshape(initial_obj_init, (1,))
-
- # Initialize the extra state.
- initial_state_vars = []
- initial_state = []
- state_shapes = []
- return_initial_state_values = reset_state
- for param in initial_tensors:
- param_state_vars, param_state = local_state_variables(
- flatten_and_sort(self._initialize_state(param)),
- return_initial_state_values)
-
- initial_state_vars.append(param_state_vars)
- initial_state.append(param_state)
- state_shapes.append([f.get_shape() for f in param_state])
-
- # Initialize any global (problem-level) state.
- initial_global_state_vars, initial_global_state = local_state_variables(
- self._initialize_global_state(), return_initial_state_values)
-
- global_shapes = []
- for item in initial_global_state:
- global_shapes.append(item.get_shape())
-
- # build the list of loop variables:
- loop_vars = [
- initial_itr,
- initial_meta_obj,
- initial_params, # Local variables.
- initial_attend_params, # Local variables.
- initial_state, # Local variables.
- initial_global_state, # Local variables.
- initial_problem_objectives,
- initial_obj, # Local variable.
- data,
- labels,
- batches,
- ]
-
- invariants = [
- initial_itr.get_shape(),
- initial_meta_obj.get_shape(),
- [t.get_shape() for t in initial_params],
- [t.get_shape() for t in initial_attend_params],
- state_shapes,
- global_shapes,
- tensor_shape.TensorShape([None]), # The problem objectives list grows
- initial_obj.get_shape(),
- tensor_shape.unknown_shape(), # Placeholder shapes are unknown
- tensor_shape.unknown_shape(),
- tensor_shape.unknown_shape(),
- ]
-
- # Initialize local variables that we will override with final tensors at the
- # next iter.
- init_loop_vars_to_override = (
- [initial_obj_var] + initial_params_vars + initial_attend_params_vars +
- initial_global_state_vars)
- init_loop_vars_to_override.extend(itertools.chain(*initial_state_vars))
-
- return loop_vars, invariants, initial_obj, init_loop_vars_to_override
-
- def scale_objective(self, total_obj, all_objs, initial_obj,
- obj_scale_eps=1e-6):
- """Normalizes the objective based on the initial objective value.
-
- Args:
- total_obj: The total accumulated objective over the training run.
- all_objs: A list of all the individual objectives over the training run.
- initial_obj: The initial objective value.
- obj_scale_eps: The epsilon value to use in computations for stability.
-
- Returns:
- The scaled objective as a single value.
- """
- if self.use_log_objective:
- if self.use_numerator_epsilon:
- scaled_problem_obj = ((all_objs + obj_scale_eps) /
- (initial_obj + obj_scale_eps))
- log_scaled_problem_obj = tf.log(scaled_problem_obj)
- else:
- scaled_problem_obj = all_objs / (initial_obj + obj_scale_eps)
- log_scaled_problem_obj = tf.log(scaled_problem_obj + obj_scale_eps)
- return tf.reduce_mean(log_scaled_problem_obj)
- else:
- return total_obj / (initial_obj + obj_scale_eps)
-
-
-def local_state_variables(init_values, return_init_values):
- """Create local variables initialized from init_values.
-
- This will create local variables from a list of init_values. Each variable
- will be named based on the value's shape and dtype.
-
- As a convenience, a boolean tensor allows you to return value from
- the created local variable or from the original init value.
-
- Args:
- init_values: iterable of tensors
- return_init_values: boolean tensor
-
- Returns:
- local_vars: list of the created local variables.
- vals: if return_init_values is true, then this returns the values of
- init_values. Otherwise it returns the values of the local_vars.
- """
- if not init_values:
- return [], []
-
- # This generates a harmless warning when saving the metagraph.
- variable_use_count = tf.get_collection_ref(_LOCAL_STATE_VARIABLE_COLLECTION)
- if not variable_use_count:
- variable_use_count.append(collections.defaultdict(int))
- variable_use_count = variable_use_count[0]
-
- local_vars = []
- with tf.variable_scope(OPTIMIZER_SCOPE):
- # We can't use the init_value as an initializer as init_value may
- # itself depend on some problem variables. This would produce
- # inter-variable initialization order dependence which TensorFlow
- # sucks at making easy.
- for init_value in init_values:
- name = create_local_state_variable_name(init_value)
- unique_name = name + "_" + str(variable_use_count[name])
- variable_use_count[name] += 1
- # The overarching idea here is to be able to reuse variables between
- # different sessions on the same TensorFlow master without errors. By
- # uniquifying based on the type and name we mirror the checks made inside
- # TensorFlow, while still allowing some memory reuse. Ultimately this is a
- # hack due to the broken Session.reset().
- local_vars.append(
- tf.get_local_variable(
- unique_name,
- initializer=tf.zeros(
- init_value.get_shape(), dtype=init_value.dtype)))
-
- # It makes things a lot simpler if we use the init_value the first
- # iteration, instead of the variable itself. It allows us to propagate
- # gradients through it as well as simplifying initialization. The variable
- # ends up assigned to after the first iteration.
- vals = tf.cond(return_init_values, lambda: init_values, lambda: local_vars)
- if len(init_values) == 1:
- # tf.cond extracts elements from singleton lists.
- vals = [vals]
- return local_vars, vals
-
-
-def create_local_state_variable_name(tensor):
- """Create a name of the variable based on its type and shape."""
- if not tensor.get_shape().is_fully_defined():
- raise ValueError("Need a fully specified shape to create a local variable.")
-
- return (_LOCAL_VARIABLE_PREFIX + "_".join(
- map(str, tensor.get_shape().as_list())) + "_" + tensor.dtype.name)
-
-
-def is_local_state_variable(op):
- """Returns if this op is a local state variable created for training."""
- return op.node_def.op in ["Variable", "VariableV2"] and op.name.startswith(
- OPTIMIZER_SCOPE + "/" + _LOCAL_VARIABLE_PREFIX)
-
-
-def flatten_and_sort(dictionary):
- """Flattens a dictionary into a list of values sorted by the keys."""
- return [dictionary[k] for k in sorted(dictionary.keys())]
diff --git a/research/learned_optimizer/optimizer/utils.py b/research/learned_optimizer/optimizer/utils.py
deleted file mode 100644
index 58744f4cb7919a84ecc8702ff1236e4c0a03f218..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/optimizer/utils.py
+++ /dev/null
@@ -1,278 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities and helper functions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-
-def make_finite(t, replacement):
- """Replaces non-finite tensor values with the replacement value."""
- return tf.where(tf.is_finite(t), t, replacement)
-
-
-def asinh(x):
- """Computes the inverse hyperbolic sine function (in tensorflow)."""
- return tf.log(x + tf.sqrt(1. + x ** 2))
-
-
-def affine(inputs, output_size, scope="Affine", scale=0.1, vec_mean=0.,
- include_bias=True, bias_init=0., random_seed=None):
- """Computes an affine function of the inputs.
-
- Creates or recalls tensorflow variables "Matrix" and "Bias"
- to generate an affine operation on the input.
-
- If the inputs are a list of tensors, they are concatenated together.
-
- Initial weights for the matrix are drawn from a Gaussian with zero
- mean and standard deviation that is the given scale divided by the
- square root of the input dimension. Initial weights for the bias are
- set to zero.
-
- Args:
- inputs: List of tensors with shape (batch_size, input_size)
- output_size: Size (dimension) of the output
- scope: Variable scope for these parameters (default: "Affine")
- scale: Initial weight scale for the matrix parameters (default: 0.1),
- this constant is divided by the sqrt of the input size to get the
- std. deviation of the initial weights
- vec_mean: The mean for the random initializer
- include_bias: Whether to include the bias term
- bias_init: The initializer bias (default 0.)
- random_seed: Random seed for random initializers. (Default: None)
-
- Returns:
- output: Tensor with shape (batch_size, output_size)
- """
-
- # Concatenate the input arguments.
- x = tf.concat(inputs, 1)
-
- with tf.variable_scope(scope):
- input_size = x.get_shape().as_list()[1]
-
- sigma = scale / np.sqrt(input_size)
- rand_init = tf.random_normal_initializer(mean=vec_mean, stddev=sigma,
- seed=random_seed)
-
- matrix = tf.get_variable("Matrix", [input_size, output_size],
- dtype=tf.float32, initializer=rand_init)
-
- if include_bias:
- bias = tf.get_variable("Bias", [output_size], dtype=tf.float32,
- initializer=tf.constant_initializer(bias_init,
- tf.float32))
- else:
- bias = 0.
- output = tf.matmul(x, matrix) + bias
-
- return output
-
-
-def project(inputs, weights, bias=0., activation=tf.identity):
- """Computes an affine or linear projection of the inputs.
-
- Projects the inputs onto the given weight vector and (optionally)
- adds a bias and passes the result through an activation function.
-
- Args:
- inputs: matrix of inputs with shape [batch_size, dim]
- weights: weight matrix with shape [dim, output_dim]
- bias: bias vector with shape [output_dim] (default: 0)
- activation: nonlinear activation function (default: tf.identity)
-
- Returns:
- outputs: an op which computes activation(inputs @ weights + bias)
- """
- return activation(tf.matmul(inputs, weights) + bias)
-
-
-def new_mean_squared(grad_vec, decay, ms):
- """Calculates the new accumulated mean squared of the gradient.
-
- Args:
- grad_vec: the vector for the current gradient
- decay: the decay term
- ms: the previous mean_squared value
-
- Returns:
- the new mean_squared value
- """
- decay_size = decay.get_shape().num_elements()
- decay_check_ops = [
- tf.assert_less_equal(decay, 1., summarize=decay_size),
- tf.assert_greater_equal(decay, 0., summarize=decay_size)]
-
- with tf.control_dependencies(decay_check_ops):
- grad_squared = tf.square(grad_vec)
-
- # If the previous mean_squared is the 0 vector, don't use the decay and just
- # return the full grad_squared. This should only happen on the first timestep.
- decay = tf.cond(tf.reduce_all(tf.equal(ms, 0.)),
- lambda: tf.zeros_like(decay, dtype=tf.float32), lambda: decay)
-
- # Update the running average of squared gradients.
- epsilon = 1e-12
- return (1. - decay) * (grad_squared + epsilon) + decay * ms
-
-
-def rms_scaling(gradient, decay, ms, update_ms=True):
- """Vectorizes and scales a tensor of gradients.
-
- Args:
- gradient: the current gradient
- decay: the current decay value.
- ms: the previous mean squared value
- update_ms: Whether to update the mean squared value (default: True)
-
- Returns:
- The scaled gradient and the new ms value if update_ms is True,
- the old ms value otherwise.
- """
-
- # Vectorize the gradients and compute the squared gradients.
- grad_vec = tf.reshape(gradient, [-1, 1])
-
- if update_ms:
- ms = new_mean_squared(grad_vec, decay, ms)
-
- # Scale the current gradients by the RMS, squashed by the asinh function.
- scaled_gradient = asinh(grad_vec / tf.sqrt(ms + 1e-16))
-
- return scaled_gradient, ms
-
-
-def accumulate_sparse_gradients(grad):
- """Accumulates repeated indices of a sparse gradient update.
-
- Args:
- grad: a tf.IndexedSlices gradient
-
- Returns:
- grad_indices: unique indices
- grad_values: gradient values corresponding to the indices
- """
-
- grad_indices, grad_segments = tf.unique(grad.indices)
- grad_values = tf.unsorted_segment_sum(grad.values, grad_segments,
- tf.shape(grad_indices)[0])
- return grad_indices, grad_values
-
-
-def slice_tensor(dense_tensor, indices, head_dims):
- """Extracts slices from a partially flattened dense tensor.
-
- indices is assumed to index into the first dimension of head_dims.
- dense_tensor is assumed to have a shape [D_0, D_1, ...] such that
- prod(head_dims) == D_0. This function will extract slices along the
- first_dimension of head_dims.
-
- Example:
-
- Consider a tensor with shape head_dims = [100, 2] and a dense_tensor with
- shape [200, 3]. Note that the first dimension of dense_tensor equals the
- product of head_dims. This function will reshape dense_tensor such that
- its shape is now [100, 2, 3] (i.e. the first dimension became head-dims)
- and then slice it along the first dimension. After slicing, the slices will
- have their initial dimensions flattened just as they were in dense_tensor
- (e.g. if there are 4 indices, the return value will have a shape of [4, 3]).
-
- Args:
- dense_tensor: a N-D dense tensor. Shape: [D_0, D_1, ...]
- indices: a 1-D integer tensor. Shape: [K]
- head_dims: True dimensions of the dense_tensor's first dimension.
-
- Returns:
- Extracted slices. Shape [K, D_1, ...]
- """
-
- tail_dims = tf.shape(dense_tensor)[1:]
- dense_tensor = tf.reshape(dense_tensor,
- tf.concat([head_dims, tail_dims], 0))
-
- slices = tf.gather(dense_tensor, indices)
- # NOTE(siege): This kills the shape annotation.
- return tf.reshape(slices, tf.concat([[-1], tail_dims], 0))
-
-
-def stack_tensor(slices, indices, dense_tensor, head_dims):
- """Reconsititutes a tensor from slices and corresponding indices.
-
- This is an inverse operation to slice_tensor. Missing slices are set to 0.
-
- Args:
- slices: a tensor. Shape [K, D_1, ...]
- indices: a 1-D integer tensor. Shape: [K]
- dense_tensor: the original tensor the slices were taken
- from. Shape: [D_0, D_1, ...]
- head_dims: True dimensions of the dense_tensor's first dimension.
-
- Returns:
- Reconsituted tensor. Shape: [D_0, D_1, ...]
- """
- # NOTE(siege): This cast shouldn't be necessary.
- indices = tf.cast(indices, tf.int32)
-
- tail_dims = tf.shape(dense_tensor)[1:]
- dense_shape = tf.concat([head_dims, tail_dims], 0)
-
- slices = tf.reshape(slices, tf.concat([[-1], dense_shape[1:]], 0))
- indices = tf.expand_dims(indices, -1)
-
- return tf.reshape(tf.scatter_nd(indices, slices, dense_shape),
- tf.shape(dense_tensor))
-
-
-def update_slices(slices, indices, dense_tensor, head_dims):
- """Reconstitutes a tensor from slices and corresponding indices.
-
- Like _stack_tensor, but instead of setting missing slices to 0, sets them to
- what they were in the original tensor. The return value is reshaped to be
- the same as dense_tensor.
-
- Args:
- slices: a tensor. Shape [K, D_1, ...]
- indices: a 1-D integer tensor. Shape: [K]
- dense_tensor: the original tensor the slices were taken
- from. Shape: [D_0, D_1, ...]
- head_dims: True dimensions of the dense_tensor's first dimension.
-
- Returns:
- Reconsituted tensor. Shape: [D_0, D_1, ...]
- """
- # NOTE(siege): This cast shouldn't be necessary.
- indices = tf.cast(indices, tf.int32)
-
- tail_dims = tf.shape(dense_tensor)[1:]
- dense_shape = tf.concat([head_dims, tail_dims], 0)
-
- update_mask_vals = tf.fill(tf.shape(indices), 1)
- reshaped_indices = tf.expand_dims(indices, -1)
- update_mask = tf.equal(
- tf.scatter_nd(reshaped_indices, update_mask_vals, head_dims[:1]), 1)
-
- reshaped_dense_slices = tf.reshape(
- stack_tensor(slices, indices, dense_tensor, head_dims), dense_shape)
- reshaped_dense_tensor = tf.reshape(dense_tensor, dense_shape)
-
- return tf.reshape(
- tf.where(update_mask, reshaped_dense_slices, reshaped_dense_tensor),
- tf.shape(dense_tensor))
diff --git a/research/learned_optimizer/problems/BUILD b/research/learned_optimizer/problems/BUILD
deleted file mode 100644
index c704618821b36ca23f221f724888cde4e5d5a5ad..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/problems/BUILD
+++ /dev/null
@@ -1,43 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-# Libraries
-# =====
-
-py_library(
- name = "datasets",
- srcs = ["datasets.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "model_adapter",
- srcs = ["model_adapter.py"],
- deps = [
- ":problem_generator",
- ],
-)
-
-py_library(
- name = "problem_generator",
- srcs = ["problem_generator.py"],
- deps = [
- ":problem_spec",
- ],
-)
-
-py_library(
- name = "problem_sets",
- srcs = ["problem_sets.py"],
- deps = [
- ":datasets",
- ":model_adapter",
- ":problem_generator",
- ],
-)
-
-py_library(
- name = "problem_spec",
- srcs = ["problem_spec.py"],
- deps = [],
-)
diff --git a/research/learned_optimizer/problems/datasets.py b/research/learned_optimizer/problems/datasets.py
deleted file mode 100644
index edf3df6532178b0e60ab93c78611d2313798e639..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/problems/datasets.py
+++ /dev/null
@@ -1,218 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Functions to generate or load datasets for supervised learning."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import namedtuple
-
-import numpy as np
-from sklearn.datasets import make_classification
-
-MAX_SEED = 4294967295
-
-
-class Dataset(namedtuple("Dataset", "data labels")):
- """Helper class for managing a supervised learning dataset.
-
- Args:
- data: an array of type float32 with N samples, each of which is the set
- of features for that sample. (Shape (N, D_i), where N is the number of
- samples and D_i is the number of features for that sample.)
- labels: an array of type int32 or int64 with N elements, indicating the
- class label for the corresponding set of features in data.
- """
- # Since this is an immutable object, we don't need to reserve slots.
- __slots__ = ()
-
- @property
- def size(self):
- """Dataset size (number of samples)."""
- return len(self.data)
-
- def batch_indices(self, num_batches, batch_size):
- """Creates indices of shuffled minibatches.
-
- Args:
- num_batches: the number of batches to generate
- batch_size: the size of each batch
-
- Returns:
- batch_indices: a list of minibatch indices, arranged so that the dataset
- is randomly shuffled.
-
- Raises:
- ValueError: if the data and labels have different lengths
- """
- if len(self.data) != len(self.labels):
- raise ValueError("Labels and data must have the same number of samples.")
-
- batch_indices = []
-
- # Follows logic in mnist.py to ensure we cover the entire dataset.
- index_in_epoch = 0
- dataset_size = len(self.data)
- dataset_indices = np.arange(dataset_size)
- np.random.shuffle(dataset_indices)
-
- for _ in range(num_batches):
- start = index_in_epoch
- index_in_epoch += batch_size
- if index_in_epoch > dataset_size:
-
- # Finished epoch, reshuffle.
- np.random.shuffle(dataset_indices)
-
- # Start next epoch.
- start = 0
- index_in_epoch = batch_size
-
- end = index_in_epoch
- batch_indices.append(dataset_indices[start:end].tolist())
-
- return batch_indices
-
-
-def noisy_parity_class(n_samples,
- n_classes=2,
- n_context_ids=5,
- noise_prob=0.25,
- random_seed=None):
- """Returns a randomly generated sparse-to-sparse dataset.
-
- The label is a parity class of a set of context classes.
-
- Args:
- n_samples: number of samples (data points)
- n_classes: number of class labels (default: 2)
- n_context_ids: how many classes to take the parity of (default: 5).
- noise_prob: how often to corrupt the label (default: 0.25)
- random_seed: seed used for drawing the random data (default: None)
- Returns:
- dataset: A Dataset namedtuple containing the generated data and labels
- """
- np.random.seed(random_seed)
- x = np.random.randint(0, n_classes, [n_samples, n_context_ids])
- noise = np.random.binomial(1, noise_prob, [n_samples])
- y = (np.sum(x, 1) + noise) % n_classes
- return Dataset(x.astype("float32"), y.astype("int32"))
-
-
-def random(n_features, n_samples, n_classes=2, sep=1.0, random_seed=None):
- """Returns a randomly generated classification dataset.
-
- Args:
- n_features: number of features (dependent variables)
- n_samples: number of samples (data points)
- n_classes: number of class labels (default: 2)
- sep: separation of the two classes, a higher value corresponds to
- an easier classification problem (default: 1.0)
- random_seed: seed used for drawing the random data (default: None)
-
- Returns:
- dataset: A Dataset namedtuple containing the generated data and labels
- """
- # Generate the problem data.
- x, y = make_classification(n_samples=n_samples,
- n_features=n_features,
- n_informative=n_features,
- n_redundant=0,
- n_classes=n_classes,
- class_sep=sep,
- random_state=random_seed)
-
- return Dataset(x.astype("float32"), y.astype("int32"))
-
-
-def random_binary(n_features, n_samples, random_seed=None):
- """Returns a randomly generated dataset of binary values.
-
- Args:
- n_features: number of features (dependent variables)
- n_samples: number of samples (data points)
- random_seed: seed used for drawing the random data (default: None)
-
- Returns:
- dataset: A Dataset namedtuple containing the generated data and labels
- """
- random_seed = (np.random.randint(MAX_SEED) if random_seed is None
- else random_seed)
- np.random.seed(random_seed)
-
- x = np.random.randint(2, size=(n_samples, n_features))
- y = np.zeros((n_samples, 1))
-
- return Dataset(x.astype("float32"), y.astype("int32"))
-
-
-def random_symmetric(n_features, n_samples, random_seed=None):
- """Returns a randomly generated dataset of values and their negatives.
-
- Args:
- n_features: number of features (dependent variables)
- n_samples: number of samples (data points)
- random_seed: seed used for drawing the random data (default: None)
-
- Returns:
- dataset: A Dataset namedtuple containing the generated data and labels
- """
- random_seed = (np.random.randint(MAX_SEED) if random_seed is None
- else random_seed)
- np.random.seed(random_seed)
-
- x1 = np.random.normal(size=(int(n_samples/2), n_features))
- x = np.concatenate((x1, -x1), axis=0)
- y = np.zeros((n_samples, 1))
-
- return Dataset(x.astype("float32"), y.astype("int32"))
-
-
-def random_mlp(n_features, n_samples, random_seed=None, n_layers=6, width=20):
- """Returns a generated output of an MLP with random weights.
-
- Args:
- n_features: number of features (dependent variables)
- n_samples: number of samples (data points)
- random_seed: seed used for drawing the random data (default: None)
- n_layers: number of layers in random MLP
- width: width of the layers in random MLP
-
- Returns:
- dataset: A Dataset namedtuple containing the generated data and labels
- """
- random_seed = (np.random.randint(MAX_SEED) if random_seed is None
- else random_seed)
- np.random.seed(random_seed)
-
- x = np.random.normal(size=(n_samples, n_features))
- y = x
- n_in = n_features
- scale_factor = np.sqrt(2.) / np.sqrt(n_features)
- for _ in range(n_layers):
- weights = np.random.normal(size=(n_in, width)) * scale_factor
- y = np.dot(y, weights).clip(min=0)
- n_in = width
-
- y = y[:, 0]
- y[y > 0] = 1
-
- return Dataset(x.astype("float32"), y.astype("int32"))
-
-
-EMPTY_DATASET = Dataset(np.array([], dtype="float32"),
- np.array([], dtype="int32"))
diff --git a/research/learned_optimizer/problems/model_adapter.py b/research/learned_optimizer/problems/model_adapter.py
deleted file mode 100644
index 8455992366dd46172e2a78471004779b1a4f091b..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/problems/model_adapter.py
+++ /dev/null
@@ -1,190 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Implementation of the ModelAdapter class."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import mock
-import tensorflow as tf
-
-from learned_optimizer.problems import problem_generator as pg
-
-
-class ModelAdapter(pg.Problem):
- """Adapts Tensorflow models/graphs into a form suitable for meta-training.
-
- This class adapts an existing TensorFlow graph into a form suitable for
- meta-training a learned optimizer.
- """
-
- def __init__(self, make_loss_and_init_fn):
- """Wraps a model in the Problem interface.
-
- make_loss_and_init argument is a callable that returns a tuple of
- two other callables as follows.
-
- The first will construct most of the graph and return the problem loss. It
- is essential that this graph contains the totality of the model's variables,
- but none of its queues.
-
- The second will return construct the model initialization graph given a list
- of parameters and return a callable that is passed an instance of
- tf.Session, and should initialize the models' parameters.
-
- An argument value function would look like this:
-
- ```python
- def make_loss_and_init_fn():
- inputs = queued_reader()
-
- def make_loss():
- return create_model_with_variables(inputs)
-
- def make_init_fn(parameters):
- saver = tf.Saver(parameters)
- def init_fn(sess):
- sess.restore(sess, ...)
- return init_fn
-
- return make_loss, make_init_fn
- ```
-
- Args:
- make_loss_and_init_fn: a callable, as described aboce
- """
- make_loss_fn, make_init_fn = make_loss_and_init_fn()
-
- self.make_loss_fn = make_loss_fn
- self.parameters, self.constants = _get_variables(make_loss_fn)
-
- if make_init_fn is not None:
- init_fn = make_init_fn(self.parameters + self.constants)
- else:
- init_op = tf.initialize_variables(self.parameters + self.constants)
- init_fn = lambda sess: sess.run(init_op)
-
- tf.logging.info("ModelAdapter parameters: %s",
- [op.name for op in self.parameters])
- tf.logging.info("ModelAdapter constants: %s",
- [op.name for op in self.constants])
-
- super(ModelAdapter, self).__init__(
- [], random_seed=None, noise_stdev=0.0, init_fn=init_fn)
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return self.parameters
-
- def init_variables(self, seed=None):
- """Returns a list of variables with the given shape."""
- # NOTE(siege): This is awkward, as these are not set as trainable.
- return self.parameters
-
- def objective(self, parameters, data=None, labels=None):
- """Computes the objective given a list of parameters.
-
- Args:
- parameters: The parameters to optimize (as a list of tensors)
- data: An optional batch of data for calculating objectives
- labels: An optional batch of corresponding labels
-
- Returns:
- A scalar tensor representing the objective value
- """
- # We need to set up a mapping based on the original parameter names, because
- # the parameters passed can be arbitrary tensors.
- parameter_mapping = {
- old_p.name: p
- for old_p, p in zip(self.parameters, parameters)
- }
-
- with tf.variable_scope(tf.get_variable_scope(), reuse=True):
- return _make_with_custom_variables(self.make_loss_fn, parameter_mapping)
-
-
-def _get_variables(func):
- """Calls func, returning any variables created.
-
- The created variables are modified to not be trainable, and are placed into
- the LOCAL_VARIABLES collection.
-
- Args:
- func: Function to be called.
-
- Returns:
- A tuple (variables, constants) where the first element is a list of
- trainable variables and the second is the non-trainable variables.
- """
- variables = []
- constants = []
-
- # We need to create these variables like normal, so grab the original
- # constructor before we mock it.
- original_init = tf.Variable.__init__
-
- def custom_init(self, *args, **kwargs):
- trainable = kwargs["trainable"]
- kwargs["trainable"] = False
- # Making these variables local keeps them out of the optimizer's checkpoints
- # somehow.
- kwargs["collections"] = [tf.GraphKeys.LOCAL_VARIABLES]
- original_init(self, *args, **kwargs)
- if trainable:
- variables.append(self)
- else:
- constants.append(self)
-
- # This name-scope is just a nicety for TensorBoard.
- with tf.name_scope("unused_graph"):
- with mock.patch.object(tf.Variable, "__init__", custom_init):
- func()
-
- return variables, constants
-
-
-def _make_with_custom_variables(func, variable_mapping):
- """Calls func and replaces the value of some variables created in it.
-
- Args:
- func: Function to be called.
- variable_mapping: A mapping of variable name to the replacement tensor or
- tf.Variable.
-
- Returns:
- The return value of func is returned.
- """
- original_value = tf.Variable.value
-
- def custom_value(self):
- if self.name in variable_mapping:
- replacement = variable_mapping[self.name]
- tf.logging.info("Replaced %s with %s" % (self.name, replacement))
-
- # value() method needs to return a tensor, we need to call value on it.
- # This has to be done manually like this otherwise we'll get an infinite
- # loop.
- if isinstance(replacement, tf.Variable):
- replacement = original_value(replacement)
-
- return replacement
- else:
- return original_value(self)
-
- with mock.patch.object(tf.Variable, "value", custom_value):
- with mock.patch.object(tf.Variable, "_AsTensor", custom_value):
- return func()
diff --git a/research/learned_optimizer/problems/problem_generator.py b/research/learned_optimizer/problems/problem_generator.py
deleted file mode 100644
index abe1008faadbb04163bc27e0b991e3ec4ba9e6bc..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/problems/problem_generator.py
+++ /dev/null
@@ -1,1016 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Generates toy optimization problems.
-
-This module contains a base class, Problem, that defines a minimal interface
-for optimization problems, and a few specific problem types that subclass it.
-
-Test functions for optimization: http://www.sfu.ca/~ssurjano/optimization.html
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-from learned_optimizer.problems import problem_spec as prob_spec
-
-tf.app.flags.DEFINE_float("l2_reg_scale", 1e-3,
- """Scaling factor for parameter value regularization
- in softmax classifier problems.""")
-FLAGS = tf.app.flags.FLAGS
-
-EPSILON = 1e-6
-MAX_SEED = 4294967295
-PARAMETER_SCOPE = "parameters"
-
-_Spec = prob_spec.Spec
-
-
-class Problem(object):
- """Base class for optimization problems.
-
- This defines an interface for optimization problems, including objective and
- gradients functions and a feed_generator function that yields data to pass to
- feed_dict in tensorflow.
-
- Subclasses of Problem must (at the minimum) override the objective method,
- which computes the objective/loss/cost to minimize, and specify the desired
- shape of the parameters in a list in the param_shapes attribute.
- """
-
- def __init__(self, param_shapes, random_seed, noise_stdev, init_fn=None):
- """Initializes a global random seed for the problem.
-
- Args:
- param_shapes: A list of tuples defining the expected shapes of the
- parameters for this problem
- random_seed: Either an integer (or None, in which case the seed is
- randomly drawn)
- noise_stdev: Strength (standard deviation) of added gradient noise
- init_fn: A function taking a tf.Session object that is used to
- initialize the problem's variables.
-
- Raises:
- ValueError: If the random_seed is not an integer and not None
- """
- if random_seed is not None and not isinstance(random_seed, int):
- raise ValueError("random_seed must be an integer or None")
-
- # Pick a random seed.
- self.random_seed = (np.random.randint(MAX_SEED) if random_seed is None
- else random_seed)
-
- # Store the noise level.
- self.noise_stdev = noise_stdev
-
- # Set the random seed to ensure any random data in the problem is the same.
- np.random.seed(self.random_seed)
-
- # Store the parameter shapes.
- self.param_shapes = param_shapes
-
- if init_fn is not None:
- self.init_fn = init_fn
- else:
- self.init_fn = lambda _: None
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_normal(shape, seed=seed) for shape in self.param_shapes]
-
- def init_variables(self, seed=None):
- """Returns a list of variables with the given shape."""
- with tf.variable_scope(PARAMETER_SCOPE):
- params = [tf.Variable(param) for param in self.init_tensors(seed)]
- return params
-
- def objective(self, parameters, data=None, labels=None):
- """Computes the objective given a list of parameters.
-
- Args:
- parameters: The parameters to optimize (as a list of tensors)
- data: An optional batch of data for calculating objectives
- labels: An optional batch of corresponding labels
-
- Returns:
- A scalar tensor representing the objective value
- """
- raise NotImplementedError
-
- def gradients(self, objective, parameters):
- """Compute gradients of the objective with respect to the parameters.
-
- Args:
- objective: The objective op (e.g. output of self.objective())
- parameters: A list of tensors (the parameters to optimize)
-
- Returns:
- A list of tensors representing the gradient for each parameter,
- returned in the same order as the given list
- """
- grads = tf.gradients(objective, list(parameters))
- noisy_grads = []
-
- for grad in grads:
- if isinstance(grad, tf.IndexedSlices):
- noise = self.noise_stdev * tf.random_normal(tf.shape(grad.values))
- new_grad = tf.IndexedSlices(grad.values + noise, grad.indices)
- else:
- new_grad = grad + self.noise_stdev * tf.random_normal(grad.get_shape())
- noisy_grads.append(new_grad)
-
- return noisy_grads
-
-
-class Quadratic(Problem):
- """Optimizes a random quadratic function.
-
- The objective is: f(x) = (1/2) ||Wx - y||_2^2
- where W is a random Gaussian matrix and y is a random Gaussian vector.
- """
-
- def __init__(self, ndim, random_seed=None, noise_stdev=0.0):
- """Initializes a random quadratic problem."""
- param_shapes = [(ndim, 1)]
- super(Quadratic, self).__init__(param_shapes, random_seed, noise_stdev)
-
- # Generate a random problem instance.
- self.w = np.random.randn(ndim, ndim).astype("float32")
- self.y = np.random.randn(ndim, 1).astype("float32")
-
- def objective(self, params, data=None, labels=None):
- """Quadratic objective (see base class for details)."""
- return tf.nn.l2_loss(tf.matmul(self.w, params[0]) - self.y)
-
-
-class SoftmaxClassifier(Problem):
- """Helper functions for supervised softmax classification problems."""
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_normal(shape, seed=seed) * 1.2 / np.sqrt(shape[0])
- for shape in self.param_shapes]
-
- def inference(self, params, data):
- """Computes logits given parameters and data.
-
- Args:
- params: List of parameter tensors or variables
- data: Batch of features with samples along the first dimension
-
- Returns:
- logits: Un-normalized logits with shape (num_samples, num_classes)
- """
- raise NotImplementedError
-
- def objective(self, params, data, labels):
- """Computes the softmax cross entropy.
-
- Args:
- params: List of parameter tensors or variables
- data: Batch of features with samples along the first dimension
- labels: Vector of labels with the same number of samples as the data
-
- Returns:
- loss: Softmax cross entropy loss averaged over the samples in the batch
-
- Raises:
- ValueError: If the objective is to be computed over >2 classes, because
- this operation is broken in tensorflow at the moment.
- """
- # Forward pass.
- logits = self.inference(params, data)
-
- # Compute the loss.
- l2reg = [tf.reduce_sum(param ** 2) for param in params]
- if int(logits.get_shape()[1]) == 2:
- labels = tf.cast(labels, tf.float32)
- losses = tf.nn.sigmoid_cross_entropy_with_logits(
- labels=labels, logits=logits[:, 0])
- else:
- raise ValueError("Unable to compute softmax cross entropy for more than"
- " 2 classes.")
-
- return tf.reduce_mean(losses) + tf.reduce_mean(l2reg) * FLAGS.l2_reg_scale
-
- def argmax(self, logits):
- """Samples the most likely class label given the logits.
-
- Args:
- logits: Un-normalized logits with shape (num_samples, num_classes)
-
- Returns:
- predictions: Predicted class labels, has shape (num_samples,)
- """
- return tf.cast(tf.argmax(tf.nn.softmax(logits), 1), tf.int32)
-
- def accuracy(self, params, data, labels):
- """Computes the accuracy (fraction of correct classifications).
-
- Args:
- params: List of parameter tensors or variables
- data: Batch of features with samples along the first dimension
- labels: Vector of labels with the same number of samples as the data
-
- Returns:
- accuracy: Fraction of correct classifications across the batch
- """
- predictions = self.argmax(self.inference(params, data))
- return tf.contrib.metrics.accuracy(predictions, tf.cast(labels, tf.int32))
-
-
-class SoftmaxRegression(SoftmaxClassifier):
- """Builds a softmax regression problem."""
-
- def __init__(self, n_features, n_classes, activation=tf.identity,
- random_seed=None, noise_stdev=0.0):
- self.activation = activation
- self.n_features = n_features
- param_shapes = [(n_features, n_classes), (n_classes,)]
- super(SoftmaxRegression, self).__init__(param_shapes,
- random_seed,
- noise_stdev)
-
- def inference(self, params, data):
- features = tf.reshape(data, (-1, self.n_features))
- return tf.matmul(features, params[0]) + params[1]
-
-
-class SparseSoftmaxRegression(SoftmaxClassifier):
- """Builds a sparse input softmax regression problem."""
-
- def __init__(self,
- n_features,
- n_classes,
- activation=tf.identity,
- random_seed=None,
- noise_stdev=0.0):
- self.activation = activation
- self.n_features = n_features
- param_shapes = [(n_classes, n_features), (n_features, n_classes), (
- n_classes,)]
- super(SparseSoftmaxRegression, self).__init__(param_shapes, random_seed,
- noise_stdev)
-
- def inference(self, params, data):
- all_embeddings, softmax_weights, softmax_bias = params
- embeddings = tf.nn.embedding_lookup(all_embeddings, tf.cast(data, tf.int32))
- embeddings = tf.reduce_sum(embeddings, 1)
- return tf.matmul(embeddings, softmax_weights) + softmax_bias
-
-
-class OneHotSparseSoftmaxRegression(SoftmaxClassifier):
- """Builds a sparse input softmax regression problem.
-
- This is identical to SparseSoftmaxRegression, but without using embedding
- ops.
- """
-
- def __init__(self,
- n_features,
- n_classes,
- activation=tf.identity,
- random_seed=None,
- noise_stdev=0.0):
- self.activation = activation
- self.n_features = n_features
- self.n_classes = n_classes
- param_shapes = [(n_classes, n_features), (n_features, n_classes), (
- n_classes,)]
- super(OneHotSparseSoftmaxRegression, self).__init__(param_shapes,
- random_seed,
- noise_stdev)
-
- def inference(self, params, data):
- all_embeddings, softmax_weights, softmax_bias = params
- num_ids = tf.shape(data)[1]
- one_hot_embeddings = tf.one_hot(tf.cast(data, tf.int32), self.n_classes)
- one_hot_embeddings = tf.reshape(one_hot_embeddings, [-1, self.n_classes])
- embeddings = tf.matmul(one_hot_embeddings, all_embeddings)
- embeddings = tf.reshape(embeddings, [-1, num_ids, self.n_features])
- embeddings = tf.reduce_sum(embeddings, 1)
- return tf.matmul(embeddings, softmax_weights) + softmax_bias
-
-
-class FullyConnected(SoftmaxClassifier):
- """Builds a multi-layer perceptron classifier."""
-
- def __init__(self, n_features, n_classes, hidden_sizes=(32, 64),
- activation=tf.nn.sigmoid, random_seed=None, noise_stdev=0.0):
- """Initializes an multi-layer perceptron classification problem."""
- # Store the number of features and activation function.
- self.n_features = n_features
- self.activation = activation
-
- # Define the network as a list of weight + bias shapes for each layer.
- param_shapes = []
- for ix, sz in enumerate(hidden_sizes + (n_classes,)):
-
- # The previous layer"s size (n_features if input).
- prev_size = n_features if ix == 0 else hidden_sizes[ix - 1]
-
- # Weight shape for this layer.
- param_shapes.append((prev_size, sz))
-
- # Bias shape for this layer.
- param_shapes.append((sz,))
-
- super(FullyConnected, self).__init__(param_shapes, random_seed, noise_stdev)
-
- def inference(self, params, data):
- # Flatten the features into a vector.
- features = tf.reshape(data, (-1, self.n_features))
-
- # Pass the data through the network.
- preactivations = tf.matmul(features, params[0]) + params[1]
-
- for layer in range(2, len(self.param_shapes), 2):
- net = self.activation(preactivations)
- preactivations = tf.matmul(net, params[layer]) + params[layer + 1]
-
- return preactivations
-
- def accuracy(self, params, data, labels):
- """Computes the accuracy (fraction of correct classifications).
-
- Args:
- params: List of parameter tensors or variables
- data: Batch of features with samples along the first dimension
- labels: Vector of labels with the same number of samples as the data
-
- Returns:
- accuracy: Fraction of correct classifications across the batch
- """
- predictions = self.argmax(self.activation(self.inference(params, data)))
- return tf.contrib.metrics.accuracy(predictions, tf.cast(labels, tf.int32))
-
-
-class ConvNet(SoftmaxClassifier):
- """Builds an N-layer convnet for image classification."""
-
- def __init__(self,
- image_shape,
- n_classes,
- filter_list,
- activation=tf.nn.relu,
- random_seed=None,
- noise_stdev=0.0):
- # Number of channels, number of pixels in x- and y- dimensions.
- n_channels, px, py = image_shape
-
- # Store the activation.
- self.activation = activation
-
- param_shapes = []
- input_size = n_channels
- for fltr in filter_list:
- # Add conv2d filters.
- param_shapes.append((fltr[0], fltr[1], input_size, fltr[2]))
- input_size = fltr[2]
-
- # Number of units in the final (dense) layer.
- self.affine_size = input_size * px * py
-
- param_shapes.append((self.affine_size, n_classes)) # affine weights
- param_shapes.append((n_classes,)) # affine bias
-
- super(ConvNet, self).__init__(param_shapes, random_seed, noise_stdev)
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_normal(shape, mean=0., stddev=0.01, seed=seed)
- for shape in self.param_shapes]
-
- def inference(self, params, data):
-
- # Unpack.
- w_conv_list = params[:-2]
- output_w, output_b = params[-2:]
-
- conv_input = data
- for w_conv in w_conv_list:
- layer = tf.nn.conv2d(conv_input, w_conv, strides=[1] * 4, padding="SAME")
- output = self.activation(layer)
- conv_input = output
-
- # Flatten.
- flattened = tf.reshape(conv_input, (-1, self.affine_size))
-
- # Fully connected layer.
- return tf.matmul(flattened, output_w) + output_b
-
-
-class Bowl(Problem):
- """A 2D quadratic bowl."""
-
- def __init__(self, condition_number, angle=0.0,
- random_seed=None, noise_stdev=0.0):
- assert condition_number > 0, "Condition number must be positive."
-
- # Define parameter shapes.
- param_shapes = [(2, 1)]
- super(Bowl, self).__init__(param_shapes, random_seed, noise_stdev)
-
- self.condition_number = condition_number
- self.angle = angle
- self._build_matrix(condition_number, angle)
-
- def _build_matrix(self, condition_number, angle):
- """Builds the Hessian matrix."""
- hessian = np.array([[condition_number, 0.], [0., 1.]], dtype="float32")
-
- # Build the rotation matrix.
- rotation_matrix = np.array([
- [np.cos(angle), -np.sin(angle)],
- [np.sin(angle), np.cos(angle)]
- ])
-
- # The objective is 0.5 * || Ax ||_2^2
- # where the data matrix (A) is: sqrt(Hessian).dot(rotation_matrix).
- self.matrix = np.sqrt(hessian).dot(rotation_matrix)
-
- def objective(self, params, data=None, labels=None):
- mtx = tf.constant(self.matrix, dtype=tf.float32)
- return tf.nn.l2_loss(tf.matmul(mtx, params[0]))
-
- def surface(self, xlim=5, ylim=5, n=50):
- xm, ym = _mesh(xlim, ylim, n)
- pts = np.vstack([xm.ravel(), ym.ravel()])
- zm = 0.5 * np.linalg.norm(self.matrix.dot(pts), axis=0) ** 2
- return xm, ym, zm.reshape(n, n)
-
-
-class Problem2D(Problem):
-
- def __init__(self, random_seed=None, noise_stdev=0.0):
- param_shapes = [(2,)]
- super(Problem2D, self).__init__(param_shapes, random_seed, noise_stdev)
-
- def surface(self, n=50, xlim=5, ylim=5):
- """Computes the objective surface over a 2d mesh."""
-
- # Create a mesh over the given coordinate ranges.
- xm, ym = _mesh(xlim, ylim, n)
-
- with tf.Graph().as_default(), tf.Session() as sess:
-
- # Ops to compute the objective at every (x, y) point.
- x = tf.placeholder(tf.float32, shape=xm.shape)
- y = tf.placeholder(tf.float32, shape=ym.shape)
- obj = self.objective([[x, y]])
-
- # Run the computation.
- zm = sess.run(obj, feed_dict={x: xm, y: ym})
-
- return xm, ym, zm
-
-
-class Rosenbrock(Problem2D):
- """See https://en.wikipedia.org/wiki/Rosenbrock_function.
-
- This function has a single global minima at [1, 1]
- The objective value at this point is zero.
- """
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_uniform(shape, minval=-5., maxval=10., seed=seed)
- for shape in self.param_shapes]
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
- obj = (1 - x)**2 + 100 * (y - x**2)**2
- return tf.squeeze(obj)
-
-
-def make_rosenbrock_loss_and_init(device=None):
- """A variable-backed version of Rosenbrock problem.
-
- See the Rosenbrock class for details.
-
- Args:
- device: Where to place the ops of this problem.
-
- Returns:
- A tuple of two callables, first of which creates the loss and the second
- creates the parameter initializer function.
- """
- def make_rosenbrock_loss():
- with tf.name_scope("optimizee"):
- with tf.device(device):
- x = tf.get_variable("x", [1])
- y = tf.get_variable("y", [1])
- c = tf.get_variable(
- "c", [1],
- initializer=tf.constant_initializer(100.0),
- trainable=False)
- obj = (1 - x)**2 + c * (y - x**2)**2
- return tf.squeeze(obj)
-
- def make_init_fn(parameters):
- with tf.device(device):
- init_op = tf.variables_initializer(parameters)
- def init_fn(sess):
- tf.logging.info("Initializing model parameters.")
- sess.run(init_op)
- return init_fn
-
- return make_rosenbrock_loss, make_init_fn
-
-
-class Saddle(Problem2D):
- """Loss surface around a saddle point."""
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
- obj = x ** 2 - y ** 2
- return tf.squeeze(obj)
-
-
-class LogSumExp(Problem2D):
- """2D function defined by the log of the sum of exponentials."""
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
- obj = tf.log(tf.exp(x + 3. * y - 0.1) +
- tf.exp(x - 3. * y - 0.1) +
- tf.exp(-x - 0.1) + 1.0)
- return tf.squeeze(obj)
-
-
-class Ackley(Problem2D):
- """Ackley's function (contains many local minima)."""
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_uniform(shape, minval=-32.768, maxval=32.768, seed=seed)
- for shape in self.param_shapes]
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
- obj = (-20 * tf.exp(-0.2 * tf.sqrt(0.5 * (x ** 2 + y ** 2))) -
- tf.exp(0.5 * (tf.cos(2 * np.pi * x) + tf.cos(2 * np.pi * y))) +
- tf.exp(1.0) + 20.)
- return tf.squeeze(obj)
-
-
-class Beale(Problem2D):
- """Beale function (a multimodal function with sharp peaks)."""
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_uniform(shape, minval=-4.5, maxval=4.5, seed=seed)
- for shape in self.param_shapes]
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
- obj = ((1.5 - x + x * y) ** 2 +
- (2.25 - x + x * y ** 2) ** 2 +
- (2.625 - x + x * y ** 3) ** 2)
- return tf.squeeze(obj)
-
-
-class Booth(Problem2D):
- """Booth's function (has a long valley along one dimension)."""
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_uniform(shape, minval=-10., maxval=10., seed=seed)
- for shape in self.param_shapes]
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
- obj = (x + 2 * y - 7) ** 2 + (2 * x + y - 5) ** 2
- return tf.squeeze(obj)
-
-
-class StyblinskiTang(Problem2D):
- """Styblinski-Tang function (a bumpy function in two dimensions)."""
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_uniform(shape, minval=-5., maxval=5., seed=seed)
- for shape in self.param_shapes]
-
- def objective(self, params, data=None, labels=None):
- params = tf.split(params[0], 2, axis=0)
- obj = 0.5 * tf.reduce_sum([x ** 4 - 16 * x ** 2 + 5 * x
- for x in params], 0) + 80.
- return tf.squeeze(obj)
-
-
-class Matyas(Problem2D):
- """Matyas function (a function with a single global minimum in a valley)."""
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_uniform(shape, minval=-10, maxval=10, seed=seed)
- for shape in self.param_shapes]
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
- obj = 0.26 * (x ** 2 + y ** 2) - 0.48 * x * y
- return tf.squeeze(obj)
-
-
-class Branin(Problem2D):
- """Branin function (a function with three global minima)."""
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- x1 = tf.random_uniform((1,), minval=-5., maxval=10.,
- seed=seed)
- x2 = tf.random_uniform((1,), minval=0., maxval=15.,
- seed=seed)
- return [tf.concat([x1, x2], 0)]
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
-
- # Define some constants.
- a = 1.
- b = 5.1 / (4. * np.pi ** 2)
- c = 5 / np.pi
- r = 6.
- s = 10.
- t = 1 / (8. * np.pi)
-
- # Evaluate the function.
- obj = a * (y - b * x ** 2 + c * x - r) ** 2 + s * (1 - t) * tf.cos(x) + s
- return tf.squeeze(obj)
-
-
-class Michalewicz(Problem2D):
- """Michalewicz function (has steep ridges and valleys)."""
-
- def init_tensors(self, seed=None):
- """Returns a list of tensors with the given shape."""
- return [tf.random_uniform(shape, minval=0., maxval=np.pi, seed=seed)
- for shape in self.param_shapes]
-
- def objective(self, params, data=None, labels=None):
- x, y = tf.split(params[0], 2, axis=0)
- m = 5 # Defines how steep the ridges are (larger m => steeper ridges).
- obj = 2. - (tf.sin(x) * tf.sin(x ** 2 / np.pi) ** (2 * m) +
- tf.sin(y) * tf.sin(2 * y ** 2 / np.pi) ** (2 * m))
- return tf.squeeze(obj)
-
-
-class Rescale(Problem):
- """Takes an existing problem, and rescales all the parameters."""
-
- def __init__(self, problem_spec, scale=10., noise_stdev=0.0):
- self.problem = problem_spec.build()
- self.param_shapes = self.problem.param_shapes
- self.scale = scale
-
- super(Rescale, self).__init__(self.param_shapes, random_seed=None,
- noise_stdev=noise_stdev)
-
- def init_tensors(self, seed=None):
- params_raw = self.problem.init_tensors(seed=seed)
- params = [t * self.scale for t in params_raw]
- return params
-
- def objective(self, params, data=None, labels=None):
- params_raw = [t/self.scale for t in params]
-
- problem_obj = self.problem.objective(params_raw, data, labels)
- return problem_obj
-
-
-class SumTask(Problem):
- """Takes a list of problems and modifies the objective to be their sum."""
-
- def __init__(self, problem_specs, noise_stdev=0.0):
- self.problems = [ps.build() for ps in problem_specs]
- self.param_shapes = []
- for prob in self.problems:
- self.param_shapes += prob.param_shapes
-
- super(SumTask, self).__init__(self.param_shapes, random_seed=None,
- noise_stdev=noise_stdev)
-
- def init_tensors(self, seed=None):
- tensors = []
- for prob in self.problems:
- tensors += prob.init_tensors(seed=seed)
- return tensors
-
- def objective(self, params, data=None, labels=None):
- obj = 0.
- index = 0
- for prob in self.problems:
- num_params = len(prob.param_shapes)
- obj += prob.objective(params[index:index + num_params])
- index += num_params
- return obj
-
-
-class IsotropicQuadratic(Problem):
- """An isotropic quadratic problem."""
-
- def objective(self, params, data=None, labels=None):
- return sum([tf.reduce_sum(param ** 2) for param in params])
-
-
-class Norm(Problem):
- """Takes an existing problem and modifies the objective to be its N-norm."""
-
- def __init__(self, ndim, random_seed=None, noise_stdev=0.0, norm_power=2.):
- param_shapes = [(ndim, 1)]
- super(Norm, self).__init__(param_shapes, random_seed, noise_stdev)
-
- # Generate a random problem instance.
- self.w = np.random.randn(ndim, ndim).astype("float32")
- self.y = np.random.randn(ndim, 1).astype("float32")
- self.norm_power = norm_power
-
- def objective(self, params, data=None, labels=None):
- diff = tf.matmul(self.w, params[0]) - self.y
- exp = 1. / self.norm_power
- loss = tf.reduce_sum((tf.abs(diff) + EPSILON) ** self.norm_power) ** exp
- return loss
-
-
-class LogObjective(Problem):
- """Takes an existing problem and modifies the objective to be its log."""
-
- def __init__(self, problem_spec):
- self.problem = problem_spec.build()
- self.param_shapes = self.problem.param_shapes
-
- super(LogObjective, self).__init__(self.param_shapes,
- random_seed=None,
- noise_stdev=0.0)
-
- def objective(self, params, data=None, labels=None):
- problem_obj = self.problem.objective(params, data, labels)
- return tf.log(problem_obj + EPSILON) - tf.log(EPSILON)
-
-
-class SparseProblem(Problem):
- """Takes a problem and sets gradients to 0 with the given probability."""
-
- def __init__(self,
- problem_spec,
- zero_probability=0.99,
- random_seed=None,
- noise_stdev=0.0):
- self.problem = problem_spec.build()
- self.param_shapes = self.problem.param_shapes
- self.zero_prob = zero_probability
-
- super(SparseProblem, self).__init__(self.param_shapes,
- random_seed=random_seed,
- noise_stdev=noise_stdev)
-
- def objective(self, parameters, data=None, labels=None):
- return self.problem.objective(parameters, data, labels)
-
- def gradients(self, objective, parameters):
- grads = tf.gradients(objective, list(parameters))
-
- new_grads = []
- for grad in grads:
- mask = tf.greater(self.zero_prob, tf.random_uniform(grad.get_shape()))
- zero_grad = tf.zeros_like(grad, dtype=tf.float32)
- noisy_grad = grad + self.noise_stdev * tf.random_normal(grad.get_shape())
- new_grads.append(tf.where(mask, zero_grad, noisy_grad))
- return new_grads
-
-
-class DependencyChain(Problem):
- """A problem in which parameters must be optimized in order.
-
- A sequence of parameters which all need to be brought to 0, but where each
- parameter in the sequence can't be brought to 0 until the preceding one
- has been. This should take a long time to optimize, with steady
- (or accelerating) progress throughout the entire process.
- """
-
- def __init__(self, ndim, random_seed=None, noise_stdev=0.):
- param_shapes = [(ndim + 1,)]
- self.ndim = ndim
- super(DependencyChain, self).__init__(
- param_shapes, random_seed, noise_stdev)
-
- def objective(self, params, data=None, labels=None):
- terms = params[0][0]**2 + params[0][1:]**2 / (params[0][:-1]**2 + EPSILON)
- return tf.reduce_sum(terms)
-
-
-class MinMaxWell(Problem):
- """Problem with global min when both the min and max (absolute) params are 1.
-
- The gradient for all but two parameters (the min and max) is zero. This
- should therefore encourage the optimizer to behave sensible even when
- parameters have zero gradients, as is common eg for some deep neural nets.
- """
-
- def __init__(self, ndim, random_seed=None, noise_stdev=0.):
- param_shapes = [(ndim,)]
- self.ndim = ndim
- super(MinMaxWell, self).__init__(param_shapes, random_seed, noise_stdev)
-
- def objective(self, params, data=None, labels=None):
- params_sqr = params[0]**2
- min_sqr = tf.reduce_min(params_sqr)
- max_sqr = tf.reduce_max(params_sqr)
- epsilon = 1e-12
-
- return max_sqr + 1./min_sqr - 2. + epsilon
-
-
-class OutwardSnake(Problem):
- """A winding path out to infinity.
-
- Ideal step length stays constant along the entire path.
- """
-
- def __init__(self, ndim, random_seed=None, noise_stdev=0.):
- param_shapes = [(ndim,)]
- self.ndim = ndim
- super(OutwardSnake, self).__init__(param_shapes, random_seed, noise_stdev)
-
- def objective(self, params, data, labels=None):
- radius = tf.sqrt(tf.reduce_sum(params[0]**2))
- rad_loss = tf.reduce_sum(1. / (radius + 1e-6) * data[:, 0])
-
- sin_dist = params[0][1:] - tf.cos(params[0][:-1]) * np.pi
- sin_loss = tf.reduce_sum((sin_dist * data[:, 1:])**2)
-
- return rad_loss + sin_loss
-
-
-class ProjectionQuadratic(Problem):
- """Dataset consists of different directions to probe. Global min is at 0."""
-
- def __init__(self, ndim, random_seed=None, noise_stdev=0.):
- param_shapes = [(1, ndim)]
- super(ProjectionQuadratic, self).__init__(
- param_shapes, random_seed, noise_stdev)
-
- def objective(self, params, data, labels=None):
- return tf.reduce_sum((params[0] * data)**2)
-
-
-class SumOfQuadratics(Problem):
-
- def __init__(self, ndim, random_seed=None, noise_stdev=0.):
- param_shapes = [(1, ndim)]
- super(SumOfQuadratics, self).__init__(
- param_shapes, random_seed, noise_stdev)
-
- def objective(self, params, data, labels=None):
- epsilon = 1e-12
- # Assume dataset is designed so that the global minimum is at params=0.
- # Subtract loss at params=0, so that global minimum has objective value
- # epsilon (added to avoid floating point issues).
- return (tf.reduce_sum((params[0] - data)**2) - tf.reduce_sum(data**2) +
- epsilon)
-
-
-class MatMulAlgorithm(Problem):
- """A 6-th order polynomial optimization problem.
-
- This problem is parametrized by n and k. A solution to this problem with
- objective value exactly zero defines a matrix multiplication algorithm of
- n x n matrices using k multiplications between matrices. When applied
- recursively, such an algorithm has complexity O(n^(log_n(k))).
-
- Given n, it is not known in general which values of k in [n^2, n^3] have a
- solution. There is always a solution with k = n^3 (this is the naive
- algorithm).
-
- In the special case n = 2, it is known that there are solutions for k = {7, 8}
- but not for k <= 6. For n = 3, it is known that there are exact solutions for
- 23 <= k <= 27, and there are asymptotic solutions for k = {21, 22}, but the
- other cases are unknown.
-
- For a given n and k, if one solution exists then infinitely many solutions
- exist due to permutation and scaling symmetries in the parameters.
-
- This is a very hard problem for some values of n and k (e.g. n = 3, k = 21),
- but very easy for other values (e.g. n = 2, k = 7).
-
- For a given n and k, the specific formulation of this problem is as follows.
- Let theta_a, theta_b, theta_c be parameter matrices with respective dimensions
- [n**2, k], [n**2, k], [k, n**2]. Then for any matrices a, b with shape [n, n],
- we can form the matrix c with shape [n, n] via the operation:
- ((vec(a) * theta_a) .* (vec(b) * theta_b)) * theta_c = vec(c), (#)
- where vec(x) is the operator that flattens a matrix with shape [n, n] into a
- row vector with shape [1, n**2], * denotes matrix multiplication and .*
- denotes elementwise multiplication.
-
- This operation, parameterized by theta_a, theta_b, theta_c, is a matrix
- multiplication algorithm iff c = a*b for all [n, n] matrices a and b. But
- actually it suffices to verify all combinations of one-hot matrices a and b,
- of which there are n**4 such combinations. This gives a batch of n**4 matrix
- triplets (a, b, c) such that equation (#) must hold for each triplet. We solve
- for theta_a, theta_b, theta_c by minimizing the sum of squares of errors
- across this batch.
-
- Finally, theta_c can be computed from theta_a and theta_b. Therefore it
- suffices to learn theta_a and theta_b, from which theta_c and therefore the
- objective value can be computed.
- """
-
- def __init__(self, n, k):
- assert isinstance(n, int), "n must be an integer"
- assert isinstance(k, int), "k must be an integer"
- assert n >= 2, "Must have n >= 2"
- assert k >= n**2 and k <= n**3, "Must have n**2 <= k <= n**3"
-
- param_shapes = [(n**2, k), (n**2, k)] # theta_a, theta_b
- super(MatMulAlgorithm, self).__init__(
- param_shapes, random_seed=None, noise_stdev=0.0)
-
- self.n = n
- self.k = k
-
- # Build a batch of all combinations of one-hot matrices a, b, and their
- # respective products c. Correctness on this batch is a necessary and
- # sufficient condition for the algorithm to be valid. The number of matrices
- # in {a, b, c}_3d is n**4 and each matrix is n x n.
- onehots = np.identity(n**2).reshape(n**2, n, n)
- a_3d = np.repeat(onehots, n**2, axis=0)
- b_3d = np.tile(onehots, [n**2, 1, 1])
- c_3d = np.matmul(a_3d, b_3d)
-
- # Convert the batch to 2D Tensors.
- self.a = tf.constant(a_3d.reshape(n**4, n**2), tf.float32, name="a")
- self.b = tf.constant(b_3d.reshape(n**4, n**2), tf.float32, name="b")
- self.c = tf.constant(c_3d.reshape(n**4, n**2), tf.float32, name="c")
-
- def init_tensors(self, seed=None):
- # Initialize params such that the columns of theta_a and theta_b have L2
- # norm 1.
- def _param_initializer(shape, seed=None):
- x = tf.random_normal(shape, dtype=tf.float32, seed=seed)
- return tf.transpose(tf.nn.l2_normalize(tf.transpose(x), 1))
-
- return [_param_initializer(shape, seed) for shape in self.param_shapes]
-
- def objective(self, parameters, data=None, labels=None):
- theta_a = parameters[0]
- theta_b = parameters[1]
-
- # Compute theta_c from theta_a and theta_b.
- p = tf.matmul(self.a, theta_a) * tf.matmul(self.b, theta_b)
- p_trans = tf.transpose(p, name="p_trans")
- p_inv = tf.matmul(
- tf.matrix_inverse(tf.matmul(p_trans, p)), p_trans, name="p_inv")
- theta_c = tf.matmul(p_inv, self.c, name="theta_c")
-
- # Compute the "predicted" value of c.
- c_hat = tf.matmul(p, theta_c, name="c_hat")
-
- # Compute the loss (sum of squared errors).
- loss = tf.reduce_sum((c_hat - self.c)**2, name="loss")
-
- return loss
-
-
-def matmul_problem_sequence(n, k_min, k_max):
- """Helper to generate a sequence of matrix multiplication problems."""
- return [(_Spec(MatMulAlgorithm, (n, k), {}), None, None)
- for k in range(k_min, k_max + 1)]
-
-
-def init_fixed_variables(arrays):
- with tf.variable_scope(PARAMETER_SCOPE):
- params = [tf.Variable(arr.astype("float32")) for arr in arrays]
- return params
-
-
-def _mesh(xlim, ylim, n):
- """Creates a 2D meshgrid covering the given ranges.
-
- Args:
- xlim: int that defines the desired x-range (-xlim, xlim)
- ylim: int that defines the desired y-range (-ylim, ylim)
- n: number of points in each dimension of the mesh
-
- Returns:
- xm: 2D array of x-values in the mesh
- ym: 2D array of y-values in the mesh
- """
- return np.meshgrid(np.linspace(-xlim, xlim, n),
- np.linspace(-ylim, ylim, n))
diff --git a/research/learned_optimizer/problems/problem_sets.py b/research/learned_optimizer/problems/problem_sets.py
deleted file mode 100644
index eaf9273b87ef69c6b3087330bdf46c8de7107a15..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/problems/problem_sets.py
+++ /dev/null
@@ -1,561 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Groups of problems of different types for optimizer training."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-from learned_optimizer.problems import datasets
-from learned_optimizer.problems import model_adapter
-from learned_optimizer.problems import problem_generator as pg
-from learned_optimizer.problems import problem_spec
-
-_Spec = problem_spec.Spec
-
-
-def quadratic_problems():
- return [
- (_Spec(pg.Quadratic, (20,), {}), None, None),
- (_Spec(pg.Quadratic, (25,), {}), None, None),
- (_Spec(pg.Quadratic, (50,), {}), None, None),
- (_Spec(pg.Quadratic, (100,), {}), None, None),
- ]
-
-
-# Note: this group contains one non-noisy problem for historical reasons. The
-# original training set before the refactor included this set of quadratics.
-def quadratic_problems_noisy():
- return [
- (_Spec(pg.Quadratic, (20,), {"noise_stdev": 0.5}), None, None),
- (_Spec(pg.Quadratic, (25,), {"noise_stdev": 0.0}), None, None),
- (_Spec(pg.Quadratic, (50,), {"noise_stdev": 1.0}), None, None),
- (_Spec(pg.Quadratic, (100,), {"noise_stdev": 2.0}), None, None),
- ]
-
-
-def quadratic_problems_large():
- return [
- (_Spec(pg.Quadratic, (784,), {}), None, None),
- (_Spec(pg.Quadratic, (1024,), {}), None, None),
- (_Spec(pg.Quadratic, (2048,), {}), None, None),
- ]
-
-
-def bowl_problems():
- return [
- (_Spec(pg.Bowl, (0.1,), {"noise_stdev": 0.0}), None, None),
- (_Spec(pg.Bowl, (1.0,), {"noise_stdev": 0.0}), None, None),
- (_Spec(pg.Bowl, (5.0,), {"noise_stdev": 0.0}), None, None),
- (_Spec(pg.Bowl, (5.0,), {"noise_stdev": 0.0, "angle": np.pi / 4.}),
- None, None),
- ]
-
-
-def bowl_problems_noisy():
- return [
- (_Spec(pg.Bowl, (0.1,), {"noise_stdev": 0.1}), None, None),
- (_Spec(pg.Bowl, (1.0,), {"noise_stdev": 0.1}), None, None),
- (_Spec(pg.Bowl, (5.0,), {"noise_stdev": 0.1}), None, None),
- (_Spec(pg.Bowl, (5.0,), {"noise_stdev": 0.1, "angle": np.pi / 4.}),
- None, None),
- ]
-
-
-def sparse_softmax_2_class_sparse_problems():
- return [(_Spec(pg.SparseSoftmaxRegression, (5, 2), {"noise_stdev": 0.0}),
- datasets.noisy_parity_class(5, random_seed=123), 23),]
-
-
-def one_hot_sparse_softmax_2_class_sparse_problems():
- return [
- (_Spec(pg.OneHotSparseSoftmaxRegression, (5, 2), {"noise_stdev": 0.0}),
- datasets.noisy_parity_class(5, random_seed=123), 23),
- ]
-
-
-def softmax_2_class_problems():
- return [
- (_Spec(pg.SoftmaxRegression, (10, 2), {}), datasets.random(
- 10, 1000, random_seed=123, sep=2.0), 100),
- (_Spec(pg.SoftmaxRegression, (100, 2), {}), datasets.random(
- 100, 1000, random_seed=123), 50),
- (_Spec(pg.SoftmaxRegression, (200, 2), {}), datasets.random(
- 200, 1000, random_seed=123, sep=1.5), 20),
- (_Spec(pg.SoftmaxRegression, (256, 2), {}), datasets.random(
- 256, 1000, random_seed=123, sep=1.5), 100),
- ]
-
-
-def softmax_2_class_problems_noisy():
- return [
- (_Spec(pg.SoftmaxRegression, (10, 2), {"noise_stdev": 0.5}),
- datasets.random(10, 1000, random_seed=123, sep=2.0), 100),
- (_Spec(pg.SoftmaxRegression, (100, 2), {"noise_stdev": 0.1}),
- datasets.random(100, 1000, random_seed=123), 50),
- (_Spec(pg.SoftmaxRegression, (200, 2), {"noise_stdev": 0.1}),
- datasets.random(200, 1000, random_seed=123, sep=1.5), 20),
- (_Spec(pg.SoftmaxRegression, (256, 2), {"noise_stdev": 0.5}),
- datasets.random(256, 1000, random_seed=123, sep=1.5), 100),
- ]
-
-
-def optimization_test_problems():
- return [
- (_Spec(pg.Ackley, (), {}), None, None),
- (_Spec(pg.Beale, (), {}), None, None),
- (_Spec(pg.Booth, (), {}), None, None),
- (_Spec(pg.Branin, (), {}), None, None),
- (_Spec(pg.LogSumExp, (), {}), None, None),
- (_Spec(pg.Matyas, (), {}), None, None),
- (_Spec(pg.Michalewicz, (), {}), None, None),
- (_Spec(pg.Rosenbrock, (), {}), None, None),
- (_Spec(pg.StyblinskiTang, (), {}), None, None),
- ]
-
-
-def optimization_test_problems_noisy():
- return [
- (_Spec(pg.Ackley, (), {"noise_stdev": 1.}), None, None),
- (_Spec(pg.Beale, (), {"noise_stdev": 1.}), None, None),
- (_Spec(pg.Booth, (), {"noise_stdev": 1.}), None, None),
- (_Spec(pg.Branin, (), {"noise_stdev": 1.}), None, None),
- (_Spec(pg.LogSumExp, (), {"noise_stdev": 1.}), None, None),
- (_Spec(pg.Matyas, (), {"noise_stdev": 1.}), None, None),
- (_Spec(pg.Michalewicz, (), {"noise_stdev": 1.}), None, None),
- (_Spec(pg.Rosenbrock, (), {"noise_stdev": 1.}), None, None),
- (_Spec(pg.StyblinskiTang, (), {"noise_stdev": 1.}), None, None),
- ]
-
-
-def fully_connected_random_2_class_problems():
- return [
- (_Spec(pg.FullyConnected, (8, 2),
- {"hidden_sizes": (8, 5,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(8, 1000), 10),
- (_Spec(pg.FullyConnected, (12, 2),
- {"hidden_sizes": (8, 5, 3), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(12, 1000), 200),
- (_Spec(pg.FullyConnected, (5, 2),
- {"hidden_sizes": (4, 4, 4, 4,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(5, 1000), 100),
- (_Spec(pg.FullyConnected, (11, 2),
- {"hidden_sizes": (4, 5, 6,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(11, 1000), 64),
- (_Spec(pg.FullyConnected, (9, 2),
- {"hidden_sizes": (8,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(9, 1000), 128),
- (_Spec(pg.FullyConnected, (7, 2),
- {"hidden_sizes": (8, 5,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(7, 1000), 16),
- (_Spec(pg.FullyConnected, (8, 2),
- {"hidden_sizes": (32, 64,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(8, 1000), 10),
- (_Spec(pg.FullyConnected, (12, 2),
- {"hidden_sizes": (16, 8, 3), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(12, 1000), 200),
- (_Spec(pg.FullyConnected, (5, 2),
- {"hidden_sizes": (8, 8, 8, 8,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(5, 1000), 100),
- (_Spec(pg.FullyConnected, (11, 2),
- {"hidden_sizes": (10, 12, 12,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(11, 1000), 64),
- (_Spec(pg.FullyConnected, (9, 2),
- {"hidden_sizes": (32,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(9, 1000), 128),
- (_Spec(pg.FullyConnected, (7, 2),
- {"hidden_sizes": (32, 64,), "activation": tf.nn.sigmoid}),
- datasets.random_mlp(7, 1000), 16),
- ]
-
-
-def matmul_problems():
- return sum([
- pg.matmul_problem_sequence(2, 5, 8),
- pg.matmul_problem_sequence(3, 19, 24)], [])
-
-
-def log_objective_problems():
- return [
- (_Spec(pg.LogObjective, [_Spec(pg.Quadratic, (20,), {})], {}),
- None, None),
- (_Spec(pg.LogObjective, [_Spec(pg.Quadratic, (50,), {})], {}),
- None, None),
- (_Spec(pg.LogObjective, [_Spec(pg.Quadratic, (100,), {})], {}),
- None, None),
- (_Spec(pg.LogObjective, [_Spec(pg.Bowl, (0.1,), {})], {}), None, None),
- (_Spec(pg.LogObjective, [_Spec(pg.Bowl, (1.0,), {})], {}), None, None),
- (_Spec(pg.LogObjective, [_Spec(pg.Bowl, (5.0,), {})], {}), None, None),
- ]
-
-
-def sparse_gradient_problems():
- return [
- (_Spec(pg.SparseProblem, [_Spec(pg.Quadratic, (20,), {})], {}),
- None, None),
- (_Spec(pg.SparseProblem, [_Spec(pg.Quadratic, (50,), {})], {}),
- None, None),
- (_Spec(pg.SparseProblem, [_Spec(pg.Quadratic, (100,), {})], {}),
- None, None),
- (_Spec(pg.SparseProblem, [_Spec(pg.Bowl, (0.1,), {})], {}), None, None),
- (_Spec(pg.SparseProblem, [_Spec(pg.Bowl, (1.0,), {})], {}), None, None),
- (_Spec(pg.SparseProblem, [_Spec(pg.Bowl, (5.0,), {})], {}), None, None),
- ]
-
-
-def sparse_gradient_problems_mlp():
- return [
- (_Spec(pg.SparseProblem, [
- _Spec(pg.FullyConnected, (8, 2), {
- "hidden_sizes": (8, 5,),
- "activation": tf.nn.sigmoid
- })
- ], {}), datasets.random_mlp(8, 1000), 10),
- (_Spec(pg.SparseProblem, [
- _Spec(pg.FullyConnected, (12, 2), {
- "hidden_sizes": (8, 5, 3),
- "activation": tf.nn.sigmoid
- })
- ], {}), datasets.random_mlp(12, 1000), 200),
- (_Spec(pg.SparseProblem, [
- _Spec(pg.FullyConnected, (5, 2), {
- "hidden_sizes": (4, 4, 4, 4,),
- "activation": tf.nn.sigmoid
- })
- ], {}), datasets.random_mlp(5, 1000), 100),
- ]
-
-
-def rescale_problems():
- return [
- (_Spec(pg.Rescale, [_Spec(pg.Norm, (18,), {"norm_power": 2.5})],
- {"scale": 0.123}), None, None),
- (_Spec(pg.Rescale, [_Spec(pg.Norm, (18,), {"norm_power": 1.5})],
- {"scale": 8}), None, None),
- (_Spec(pg.Rescale, [_Spec(pg.Norm, (18,), {"norm_power": 2.})],
- {"scale": 50}), None, None),
- (_Spec(pg.Rescale, [_Spec(pg.Norm, (18,), {"norm_power": 3.})],
- {"scale": 200}), None, None),
- (_Spec(pg.Rescale, [_Spec(pg.Norm, (18,), {"norm_power": 1.})],
- {"scale": 1000}), None, None),
- (_Spec(pg.Rescale, [_Spec(pg.Quadratic, (20,), {})], {"scale": 0.1}),
- None, None),
- (_Spec(pg.Rescale, [_Spec(pg.Quadratic, (25,), {})], {"scale": 10.}),
- None, None),
- (_Spec(pg.Rescale, [_Spec(pg.Quadratic, (50,), {})], {"scale": 350.}),
- None, None),
- (_Spec(pg.Rescale, [_Spec(pg.Quadratic, (100,), {})], {"scale": 132}),
- None, None),
- ]
-
-
-def norm_problems():
- return [
- # < 1 Norm causes NaN gradients early in training.
- (_Spec(pg.Norm, (27,), {"norm_power": 1.}), None, None),
- (_Spec(pg.Norm, (25,), {"norm_power": 2.}), None, None),
- (_Spec(pg.Norm, (22,), {"norm_power": 3.}), None, None),
- ]
-
-
-def norm_problems_noisy():
- return [
- # < 1 Norm causes NaN gradients early in training.
- (_Spec(pg.Norm, (19,), {"noise_stdev": .1, "norm_power": 1.}),
- None, None),
- (_Spec(pg.Norm, (26,), {"noise_stdev": .1, "norm_power": 2.}),
- None, None),
- (_Spec(pg.Norm, (23,), {"noise_stdev": .1, "norm_power": 3.}),
- None, None),
- ]
-
-
-def sum_problems():
- return [
- (_Spec(pg.SumTask, [[
- _Spec(pg.Quadratic, (11,), {}),
- _Spec(pg.Quadratic, (3,), {}),
- _Spec(pg.Quadratic, (9,), {}),
- _Spec(pg.Quadratic, (7,), {}),
- _Spec(pg.Quadratic, (5,), {}),
- _Spec(pg.Quadratic, (13,), {}),
- _Spec(pg.Quadratic, (12,), {})
- ]], {}), None, None),
- (_Spec(pg.SumTask, [[
- _Spec(pg.Norm, (18,), {"norm_power": 3}),
- _Spec(pg.Quadratic, (25,), {}),
- _Spec(pg.Rosenbrock, (), {})
- ]], {}), None, None),
- (_Spec(pg.SumTask, [[
- _Spec(pg.Rosenbrock, (), {}),
- _Spec(pg.LogSumExp, (), {}),
- _Spec(pg.Ackley, (), {}),
- _Spec(pg.Beale, (), {}),
- _Spec(pg.Booth, (), {}),
- _Spec(pg.StyblinskiTang, (), {}),
- _Spec(pg.Matyas, (), {}),
- _Spec(pg.Branin, (), {}),
- _Spec(pg.Michalewicz, (), {})
- ]], {}), None, None),
- (_Spec(pg.SumTask, [[
- _Spec(pg.Rosenbrock, (), {}),
- _Spec(pg.LogSumExp, (), {}),
- _Spec(pg.Ackley, (), {}),
- _Spec(pg.Beale, (), {}),
- _Spec(pg.Booth, (), {}),
- _Spec(pg.StyblinskiTang, (), {}),
- _Spec(pg.Matyas, (), {}),
- _Spec(pg.Branin, (), {}),
- _Spec(pg.Michalewicz, (), {}),
- _Spec(pg.Quadratic, (5,), {}),
- _Spec(pg.Quadratic, (13,), {})
- ]], {}), None, None),
- (_Spec(pg.SumTask, [[
- _Spec(pg.Quadratic, (11,), {}),
- _Spec(pg.Quadratic, (3,), {})
- ]], {}), None, None),
- (_Spec(pg.SumTask, [[
- _Spec(pg.Rosenbrock, (), {}),
- _Spec(pg.LogSumExp, (), {}),
- _Spec(pg.Ackley, (), {})
- ]], {}), None, None),
- ]
-
-
-def sum_problems_noisy():
- return [
- (_Spec(pg.SumTask, [[
- _Spec(pg.Quadratic, (11,), {"noise_stdev": 0.1}),
- _Spec(pg.Quadratic, (3,), {"noise_stdev": 0.1}),
- _Spec(pg.Quadratic, (9,), {"noise_stdev": 0.1}),
- _Spec(pg.Quadratic, (7,), {"noise_stdev": 0.1}),
- _Spec(pg.Quadratic, (5,), {"noise_stdev": 0.1}),
- _Spec(pg.Quadratic, (13,), {"noise_stdev": 0.1}),
- _Spec(pg.Quadratic, (12,), {"noise_stdev": 0.1})
- ]], {}), None, None),
- (_Spec(pg.SumTask, [[
- _Spec(pg.Rosenbrock, (), {}),
- _Spec(pg.LogSumExp, (), {}),
- _Spec(pg.Ackley, (), {}),
- _Spec(pg.Beale, (), {}),
- _Spec(pg.Booth, (), {}),
- _Spec(pg.StyblinskiTang, (), {}),
- _Spec(pg.Matyas, (), {}),
- _Spec(pg.Branin, (), {}),
- _Spec(pg.Michalewicz, (), {}),
- _Spec(pg.Quadratic, (5,), {}),
- _Spec(pg.Quadratic, (13,), {"noise_stdev": 0.5})
- ]], {}), None, None),
- ]
-
-
-def dependency_chain_problems():
- return [
- (_Spec(pg.DependencyChain, (20,), {}), datasets.random_binary(
- 20, 1000), 100),
- (_Spec(pg.DependencyChain, (12,), {}), datasets.random_binary(
- 12, 200), 10),
- (_Spec(pg.DependencyChain, (56,), {}), datasets.random_binary(
- 56, 5000), 100),
- (_Spec(pg.DependencyChain, (64,), {}), datasets.random_binary(
- 64, 1000), 50),
- (_Spec(pg.DependencyChain, (13,), {}), datasets.random_binary(
- 13, 10000), 50),
- (_Spec(pg.DependencyChain, (20,), {}), datasets.random_binary(
- 20, 1000), 128),
- (_Spec(pg.DependencyChain, (12,), {}), datasets.random_binary(
- 12, 300), 16),
- (_Spec(pg.DependencyChain, (56,), {}), datasets.random_binary(
- 56, 5000), 128),
- (_Spec(pg.DependencyChain, (64,), {}), datasets.random_binary(
- 64, 1000), 64),
- (_Spec(pg.DependencyChain, (13,), {}), datasets.random_binary(
- 13, 10000), 32),
- ]
-
-
-def outward_snake_problems():
- return [
- (_Spec(pg.OutwardSnake, (20,), {}), datasets.random_binary(
- 20, 1000), 100),
- (_Spec(pg.OutwardSnake, (12,), {}), datasets.random_binary(
- 12, 200), 10),
- (_Spec(pg.OutwardSnake, (56,), {}), datasets.random_binary(
- 56, 5000), 100),
- (_Spec(pg.OutwardSnake, (64,), {}), datasets.random_binary(
- 64, 1000), 50),
- (_Spec(pg.OutwardSnake, (13,), {}), datasets.random_binary(
- 13, 10000), 50),
- (_Spec(pg.OutwardSnake, (20,), {}), datasets.random_binary(
- 20, 1000), 128),
- (_Spec(pg.OutwardSnake, (12,), {}), datasets.random_binary(
- 12, 300), 16),
- (_Spec(pg.OutwardSnake, (56,), {}), datasets.random_binary(
- 56, 5000), 128),
- (_Spec(pg.OutwardSnake, (64,), {}), datasets.random_binary(
- 64, 1000), 64),
- (_Spec(pg.OutwardSnake, (13,), {}), datasets.random_binary(
- 13, 10000), 32),
- ]
-
-
-def min_max_well_problems():
- return [
- (_Spec(pg.MinMaxWell, (20,), {}), None, None),
- (_Spec(pg.MinMaxWell, (12,), {}), None, None),
- (_Spec(pg.MinMaxWell, (56,), {}), None, None),
- (_Spec(pg.MinMaxWell, (64,), {}), None, None),
- (_Spec(pg.MinMaxWell, (13,), {}), None, None),
- ]
-
-
-def sum_of_quadratics_problems():
- return [
- (_Spec(pg.SumOfQuadratics, (20,), {}),
- datasets.random_symmetric(20, 1000), 100),
- (_Spec(pg.SumOfQuadratics, (12,), {}),
- datasets.random_symmetric(12, 100), 10),
- (_Spec(pg.SumOfQuadratics, (56,), {}),
- datasets.random_symmetric(56, 5000), 100),
- (_Spec(pg.SumOfQuadratics, (64,), {}),
- datasets.random_symmetric(64, 1000), 50),
- (_Spec(pg.SumOfQuadratics, (13,), {}),
- datasets.random_symmetric(13, 10000), 50),
- (_Spec(pg.SumOfQuadratics, (20,), {}),
- datasets.random_symmetric(20, 1000), 128),
- (_Spec(pg.SumOfQuadratics, (12,), {}),
- datasets.random_symmetric(12, 100), 16),
- (_Spec(pg.SumOfQuadratics, (56,), {}),
- datasets.random_symmetric(56, 5000), 128),
- (_Spec(pg.SumOfQuadratics, (64,), {}),
- datasets.random_symmetric(64, 1000), 64),
- (_Spec(pg.SumOfQuadratics, (13,), {}),
- datasets.random_symmetric(13, 10000), 32),
- ]
-
-
-def projection_quadratic_problems():
- return [
- (_Spec(pg.ProjectionQuadratic, (20,), {}),
- datasets.random_symmetric(20, 1000), 100),
- (_Spec(pg.ProjectionQuadratic, (12,), {}),
- datasets.random_symmetric(12, 100), 10),
- (_Spec(pg.ProjectionQuadratic, (56,), {}),
- datasets.random_symmetric(56, 5000), 100),
- (_Spec(pg.ProjectionQuadratic, (64,), {}),
- datasets.random_symmetric(64, 1000), 50),
- (_Spec(pg.ProjectionQuadratic, (13,), {}),
- datasets.random_symmetric(13, 10000), 50),
- (_Spec(pg.ProjectionQuadratic, (20,), {}),
- datasets.random_symmetric(20, 1000), 128),
- (_Spec(pg.ProjectionQuadratic, (12,), {}),
- datasets.random_symmetric(12, 100), 16),
- (_Spec(pg.ProjectionQuadratic, (56,), {}),
- datasets.random_symmetric(56, 5000), 128),
- (_Spec(pg.ProjectionQuadratic, (64,), {}),
- datasets.random_symmetric(64, 1000), 64),
- (_Spec(pg.ProjectionQuadratic, (13,), {}),
- datasets.random_symmetric(13, 10000), 32),
- ]
-
-
-def adapter_rosenbrock_local():
- return [(_Spec(model_adapter.ModelAdapter,
- (pg.make_rosenbrock_loss_and_init,), {}), None, None),]
-
-
-def adapter_rosenbrock_worker():
- return [(_Spec(model_adapter.ModelAdapter,
- (pg.make_rosenbrock_loss_and_init,),
- {"device": "/job:worker"}), None, None),]
-
-
-def _test_problem_mlp_scaled_init_small():
- return [
- np.random.randn(10, 32) * np.sqrt(2./10),
- np.random.randn(32,) * 0.1,
- np.random.randn(32, 64) * np.sqrt(2./32.),
- np.random.randn(64,) * 0.1,
- np.random.randn(64, 2) * np.sqrt(2./64.),
- np.random.randn(2,) * 0.1
- ]
-
-
-def _test_problem_mlp_scaled_init_large():
- return [
- np.random.randn(20, 32) * np.sqrt(2./20),
- np.random.randn(32,) * 0.1,
- np.random.randn(32, 64) * np.sqrt(2./32.),
- np.random.randn(64,) * 0.1,
- np.random.randn(64, 10) * np.sqrt(2./64.),
- np.random.randn(10,) * 0.1
- ]
-
-
-def _test_problem_mlp_scaled_init_mnist():
- return [
- np.random.randn(784, 64) * np.sqrt(2./784.),
- np.random.randn(64,) * 0.1,
- np.random.randn(64, 10) * np.sqrt(2./ 64.),
- np.random.randn(10,) * 0.1,
- ]
-
-
-# Wrap this construction in a function to avoid UnparsedFlagAccessError
-def test_problems():
- """Test problems for visualizations."""
- # Unlike the training problem sets, these test problems are made up of
- # length-5 tuples. The final items in the tuple are the name of the problem
- # and the initialization random_seed for testing consistency.
- tp = [
- (_Spec(pg.Quadratic, (20,), {"random_seed": 1234}), None, None,
- "quad_problem", 5678),
- (_Spec(pg.Quadratic, (20,), {"noise_stdev": 1.0, "random_seed": 1234}),
- None, None, "quad_problem_noise", 5678),
- (_Spec(pg.Rosenbrock, (), {"random_seed": 1234}), None, None,
- "rosenbrock", 5678),
- (_Spec(pg.Rosenbrock, (), {"random_seed": 1234, "noise_stdev": 1.0}),
- None, None, "rosenbrock_noise", 5678),
- (_Spec(pg.SoftmaxRegression, (10, 2), {}), datasets.random(
- 10, 10000, random_seed=1234), 100, "softmax", 5678),
- (_Spec(pg.SoftmaxRegression, (10, 2), {"noise_stdev": 1.0}),
- datasets.random(10, 10000, random_seed=1234), 100, "softmax_noise",
- 5678),
- (_Spec(pg.FullyConnected, (10, 2), {}), datasets.random(
- 10, 10000, random_seed=1234), 100, "mlp_small",
- _test_problem_mlp_scaled_init_small()),
- (_Spec(pg.FullyConnected, (20, 10), {}), datasets.random(
- 20, 10000, n_classes=10, random_seed=1234), 100, "mlp_large",
- _test_problem_mlp_scaled_init_large()),
- (_Spec(pg.FullyConnected, (784, 10),
- {"hidden_sizes": (64,), "activation": tf.nn.sigmoid}),
- datasets.mnist(), 64, "mlp_mnist_sigmoid",
- _test_problem_mlp_scaled_init_mnist()),
- (_Spec(pg.FullyConnected, (784, 10),
- {"hidden_sizes": (64,), "activation": tf.nn.relu}),
- datasets.mnist(), 64, "mlp_mnist_relu",
- _test_problem_mlp_scaled_init_mnist()),
- (_Spec(pg.ConvNet, ((1, 28, 28), 10, [(3, 3, 8), (5, 5, 8)]),
- {"activation": tf.nn.sigmoid}), datasets.mnist(), 64,
- "convnet_mnist_sigmoid", None),
- (_Spec(pg.ConvNet, ((1, 28, 28), 10, [(3, 3, 8), (5, 5, 8)]),
- {"activation": tf.nn.relu}), datasets.mnist(), 64,
- "convnet_mnist_relu", None),
- ]
- return tp
diff --git a/research/learned_optimizer/problems/problem_spec.py b/research/learned_optimizer/problems/problem_spec.py
deleted file mode 100644
index e30c47b277e5c8b3b8aba3b8d691a2af3a595ef6..0000000000000000000000000000000000000000
--- a/research/learned_optimizer/problems/problem_spec.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Copyright 2017 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Wrapper around a training problem."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import namedtuple
-
-
-class Spec(namedtuple("Spec", "callable args kwargs")):
- """Syntactic sugar for keeping track of a function/class + args."""
-
- # Since this is an immutable object, we don't need to reserve slots.
- __slots__ = ()
-
- def build(self):
- """Returns the output of the callable."""
- return self.callable(*self.args, **self.kwargs)
diff --git a/research/learning_to_remember_rare_events/README.md b/research/learning_to_remember_rare_events/README.md
deleted file mode 100644
index 2eeadea784d4d22efc88c56e482c5d5374c90e24..0000000000000000000000000000000000000000
--- a/research/learning_to_remember_rare_events/README.md
+++ /dev/null
@@ -1,61 +0,0 @@
-
-
-
-
----
-
-Code for the Memory Module as described
-in "Learning to Remember Rare Events" by
-Lukasz Kaiser, Ofir Nachum, Aurko Roy, and Samy Bengio
-published as a conference paper at ICLR 2017.
-
-Requirements:
-* TensorFlow (see tensorflow.org for how to install)
-* Some basic command-line utilities (git, unzip).
-
-Description:
-
-The general memory module is located in memory.py.
-Some code is provided to see the memory module in
-action on the standard Omniglot dataset.
-Download and setup the dataset using data_utils.py
-and then run the training script train.py
-(see example commands below).
-
-Note that the structure and parameters of the model
-are optimized for the data preparation as provided.
-
-Quick Start:
-
-First download and set-up Omniglot data by running
-
-```
-python data_utils.py
-```
-
-Then run the training script:
-
-```
-python train.py --memory_size=8192 \
- --batch_size=16 --validation_length=50 \
- --episode_width=5 --episode_length=30
-```
-
-The first validation batch may look like this (although it is noisy):
-```
-0-shot: 0.040, 1-shot: 0.404, 2-shot: 0.516, 3-shot: 0.604,
- 4-shot: 0.656, 5-shot: 0.684
-```
-At step 500 you may see something like this:
-```
-0-shot: 0.036, 1-shot: 0.836, 2-shot: 0.900, 3-shot: 0.940,
- 4-shot: 0.944, 5-shot: 0.916
-```
-At step 4000 you may see something like this:
-```
-0-shot: 0.044, 1-shot: 0.960, 2-shot: 1.000, 3-shot: 0.988,
- 4-shot: 0.972, 5-shot: 0.992
-```
-
-Maintained by Ofir Nachum (ofirnachum) and
-Lukasz Kaiser (lukaszkaiser).
diff --git a/research/learning_to_remember_rare_events/data_utils.py b/research/learning_to_remember_rare_events/data_utils.py
deleted file mode 100644
index 03d5dafb251d4e058a6780b447aabdcd1a84a1d4..0000000000000000000000000000000000000000
--- a/research/learning_to_remember_rare_events/data_utils.py
+++ /dev/null
@@ -1,243 +0,0 @@
-# Copyright 2017 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# ==============================================================================
-"""Data loading and other utilities.
-
-Use this file to first copy over and pre-process the Omniglot dataset.
-Simply call
- python data_utils.py
-"""
-
-import logging
-import os
-import subprocess
-from six.moves import cPickle as pickle
-
-import numpy as np
-from scipy.misc import imresize
-from scipy.misc import imrotate
-from scipy.ndimage import imread
-from six.moves import xrange
-import tensorflow as tf
-
-
-MAIN_DIR = ''
-REPO_LOCATION = 'https://github.com/brendenlake/omniglot.git'
-REPO_DIR = os.path.join(MAIN_DIR, 'omniglot')
-DATA_DIR = os.path.join(REPO_DIR, 'python')
-TRAIN_DIR = os.path.join(DATA_DIR, 'images_background')
-TEST_DIR = os.path.join(DATA_DIR, 'images_evaluation')
-DATA_FILE_FORMAT = os.path.join(MAIN_DIR, '%s_omni.pkl')
-
-TRAIN_ROTATIONS = True # augment training data with rotations
-TEST_ROTATIONS = False # augment testing data with rotations
-IMAGE_ORIGINAL_SIZE = 105
-IMAGE_NEW_SIZE = 28
-
-
-def get_data():
- """Get data in form suitable for episodic training.
-
- Returns:
- Train and test data as dictionaries mapping
- label to list of examples.
- """
- with tf.gfile.GFile(DATA_FILE_FORMAT % 'train', 'rb') as f:
- processed_train_data = pickle.load(f)
- with tf.gfile.GFile(DATA_FILE_FORMAT % 'test', 'rb') as f:
- processed_test_data = pickle.load(f)
-
- train_data = {}
- test_data = {}
-
- for data, processed_data in zip([train_data, test_data],
- [processed_train_data, processed_test_data]):
- for image, label in zip(processed_data['images'],
- processed_data['labels']):
- if label not in data:
- data[label] = []
- data[label].append(image.reshape([-1]).astype('float32'))
-
- intersection = set(train_data.keys()) & set(test_data.keys())
- assert not intersection, 'Train and test data intersect.'
- ok_num_examples = [len(ll) == 20 for _, ll in train_data.items()]
- assert all(ok_num_examples), 'Bad number of examples in train data.'
- ok_num_examples = [len(ll) == 20 for _, ll in test_data.items()]
- assert all(ok_num_examples), 'Bad number of examples in test data.'
-
- logging.info('Number of labels in train data: %d.', len(train_data))
- logging.info('Number of labels in test data: %d.', len(test_data))
-
- return train_data, test_data
-
-
-def crawl_directory(directory, augment_with_rotations=False,
- first_label=0):
- """Crawls data directory and returns stuff."""
- label_idx = first_label
- images = []
- labels = []
- info = []
-
- # traverse root directory
- for root, _, files in os.walk(directory):
- logging.info('Reading files from %s', root)
- fileflag = 0
- for file_name in files:
- full_file_name = os.path.join(root, file_name)
- img = imread(full_file_name, flatten=True)
- for i, angle in enumerate([0, 90, 180, 270]):
- if not augment_with_rotations and i > 0:
- break
-
- images.append(imrotate(img, angle))
- labels.append(label_idx + i)
- info.append(full_file_name)
-
- fileflag = 1
-
- if fileflag:
- label_idx += 4 if augment_with_rotations else 1
-
- return images, labels, info
-
-
-def resize_images(images, new_width, new_height):
- """Resize images to new dimensions."""
- resized_images = np.zeros([images.shape[0], new_width, new_height],
- dtype=np.float32)
-
- for i in range(images.shape[0]):
- resized_images[i, :, :] = imresize(images[i, :, :],
- [new_width, new_height],
- interp='bilinear',
- mode=None)
- return resized_images
-
-
-def write_datafiles(directory, write_file,
- resize=True, rotate=False,
- new_width=IMAGE_NEW_SIZE, new_height=IMAGE_NEW_SIZE,
- first_label=0):
- """Load and preprocess images from a directory and write them to a file.
-
- Args:
- directory: Directory of alphabet sub-directories.
- write_file: Filename to write to.
- resize: Whether to resize the images.
- rotate: Whether to augment the dataset with rotations.
- new_width: New resize width.
- new_height: New resize height.
- first_label: Label to start with.
-
- Returns:
- Number of new labels created.
- """
-
- # these are the default sizes for Omniglot:
- imgwidth = IMAGE_ORIGINAL_SIZE
- imgheight = IMAGE_ORIGINAL_SIZE
-
- logging.info('Reading the data.')
- images, labels, info = crawl_directory(directory,
- augment_with_rotations=rotate,
- first_label=first_label)
-
- images_np = np.zeros([len(images), imgwidth, imgheight], dtype=np.bool)
- labels_np = np.zeros([len(labels)], dtype=np.uint32)
- for i in xrange(len(images)):
- images_np[i, :, :] = images[i]
- labels_np[i] = labels[i]
-
- if resize:
- logging.info('Resizing images.')
- resized_images = resize_images(images_np, new_width, new_height)
-
- logging.info('Writing resized data in float32 format.')
- data = {'images': resized_images,
- 'labels': labels_np,
- 'info': info}
- with tf.gfile.GFile(write_file, 'w') as f:
- pickle.dump(data, f)
- else:
- logging.info('Writing original sized data in boolean format.')
- data = {'images': images_np,
- 'labels': labels_np,
- 'info': info}
- with tf.gfile.GFile(write_file, 'w') as f:
- pickle.dump(data, f)
-
- return len(np.unique(labels_np))
-
-
-def maybe_download_data():
- """Download Omniglot repo if it does not exist."""
- if os.path.exists(REPO_DIR):
- logging.info('It appears that Git repo already exists.')
- else:
- logging.info('It appears that Git repo does not exist.')
- logging.info('Cloning now.')
-
- subprocess.check_output('git clone %s' % REPO_LOCATION, shell=True)
-
- if os.path.exists(TRAIN_DIR):
- logging.info('It appears that train data has already been unzipped.')
- else:
- logging.info('It appears that train data has not been unzipped.')
- logging.info('Unzipping now.')
-
- subprocess.check_output('unzip %s.zip -d %s' % (TRAIN_DIR, DATA_DIR),
- shell=True)
-
- if os.path.exists(TEST_DIR):
- logging.info('It appears that test data has already been unzipped.')
- else:
- logging.info('It appears that test data has not been unzipped.')
- logging.info('Unzipping now.')
-
- subprocess.check_output('unzip %s.zip -d %s' % (TEST_DIR, DATA_DIR),
- shell=True)
-
-
-def preprocess_omniglot():
- """Download and prepare raw Omniglot data.
-
- Downloads the data from GitHub if it does not exist.
- Then load the images, augment with rotations if desired.
- Resize the images and write them to a pickle file.
- """
-
- maybe_download_data()
-
- directory = TRAIN_DIR
- write_file = DATA_FILE_FORMAT % 'train'
- num_labels = write_datafiles(
- directory, write_file, resize=True, rotate=TRAIN_ROTATIONS,
- new_width=IMAGE_NEW_SIZE, new_height=IMAGE_NEW_SIZE)
-
- directory = TEST_DIR
- write_file = DATA_FILE_FORMAT % 'test'
- write_datafiles(directory, write_file, resize=True, rotate=TEST_ROTATIONS,
- new_width=IMAGE_NEW_SIZE, new_height=IMAGE_NEW_SIZE,
- first_label=num_labels)
-
-
-def main(unused_argv):
- logging.basicConfig(level=logging.INFO)
- preprocess_omniglot()
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/learning_to_remember_rare_events/memory.py b/research/learning_to_remember_rare_events/memory.py
deleted file mode 100644
index 2f40ff57f9434994f08b1ad97dc23142bb23daaa..0000000000000000000000000000000000000000
--- a/research/learning_to_remember_rare_events/memory.py
+++ /dev/null
@@ -1,392 +0,0 @@
-# Copyright 2017 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# ==============================================================================
-"""Memory module for storing "nearest neighbors".
-
-Implements a key-value memory for generalized one-shot learning
-as described in the paper
-"Learning to Remember Rare Events"
-by Lukasz Kaiser, Ofir Nachum, Aurko Roy, Samy Bengio,
-published as a conference paper at ICLR 2017.
-"""
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-
-class Memory(object):
- """Memory module."""
-
- def __init__(self, key_dim, memory_size, vocab_size,
- choose_k=256, alpha=0.1, correct_in_top=1, age_noise=8.0,
- var_cache_device='', nn_device=''):
- self.key_dim = key_dim
- self.memory_size = memory_size
- self.vocab_size = vocab_size
- self.choose_k = min(choose_k, memory_size)
- self.alpha = alpha
- self.correct_in_top = correct_in_top
- self.age_noise = age_noise
- self.var_cache_device = var_cache_device # Variables are cached here.
- self.nn_device = nn_device # Device to perform nearest neighbour matmul.
-
- caching_device = var_cache_device if var_cache_device else None
- self.update_memory = tf.constant(True) # Can be fed "false" if needed.
- self.mem_keys = tf.get_variable(
- 'memkeys', [self.memory_size, self.key_dim], trainable=False,
- initializer=tf.random_uniform_initializer(-0.0, 0.0),
- caching_device=caching_device)
- self.mem_vals = tf.get_variable(
- 'memvals', [self.memory_size], dtype=tf.int32, trainable=False,
- initializer=tf.constant_initializer(0, tf.int32),
- caching_device=caching_device)
- self.mem_age = tf.get_variable(
- 'memage', [self.memory_size], dtype=tf.float32, trainable=False,
- initializer=tf.constant_initializer(0.0), caching_device=caching_device)
- self.recent_idx = tf.get_variable(
- 'recent_idx', [self.vocab_size], dtype=tf.int32, trainable=False,
- initializer=tf.constant_initializer(0, tf.int32))
-
- # variable for projecting query vector into memory key
- self.query_proj = tf.get_variable(
- 'memory_query_proj', [self.key_dim, self.key_dim], dtype=tf.float32,
- initializer=tf.truncated_normal_initializer(0, 0.01),
- caching_device=caching_device)
-
- def get(self):
- return self.mem_keys, self.mem_vals, self.mem_age, self.recent_idx
-
- def set(self, k, v, a, r=None):
- return tf.group(
- self.mem_keys.assign(k),
- self.mem_vals.assign(v),
- self.mem_age.assign(a),
- (self.recent_idx.assign(r) if r is not None else tf.group()))
-
- def clear(self):
- return tf.variables_initializer([self.mem_keys, self.mem_vals, self.mem_age,
- self.recent_idx])
-
- def get_hint_pool_idxs(self, normalized_query):
- """Get small set of idxs to compute nearest neighbor queries on.
-
- This is an expensive look-up on the whole memory that is used to
- avoid more expensive operations later on.
-
- Args:
- normalized_query: A Tensor of shape [None, key_dim].
-
- Returns:
- A Tensor of shape [None, choose_k] of indices in memory
- that are closest to the queries.
-
- """
- # look up in large memory, no gradients
- with tf.device(self.nn_device):
- similarities = tf.matmul(tf.stop_gradient(normalized_query),
- self.mem_keys, transpose_b=True, name='nn_mmul')
- _, hint_pool_idxs = tf.nn.top_k(
- tf.stop_gradient(similarities), k=self.choose_k, name='nn_topk')
- return hint_pool_idxs
-
- def make_update_op(self, upd_idxs, upd_keys, upd_vals,
- batch_size, use_recent_idx, intended_output):
- """Function that creates all the update ops."""
- mem_age_incr = self.mem_age.assign_add(tf.ones([self.memory_size],
- dtype=tf.float32))
- with tf.control_dependencies([mem_age_incr]):
- mem_age_upd = tf.scatter_update(
- self.mem_age, upd_idxs, tf.zeros([batch_size], dtype=tf.float32))
-
- mem_key_upd = tf.scatter_update(
- self.mem_keys, upd_idxs, upd_keys)
- mem_val_upd = tf.scatter_update(
- self.mem_vals, upd_idxs, upd_vals)
-
- if use_recent_idx:
- recent_idx_upd = tf.scatter_update(
- self.recent_idx, intended_output, upd_idxs)
- else:
- recent_idx_upd = tf.group()
-
- return tf.group(mem_age_upd, mem_key_upd, mem_val_upd, recent_idx_upd)
-
- def query(self, query_vec, intended_output, use_recent_idx=True):
- """Queries memory for nearest neighbor.
-
- Args:
- query_vec: A batch of vectors to query (embedding of input to model).
- intended_output: The values that would be the correct output of the
- memory.
- use_recent_idx: Whether to always insert at least one instance of a
- correct memory fetch.
-
- Returns:
- A tuple (result, mask, teacher_loss).
- result: The result of the memory look up.
- mask: The affinity of the query to the result.
- teacher_loss: The loss for training the memory module.
- """
-
- batch_size = tf.shape(query_vec)[0]
- output_given = intended_output is not None
-
- # prepare query for memory lookup
- query_vec = tf.matmul(query_vec, self.query_proj)
- normalized_query = tf.nn.l2_normalize(query_vec, dim=1)
-
- hint_pool_idxs = self.get_hint_pool_idxs(normalized_query)
-
- if output_given and use_recent_idx: # add at least one correct memory
- most_recent_hint_idx = tf.gather(self.recent_idx, intended_output)
- hint_pool_idxs = tf.concat(
- axis=1,
- values=[hint_pool_idxs, tf.expand_dims(most_recent_hint_idx, 1)])
- choose_k = tf.shape(hint_pool_idxs)[1]
-
- with tf.device(self.var_cache_device):
- # create small memory and look up with gradients
- my_mem_keys = tf.stop_gradient(tf.gather(self.mem_keys, hint_pool_idxs,
- name='my_mem_keys_gather'))
- similarities = tf.matmul(tf.expand_dims(normalized_query, 1),
- my_mem_keys, adjoint_b=True, name='batch_mmul')
- hint_pool_sims = tf.squeeze(similarities, [1], name='hint_pool_sims')
- hint_pool_mem_vals = tf.gather(self.mem_vals, hint_pool_idxs,
- name='hint_pool_mem_vals')
- # Calculate softmax mask on the top-k if requested.
- # Softmax temperature. Say we have K elements at dist x and one at (x+a).
- # Softmax of the last is e^tm(x+a)/Ke^tm*x + e^tm(x+a) = e^tm*a/K+e^tm*a.
- # To make that 20% we'd need to have e^tm*a ~= 0.2K, so tm = log(0.2K)/a.
- softmax_temp = max(1.0, np.log(0.2 * self.choose_k) / self.alpha)
- mask = tf.nn.softmax(hint_pool_sims[:, :choose_k - 1] * softmax_temp)
-
- # prepare returned values
- nearest_neighbor = tf.to_int32(
- tf.argmax(hint_pool_sims[:, :choose_k - 1], 1))
-
- no_teacher_idxs = tf.gather(
- tf.reshape(hint_pool_idxs, [-1]),
- nearest_neighbor + choose_k * tf.range(batch_size))
-
- with tf.device(self.var_cache_device):
- result = tf.gather(self.mem_vals, tf.reshape(no_teacher_idxs, [-1]))
-
- if not output_given:
- teacher_loss = None
- return result, mask, teacher_loss
-
- # prepare hints from the teacher on hint pool
- teacher_hints = tf.to_float(
- tf.abs(tf.expand_dims(intended_output, 1) - hint_pool_mem_vals))
- teacher_hints = 1.0 - tf.minimum(1.0, teacher_hints)
-
- teacher_vals, teacher_hint_idxs = tf.nn.top_k(
- hint_pool_sims * teacher_hints, k=1)
- neg_teacher_vals, _ = tf.nn.top_k(
- hint_pool_sims * (1 - teacher_hints), k=1)
-
- # bring back idxs to full memory
- teacher_idxs = tf.gather(
- tf.reshape(hint_pool_idxs, [-1]),
- teacher_hint_idxs[:, 0] + choose_k * tf.range(batch_size))
-
- # zero-out teacher_vals if there are no hints
- teacher_vals *= (
- 1 - tf.to_float(tf.equal(0.0, tf.reduce_sum(teacher_hints, 1))))
-
- # we'll determine whether to do an update to memory based on whether
- # memory was queried correctly
- sliced_hints = tf.slice(teacher_hints, [0, 0], [-1, self.correct_in_top])
- incorrect_memory_lookup = tf.equal(0.0, tf.reduce_sum(sliced_hints, 1))
-
- # loss based on triplet loss
- teacher_loss = (tf.nn.relu(neg_teacher_vals - teacher_vals + self.alpha)
- - self.alpha)
-
- # prepare memory updates
- update_keys = normalized_query
- update_vals = intended_output
-
- fetched_idxs = teacher_idxs # correctly fetched from memory
- with tf.device(self.var_cache_device):
- fetched_keys = tf.gather(self.mem_keys, fetched_idxs, name='fetched_keys')
- fetched_vals = tf.gather(self.mem_vals, fetched_idxs, name='fetched_vals')
-
- # do memory updates here
- fetched_keys_upd = update_keys + fetched_keys # Momentum-like update
- fetched_keys_upd = tf.nn.l2_normalize(fetched_keys_upd, dim=1)
- # Randomize age a bit, e.g., to select different ones in parallel workers.
- mem_age_with_noise = self.mem_age + tf.random_uniform(
- [self.memory_size], - self.age_noise, self.age_noise)
-
- _, oldest_idxs = tf.nn.top_k(mem_age_with_noise, k=batch_size, sorted=False)
-
- with tf.control_dependencies([result]):
- upd_idxs = tf.where(incorrect_memory_lookup,
- oldest_idxs,
- fetched_idxs)
- # upd_idxs = tf.Print(upd_idxs, [upd_idxs], "UPD IDX", summarize=8)
- upd_keys = tf.where(incorrect_memory_lookup,
- update_keys,
- fetched_keys_upd)
- upd_vals = tf.where(incorrect_memory_lookup,
- update_vals,
- fetched_vals)
-
- def make_update_op():
- return self.make_update_op(upd_idxs, upd_keys, upd_vals,
- batch_size, use_recent_idx, intended_output)
-
- update_op = tf.cond(self.update_memory, make_update_op, tf.no_op)
-
- with tf.control_dependencies([update_op]):
- result = tf.identity(result)
- mask = tf.identity(mask)
- teacher_loss = tf.identity(teacher_loss)
-
- return result, mask, tf.reduce_mean(teacher_loss)
-
-
-class LSHMemory(Memory):
- """Memory employing locality sensitive hashing.
-
- Note: Not fully tested.
- """
-
- def __init__(self, key_dim, memory_size, vocab_size,
- choose_k=256, alpha=0.1, correct_in_top=1, age_noise=8.0,
- var_cache_device='', nn_device='',
- num_hashes=None, num_libraries=None):
- super(LSHMemory, self).__init__(
- key_dim, memory_size, vocab_size,
- choose_k=choose_k, alpha=alpha, correct_in_top=1, age_noise=age_noise,
- var_cache_device=var_cache_device, nn_device=nn_device)
-
- self.num_libraries = num_libraries or int(self.choose_k ** 0.5)
- self.num_per_hash_slot = max(1, self.choose_k // self.num_libraries)
- self.num_hashes = (num_hashes or
- int(np.log2(self.memory_size / self.num_per_hash_slot)))
- self.num_hashes = min(max(self.num_hashes, 1), 20)
- self.num_hash_slots = 2 ** self.num_hashes
-
- # hashing vectors
- self.hash_vecs = [
- tf.get_variable(
- 'hash_vecs%d' % i, [self.num_hashes, self.key_dim],
- dtype=tf.float32, trainable=False,
- initializer=tf.truncated_normal_initializer(0, 1))
- for i in xrange(self.num_libraries)]
-
- # map representing which hash slots map to which mem keys
- self.hash_slots = [
- tf.get_variable(
- 'hash_slots%d' % i, [self.num_hash_slots, self.num_per_hash_slot],
- dtype=tf.int32, trainable=False,
- initializer=tf.random_uniform_initializer(maxval=self.memory_size,
- dtype=tf.int32))
- for i in xrange(self.num_libraries)]
-
- def get(self): # not implemented
- return self.mem_keys, self.mem_vals, self.mem_age, self.recent_idx
-
- def set(self, k, v, a, r=None): # not implemented
- return tf.group(
- self.mem_keys.assign(k),
- self.mem_vals.assign(v),
- self.mem_age.assign(a),
- (self.recent_idx.assign(r) if r is not None else tf.group()))
-
- def clear(self):
- return tf.variables_initializer([self.mem_keys, self.mem_vals, self.mem_age,
- self.recent_idx] + self.hash_slots)
-
- def get_hash_slots(self, query):
- """Gets hashed-to buckets for batch of queries.
-
- Args:
- query: 2-d Tensor of query vectors.
-
- Returns:
- A list of hashed-to buckets for each hash function.
- """
-
- binary_hash = [
- tf.less(tf.matmul(query, self.hash_vecs[i], transpose_b=True), 0)
- for i in xrange(self.num_libraries)]
- hash_slot_idxs = [
- tf.reduce_sum(
- tf.to_int32(binary_hash[i]) *
- tf.constant([[2 ** i for i in xrange(self.num_hashes)]],
- dtype=tf.int32), 1)
- for i in xrange(self.num_libraries)]
- return hash_slot_idxs
-
- def get_hint_pool_idxs(self, normalized_query):
- """Get small set of idxs to compute nearest neighbor queries on.
-
- This is an expensive look-up on the whole memory that is used to
- avoid more expensive operations later on.
-
- Args:
- normalized_query: A Tensor of shape [None, key_dim].
-
- Returns:
- A Tensor of shape [None, choose_k] of indices in memory
- that are closest to the queries.
-
- """
- # get hash of query vecs
- hash_slot_idxs = self.get_hash_slots(normalized_query)
-
- # grab mem idxs in the hash slots
- hint_pool_idxs = [
- tf.maximum(tf.minimum(
- tf.gather(self.hash_slots[i], idxs),
- self.memory_size - 1), 0)
- for i, idxs in enumerate(hash_slot_idxs)]
-
- return tf.concat(axis=1, values=hint_pool_idxs)
-
- def make_update_op(self, upd_idxs, upd_keys, upd_vals,
- batch_size, use_recent_idx, intended_output):
- """Function that creates all the update ops."""
- base_update_op = super(LSHMemory, self).make_update_op(
- upd_idxs, upd_keys, upd_vals,
- batch_size, use_recent_idx, intended_output)
-
- # compute hash slots to be updated
- hash_slot_idxs = self.get_hash_slots(upd_keys)
-
- # make updates
- update_ops = []
- with tf.control_dependencies([base_update_op]):
- for i, slot_idxs in enumerate(hash_slot_idxs):
- # for each slot, choose which entry to replace
- entry_idx = tf.random_uniform([batch_size],
- maxval=self.num_per_hash_slot,
- dtype=tf.int32)
- entry_mul = 1 - tf.one_hot(entry_idx, self.num_per_hash_slot,
- dtype=tf.int32)
- entry_add = (tf.expand_dims(upd_idxs, 1) *
- tf.one_hot(entry_idx, self.num_per_hash_slot,
- dtype=tf.int32))
-
- mul_op = tf.scatter_mul(self.hash_slots[i], slot_idxs, entry_mul)
- with tf.control_dependencies([mul_op]):
- add_op = tf.scatter_add(self.hash_slots[i], slot_idxs, entry_add)
- update_ops.append(add_op)
-
- return tf.group(*update_ops)
diff --git a/research/learning_to_remember_rare_events/model.py b/research/learning_to_remember_rare_events/model.py
deleted file mode 100644
index 7a6b460047fda3349c04d0e024c035f69a300461..0000000000000000000000000000000000000000
--- a/research/learning_to_remember_rare_events/model.py
+++ /dev/null
@@ -1,302 +0,0 @@
-# Copyright 2017 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# ==============================================================================
-"""Model using memory component.
-
-The model embeds images using a standard CNN architecture.
-These embeddings are used as keys to the memory component,
-which returns nearest neighbors.
-"""
-
-import tensorflow as tf
-
-import memory
-
-FLAGS = tf.flags.FLAGS
-
-
-class BasicClassifier(object):
-
- def __init__(self, output_dim):
- self.output_dim = output_dim
-
- def core_builder(self, memory_val, x, y):
- del x, y
- y_pred = memory_val
- loss = 0.0
-
- return loss, y_pred
-
-
-class LeNet(object):
- """Standard CNN architecture."""
-
- def __init__(self, image_size, num_channels, hidden_dim):
- self.image_size = image_size
- self.num_channels = num_channels
- self.hidden_dim = hidden_dim
- self.matrix_init = tf.truncated_normal_initializer(stddev=0.1)
- self.vector_init = tf.constant_initializer(0.0)
-
- def core_builder(self, x):
- """Embeds x using standard CNN architecture.
-
- Args:
- x: Batch of images as a 2-d Tensor [batch_size, -1].
-
- Returns:
- A 2-d Tensor [batch_size, hidden_dim] of embedded images.
- """
-
- ch1 = 32 * 2 # number of channels in 1st layer
- ch2 = 64 * 2 # number of channels in 2nd layer
- conv1_weights = tf.get_variable('conv1_w',
- [3, 3, self.num_channels, ch1],
- initializer=self.matrix_init)
- conv1_biases = tf.get_variable('conv1_b', [ch1],
- initializer=self.vector_init)
- conv1a_weights = tf.get_variable('conv1a_w',
- [3, 3, ch1, ch1],
- initializer=self.matrix_init)
- conv1a_biases = tf.get_variable('conv1a_b', [ch1],
- initializer=self.vector_init)
-
- conv2_weights = tf.get_variable('conv2_w', [3, 3, ch1, ch2],
- initializer=self.matrix_init)
- conv2_biases = tf.get_variable('conv2_b', [ch2],
- initializer=self.vector_init)
- conv2a_weights = tf.get_variable('conv2a_w', [3, 3, ch2, ch2],
- initializer=self.matrix_init)
- conv2a_biases = tf.get_variable('conv2a_b', [ch2],
- initializer=self.vector_init)
-
- # fully connected
- fc1_weights = tf.get_variable(
- 'fc1_w', [self.image_size // 4 * self.image_size // 4 * ch2,
- self.hidden_dim], initializer=self.matrix_init)
- fc1_biases = tf.get_variable('fc1_b', [self.hidden_dim],
- initializer=self.vector_init)
-
- # define model
- x = tf.reshape(x,
- [-1, self.image_size, self.image_size, self.num_channels])
- batch_size = tf.shape(x)[0]
-
- conv1 = tf.nn.conv2d(x, conv1_weights,
- strides=[1, 1, 1, 1], padding='SAME')
- relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))
- conv1 = tf.nn.conv2d(relu1, conv1a_weights,
- strides=[1, 1, 1, 1], padding='SAME')
- relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1a_biases))
-
- pool1 = tf.nn.max_pool(relu1, ksize=[1, 2, 2, 1],
- strides=[1, 2, 2, 1], padding='SAME')
-
- conv2 = tf.nn.conv2d(pool1, conv2_weights,
- strides=[1, 1, 1, 1], padding='SAME')
- relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))
- conv2 = tf.nn.conv2d(relu2, conv2a_weights,
- strides=[1, 1, 1, 1], padding='SAME')
- relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2a_biases))
-
- pool2 = tf.nn.max_pool(relu2, ksize=[1, 2, 2, 1],
- strides=[1, 2, 2, 1], padding='SAME')
-
- reshape = tf.reshape(pool2, [batch_size, -1])
- hidden = tf.matmul(reshape, fc1_weights) + fc1_biases
-
- return hidden
-
-
-class Model(object):
- """Model for coordinating between CNN embedder and Memory module."""
-
- def __init__(self, input_dim, output_dim, rep_dim, memory_size, vocab_size,
- learning_rate=0.0001, use_lsh=False):
- self.input_dim = input_dim
- self.output_dim = output_dim
- self.rep_dim = rep_dim
- self.memory_size = memory_size
- self.vocab_size = vocab_size
- self.learning_rate = learning_rate
- self.use_lsh = use_lsh
-
- self.embedder = self.get_embedder()
- self.memory = self.get_memory()
- self.classifier = self.get_classifier()
-
- self.global_step = tf.train.get_or_create_global_step()
-
- def get_embedder(self):
- return LeNet(int(self.input_dim ** 0.5), 1, self.rep_dim)
-
- def get_memory(self):
- cls = memory.LSHMemory if self.use_lsh else memory.Memory
- return cls(self.rep_dim, self.memory_size, self.vocab_size)
-
- def get_classifier(self):
- return BasicClassifier(self.output_dim)
-
- def core_builder(self, x, y, keep_prob, use_recent_idx=True):
- embeddings = self.embedder.core_builder(x)
- if keep_prob < 1.0:
- embeddings = tf.nn.dropout(embeddings, keep_prob)
- memory_val, _, teacher_loss = self.memory.query(
- embeddings, y, use_recent_idx=use_recent_idx)
- loss, y_pred = self.classifier.core_builder(memory_val, x, y)
-
- return loss + teacher_loss, y_pred
-
- def train(self, x, y):
- loss, _ = self.core_builder(x, y, keep_prob=0.3)
- gradient_ops = self.training_ops(loss)
- return loss, gradient_ops
-
- def eval(self, x, y):
- _, y_preds = self.core_builder(x, y, keep_prob=1.0,
- use_recent_idx=False)
- return y_preds
-
- def get_xy_placeholders(self):
- return (tf.placeholder(tf.float32, [None, self.input_dim]),
- tf.placeholder(tf.int32, [None]))
-
- def setup(self):
- """Sets up all components of the computation graph."""
-
- self.x, self.y = self.get_xy_placeholders()
-
- # This context creates variables
- with tf.variable_scope('core', reuse=None):
- self.loss, self.gradient_ops = self.train(self.x, self.y)
- # And this one re-uses them (thus the `reuse=True`)
- with tf.variable_scope('core', reuse=True):
- self.y_preds = self.eval(self.x, self.y)
-
- def training_ops(self, loss):
- opt = self.get_optimizer()
- params = tf.trainable_variables()
- gradients = tf.gradients(loss, params)
- clipped_gradients, _ = tf.clip_by_global_norm(gradients, 5.0)
- return opt.apply_gradients(zip(clipped_gradients, params),
- global_step=self.global_step)
-
- def get_optimizer(self):
- return tf.train.AdamOptimizer(learning_rate=self.learning_rate,
- epsilon=1e-4)
-
- def one_step(self, sess, x, y):
- outputs = [self.loss, self.gradient_ops]
- return sess.run(outputs, feed_dict={self.x: x, self.y: y})
-
- def episode_step(self, sess, x, y, clear_memory=False):
- """Performs training steps on episodic input.
-
- Args:
- sess: A Tensorflow Session.
- x: A list of batches of images defining the episode.
- y: A list of batches of labels corresponding to x.
- clear_memory: Whether to clear the memory before the episode.
-
- Returns:
- List of losses the same length as the episode.
- """
-
- outputs = [self.loss, self.gradient_ops]
-
- if clear_memory:
- self.clear_memory(sess)
-
- losses = []
- for xx, yy in zip(x, y):
- out = sess.run(outputs, feed_dict={self.x: xx, self.y: yy})
- loss = out[0]
- losses.append(loss)
-
- return losses
-
- def predict(self, sess, x, y=None):
- """Predict the labels on a single batch of examples.
-
- Args:
- sess: A Tensorflow Session.
- x: A batch of images.
- y: The labels for the images in x.
- This allows for updating the memory.
-
- Returns:
- Predicted y.
- """
-
- # Storing current memory state to restore it after prediction
- mem_keys, mem_vals, mem_age, _ = self.memory.get()
- cur_memory = (
- tf.identity(mem_keys),
- tf.identity(mem_vals),
- tf.identity(mem_age),
- None,
- )
-
- outputs = [self.y_preds]
- if y is None:
- ret = sess.run(outputs, feed_dict={self.x: x})
- else:
- ret = sess.run(outputs, feed_dict={self.x: x, self.y: y})
-
- # Restoring memory state
- self.memory.set(*cur_memory)
-
- return ret
-
- def episode_predict(self, sess, x, y, clear_memory=False):
- """Predict the labels on an episode of examples.
-
- Args:
- sess: A Tensorflow Session.
- x: A list of batches of images.
- y: A list of labels for the images in x.
- This allows for updating the memory.
- clear_memory: Whether to clear the memory before the episode.
-
- Returns:
- List of predicted y.
- """
-
- # Storing current memory state to restore it after prediction
- mem_keys, mem_vals, mem_age, _ = self.memory.get()
- cur_memory = (
- tf.identity(mem_keys),
- tf.identity(mem_vals),
- tf.identity(mem_age),
- None,
- )
-
- if clear_memory:
- self.clear_memory(sess)
-
- outputs = [self.y_preds]
- y_preds = []
- for xx, yy in zip(x, y):
- out = sess.run(outputs, feed_dict={self.x: xx, self.y: yy})
- y_pred = out[0]
- y_preds.append(y_pred)
-
- # Restoring memory state
- self.memory.set(*cur_memory)
-
- return y_preds
-
- def clear_memory(self, sess):
- sess.run([self.memory.clear()])
diff --git a/research/learning_to_remember_rare_events/train.py b/research/learning_to_remember_rare_events/train.py
deleted file mode 100644
index c5c6d06b5ee02e73128ee2b23f3b399d29b1e212..0000000000000000000000000000000000000000
--- a/research/learning_to_remember_rare_events/train.py
+++ /dev/null
@@ -1,242 +0,0 @@
-# Copyright 2017 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# ==============================================================================
-r"""Script for training model.
-
-Simple command to get up and running:
- python train.py --memory_size=8192 \
- --batch_size=16 --validation_length=50 \
- --episode_width=5 --episode_length=30
-"""
-
-import logging
-import os
-import random
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import data_utils
-import model
-
-FLAGS = tf.flags.FLAGS
-
-tf.flags.DEFINE_integer('rep_dim', 128,
- 'dimension of keys to use in memory')
-tf.flags.DEFINE_integer('episode_length', 100, 'length of episode')
-tf.flags.DEFINE_integer('episode_width', 5,
- 'number of distinct labels in a single episode')
-tf.flags.DEFINE_integer('memory_size', None, 'number of slots in memory. '
- 'Leave as None to default to episode length')
-tf.flags.DEFINE_integer('batch_size', 16, 'batch size')
-tf.flags.DEFINE_integer('num_episodes', 100000, 'number of training episodes')
-tf.flags.DEFINE_integer('validation_frequency', 20,
- 'every so many training episodes, '
- 'assess validation accuracy')
-tf.flags.DEFINE_integer('validation_length', 10,
- 'number of episodes to use to compute '
- 'validation accuracy')
-tf.flags.DEFINE_integer('seed', 888, 'random seed for training sampling')
-tf.flags.DEFINE_string('save_dir', '', 'directory to save model to')
-tf.flags.DEFINE_bool('use_lsh', False,
- 'use locality-sensitive hashing '
- '(NOTE: not fully tested)')
-
-
-class Trainer(object):
- """Class that takes care of training, validating, and checkpointing model."""
-
- def __init__(self, train_data, valid_data, input_dim, output_dim=None):
- self.train_data = train_data
- self.valid_data = valid_data
- self.input_dim = input_dim
-
- self.rep_dim = FLAGS.rep_dim
- self.episode_length = FLAGS.episode_length
- self.episode_width = FLAGS.episode_width
- self.batch_size = FLAGS.batch_size
- self.memory_size = (self.episode_length * self.batch_size
- if FLAGS.memory_size is None else FLAGS.memory_size)
- self.use_lsh = FLAGS.use_lsh
-
- self.output_dim = (output_dim if output_dim is not None
- else self.episode_width)
-
- def get_model(self):
- # vocab size is the number of distinct values that
- # could go into the memory key-value storage
- vocab_size = self.episode_width * self.batch_size
- return model.Model(
- self.input_dim, self.output_dim, self.rep_dim, self.memory_size,
- vocab_size, use_lsh=self.use_lsh)
-
- def sample_episode_batch(self, data,
- episode_length, episode_width, batch_size):
- """Generates a random batch for training or validation.
-
- Structures each element of the batch as an 'episode'.
- Each episode contains episode_length examples and
- episode_width distinct labels.
-
- Args:
- data: A dictionary mapping label to list of examples.
- episode_length: Number of examples in each episode.
- episode_width: Distinct number of labels in each episode.
- batch_size: Batch size (number of episodes).
-
- Returns:
- A tuple (x, y) where x is a list of batches of examples
- with size episode_length and y is a list of batches of labels.
- """
-
- episodes_x = [[] for _ in xrange(episode_length)]
- episodes_y = [[] for _ in xrange(episode_length)]
- assert len(data) >= episode_width
- keys = data.keys()
- for b in xrange(batch_size):
- episode_labels = random.sample(keys, episode_width)
- remainder = episode_length % episode_width
- remainders = [0] * (episode_width - remainder) + [1] * remainder
- episode_x = [
- random.sample(data[lab],
- r + (episode_length - remainder) // episode_width)
- for lab, r in zip(episode_labels, remainders)]
- episode = sum([[(x, i, ii) for ii, x in enumerate(xx)]
- for i, xx in enumerate(episode_x)], [])
- random.shuffle(episode)
- # Arrange episode so that each distinct label is seen before moving to
- # 2nd showing
- episode.sort(key=lambda elem: elem[2])
- assert len(episode) == episode_length
- for i in xrange(episode_length):
- episodes_x[i].append(episode[i][0])
- episodes_y[i].append(episode[i][1] + b * episode_width)
-
- return ([np.array(xx).astype('float32') for xx in episodes_x],
- [np.array(yy).astype('int32') for yy in episodes_y])
-
- def compute_correct(self, ys, y_preds):
- return np.mean(np.equal(y_preds, np.array(ys)))
-
- def individual_compute_correct(self, y, y_pred):
- return y_pred == y
-
- def run(self):
- """Performs training.
-
- Trains a model using episodic training.
- Every so often, runs some evaluations on validation data.
- """
-
- train_data, valid_data = self.train_data, self.valid_data
- input_dim, output_dim = self.input_dim, self.output_dim
- rep_dim, episode_length = self.rep_dim, self.episode_length
- episode_width, memory_size = self.episode_width, self.memory_size
- batch_size = self.batch_size
-
- train_size = len(train_data)
- valid_size = len(valid_data)
- logging.info('train_size (number of labels) %d', train_size)
- logging.info('valid_size (number of labels) %d', valid_size)
- logging.info('input_dim %d', input_dim)
- logging.info('output_dim %d', output_dim)
- logging.info('rep_dim %d', rep_dim)
- logging.info('episode_length %d', episode_length)
- logging.info('episode_width %d', episode_width)
- logging.info('memory_size %d', memory_size)
- logging.info('batch_size %d', batch_size)
-
- assert all(len(v) >= float(episode_length) / episode_width
- for v in train_data.values())
- assert all(len(v) >= float(episode_length) / episode_width
- for v in valid_data.values())
-
- output_dim = episode_width
- self.model = self.get_model()
- self.model.setup()
-
- sess = tf.Session()
- sess.run(tf.global_variables_initializer())
-
- saver = tf.train.Saver(max_to_keep=10)
- ckpt = None
- if FLAGS.save_dir:
- ckpt = tf.train.get_checkpoint_state(FLAGS.save_dir)
- if ckpt and ckpt.model_checkpoint_path:
- logging.info('restoring from %s', ckpt.model_checkpoint_path)
- saver.restore(sess, ckpt.model_checkpoint_path)
-
- logging.info('starting now')
- losses = []
- random.seed(FLAGS.seed)
- np.random.seed(FLAGS.seed)
- for i in xrange(FLAGS.num_episodes):
- x, y = self.sample_episode_batch(
- train_data, episode_length, episode_width, batch_size)
- outputs = self.model.episode_step(sess, x, y, clear_memory=True)
- loss = outputs
- losses.append(loss)
-
- if i % FLAGS.validation_frequency == 0:
- logging.info('episode batch %d, avg train loss %f',
- i, np.mean(losses))
- losses = []
-
- # validation
- correct = []
- num_shots = episode_length // episode_width
- correct_by_shot = dict((k, []) for k in xrange(num_shots))
- for _ in xrange(FLAGS.validation_length):
- x, y = self.sample_episode_batch(
- valid_data, episode_length, episode_width, 1)
- outputs = self.model.episode_predict(
- sess, x, y, clear_memory=True)
- y_preds = outputs
- correct.append(self.compute_correct(np.array(y), y_preds))
-
- # compute per-shot accuracies
- seen_counts = [0] * episode_width
- # loop over episode steps
- for yy, yy_preds in zip(y, y_preds):
- # loop over batch examples
- yyy, yyy_preds = int(yy[0]), int(yy_preds[0])
- count = seen_counts[yyy % episode_width]
- if count in correct_by_shot:
- correct_by_shot[count].append(
- self.individual_compute_correct(yyy, yyy_preds))
- seen_counts[yyy % episode_width] = count + 1
-
- logging.info('validation overall accuracy %f', np.mean(correct))
- logging.info('%d-shot: %.3f, ' * num_shots,
- *sum([[k, np.mean(correct_by_shot[k])]
- for k in xrange(num_shots)], []))
-
- if saver and FLAGS.save_dir:
- saved_file = saver.save(sess,
- os.path.join(FLAGS.save_dir, 'model.ckpt'),
- global_step=self.model.global_step)
- logging.info('saved model to %s', saved_file)
-
-
-def main(unused_argv):
- train_data, valid_data = data_utils.get_data()
- trainer = Trainer(train_data, valid_data, data_utils.IMAGE_NEW_SIZE ** 2)
- trainer.run()
-
-
-if __name__ == '__main__':
- logging.basicConfig(level=logging.INFO)
- tf.app.run()
diff --git a/research/learning_unsupervised_learning/.gitignore b/research/learning_unsupervised_learning/.gitignore
deleted file mode 100644
index 0d20b6487c61e7d1bde93acf4a14b7a89083a16d..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.pyc
diff --git a/research/learning_unsupervised_learning/README.md b/research/learning_unsupervised_learning/README.md
deleted file mode 100644
index 0e38717f5de29df28959062889abeb1ce578feea..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/README.md
+++ /dev/null
@@ -1,40 +0,0 @@
-
-
-
-
-# Learning Unsupervised Learning Rules
-This repository contains code and weights for the learned update rule
-presented in "Learning Unsupervised Learning Rules." At this time, this
-code can not meta-train the update rule.
-
-### Structure
-`run_eval.py` contains the main training loop. This constructs an op
-that runs one iteration of the learned update rule and assigns the
-results to variables. Additionally, it loads the weights from our
-pre-trained model.
-
-The base model and the update rule architecture definition can be found in
-`architectures/more_local_weight_update.py`. For a complete description
-of the model, see our [paper](https://arxiv.org/abs/1804.00222).
-
-### Dependencies
-[absl]([https://github.com/abseil/abseil-py), [tensorflow](https://tensorflow.org), [sonnet](https://github.com/deepmind/sonnet)
-
-### Usage
-
-First, download the [pre-trained optimizer model weights](https://storage.googleapis.com/learning_unsupervised_learning/200_tf_graph.zip) and extract it.
-
-```bash
-# move to the folder above this folder
-cd path_to/research/learning_unsupervised_learning/../
-
-# launch the eval script
-python -m learning_unsupervised_learning.run_eval \
---train_log_dir="/tmp/learning_unsupervised_learning" \
---checkpoint_dir="/path/to/downloaded/model/tf_graph_data.ckpt"
-```
-
-### Contact
-Luke Metz, Niru Maheswaranathan, Github: @lukemetz, @nirum. Email: {lmetz, nirum}@google.com
-
-
diff --git a/research/learning_unsupervised_learning/__init__.py b/research/learning_unsupervised_learning/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/learning_unsupervised_learning/architectures/__init__.py b/research/learning_unsupervised_learning/architectures/__init__.py
deleted file mode 100644
index af9545f26da538aa986b19a96b6cfa2bc7459227..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/architectures/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-import more_local_weight_update
diff --git a/research/learning_unsupervised_learning/architectures/common.py b/research/learning_unsupervised_learning/architectures/common.py
deleted file mode 100644
index 43a2d4f8965ecd337abd3a072a7ecb789df21910..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/architectures/common.py
+++ /dev/null
@@ -1,153 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import sonnet as snt
-import tensorflow as tf
-import numpy as np
-import collections
-from learning_unsupervised_learning import utils
-
-from tensorflow.python.util import nest
-
-from learning_unsupervised_learning import variable_replace
-
-
-class LinearBatchNorm(snt.AbstractModule):
- """Module that does a Linear layer then a BatchNorm followed by an activation fn"""
- def __init__(self, size, activation_fn=tf.nn.relu, name="LinearBatchNorm"):
- self.size = size
- self.activation_fn = activation_fn
- super(LinearBatchNorm, self).__init__(name=name)
-
- def _build(self, x):
- x = tf.to_float(x)
- initializers={"w": tf.truncated_normal_initializer(stddev=0.01)}
- lin = snt.Linear(self.size, use_bias=False, initializers=initializers)
- z = lin(x)
-
- scale = tf.constant(1., dtype=tf.float32)
- offset = tf.get_variable(
- "b",
- shape=[1, z.shape.as_list()[1]],
- initializer=tf.truncated_normal_initializer(stddev=0.1),
- dtype=tf.float32
- )
-
- mean, var = tf.nn.moments(z, [0], keep_dims=True)
- z = ((z - mean) * tf.rsqrt(var + 1e-6)) * scale + offset
-
- x_p = self.activation_fn(z)
-
- return z, x_p
-
- # This needs to work by string name sadly due to how the variable replace
- # works and would also work even if the custom getter approuch was used.
- # This is verbose, but it should atleast be clear as to what is going on.
- # TODO(lmetz) a better way to do this (the next 3 functions:
- # _raw_name, w(), b() )
- def _raw_name(self, var_name):
- """Return just the name of the variable, not the scopes."""
- return var_name.split("/")[-1].split(":")[0]
-
-
- @property
- def w(self):
- var_list = snt.get_variables_in_module(self)
- w = [x for x in var_list if self._raw_name(x.name) == "w"]
- assert len(w) == 1
- return w[0]
-
- @property
- def b(self):
- var_list = snt.get_variables_in_module(self)
- b = [x for x in var_list if self._raw_name(x.name) == "b"]
- assert len(b) == 1
- return b[0]
-
-
-
-class Linear(snt.AbstractModule):
- def __init__(self, size, use_bias=True, init_const_mag=True):
- self.size = size
- self.use_bias = use_bias
- self.init_const_mag = init_const_mag
- super(Linear, self).__init__(name="commonLinear")
-
- def _build(self, x):
- if self.init_const_mag:
- initializers={"w": tf.truncated_normal_initializer(stddev=0.01)}
- else:
- initializers={}
- lin = snt.Linear(self.size, use_bias=self.use_bias, initializers=initializers)
- z = lin(x)
- return z
-
- # This needs to work by string name sadly due to how the variable replace
- # works and would also work even if the custom getter approuch was used.
- # This is verbose, but it should atleast be clear as to what is going on.
- # TODO(lmetz) a better way to do this (the next 3 functions:
- # _raw_name, w(), b() )
- def _raw_name(self, var_name):
- """Return just the name of the variable, not the scopes."""
- return var_name.split("/")[-1].split(":")[0]
-
- @property
- def w(self):
- var_list = snt.get_variables_in_module(self)
- if self.use_bias:
- assert len(var_list) == 2, "Found not 2 but %d" % len(var_list)
- else:
- assert len(var_list) == 1, "Found not 1 but %d" % len(var_list)
- w = [x for x in var_list if self._raw_name(x.name) == "w"]
- assert len(w) == 1
- return w[0]
-
- @property
- def b(self):
- var_list = snt.get_variables_in_module(self)
- assert len(var_list) == 2, "Found not 2 but %d" % len(var_list)
- b = [x for x in var_list if self._raw_name(x.name) == "b"]
- assert len(b) == 1
- return b[0]
-
-
-def transformer_at_state(base_model, new_variables):
- """Get the base_model that has been transformed to use the variables
- in final_state.
- Args:
- base_model: snt.Module
- Goes from batch to features
- new_variables: list
- New list of variables to use
- Returns:
- func: callable of same api as base_model.
- """
- assert not variable_replace.in_variable_replace_scope()
-
- def _feature_transformer(input_data):
- """Feature transformer at the end of training."""
- initial_variables = base_model.get_variables()
- replacement = collections.OrderedDict(
- utils.eqzip(initial_variables, new_variables))
- with variable_replace.variable_replace(replacement):
- features = base_model(input_data)
- return features
-
- return _feature_transformer
diff --git a/research/learning_unsupervised_learning/architectures/more_local_weight_update.py b/research/learning_unsupervised_learning/architectures/more_local_weight_update.py
deleted file mode 100644
index 117549af0f21f9e5148435b73f664a08013f8786..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/architectures/more_local_weight_update.py
+++ /dev/null
@@ -1,861 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import numpy as np
-import sonnet as snt
-import tensorflow as tf
-
-from learning_unsupervised_learning.architectures import common
-from learning_unsupervised_learning import optimizers
-from learning_unsupervised_learning import utils
-from learning_unsupervised_learning import summary_utils
-
-OptState = collections.namedtuple('OptState',
- ['variables', 'opt_state', 'index'])
-
-BaseModelOutputs = collections.namedtuple(
- 'BaseModelOutputs', ['xs', 'zs', 'mods', 'batch', 'backward_mods'])
-
-
-class GradChannelReadout(snt.AbstractModule):
- """Perform a linear readout and reshape from input 3 tensor."""
-
- def __init__(self,
- num_grad_channels,
- device,
- perm=(2, 0, 1),
- name='GradChannelReadout'):
- """Args:
-
- num_grad_channels: int
- number of channels to readout to.
- device: str or callable
- devicwe to place weights.
- perm: list or tuple
- transpose applied.
- """
-
- self.num_grad_channels = num_grad_channels
- self.device = device
- self.perm = perm
- super(GradChannelReadout, self).__init__(name=name)
-
- def _build(self, h):
- with tf.device(self.device):
- mod = snt.Linear(self.num_grad_channels)
- ret = snt.BatchApply(mod)(h)
- # return as [num_grad_channels] x [bs] x [num units]
- return tf.transpose(ret, perm=self.perm)
-
-
-def get_weight_stats(x, axis):
- """ Compute weight statistics over the given axis.
-
- Args:
- x: tf.Tensor
- a batch of activations.
- axis: int
- axis to perform statistics over.
- Returns:
- tf.Tensor
- a 3-D tensor with statistics.
- """
- if x is None:
- return []
-
- stats = []
- l1 = tf.reduce_mean(tf.abs(x), axis=axis)
- l2 = tf.sqrt(tf.reduce_mean(x**2, axis=axis) + 1e-6)
-
- mean, var = tf.nn.moments(x, [axis])
- stats.extend([l1, l2, mean, tf.sqrt(var + 1e-8)])
-
- stats = [tf.reshape(s, [-1, 1, 1]) for s in stats]
-
- return stats
-
-
-class AddUnitBatchStatistics(snt.AbstractModule):
- """Compute some number of statistics over units and concat them on."""
-
- def __init__(self, name='AddUnitBatchStatistics'):
- super(AddUnitBatchStatistics, self).__init__(name=name)
-
- def _build(self, x):
- # [channel, bs, 1]
- output = x
- for d in [0, 1]:
- stats = []
- l1 = tf.reduce_mean(tf.abs(x), axis=d, keepdims=True)
- l2 = tf.sqrt(tf.reduce_mean(x**2, axis=d, keepdims=True) + 1e-6)
-
- mean, var = tf.nn.moments(x, [d], keepdims=True)
- stats.extend([l1, l2, mean, tf.sqrt(var + 1e-8)])
-
- to_add = tf.concat(stats, axis=2) # [channels/1, units/1, stats]
- output += snt.BatchApply(snt.Linear(x.shape.as_list()[2]))(to_add)
- return output
-
-
-class ConcatUnitConv(snt.AbstractModule):
- """Do a small number of convolutions over units and concat / add them on."""
-
- def __init__(self, add=True):
- self.add = add
- super(ConcatUnitConv, self).__init__(name='ConcatUnitConv')
-
- def _build(self, x):
- # x is [units, bs, 1]
- net = tf.transpose(x, [1, 0, 2]) # now [bs x units x 1]
- channels = x.shape.as_list()[2]
- mod = snt.Conv1D(output_channels=channels, kernel_shape=[3])
- net = mod(net)
- net = snt.BatchNorm(axis=[0, 1])(net, is_training=False)
- net = tf.nn.relu(net)
- mod = snt.Conv1D(output_channels=channels, kernel_shape=[3])
- net = mod(net)
- net = snt.BatchNorm(axis=[0, 1])(net, is_training=False)
- net = tf.nn.relu(net)
- to_concat = tf.transpose(net, [1, 0, 2])
- if self.add:
- return x + to_concat
- else:
- return tf.concat([x, to_concat], 2)
-
-
-class MoreLocalWeightUpdateProcess(snt.AbstractModule):
-
- def __init__(
- self,
- remote_device,
- local_device,
- top_delta_size=64,
- top_delta_layers=2,
- compute_h_size=64,
- compute_h_layers=1,
- delta_dim=32,
- num_grad_channels=4,
- normalize_epsilon=1.,
- ):
- self.local_device = local_device
- self.remote_device = remote_device
- self.top_delta_size = top_delta_size
- self.top_delta_layers = top_delta_layers
- self.compute_h_size = compute_h_size
- self.compute_h_layers = compute_h_layers
- self.delta_dim = delta_dim
- self.num_grad_channels = num_grad_channels
- self.normalize_epsilon = normalize_epsilon,
-
- with tf.device(local_device):
- self.opt = optimizers.UnrollableGradientDescentRollingOptimizer(
- learning_rate=1e-4)
-
- # lazily initialized for readouts
- self.readout_mods = {}
-
- super(MoreLocalWeightUpdateProcess,
- self).__init__(name='MoreLocalWeightUpdateProcess')
-
- with tf.device(remote_device):
- self()
-
- def normalize(self, change_w, normalize_epsilon=None):
- if normalize_epsilon is None:
- normalize_epsilon = self.normalize_epsilon
-
- # normalize the weights per receptive-field, rather than per-matrix
- var = tf.reduce_mean(tf.square(change_w), axis=0, keepdims=True)
- change_w = (change_w) / tf.sqrt(normalize_epsilon + var)
- return change_w
-
- def _build(self):
- pass
-
- @snt.reuse_variables
- def compute_top_delta(self, z):
- """ parameterization of topD. This converts the top level activation
- to an error signal.
- Args:
- z: tf.Tensor
- batch of final layer post activations
- Returns
- delta: tf.Tensor
- the error signal
- """
- s_idx = 0
- with tf.variable_scope('compute_top_delta'), tf.device(self.remote_device):
- # typically this takes [BS, length, input_channels],
- # We are applying this such that we convolve over the batch dimension.
- act = tf.expand_dims(tf.transpose(z, [1, 0]), 2) # [channels, BS, 1]
-
- mod = snt.Conv1D(output_channels=self.top_delta_size, kernel_shape=[5])
- act = mod(act)
-
- act = snt.BatchNorm(axis=[0, 1])(act, is_training=False)
- act = tf.nn.relu(act)
-
- bs = act.shape.as_list()[0]
- act = tf.transpose(act, [2, 1, 0])
- act = snt.Conv1D(output_channels=bs, kernel_shape=[3])(act)
- act = snt.BatchNorm(axis=[0, 1])(act, is_training=False)
- act = tf.nn.relu(act)
- act = snt.Conv1D(output_channels=bs, kernel_shape=[3])(act)
- act = snt.BatchNorm(axis=[0, 1])(act, is_training=False)
- act = tf.nn.relu(act)
- act = tf.transpose(act, [2, 1, 0])
-
- prev_act = act
- for i in range(self.top_delta_layers):
- mod = snt.Conv1D(output_channels=self.top_delta_size, kernel_shape=[3])
- act = mod(act)
-
- act = snt.BatchNorm(axis=[0, 1])(act, is_training=False)
- act = tf.nn.relu(act)
-
- prev_act = act
-
- mod = snt.Conv1D(output_channels=self.delta_dim, kernel_shape=[3])
- act = mod(act)
-
- # [bs, feature_channels, delta_channels]
- act = tf.transpose(act, [1, 0, 2])
- return act
-
- @snt.reuse_variables
- def compute_h(self,
- x,
- z,
- d,
- bias,
- W_bot,
- W_top,
- compute_perc=1.0,
- compute_units=None):
- """z = [BS, n_units] a = [BS, n_units] b = [BS, n_units] d = [BS, n_units, delta_channels]
-
- """
-
- s_idx = 0
- if compute_perc != 1.0:
- assert compute_units is None
-
- with tf.device(self.remote_device):
- inp_feat = [x, z]
- inp_feat = [tf.transpose(f, [1, 0]) for f in inp_feat]
-
- units = x.shape.as_list()[1]
- bs = x.shape.as_list()[0]
-
- # add unit ID, to help the network differentiate units
- id_theta = tf.linspace(0., (4) * np.pi, units)
- assert bs is not None
- id_theta_bs = tf.reshape(id_theta, [-1, 1]) * tf.ones([1, bs])
- inp_feat += [tf.sin(id_theta_bs), tf.cos(id_theta_bs)]
-
- # list of [units, BS, 1]
- inp_feat = [tf.expand_dims(f, 2) for f in inp_feat]
-
- d_trans = tf.transpose(d, [1, 0, 2])
-
- if compute_perc != 1.0:
- compute_units = int(compute_perc * inp_feat.shape.as_list()[0])
-
- # add weight matrix statistics, both from above and below
- w_stats_bot = get_weight_stats(W_bot, 0)
- w_stats_top = get_weight_stats(W_top, 1)
- w_stats = w_stats_bot + w_stats_top
- if W_bot is None or W_top is None:
- # if it's an edge layer (top or bottom), just duplicate the stats for
- # the weight matrix that does exist
- w_stats = w_stats + w_stats
- w_stats = [tf.ones([1, x.shape[0], 1]) * ww for ww in w_stats]
- # w_stats is a list, with entries with shape UNITS x 1 x channels
-
- if compute_units is None:
- inp_feat_in = inp_feat
- d_trans_in = d_trans
- w_stats_in = w_stats
- bias_in = tf.transpose(bias)
- else:
- # only run on a subset of the activations.
- mask = tf.random_uniform(
- minval=0,
- maxval=1,
- dtype=tf.float32,
- shape=inp_feat[0].shape.as_list()[0:1])
- _, ind = tf.nn.top_k(mask, k=compute_units)
- ind = tf.reshape(ind, [-1, 1])
-
- inp_feat_in = [tf.gather_nd(xx, ind) for xx in inp_feat]
- w_stats_in = [tf.gather_nd(xx, ind) for xx in w_stats]
- d_trans_in = tf.gather_nd(d_trans, ind)
- bias_in = tf.gather_nd(tf.transpose(bias), ind)
-
- w_stats_in = tf.concat(w_stats_in, 2)
- w_stats_in_norm = w_stats_in * tf.rsqrt(
- tf.reduce_mean(w_stats_in**2) + 1e-6)
-
- act = tf.concat(inp_feat_in + [d_trans_in], 2)
- act = snt.BatchNorm(axis=[0, 1])(act, is_training=True)
-
- bias_dense = tf.reshape(bias_in, [-1, 1, 1]) * tf.ones([1, bs, 1])
- act = tf.concat([w_stats_in_norm, bias_dense, act], 2)
-
- mod = snt.Conv1D(output_channels=self.compute_h_size, kernel_shape=[3])
- act = mod(act)
-
- act = snt.BatchNorm(axis=[0, 1])(act, is_training=True)
- act = tf.nn.relu(act)
-
- act2 = ConcatUnitConv()(act)
- act = act2
-
- prev_act = act
- for i in range(self.compute_h_layers):
- mod = snt.Conv1D(output_channels=self.compute_h_size, kernel_shape=[3])
- act = mod(act)
-
- act = snt.BatchNorm(axis=[0, 1])(act, is_training=True)
- act = tf.nn.relu(act)
-
- act = ConcatUnitConv()(act)
-
- prev_act = act
-
- h = act
- if compute_units is not None:
- shape = inp_feat[0].shape.as_list()[:1] + h.shape.as_list()[1:]
- h = tf.scatter_nd(ind, h, shape=shape)
-
- h = tf.transpose(h, [1, 0, 2]) # [bs, units, channels]
-
- return h
-
- ## wrappers to allow forward and backward to have different variables
- @snt.reuse_variables
- def merge_change_w_forward(self, change_w_terms, global_prefix='', prefix=''):
- return self.merge_change_w(
- change_w_terms, global_prefix=global_prefix, prefix=prefix)
-
- @snt.reuse_variables
- def merge_change_w_backward(self, change_w_terms, global_prefix='',
- prefix=''):
- return self.merge_change_w(
- change_w_terms, global_prefix=global_prefix, prefix=prefix)
-
- def merge_change_w(self, change_w_terms, global_prefix='', prefix=''):
- with tf.device(
- self.remote_device), tf.name_scope(global_prefix + '_merge_change_w'):
- w_base = change_w_terms['w_base']
-
- for kk in sorted(change_w_terms.keys()):
- name = global_prefix + 'change_w_plane_%s' % kk
- delta_w = change_w_terms[kk]
- mean, var = tf.nn.moments(delta_w, [0, 1])
- root_mean_square = tf.sqrt(tf.reduce_mean(delta_w**2) + 1e-6)
-
- for kk in sorted(change_w_terms.keys()):
- change_w_terms[kk] = self.normalize(change_w_terms[kk])
-
- initializers = {
- 'w': tf.constant_initializer(0.1),
- 'b': tf.zeros_initializer()
- }
- mod = snt.Linear(
- 1,
- name=global_prefix + '_weight_readout_coeffs',
- initializers=initializers)
-
- change_w_terms_list = [
- change_w_terms[kk] for kk in sorted(change_w_terms.keys())
- ]
- stack_terms = tf.stack(change_w_terms_list, axis=-1)
- change_w = tf.squeeze(
- snt.BatchApply(mod)(stack_terms), axis=-1) / len(change_w_terms)
-
- # only allow perpendicular updates, or updates which grow length. don't
- # allow length to decay towards zero.
- ip = tf.reduce_mean(change_w * w_base)
- # zero out any updates that shrink length
- ip = tf.nn.relu(ip)
- change_w -= w_base * ip
- change_w /= tf.sqrt(len(change_w_terms) * 1.)
-
- change_w = self.normalize(change_w)
-
- # encourage the receptive field to not collapse to 0
- change_w -= w_base / 7. # This is an arbitrary scale choice
-
- return tf.identity(change_w)
-
- @snt.reuse_variables
- def bias_readout(self, h):
- with tf.device(self.remote_device):
- mod = snt.Linear(1, name='bias_readout')
- ret = snt.BatchApply(mod)(h)
- return tf.squeeze(ret, 2)
-
- @snt.reuse_variables
- def next_delta(self, z, h, d):
- with tf.device(self.remote_device):
- return d * tf.expand_dims(tf.nn.sigmoid(z), 2) + self.to_delta_size(h)
-
- @utils.create_variables_in_class_scope
- def get_readout_mod(self, name):
- if name not in self.readout_mods:
- self.readout_mods[name] = GradChannelReadout(
- self.num_grad_channels, device=self.remote_device, name=name)
-
- return self.readout_mods[name]
-
- @utils.create_variables_in_class_scope
- def low_rank_readout(self, name, h1, h2, psd=False):
- BS = h1.shape.as_list()[0]
- r_t = self.get_readout_mod(name + '_top')(h1)
- if psd:
- r_b = r_t
- else:
- r_b = self.get_readout_mod(name + '_bottom')(h2)
- return tf.reduce_mean(tf.matmul(r_b, r_t, transpose_a=True), axis=0) / BS
-
- @snt.reuse_variables
- def to_delta_size(self, h):
- with tf.device(self.remote_device):
- mod = snt.Linear(self.delta_dim)
- return snt.BatchApply(mod)(h)
-
- @snt.reuse_variables
- def initial_state(self, variables):
- """The inner optimization state.
-
- Args:
- variables: list of tf.Variable
- list of variables to get the initial state of.
- Returns:
- opt_state: OptState
- """
-
- with tf.device(self.local_device):
- initial_opt_state = self.opt.get_state(variables)
-
- return OptState(
- variables=variables, opt_state=initial_opt_state, index=tf.constant(0))
-
- @snt.reuse_variables
- def compute_next_state(self, grads, learning_rate, cur_state,
- cur_transformer):
-
- summaries = []
- with tf.device(self.local_device):
- with tf.control_dependencies(summaries):
- new_vars, new_state = self.opt.compute_updates(
- cur_state.variables, grads, learning_rate, cur_state.opt_state)
- pass
-
- return OptState(
- variables=tuple(new_vars),
- opt_state=new_state,
- index=cur_state.index + 1)
-
- def assign_state(self, base_model, next_state):
- var_ups = [
- v.assign(nv) for v, nv in utils.eqzip(base_model.get_variables(),
- next_state.variables)
- ]
-
- opt_ups = self.opt.assign_state(next_state.opt_state)
-
- return tf.group(opt_ups, *var_ups)
-
- def local_variables(self):
- return list(self.opt.get_variables())
-
- def remote_variables(self):
- train = list(
- snt.get_variables_in_module(self, tf.GraphKeys.TRAINABLE_VARIABLES))
- train += list(
- snt.get_variables_in_module(self,
- tf.GraphKeys.MOVING_AVERAGE_VARIABLES))
- return train
-
-
-class MoreLocalWeightUpdateWLearner(snt.AbstractModule):
- """The BaseModel that the UnsupervisedUpdateRule acts on.
- """
-
- def __init__(self,
- remote_device,
- local_device,
- inner_size=128,
- output_size=32,
- n_layers=4,
- shuffle_input=True,
- activation_fn=tf.nn.relu,
- identical_updates=True,
- **kwargs):
- self.local_device = local_device
- self.remote_device = remote_device
- self.inner_size = inner_size
- self.n_layers = n_layers
- self.shuffle_input = shuffle_input
- self.activation_fn = activation_fn
- self.identical_updates = identical_updates
-
- self.output_size = output_size
- if output_size == None:
- self.output_size = inner_size
-
- self.shuffle_ind = None
-
- super(MoreLocalWeightUpdateWLearner, self).__init__(
- name='LocalWeightUpdateWLearner', **kwargs)
-
- @snt.reuse_variables
- def get_shuffle_ind(self, size):
- if self.shuffle_ind is None:
- # put the shuffle in tf memory to make the eval jobs
- # re-entrant.
- shuffle_ind_val = np.random.permutation(size)
- shuffle_ind = tf.get_variable(
- name='shuffle_ind', dtype=tf.int64, initializer=shuffle_ind_val)
- unshuffle_ind = tf.scatter_nd(
- tf.reshape(shuffle_ind, [-1, 1]), tf.range(size), [size])
-
- return shuffle_ind, unshuffle_ind
-
- def _build(self, batch):
- image = batch.image
- x0 = snt.BatchFlatten()(image)
- if self.shuffle_input:
- size = x0.shape.as_list()[1]
- shuffle_ind, unshuffle_ind = self.get_shuffle_ind(size)
- x0 = tf.gather(x0, shuffle_ind, axis=1)
-
- xs = [x0]
- mods = []
- zs = []
- init = {}
-
- for i in range(self.n_layers):
- mod = common.LinearBatchNorm(
- self.inner_size, activation_fn=self.activation_fn)
- z, x = mod(xs[i])
- xs.append(x)
- zs.append(z)
- mods.append(mod)
-
- mod = common.LinearBatchNorm(
- self.output_size, activation_fn=self.activation_fn)
- z, x = mod(xs[-1])
- mods.append(mod)
-
- xs.append(x)
- zs.append(z)
-
- embedding_x = xs[-1]
-
- # make a random set of backward mods
- backward_mods = []
- for i, (x, x_p1) in enumerate(zip(xs[0:-1], xs[1:])):
- m = common.LinearBatchNorm(
- x_p1.shape.as_list()[1], activation_fn=tf.identity)
- _ = m(x)
- backward_mods.append(m)
-
- shape = image.shape.as_list()[1:4]
-
- for mods_p, prefix in [(mods, 'forward'), (backward_mods, 'backward')]:
- if self.shuffle_input:
- unshuf_w = tf.gather(mods_p[0].w, unshuffle_ind, axis=0)
- else:
- unshuf_w = mods_p[0].w
- img = summary_utils.first_layer_weight_image(unshuf_w, shape)
- tf.summary.image(prefix + '_w0_receptive_field', img)
-
- for i, m in enumerate(mods_p[0:]):
- img = summary_utils.inner_layer_weight_image(m.w)
- tf.summary.image(prefix + '_w%d' % (i + 1), img)
-
- img = summary_utils.sorted_images(image, batch.label_onehot)
- tf.summary.image('inputs', img)
-
- # log out pre-activations and activations
- for all_vis, base_name in [(xs, 'x'), (zs, 'z')]:
- for i, x_vis in enumerate(all_vis):
- img = summary_utils.activation_image(x_vis, batch.label_onehot)
- tf.summary.image('%s%d' % (base_name, i), img)
-
- embedding_x = tf.identity(embedding_x)
-
- outputs = BaseModelOutputs(
- xs=xs, zs=zs, mods=mods, batch=batch, backward_mods=backward_mods)
-
- return embedding_x, outputs
-
- def compute_next_h_d(self, meta_opt, w_bot, w_top, bias, x, z, d, backward_w):
- """ Propogate error back down the network while computing hidden state.
- """
- if z is None:
- z = x
-
- h = meta_opt.compute_h(x, z, d, bias, w_bot,
- w_top) # [bs x 60 x h_channels]
-
- # compute the next d
- delta = meta_opt.next_delta(z, h, d)
-
- if backward_w is not None:
-
- def delta_matmul(w, delta):
- d = tf.transpose(delta, [0, 2, 1]) # [bs x delta_channels x n_units)
- d = snt.BatchApply(lambda x: tf.matmul(x, w, transpose_b=True))(d)
- d = tf.transpose(d, [0, 2, 1])
- return d
-
- # replace the "backward pass" with a random matrix.
- d = delta_matmul(backward_w, delta) # [bs x 60 x delta_channels]
- var = tf.reduce_mean(tf.square(d), [2], keepdims=True)
- d = d * tf.rsqrt(1e-6 + var)
-
- return h, d
-
- def weight_change_for_layer(self, meta_opt, l_idx, w_base, b_base, upper_h,
- lower_h, upper_x, lower_x, prefix, include_bias):
- """Compute the change in weights for each layer.
- This computes something roughly analagous to a gradient.
- """
- reduce_upper_h = upper_h
- reduce_lower_h = lower_h
-
- BS = lower_x.shape.as_list()[0]
-
- change_w_terms = dict()
-
- # initial weight value normalized
- # normalize the weights per receptive-field, rather than per-matrix
- weight_scale = tf.rsqrt(
- tf.reduce_mean(w_base**2, axis=0, keepdims=True) + 1e-6)
- w_base *= weight_scale
-
- change_w_terms['w_base'] = w_base
-
- # this will act to decay larger weights towards zero
- change_w_terms['large_decay'] = w_base**2 * tf.sign(w_base)
-
- # term based on activations
- ux0 = upper_x - tf.reduce_mean(upper_x, axis=0, keepdims=True)
- uxs0 = ux0 * tf.rsqrt(tf.reduce_mean(ux0**2, axis=0, keepdims=True) + 1e-6)
- change_U = tf.matmul(uxs0, uxs0, transpose_a=True) / BS
- change_U /= tf.sqrt(float(change_U.shape.as_list()[0]))
-
- cw = tf.matmul(w_base, change_U)
- cw_scale = tf.rsqrt(tf.reduce_mean(cw**2 + 1e-8))
- cw *= cw_scale
- change_w_terms['decorr_x'] = cw
-
- # hebbian term
- lx0 = lower_x - tf.reduce_mean(lower_x, axis=0, keepdims=True)
- lxs0 = lx0 * tf.rsqrt(tf.reduce_mean(lx0**2, axis=0, keepdims=True) + 1e-6)
- cw = tf.matmul(lxs0, uxs0, transpose_a=True) / BS
- change_w_terms['hebb'] = -cw
-
- # 0th order term
- w_term = meta_opt.low_rank_readout(prefix + 'weight_readout_0', upper_h,
- lower_h)
- change_w_terms['0_order'] = w_term
-
- # # rbf term (weight update scaled by distance from 0)
- w_term = meta_opt.low_rank_readout(prefix + 'weight_readout_rbf',
- reduce_upper_h, reduce_lower_h)
- change_w_terms['rbf'] = tf.exp(-w_base**2) * w_term
-
- # 1st order term (weight dependent update to weights)
- w_term = meta_opt.low_rank_readout(prefix + 'weight_readout_1',
- reduce_upper_h, reduce_lower_h)
- change_w_terms['1_order'] = w_base * w_term
-
- # more terms based on single layer readouts.
- for update_type in ['lin', 'sqr']:
- for h_source, h_source_name in [(reduce_upper_h, 'upper'),
- (reduce_lower_h, 'lower')]:
- structures = ['symm']
- if update_type == 'lin' and h_source_name == 'upper':
- structures += ['psd']
- for structure in structures:
- name = update_type + '_' + h_source_name + '_' + structure
- if structure == 'symm':
- change_U = meta_opt.low_rank_readout(prefix + name, h_source,
- h_source)
- change_U = (change_U + tf.transpose(change_U)) / tf.sqrt(2.)
- change_U = tf.matrix_set_diag(change_U,
- tf.zeros(
- [change_U.shape.as_list()[0]]))
- elif structure == 'psd':
- change_U = meta_opt.low_rank_readout(
- prefix + name, h_source, None, psd=True)
- else:
- assert False
- change_U /= tf.sqrt(float(change_U.shape.as_list()[0]))
-
- if update_type == 'lin':
- sign_multiplier = tf.ones_like(w_base)
- w_base_l = w_base
- elif update_type == 'sqr':
- sign_multiplier = tf.sign(w_base)
- w_base_l = tf.sqrt(1. + w_base**2) - 1.
-
- if h_source_name == 'upper':
- cw = tf.matmul(w_base_l, change_U) # [N^l-1 x N^l]
- elif h_source_name == 'lower':
- cw = tf.matmul(change_U, w_base_l)
- change_w_terms[name] = cw * sign_multiplier
-
-
- if prefix == 'forward':
- change_w = meta_opt.merge_change_w_forward(
- change_w_terms, global_prefix=prefix, prefix='l%d' % l_idx)
- elif prefix == 'backward':
- change_w = meta_opt.merge_change_w_backward(
- change_w_terms, global_prefix=prefix, prefix='l%d' % l_idx)
- else:
- assert (False)
-
- if not include_bias:
- return change_w
-
- change_b = tf.reduce_mean(meta_opt.bias_readout(upper_h), [0])
-
- # force nonlinearities to be exercised -- biases can't all be increased without bound
- change_b_mean = tf.reduce_mean(change_b)
- offset = -tf.nn.relu(-change_b_mean)
- change_b -= offset
-
- var = tf.reduce_mean(tf.square(change_b), [0], keepdims=True)
- change_b = (change_b) / tf.sqrt(0.5 + var)
- return change_w, change_b
-
- def compute_next_state(self, outputs, meta_opt, previous_state):
- zs = outputs.zs
- xs = outputs.xs
- batch = outputs.batch
- mods = outputs.mods
- backward_mods = outputs.backward_mods
- variables = self.get_variables()
-
- rev_mods = mods[::-1]
- rev_backward_mods = backward_mods[::-1]
- rev_xs = xs[::-1]
- rev_zs = zs[::-1] + [None]
-
- to_top = xs[-1]
-
- # variables that change in the loop
- hs = []
- d = meta_opt.compute_top_delta(to_top) # [bs x 32 x delta_channels]
-
- iterator = utils.eqzip(rev_backward_mods + [None], rev_mods + [None],
- [None] + rev_mods, rev_xs, rev_zs)
- for (backward_mod, lower_mod, upper_mod, x, z) in iterator:
- w_bot = None
- if not lower_mod is None:
- w_bot = previous_state.variables[variables.index(lower_mod.w)]
- w_top = None
- if not upper_mod is None:
- w_top = previous_state.variables[variables.index(upper_mod.w)]
- backward_w = None
- if backward_mod is not None:
- backward_w = previous_state.variables[variables.index(backward_mod.w)]
- if lower_mod is not None:
- bias = previous_state.variables[variables.index(lower_mod.b)]
- else:
- bias = tf.zeros([x.shape[1]])
-
- h, d = self.compute_next_h_d(
- meta_opt=meta_opt,
- w_bot=w_bot,
- w_top=w_top,
- bias=bias,
- backward_w=backward_w,
- x=x,
- z=z,
- d=d)
- hs.append(h)
-
- w_forward_var_idx = [variables.index(mod.w) for mod in rev_mods]
- w_backward_var_idx = [variables.index(mod.w) for mod in rev_backward_mods]
- b_var_idx = [variables.index(mod.b) for mod in rev_mods]
-
- # storage location for outputs of below loop
- grads = [None for _ in previous_state.variables]
-
- # over-ride learning rate for perturbation variables
- learning_rate = [None for _ in previous_state.variables]
-
- # This is a map -- no state is shared cross loop
- for l_idx, w_forward_idx, w_backward_idx, b_idx, upper_h, lower_h, lower_x, upper_x in utils.eqzip(
- range(len(w_forward_var_idx)), w_forward_var_idx, w_backward_var_idx,
- b_var_idx, hs[:-1], hs[1:], xs[::-1][1:], xs[::-1][:-1]):
-
- b_base = previous_state.variables[b_idx]
- change_w_forward, change_b = self.weight_change_for_layer(
- meta_opt=meta_opt,
- l_idx=l_idx,
- w_base=previous_state.variables[w_forward_idx],
- b_base=b_base,
- upper_h=upper_h,
- lower_h=lower_h,
- upper_x=upper_x,
- lower_x=lower_x,
- prefix='forward',
- include_bias=True)
-
- if self.identical_updates:
- change_w_backward = change_w_forward
- else:
- change_w_backward = self.weight_change_for_layer(
- meta_opt=meta_opt,
- l_idx=l_idx,
- w_base=previous_state.variables[w_backward_idx],
- b_base=b_base,
- upper_h=upper_h,
- lower_h=lower_h,
- upper_x=upper_x,
- lower_x=lower_x,
- prefix='backward',
- include_bias=False)
-
- grads[w_forward_idx] = change_w_forward
-
- grads[w_backward_idx] = change_w_backward
-
- grads[b_idx] = change_b
-
- cur_transformer = common.transformer_at_state(self,
- previous_state.variables)
- next_state = meta_opt.compute_next_state(
- grads,
- learning_rate=learning_rate,
- cur_state=previous_state,
- cur_transformer=lambda x: cur_transformer(x)[0])
- return next_state
-
- def initial_state(self, meta_opt):
- return meta_opt.initial_state(self.get_variables())
diff --git a/research/learning_unsupervised_learning/datasets/__init__.py b/research/learning_unsupervised_learning/datasets/__init__.py
deleted file mode 100644
index 9949cd96ca8f2fe1c39705a5ca8570de9cad5a66..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/datasets/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import mnist
diff --git a/research/learning_unsupervised_learning/datasets/common.py b/research/learning_unsupervised_learning/datasets/common.py
deleted file mode 100644
index 11f65ceab57a4114ca3876b3cb6eed86e2263745..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/datasets/common.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-
-import tensorflow as tf
-import numpy as np
-
-ImageLabelOnehot = collections.namedtuple('ImageLabelOnehot',
- ['image', 'label', 'label_onehot'])
-ImageLabelOnehotRegression = collections.namedtuple(
- "ImageLabelOnehotRegression",
- ["image", "label", "label_onehot", "regression_target"])
diff --git a/research/learning_unsupervised_learning/datasets/mnist.py b/research/learning_unsupervised_learning/datasets/mnist.py
deleted file mode 100644
index 6ee595d99ad2523042454f038b4665095f501caf..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/datasets/mnist.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-import sonnet as snt
-import tensorflow as tf
-from tensorflow.python.keras.datasets import mnist
-from learning_unsupervised_learning.datasets import common
-
-class Mnist(snt.AbstractModule):
- def __init__(self, device, batch_size=128, name="Mnist"):
- self.device = device
- self.batch_size = batch_size
-
- self._make_dataset()
- self.iterator = None
-
- super(Mnist, self).__init__(name=name)
-
- def _make_dataset(self):
- (x_train, y_train), (x_test, y_test) = mnist.load_data()
-
- x_train = x_train.reshape(60000, 784)
- x_test = x_test.reshape(10000, 784)
-
- dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
- dataset = dataset.repeat()
- dataset = dataset.shuffle(self.batch_size * 3)
- dataset = dataset.batch(self.batch_size)
- def _map_fn(image, label):
- image = tf.to_float(image) / 255.
- label.set_shape([self.batch_size])
- label = tf.cast(label, dtype=tf.int32)
- label_onehot = tf.one_hot(label, 10)
- image = tf.reshape(image, [self.batch_size, 28, 28, 1])
- return common.ImageLabelOnehot(
- image=image, label=label, label_onehot=label_onehot)
-
- self.dataset = dataset.map(_map_fn)
-
- def _build(self):
- if self.iterator is None:
- self.iterator = self.dataset.make_one_shot_iterator()
- batch = self.iterator.get_next()
- [b.set_shape([self.batch_size] + b.shape.as_list()[1:]) for b in batch]
- return batch
-
-
-class TinyMnist(Mnist):
- def __init__(self, *args, **kwargs):
- kwargs.setdefault("name", "TinyMnist")
- super(TinyMnist, self).__init__(*args, **kwargs)
-
- def _make_dataset(self):
- super(TinyMnist, self)._make_dataset()
-
- def _map_fn(batch):
- new_img = tf.image.resize_images(batch.image, [14, 14])
- return common.ImageLabelOnehot(
- image=new_img, label=batch.label, label_onehot=batch.label_onehot)
-
- self.dataset = self.dataset.map(_map_fn)
diff --git a/research/learning_unsupervised_learning/evaluation.py b/research/learning_unsupervised_learning/evaluation.py
deleted file mode 100644
index 2ec40e99a672f9420200653b92818374e0e84d78..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/evaluation.py
+++ /dev/null
@@ -1,76 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-"""Evaluation job.
-
-This sits on the side and performs evaluation on a saved model.
-This is a separate process for ease of use and stability of numbers.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from learning_unsupervised_learning import utils
-
-
-def construct_evaluation_graph(theta_process_fn=None,
- w_learner_fn=None,
- dataset_fn=None,
- meta_objectives=None,
- ):
- """Construct the evaluation graph.
- """
- if meta_objectives is None:
- meta_objectives = []
-
- tf.train.create_global_step()
-
- local_device = ""
- remote_device = ""
-
- meta_opt = theta_process_fn(
- remote_device=remote_device, local_device=local_device)
-
- base_model = w_learner_fn(
- remote_device=remote_device, local_device=local_device)
-
- train_dataset = dataset_fn(device=local_device)
-
- # construct variables
- x, outputs = base_model(train_dataset())
- initial_state = base_model.initial_state(meta_opt, max_steps=10)
- next_state = base_model.compute_next_state(outputs, meta_opt, initial_state)
- with utils.state_barrier_context(next_state):
- train_one_step_op = meta_opt.assign_state(base_model, next_state)
-
- meta_objs = []
- for meta_obj_fn in meta_objectives:
- meta_obj = meta_obj_fn(local_device="", remote_device="")
- meta_objs.append(meta_obj)
- J = meta_obj(train_dataset, lambda x: base_model(x)[0])
- tf.summary.scalar(str(meta_obj.__class__.__name__)+"_J", tf.reduce_mean(J))
-
- # TODO(lmetz) this is kinda error prone.
- # We should share the construction of the global variables across train and
- # make sure both sets of savable variables are the same
- checkpoint_vars = meta_opt.remote_variables() + [tf.train.get_global_step()]
- for meta_obj in meta_objs:
- checkpoint_vars.extend(meta_obj.remote_variables())
-
- return checkpoint_vars, train_one_step_op, (base_model, train_dataset)
diff --git a/research/learning_unsupervised_learning/meta_objective/__init__.py b/research/learning_unsupervised_learning/meta_objective/__init__.py
deleted file mode 100644
index 54c46145e3c3a9f19110f92197f1d3cb2afe31fb..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/meta_objective/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-import sklearn
-import linear_regression
diff --git a/research/learning_unsupervised_learning/meta_objective/linear_regression.py b/research/learning_unsupervised_learning/meta_objective/linear_regression.py
deleted file mode 100644
index b49fc2529ccba08a6b47019cd7546f8fb409b28b..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/meta_objective/linear_regression.py
+++ /dev/null
@@ -1,258 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-
-"""Closed form linear regression.
-
-Can be differentiated through.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import numpy as np
-import sonnet as snt
-import tensorflow as tf
-
-from learning_unsupervised_learning import utils
-from learning_unsupervised_learning import variable_replace
-
-
-def solve_ridge(x, y, ridge_factor):
- with tf.name_scope("solve_ridge"):
- # Added a column of ones to the end of the feature matrix for bias
- A = tf.concat([x, tf.ones((x.shape.as_list()[0], 1))], axis=1)
-
- # Analytic solution for the ridge regression loss
- inv_target = tf.matmul(A, A, transpose_a=True)
- np_diag_penalty = ridge_factor * np.ones(
- A.shape.as_list()[1], dtype="float32")
- # Remove penalty on bias component of weights
- np_diag_penalty[-1] = 0.
- diag_penalty = tf.constant(np_diag_penalty)
- inv_target += tf.diag(diag_penalty)
-
- inv = tf.matrix_inverse(inv_target)
- w = tf.matmul(inv, tf.matmul(A, y, transpose_a=True))
- return w
-
-
-class LinearRegressionMetaObjective(snt.AbstractModule):
- """A meta objective based on training Ridge Regression with analytic solution.
-
- This is used to evaluate the performance of a given feature set trained in
- some other manner.
- """
-
- def __init__(self,
- local_device=None,
- remote_device=None,
- zero_one_labels=True,
- normalize_y_hat=True,
- normalize_act=False,
- averages=1,
- ridge_factor=0.1,
- center_y=True,
- hinge_loss=False,
- samples_per_class=10,
- test_train_scalar=1.0,
- ):
- self._local_device = local_device
- self._remote_device = remote_device
- self.zero_one_labels = zero_one_labels
- self.normalize_y_hat = normalize_y_hat
- self.normalize_act = normalize_act
- self.ridge_factor = ridge_factor
- self.averages = averages
- self.samples_per_class = samples_per_class
- self.center_y=center_y
- self.test_train_scalar=test_train_scalar
- self.hinge_loss = hinge_loss
-
- self.dataset_map = {}
-
- super(LinearRegressionMetaObjective,
- self).__init__(name="LinearRegressionMetaObjective")
-
- def _build(self, dataset, feature_transformer):
- if self.samples_per_class is not None:
- if dataset not in self.dataset_map:
- # datasets are outside of frames from while loops
- with tf.control_dependencies(None):
- self.dataset_map[dataset] = utils.sample_n_per_class(
- dataset, self.samples_per_class)
-
- dataset = self.dataset_map[dataset]
-
- stats = collections.defaultdict(list)
- losses = []
- # TODO(lmetz) move this to ingraph control flow?
- for _ in xrange(self.averages):
- loss, stat = self._build_once(dataset, feature_transformer)
- losses.append(loss)
- for k, v in stat.items():
- stats[k].append(v)
- stats = {k: tf.add_n(v) / float(len(v)) for k, v in stats.items()}
-
- summary_updates = []
- for k, v in stats.items():
- tf.summary.scalar(k, v)
-
- with tf.control_dependencies(summary_updates):
- return tf.add_n(losses) / float(len(losses))
-
- def _build_once(self, dataset, feature_transformer):
- with tf.device(self._local_device):
- batch = dataset()
- num_classes = batch.label_onehot.shape.as_list()[1]
-
- regression_mod = snt.Linear(num_classes)
-
- if self.normalize_act:
-
- def normalize_transformer(x):
- unnorm_x = feature_transformer(x)
- return tf.nn.l2_normalize(unnorm_x, 0)
-
- feature_transformer_wrap = normalize_transformer
- else:
- feature_transformer_wrap = feature_transformer
-
- # construct the variables of the right shape in the sonnet module by
- # calling a forward pass through the regressor.
- with utils.assert_no_new_variables():
- dummy_features = feature_transformer_wrap(batch)
- regression_mod(dummy_features)
- reg_w = regression_mod.w
- reg_b = regression_mod.b
-
- batch_test = dataset()
- all_batch = utils.structure_map_multi(lambda x: tf.concat(x, 0), [batch, batch_test])
- #all_batch = tf.concat([batch, batch_test], 0)
- # Grab a new batch of data from the dataset.
- features = feature_transformer_wrap(all_batch)
- features, features_test = utils.structure_map_split(lambda x: tf.split(x, 2, axis=0), features)
-
- def center_y(y):
- y -= tf.reduce_mean(y)
- y *= tf.rsqrt(tf.reduce_mean(tf.reduce_sum(y**2, axis=[1], keep_dims=True)))
- return y
- def get_y_vec(batch):
- y_pieces = []
- if hasattr(batch, "label_onehot"):
- if self.zero_one_labels:
- y_pieces += [batch.label_onehot]
- else:
- y_pieces += [2. * batch.label_onehot - 1.]
- if hasattr(batch, "regression_target"):
- y_pieces += [batch.regression_target]
- y = tf.concat(y_pieces, 1)
- if self.center_y:
- y = center_y(y)
- return y
-
- y_train = get_y_vec(batch)
-
- w = solve_ridge(features, y_train, self.ridge_factor)
-
- # Generate features from another batch to evaluate loss on the validation
- # set. This provide a less overfit signal to the learned optimizer.
- y_test = get_y_vec(batch_test)
-
- def compute_logit(features):
- # We have updated the classifier mod in previous steps, we need to
- # substitute out those variables to get new values.
- replacement = collections.OrderedDict([(reg_w, w[:-1]), (reg_b, w[-1])])
- with variable_replace.variable_replace(replacement):
- logits = regression_mod(features)
-
- return logits
-
- batch_size = y_train.shape.as_list()[0]
-
- logit_train = compute_logit(features)
- logit_test_unnorm = compute_logit(features_test)
- if self.normalize_y_hat:
- logit_test = logit_test_unnorm / tf.sqrt(
- tf.reduce_sum(logit_test_unnorm**2, axis=[1], keep_dims=True))
- else:
- logit_test = logit_test_unnorm
-
- stats = {}
-
- if self.hinge_loss:
- # slightly closer to the true classification loss
- # any distance smaller than 1 is guaranteed to map to the correct class
- mse_test = tf.reduce_sum(tf.nn.relu(tf.reduce_sum(tf.square(logit_test - y_test), axis=1)-1.)) / batch_size
- else:
- mse_test = tf.reduce_sum(tf.square(logit_test - y_test)) / batch_size
-
- stats["mse_test"] = mse_test
-
- mse_train = tf.reduce_sum(tf.square(logit_train - y_train)) / batch_size
- stats["mse_train"] = mse_train
-
- is_correct_test = tf.equal(tf.argmax(logit_test, 1), tf.argmax(y_test, 1))
- accuracy_test = tf.reduce_mean(tf.cast(is_correct_test, tf.float32))
- stats["accuracy_test"] = accuracy_test
-
- def test_confusion_fn():
- test_confusion = tf.confusion_matrix(tf.argmax(y_test, 1), tf.argmax(logit_test, 1))
- test_confusion = tf.to_float(test_confusion) / tf.constant((logit_test.shape.as_list()[0] / float(logit_test.shape.as_list()[1])), dtype=tf.float32)
- test_confusion = tf.expand_dims(tf.expand_dims(test_confusion, 0), 3)
- return test_confusion
- tf.summary.image("test_confusion", test_confusion_fn())
-
- def train_confusion_fn():
- train_confusion = tf.confusion_matrix(tf.argmax(y_train, 1), tf.argmax(logit_train, 1))
- train_confusion = tf.to_float(train_confusion) / tf.constant((logit_train.shape.as_list()[0] / float(logit_train.shape.as_list()[1])), dtype=tf.float32)
- train_confusion = tf.expand_dims(tf.expand_dims(train_confusion, 0), 3)
- return train_confusion
- tf.summary.image("train_confusion", train_confusion_fn())
-
- is_correct = tf.equal(tf.argmax(logit_train, 1), tf.argmax(y_train, 1))
- accuracy_train = tf.reduce_mean(tf.cast(is_correct, tf.float32))
- stats["accuracy_train"] = accuracy_train
-
- reg = self.ridge_factor * tf.reduce_sum(tf.square(w[:-1])) / batch_size
- stats["ridge_component"] = reg
-
- stats["total_loss"] = mse_test + reg
-
- loss_to_train_at = (reg+ mse_test) * self.test_train_scalar + (mse_train + reg)*(1 - self.test_train_scalar)
-
- loss_to_train_at = tf.identity(loss_to_train_at)
-
- # Minimizing the test loss should not require regurization because the
- # metaobjective is solved for the training loss
- return loss_to_train_at, stats
-
- def local_variables(self):
- """List of variables that need to be updated for each evaluation.
-
- These variables should not be stored on a parameter server and
- should be reset every computation of a meta_objective loss.
-
- Returns:
- vars: list of tf.Variable
- """
- return list(
- snt.get_variables_in_module(self, tf.GraphKeys.TRAINABLE_VARIABLES))
-
- def remote_variables(self):
- return []
diff --git a/research/learning_unsupervised_learning/meta_objective/sklearn.py b/research/learning_unsupervised_learning/meta_objective/sklearn.py
deleted file mode 100644
index 4f1f2d59102c511fd42ad323c32ab1709bd60c90..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/meta_objective/sklearn.py
+++ /dev/null
@@ -1,167 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-"""
-
-Can NOT be differentiated through.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import numpy as np
-import sonnet as snt
-import tensorflow as tf
-from tensorflow.python.framework import function
-
-from learning_unsupervised_learning import utils
-
-from learning_unsupervised_learning.meta_objective import utils as meta_obj_utils
-
-from sklearn import svm
-from sklearn import linear_model
-
-
-def build_fit(device, model_fn, num_classes, probs=True):
-
- def _py_fit_predict(trX, trY, teX):
- assert len(np.unique(trY)) == num_classes
- model = model_fn()
- model.fit(trX, trY)
- trP = model.predict(trX)
- teP = model.predict(teX)
- if probs:
- teP_probs = model.predict_log_proba(teX)
- return trP.astype(np.int64), teP.astype(np.int64), teP_probs.astype(
- np.float32)
- else:
- teP = model.predict(teX)
- return trP.astype(np.int64), teP.astype(np.int64)
-
- def return_fn(trX, trY, teX):
- with tf.device(device):
- with tf.device("/cpu:0"):
- if probs:
- return tf.py_func(
- _py_fit_predict,
- [tf.identity(trX),
- tf.identity(trY),
- tf.identity(teX)], [tf.int64, tf.int64, tf.float32])
- else:
- return tf.py_func(
- _py_fit_predict,
- [tf.identity(trX),
- tf.identity(trY),
- tf.identity(teX)], [tf.int64, tf.int64])
-
- return return_fn
-
-
-class SKLearn(meta_obj_utils.MultiTrialMetaObjective):
-
- def __init__(
- self,
- local_device=None,
- remote_device=None,
- averages=1,
- samples_per_class=10,
- probs=False,
- stddev=0.01,
- n_samples=10,
- name="SKLearn",
- ):
- self._local_device = local_device
- self._remote_device = remote_device
- self.name = name
- self.probs = probs
- self.n_samples = n_samples
- self.stddev = stddev
-
- super(SKLearn, self).__init__(
- name=name, samples_per_class=samples_per_class, averages=averages)
-
- def _get_model(self):
- raise NotImplemented()
-
- def _build_once(self, dataset, feature_transformer):
- with tf.device(self._local_device):
- tr_batch = dataset()
- te_batch = dataset()
- num_classes = tr_batch.label_onehot.shape.as_list()[1]
- all_batch = utils.structure_map_multi(lambda x: tf.concat(x, 0),
- [tr_batch, te_batch])
- features = feature_transformer(all_batch)
- trX, teX = utils.structure_map_split(lambda x: tf.split(x, 2, axis=0),
- features)
- trY = tf.to_int64(tr_batch.label)
- trY_onehot = tf.to_int32(tr_batch.label_onehot)
- teY = tf.to_int64(te_batch.label)
- teY_shape = teY.shape.as_list()
-
- def blackbox((trX, trY, teX, teY)):
- trY = tf.to_int32(tf.rint(trY))
- teY = tf.to_int32(tf.rint(teY))
- tf_fn = build_fit(
- self._local_device,
- self._get_model,
- num_classes=num_classes,
- probs=self.probs)
- if self.probs:
- trP, teP, teP_probs = tf_fn(trX, trY, teX)
- else:
- trP, teP = tf_fn(trX, trY, teX)
-
- teY.set_shape(teY_shape)
- if self.probs:
- onehot = tf.one_hot(teY, num_classes)
- crossent = -tf.reduce_sum(onehot * teP_probs, [1])
- return tf.reduce_mean(crossent)
- else:
- # use error rate as the loss if no surrogate is avalible.
- return 1 - tf.reduce_mean(
- tf.to_float(tf.equal(teY, tf.to_int32(teP))))
-
- test_loss = blackbox((trX, tf.to_float(trY), teX, tf.to_float(teY)))
-
- stats = {}
-
- tf_fn = build_fit(
- self._local_device,
- self._get_model,
- num_classes=num_classes,
- probs=self.probs)
- if self.probs:
- trP, teP, teP_probs = tf_fn(trX, trY, teX)
- else:
- trP, teP = tf_fn(trX, trY, teX)
- stats["%s/accuracy_train" % self.name] = tf.reduce_mean(
- tf.to_float(tf.equal(tf.to_int32(trY), tf.to_int32(trP))))
- stats["%s/accuracy_test" % self.name] = tf.reduce_mean(
- tf.to_float(tf.equal(tf.to_int32(teY), tf.to_int32(teP))))
- stats["%s/test_loss" % self.name] = test_loss
- return test_loss, stats
-
-
-class LogisticRegression(SKLearn):
-
- def __init__(self, C=1.0, name="LogisticRegression", probs=True, **kwargs):
- self.C = C
- super(LogisticRegression, self).__init__(name=name, probs=probs, **kwargs)
-
- def _get_model(self):
- return linear_model.LogisticRegression(C=self.C)
diff --git a/research/learning_unsupervised_learning/meta_objective/utils.py b/research/learning_unsupervised_learning/meta_objective/utils.py
deleted file mode 100644
index a29197d1d0cb7f0fdcebac3980027640651f185b..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/meta_objective/utils.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import numpy as np
-import sonnet as snt
-import tensorflow as tf
-
-from learning_unsupervised_learning import optimizers
-from learning_unsupervised_learning import utils
-from learning_unsupervised_learning import summary_utils
-from learning_unsupervised_learning import variable_replace
-
-class MultiTrialMetaObjective(snt.AbstractModule):
- def __init__(self, samples_per_class, averages, **kwargs):
- self.samples_per_class = samples_per_class
- self.averages = averages
- self.dataset_map = {}
-
- super(MultiTrialMetaObjective,
- self).__init__(**kwargs)
-
- def _build(self, dataset, feature_transformer):
- if self.samples_per_class is not None:
- if dataset not in self.dataset_map:
- # datasets are outside of frames from while loops
- with tf.control_dependencies(None):
- self.dataset_map[dataset] = utils.sample_n_per_class(
- dataset, self.samples_per_class)
-
- dataset = self.dataset_map[dataset]
-
- stats = collections.defaultdict(list)
- losses = []
- # TODO(lmetz) move this to ingraph control flow?
- for _ in xrange(self.averages):
- loss, stat = self._build_once(dataset, feature_transformer)
- losses.append(loss)
- for k, v in stat.items():
- stats[k].append(v)
- stats = {k: tf.add_n(v) / float(len(v)) for k, v in stats.items()}
-
- for k, v in stats.items():
- tf.summary.scalar(k, v)
-
- return tf.add_n(losses) / float(len(losses))
-
- def local_variables(self):
- """List of variables that need to be updated for each evaluation.
-
- These variables should not be stored on a parameter server and
- should be reset every computation of a meta_objective loss.
-
- Returns:
- vars: list of tf.Variable
- """
- return list(
- snt.get_variables_in_module(self, tf.GraphKeys.TRAINABLE_VARIABLES))
-
- def remote_variables(self):
- return []
diff --git a/research/learning_unsupervised_learning/optimizers.py b/research/learning_unsupervised_learning/optimizers.py
deleted file mode 100644
index 02c6106b19d1255907beb0ade07c46c5b065f701..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/optimizers.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-
-"""Optimizers for use in unrolled optimization.
-
-These optimizers contain a compute_updates function and its own ability to keep
-track of internal state.
-These functions can be used with a tf.while_loop to perform multiple training
-steps per sess.run.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import abc
-import collections
-import tensorflow as tf
-import sonnet as snt
-
-from learning_unsupervised_learning import utils
-
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.training import optimizer
-from tensorflow.python.training import training_ops
-
-
-class UnrollableOptimizer(snt.AbstractModule):
- """Interface for optimizers that can be used in unrolled computation.
- apply_gradients is derrived from compute_update and assign_state.
- """
-
- def __init__(self, *args, **kwargs):
- super(UnrollableOptimizer, self).__init__(*args, **kwargs)
- self()
-
- @abc.abstractmethod
- def compute_updates(self, xs, gs, state=None):
- """Compute next step updates for a given variable list and state.
-
- Args:
- xs: list of tensors
- The "variables" to perform an update on.
- Note these must match the same order for which get_state was originally
- called.
- gs: list of tensors
- Gradients of `xs` with respect to some loss.
- state: Any
- Optimizer specific state to keep track of accumulators such as momentum
- terms
- """
- raise NotImplementedError()
-
- def _build(self):
- pass
-
- @abc.abstractmethod
- def get_state(self, var_list):
- """Get the state value associated with a list of tf.Variables.
-
- This state is commonly going to be a NamedTuple that contains some
- mapping between variables and the state associated with those variables.
- This state could be a moving momentum variable tracked by the optimizer.
-
- Args:
- var_list: list of tf.Variable
- Returns:
- state: Any
- Optimizer specific state
- """
- raise NotImplementedError()
-
- def assign_state(self, state):
- """Assigns the state to the optimizers internal variables.
-
- Args:
- state: Any
- Returns:
- op: tf.Operation
- The operation that performs the assignment.
- """
- raise NotImplementedError()
-
- def apply_gradients(self, grad_vars):
- gradients, variables = zip(*grad_vars)
- state = self.get_state(variables)
- new_vars, new_state = self.compute_updates(variables, gradients, state)
- assign_op = self.assign_state(new_state)
- op = utils.assign_variables(variables, new_vars)
- return tf.group(assign_op, op, name="apply_gradients")
-
-
-class UnrollableGradientDescentRollingOptimizer(UnrollableOptimizer):
-
- def __init__(self,
- learning_rate,
- name="UnrollableGradientDescentRollingOptimizer"):
- self.learning_rate = learning_rate
- super(UnrollableGradientDescentRollingOptimizer, self).__init__(name=name)
-
-
- def compute_updates(self, xs, gs, learning_rates, state):
- new_vars = []
- for x, g, lr in utils.eqzip(xs, gs, learning_rates):
- if lr is None:
- lr = self.learning_rate
- if g is not None:
- new_vars.append((x * (1 - lr) - g * lr))
- else:
- new_vars.append(x)
- return new_vars, state
-
- def get_state(self, var_list):
- return tf.constant(0.0)
-
- def assign_state(self, state, var_list=None):
- return tf.no_op()
diff --git a/research/learning_unsupervised_learning/run_eval.py b/research/learning_unsupervised_learning/run_eval.py
deleted file mode 100644
index dcb2529dd4cc5354012befd5790c8d402f4caafd..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/run_eval.py
+++ /dev/null
@@ -1,122 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-""" Script that iteratively applies the unsupervised update rule and evaluates the
-
-meta-objective performance.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from absl import flags
-from absl import app
-
-from learning_unsupervised_learning import evaluation
-from learning_unsupervised_learning import datasets
-from learning_unsupervised_learning import architectures
-from learning_unsupervised_learning import summary_utils
-from learning_unsupervised_learning import meta_objective
-
-import tensorflow as tf
-import sonnet as snt
-
-from tensorflow.contrib.framework.python.framework import checkpoint_utils
-
-flags.DEFINE_string("checkpoint_dir", None, "Dir to load pretrained update rule from")
-flags.DEFINE_string("train_log_dir", None, "Training log directory")
-
-FLAGS = flags.FLAGS
-
-
-def train(train_log_dir, checkpoint_dir, eval_every_n_steps=10, num_steps=3000):
- dataset_fn = datasets.mnist.TinyMnist
- w_learner_fn = architectures.more_local_weight_update.MoreLocalWeightUpdateWLearner
- theta_process_fn = architectures.more_local_weight_update.MoreLocalWeightUpdateProcess
-
- meta_objectives = []
- meta_objectives.append(
- meta_objective.linear_regression.LinearRegressionMetaObjective)
- meta_objectives.append(meta_objective.sklearn.LogisticRegression)
-
- checkpoint_vars, train_one_step_op, (
- base_model, dataset) = evaluation.construct_evaluation_graph(
- theta_process_fn=theta_process_fn,
- w_learner_fn=w_learner_fn,
- dataset_fn=dataset_fn,
- meta_objectives=meta_objectives)
- batch = dataset()
- pre_logit, outputs = base_model(batch)
-
- global_step = tf.train.get_or_create_global_step()
- var_list = list(
- snt.get_variables_in_module(base_model, tf.GraphKeys.TRAINABLE_VARIABLES))
-
- tf.logging.info("all vars")
- for v in tf.all_variables():
- tf.logging.info(" %s" % str(v))
- global_step = tf.train.get_global_step()
- accumulate_global_step = global_step.assign_add(1)
- reset_global_step = global_step.assign(0)
-
- train_op = tf.group(
- train_one_step_op, accumulate_global_step, name="train_op")
-
- summary_op = tf.summary.merge_all()
-
- file_writer = summary_utils.LoggingFileWriter(train_log_dir, regexes=[".*"])
- if checkpoint_dir:
- str_var_list = checkpoint_utils.list_variables(checkpoint_dir)
- name_to_v_map = {v.op.name: v for v in tf.all_variables()}
- var_list = [
- name_to_v_map[vn] for vn, _ in str_var_list if vn in name_to_v_map
- ]
- saver = tf.train.Saver(var_list)
- missed_variables = [
- v.op.name for v in set(
- snt.get_variables_in_scope("LocalWeightUpdateProcess",
- tf.GraphKeys.GLOBAL_VARIABLES)) -
- set(var_list)
- ]
- assert len(missed_variables) == 0, "Missed a theta variable."
-
- hooks = []
-
- with tf.train.SingularMonitoredSession(master="", hooks=hooks) as sess:
-
- # global step should be restored from the evals job checkpoint or zero for fresh.
- step = sess.run(global_step)
-
- if step == 0 and checkpoint_dir:
- tf.logging.info("force restore")
- saver.restore(sess, checkpoint_dir)
- tf.logging.info("force restore done")
- sess.run(reset_global_step)
- step = sess.run(global_step)
-
- while step < num_steps:
- if step % eval_every_n_steps == 0:
- s, _, step = sess.run([summary_op, train_op, global_step])
- file_writer.add_summary(s, step)
- else:
- _, step = sess.run([train_op, global_step])
-
-
-def main(argv):
- train(FLAGS.train_log_dir, FLAGS.checkpoint_dir)
-
-
-if __name__ == "__main__":
- app.run(main)
diff --git a/research/learning_unsupervised_learning/summary_utils.py b/research/learning_unsupervised_learning/summary_utils.py
deleted file mode 100644
index d5c0fdd9186bdef0b4e25ca10978e22ab910d276..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/summary_utils.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-
-import collections
-import functools
-import threading
-import tensorflow as tf
-import matplotlib
-import numpy as np
-import time
-import re
-import math
-matplotlib.use("Agg")
-
-import matplotlib.pyplot as plt
-import scipy.signal
-
-from tensorflow.python.util import tf_should_use
-from tensorflow.contrib.summary import summary_ops
-from tensorflow.python.ops import summary_op_util
-from tensorflow.contrib.summary import gen_summary_ops
-
-_DEBUG_DISABLE_SUMMARIES=False
-
-class LoggingFileWriter(tf.summary.FileWriter):
- """A FileWriter that also logs things out.
-
- This is entirely for ease of debugging / not having to open up Tensorboard
- a lot.
- """
-
- def __init__(self, logdir, regexes=[], **kwargs):
- self.regexes = regexes
- super(LoggingFileWriter, self).__init__(logdir, **kwargs)
-
- def add_summary(self, summary, global_step):
- if type(summary) != tf.Summary:
- summary_p = tf.Summary()
- summary_p.ParseFromString(summary)
- summary = summary_p
- for s in summary.value:
- for exists in [re.match(p, s.tag) for p in self.regexes]:
- if exists is not None:
- tf.logging.info("%d ] %s : %f", global_step, s.tag, s.simple_value)
- break
- super(LoggingFileWriter, self).add_summary(summary, global_step)
-
-
-def image_grid(images, max_grid_size=4, border=1):
- """Given images and N, return first N^2 images as an NxN image grid.
-
- Args:
- images: a `Tensor` of size [batch_size, height, width, channels]
- max_grid_size: Maximum image grid height/width
-
- Returns:
- Single image batch, of dim [1, h*n, w*n, c]
- """
- batch_size = images.shape.as_list()[0]
- to_pad = int((np.ceil(np.sqrt(batch_size)))**2 - batch_size)
- images = tf.pad(images, [[0, to_pad], [0, border], [0, border], [0, 0]])
-
- batch_size = images.shape.as_list()[0]
- grid_size = min(int(np.sqrt(batch_size)), max_grid_size)
- assert images.shape.as_list()[0] >= grid_size * grid_size
-
- # If we have a depth channel
- if images.shape.as_list()[-1] == 4:
- images = images[:grid_size * grid_size, :, :, 0:3]
- depth = tf.image.grayscale_to_rgb(images[:grid_size * grid_size, :, :, 3:4])
-
- images = tf.reshape(images, [-1, images.shape.as_list()[2], 3])
- split = tf.split(images, grid_size, axis=0)
- depth = tf.reshape(depth, [-1, images.shape.as_list()[2], 3])
- depth_split = tf.split(depth, grid_size, axis=0)
- grid = tf.concat(split + depth_split, 1)
- return tf.expand_dims(grid, 0)
- else:
- images = images[:grid_size * grid_size, :, :, :]
- images = tf.reshape(
- images, [-1, images.shape.as_list()[2],
- images.shape.as_list()[3]])
- split = tf.split(value=images, num_or_size_splits=grid_size, axis=0)
- grid = tf.concat(split, 1)
- return tf.expand_dims(grid, 0)
-
-
-def first_layer_weight_image(weight, shape):
- weight_image = tf.reshape(weight,
- shape + [tf.identity(weight).shape.as_list()[1]])
- # [winx, winy, wout]
- mean, var = tf.nn.moments(weight_image, [0,1,2], keep_dims=True)
- #mean, var = tf.nn.moments(weight_image, [0,1], keep_dims=True)
- weight_image = (weight_image - mean) / tf.sqrt(var + 1e-5)
- weight_image = (weight_image + 1.0) / 2.0
- weight_image = tf.clip_by_value(weight_image, 0, 1)
- weight_image = tf.transpose(weight_image, (3, 0, 1, 2))
- grid = image_grid(weight_image, max_grid_size=10)
- return grid
-
-def inner_layer_weight_image(weight):
- """Visualize a weight matrix of an inner layer.
- Add padding to make it square, then visualize as a gray scale image
- """
- weight = tf.identity(weight) # turn into a tensor
- weight = weight / (tf.reduce_max(tf.abs(weight), [0], keep_dims=True))
- weight = tf.reshape(weight, [1]+weight.shape.as_list() + [1])
- return weight
-
-
-def activation_image(activations, label_onehot):
- """Make a row sorted by class for each activation. Put a black line around the activations."""
- labels = tf.argmax(label_onehot, axis=1)
- _, n_classes = label_onehot.shape.as_list()
- mean, var = tf.nn.moments(activations, [0, 1])
- activations = (activations - mean)/tf.sqrt(var+1e-5)
-
- activations = tf.clip_by_value(activations, -1, 1)
- activations = (activations + 1.0) / 2.0 # shift to [0, 1]
-
- canvas = []
- for i in xrange(n_classes):
- inds = tf.where(tf.equal(labels, i))
-
- def _gather():
- return tf.squeeze(tf.gather(activations, inds), 1)
-
- def _empty():
- return tf.zeros([0, activations.shape.as_list()[1]], dtype=tf.float32)
-
- assert inds.shape.as_list()[0] is None
- x = tf.cond(tf.equal(tf.shape(inds)[0], 0), _empty, _gather)
- canvas.append(x)
- canvas.append(tf.zeros([1, activations.shape.as_list()[1]]))
- canvas = tf.concat(canvas, 0)
- canvas = tf.reshape(canvas, [1, activations.shape.as_list()[0]+n_classes, canvas.shape.as_list()[1], 1])
- return canvas
-
-
-def sorted_images(images, label_onehot):
- # images is [bs, x, y, c]
- labels = tf.argmax(label_onehot, axis=1)
- _, n_classes = label_onehot.shape.as_list()
- to_stack = []
- for i in xrange(n_classes):
- inds = tf.where(tf.equal(labels, i))
-
- def _gather():
- return tf.squeeze(tf.gather(images, inds), 1)
-
- def _empty():
- return tf.zeros([0] + images.shape.as_list()[1:], dtype=tf.float32)
-
- assert inds.shape.as_list()[0] is None
- x = tf.cond(tf.equal(tf.shape(inds)[0], 0), _empty, _gather)
- to_stack.append(x)
- # pad / trim all up to 10.
- padded = []
- for t in to_stack:
- n_found = tf.shape(t)[0]
- pad = tf.pad(t[0:10], tf.stack([tf.stack([0,tf.maximum(0, 10-n_found)]), [0,0], [0,0], [0,0]]))
- padded.append(pad)
-
- xs = [tf.concat(tf.split(p, 10), axis=1) for p in padded]
- ys = tf.concat(xs, axis=2)
- ys = tf.cast(tf.clip_by_value(ys, 0., 1.) * 255., tf.uint8)
- return ys
diff --git a/research/learning_unsupervised_learning/utils.py b/research/learning_unsupervised_learning/utils.py
deleted file mode 100644
index ca56ca93181df1ed9c403fef79e8154c3c9515b4..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/utils.py
+++ /dev/null
@@ -1,287 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import contextlib
-import tensorflow as tf
-import sonnet as snt
-import itertools
-import functools
-
-from tensorflow.core.framework import node_def_pb2
-from tensorflow.python.framework import device as pydev
-from tensorflow.python.framework import errors
-from tensorflow.python.ops import variable_scope as variable_scope_ops
-from sonnet.python.modules import util as snt_util
-
-from tensorflow.python.util import nest
-
-
-def eqzip(*args):
- """Zip but raises error if lengths don't match.
-
- Args:
- *args: list of lists or tuples
- Returns:
- list: the result of zip
- Raises:
- ValueError: when the lengths don't match
- """
-
- sizes = [len(x) for x in args]
- if not all([sizes[0] == x for x in sizes]):
- raise ValueError("Lists are of different sizes. \n %s"%str(sizes))
- return zip(*args)
-
-
-@contextlib.contextmanager
-def assert_no_new_variables():
- """Ensure that no tf.Variables are constructed inside the context.
-
- Yields:
- None
- Raises:
- ValueError: if there is a variable created.
- """
- num_vars = len(tf.global_variables())
- old_variables = tf.global_variables()
- yield
- if len(tf.global_variables()) != num_vars:
- new_vars = set(tf.global_variables()) - set(old_variables)
- tf.logging.error("NEW VARIABLES CREATED")
- tf.logging.error(10*"=")
- for v in new_vars:
- tf.logging.error(v)
-
- raise ValueError("Variables created inside an "
- "assert_no_new_variables context")
- if old_variables != tf.global_variables():
- raise ValueError("Variables somehow changed inside an "
- "assert_no_new_variables context."
- "This means something modified the tf.global_variables()")
-
-
-def get_variables_in_modules(module_list):
- var_list = []
- for m in module_list:
- var_list.extend(snt.get_variables_in_module(m))
- return var_list
-
-
-def state_barrier_context(state):
- """Return a context manager that prevents interior ops from running
- unless the whole state has been computed.
-
- This is to prevent assign race conditions.
- """
- tensors = [x for x in nest.flatten(state) if type(x) == tf.Tensor]
- tarray = [x.flow for x in nest.flatten(state) if hasattr(x, "flow")]
- return tf.control_dependencies(tensors + tarray)
-
-
-def _identity_fn(tf_entity):
- if hasattr(tf_entity, "identity"):
- return tf_entity.identity()
- else:
- return tf.identity(tf_entity)
-
-
-def state_barrier_result(state):
- """Return the same state, but with a control dependency to prevent it from
- being partially computed
- """
- with state_barrier_context(state):
- return nest.map_structure(_identity_fn, state)
-
-
-def train_iterator(num_iterations):
- """Iterator that returns an index of the current step.
- This iterator runs forever if num_iterations is None
- otherwise it runs for some fixed amount of steps.
- """
- if num_iterations is None:
- return itertools.count()
- else:
- return xrange(num_iterations)
-
-
-def print_op(op, msg):
- """Print a string and return an op wrapped in a control dependency to make
- sure it ran."""
- print_op = tf.Print(tf.constant(0), [tf.constant(0)], msg)
- return tf.group(op, print_op)
-
-
-class MultiQueueRunner(tf.train.QueueRunner):
- """A QueueRunner with multiple queues """
- def __init__(self, queues, enqueue_ops):
- close_op = tf.group(* [q.close() for q in queues])
- cancel_op = tf.group(
- * [q.close(cancel_pending_enqueues=True) for q in queues])
- queue_closed_exception_types = (errors.OutOfRangeError,)
-
- enqueue_op = tf.group(*enqueue_ops, name="multi_enqueue")
-
- super(MultiQueueRunner, self).__init__(
- queues[0],
- enqueue_ops=[enqueue_op],
- close_op=close_op,
- cancel_op=cancel_op,
- queue_closed_exception_types=queue_closed_exception_types)
-
-
-# This function is not elegant, but I tried so many other ways to get this to
-# work and this is the only one that ended up not incuring significant overhead
-# or obscure tensorflow bugs.
-def sample_n_per_class(dataset, samples_per_class):
- """Create a new callable / dataset object that returns batches of each with
- samples_per_class per label.
-
- Args:
- dataset: fn
- samples_per_class: int
- Returns:
- function, [] -> batch where batch is the same type as the return of
- dataset().
- """
-
- with tf.control_dependencies(None), tf.name_scope(None):
- with tf.name_scope("queue_runner/sample_n_per_class"):
- batch = dataset()
- num_classes = batch.label_onehot.shape.as_list()[1]
- batch_size = num_classes * samples_per_class
-
- flatten = nest.flatten(batch)
- queues = []
- enqueue_ops = []
- capacity = samples_per_class * 20
- for i in xrange(num_classes):
- queue = tf.FIFOQueue(
- capacity=capacity,
- shapes=[f.shape.as_list()[1:] for f in flatten],
- dtypes=[f.dtype for f in flatten])
- queues.append(queue)
-
- idx = tf.where(tf.equal(batch.label, i))
- sub_batch = []
- to_enqueue = []
- for elem in batch:
- new_e = tf.gather(elem, idx)
- new_e = tf.squeeze(new_e, 1)
- to_enqueue.append(new_e)
-
- remaining = (capacity - queue.size())
- to_add = tf.minimum(tf.shape(idx)[0], remaining)
-
- def _enqueue():
- return queue.enqueue_many([t[:to_add] for t in to_enqueue])
-
- enqueue_op = tf.cond(
- tf.equal(to_add, 0), tf.no_op, _enqueue)
- enqueue_ops.append(enqueue_op)
-
- # This has caused many deadlocks / issues. This is some logging to at least
- # shed light to what is going on.
- print_lam = lambda: tf.Print(tf.constant(0.0), [q.size() for q in queues], "MultiQueueRunner queues status. Has capacity %d"%capacity)
- some_percent_of_time = tf.less(tf.random_uniform([]), 0.0005)
- maybe_print = tf.cond(some_percent_of_time, print_lam, lambda: tf.constant(0.0))
- with tf.control_dependencies([maybe_print]):
- enqueue_ops = [tf.group(e) for e in enqueue_ops]
- qr = MultiQueueRunner(queues=queues, enqueue_ops=enqueue_ops)
- tf.train.add_queue_runner(qr)
-
- def dequeue_batch():
- with tf.name_scope("sample_n_per_batch/dequeue/"):
- entries = []
- for q in queues:
- entries.append(q.dequeue_many(samples_per_class))
-
- flat_batch = [tf.concat(x, 0) for x in zip(*entries)]
- idx = tf.random_shuffle(tf.range(batch_size))
- flat_batch = [tf.gather(f, idx, axis=0) for f in flat_batch]
- return nest.pack_sequence_as(batch, flat_batch)
-
- return dequeue_batch
-
-def structure_map_multi(func, values):
- all_values = [nest.flatten(v) for v in values]
- rets = []
- for pair in zip(*all_values):
- rets.append(func(pair))
- return nest.pack_sequence_as(values[0], rets)
-
-def structure_map_split(func, value):
- vv = nest.flatten(value)
- rets = []
- for v in vv:
- rets.append(func(v))
- return [nest.pack_sequence_as(value, r) for r in zip(*rets)]
-
-def assign_variables(targets, values):
- return tf.group(*[t.assign(v) for t,v in eqzip(targets, values)],
- name="assign_variables")
-
-
-def create_variables_in_class_scope(method):
- """Force the variables constructed in this class to live in the sonnet module.
- Wraps a method on a sonnet module.
-
- For example the following will create two different variables.
- ```
- class Mod(snt.AbstractModule):
- @create_variables_in_class_scope
- def dynamic_thing(self, input, name):
- return snt.Linear(name)(input)
- mod.dynamic_thing(x, name="module_nameA")
- mod.dynamic_thing(x, name="module_nameB")
- # reuse
- mod.dynamic_thing(y, name="module_nameA")
- ```
- """
- @functools.wraps(method)
- def wrapper(obj, *args, **kwargs):
- def default_context_manager(reuse=None):
- variable_scope = obj.variable_scope
- return tf.variable_scope(variable_scope, reuse=reuse)
-
- variable_scope_context_manager = getattr(obj, "_enter_variable_scope",
- default_context_manager)
- graph = tf.get_default_graph()
-
- # Temporarily enter the variable scope to capture it
- with variable_scope_context_manager() as tmp_variable_scope:
- variable_scope = tmp_variable_scope
-
- with variable_scope_ops._pure_variable_scope(
- variable_scope, reuse=tf.AUTO_REUSE) as pure_variable_scope:
-
- name_scope = variable_scope.original_name_scope
- if name_scope[-1] != "/":
- name_scope += "/"
-
- with tf.name_scope(name_scope):
- sub_scope = snt_util.to_snake_case(method.__name__)
- with tf.name_scope(sub_scope) as scope:
- out_ops = method(obj, *args, **kwargs)
- return out_ops
-
- return wrapper
-
diff --git a/research/learning_unsupervised_learning/variable_replace.py b/research/learning_unsupervised_learning/variable_replace.py
deleted file mode 100644
index ebfbeadc8aba7f8a09e1392f1de8d7b33f10d43c..0000000000000000000000000000000000000000
--- a/research/learning_unsupervised_learning/variable_replace.py
+++ /dev/null
@@ -1,112 +0,0 @@
-# Copyright 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-from __future__ import absolute_import
-from __future__ import division
-
-import tensorflow as tf
-from contextlib import contextmanager
-
-from tensorflow.python.ops import variable_scope
-
-# sanity global state to ensure non recursive.
-_is_variable_replacing = [False]
-
-def in_variable_replace_scope():
- return _is_variable_replacing[0]
-
-@contextmanager
-def variable_replace(replacements, no_new=True):
- """ A context manager that replaces variables.
-
- This is a context manager that replaces all calls to
- get_variable with the variable in replacements.
- This function does not support recursive application.
-
- Args:
- replacements: dict
- dictionary mapping a variable to replace (the key), with
- the variable one wants to replace this variable with (the value).
- no_new: bool
- raise an error if variables were created.
- This is for sanity checking.
- Raises:
- ValueError: if a new variable or not all the replacements are used.
- """
- # TODO(lmetz) This function is a bit scary, as it relies on monkey patching
- # the call to get_variable. Ideally this can be done with variable_scope's
- # custom_getter attribute, but when initially writing this that was not
- # avalible.
-
- replacements = {k: v for k, v in replacements.items() if not k == v}
-
- init_vars = tf.trainable_variables()
- old_get_variable = variable_scope.get_variable
- old_tf_get_variable = tf.get_variable
-
- names_replace = {}
- has_replaced_names = []
- tf.logging.vlog(2, "Trying to replace")
- for k, v in replacements.items():
- tf.logging.vlog(2, k.name + " >> " + v.name)
- tf.logging.vlog(2, "===")
-
- for k, v in replacements.items():
- strip_name = k.name.replace("/read:0", "")
- strip_name = strip_name.replace(":0", "")
- names_replace[strip_name] = v
- # TODO(lmetz) is there a cleaner way to do this?
- def new_get_variable(name, *args, **kwargs):
- #print "Monkeypatch get variable run with name:", name
- n = tf.get_variable_scope().name + "/" + name
- #print "Monkeypatch get variable run with name:", n
- if n in names_replace:
- has_replaced_names.append(n)
- return names_replace[n]
- else:
- return old_get_variable(name, *args, **kwargs)
-
- # perform the monkey patch
- if _is_variable_replacing[0] == True:
- raise ValueError("No recursive calling to variable replace allowed.")
-
- variable_scope.get_variable = new_get_variable
- tf.get_variable = new_get_variable
-
- _is_variable_replacing[0] = True
-
- yield
-
- if set(has_replaced_names) != set(names_replace.keys()):
- print "Didn't use all replacements"
- print "replaced variables that are not requested??"
- print "==="
- for n in list(set(has_replaced_names) - set(names_replace.keys())):
- print n
- print "Missed replacing variables"
- print "==="
- for n in list(set(names_replace.keys()) - set(has_replaced_names)):
- print n, "==>", names_replace[n].name
- raise ValueError("Fix this -- see stderr")
-
- # undo the monkey patch
- tf.get_variable = old_tf_get_variable
- variable_scope.get_variable = old_get_variable
-
- _is_variable_replacing[0] = False
-
- final_vars = tf.trainable_variables()
- assert set(init_vars) == set(final_vars), "trainable variables changed"
diff --git a/research/lexnet_nc/README.md b/research/lexnet_nc/README.md
deleted file mode 100644
index 4ecb5d39867c2ebf7280b9d19bbabb41957b9465..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/README.md
+++ /dev/null
@@ -1,215 +0,0 @@
-
-
-
-
-# LexNET for Noun Compound Relation Classification
-
-This is a [Tensorflow](http://www.tensorflow.org/) implementation of the LexNET
-algorithm for classifying relationships, specifically applied to classifying the
-relationships that hold between noun compounds:
-
-* *olive oil* is oil that is *made from* olives
-* *cooking oil* which is oil that is *used for* cooking
-* *motor oil* is oil that is *contained in* a motor
-
-The model is a supervised classifier that predicts the relationship that holds
-between the constituents of a two-word noun compound using:
-
-1. A neural "paraphrase" of each syntactic dependency path that connects the
- constituents in a large corpus. For example, given a sentence like *This fine
- oil is made from first-press olives*, the dependency path is something like
- `oil from POBJ> olive`.
-2. The distributional information provided by the individual words; i.e., the
- word embeddings of the two consituents.
-3. The distributional signal provided by the compound itself; i.e., the
- embedding of the noun compound in context.
-
-The model includes several variants: *path-based model* uses (1) alone, the
-*distributional model* uses (2) alone, and the *integrated model* uses (1) and
-(2). The *distributional-nc model* and the *integrated-nc* model each add (3).
-
-Training a model requires the following:
-
-1. A collection of noun compounds that have been labeled using a *relation
- inventory*. The inventory describes the specific relationships that you'd
- like the model to differentiate (e.g. *part of* versus *composed of* versus
- *purpose*), and generally may consist of tens of classes. You can download
- the dataset used in the paper from
- [here](https://vered1986.github.io/papers/Tratz2011_Dataset.tar.gz).
-2. A collection of word embeddings: the path-based model uses the word
- embeddings as part of the path representation, and the distributional models
- use the word embeddings directly as prediction features.
-3. The path-based model requires a collection of syntactic dependency parses
- that connect the constituents for each noun compound. To generate these,
- you'll need a corpus from which to train this data; we used Wikipedia and the
- [LDC GigaWord5](https://catalog.ldc.upenn.edu/LDC2011T07) corpora.
-
-# Contents
-
-The following source code is included here:
-
-* `learn_path_embeddings.py` is a script that trains and evaluates a path-based
- model to predict a noun-compound relationship given labeled noun-compounds and
- dependency parse paths.
-* `learn_classifier.py` is a script that trains and evaluates a classifier based
- on any combination of paths, word embeddings, and noun-compound embeddings.
-* `get_indicative_paths.py` is a script that generates the most indicative
- syntactic dependency paths for a particular relationship.
-
-Also included are utilities for preparing data for training:
-
-* `text_embeddings_to_binary.py` converts a text file containing word embeddings
- into a binary file that is quicker to load.
-* `extract_paths.py` finds all the dependency paths that connect words in a
- corpus.
-* `sorted_paths_to_examples.py` processes the output of `extract_paths.py` to
- produce summarized training data.
-
-This code (in particular, the utilities used to prepare the data) differs from
-the code that was used to prepare data for the paper. Notably, we used a
-proprietary dependency parser instead of spaCy, which is used here.
-
-# Dependencies
-
-* [TensorFlow](http://www.tensorflow.org/): see detailed installation
- instructions at that site.
-* [SciKit Learn](http://scikit-learn.org/): you can probably just install this
- with `pip install sklearn`.
-* [SpaCy](https://spacy.io/): `pip install spacy` ought to do the trick, along
- with the English model.
-
-# Creating the Model
-
-This sections described the steps necessary to create and evaluate the model
-described in the paper.
-
-## Generate Path Data
-
-To begin, you need three text files:
-
-1. **Corpus**. This file should contain natural language sentences, written with
- one sentence per line. For purposes of exposition, we'll assume that you
- have English Wikipedia serialized this way in `${HOME}/data/wiki.txt`.
-2. **Labeled Noun Compound Pairs**. This file contain (modfier, head, label)
- tuples, tab-separated, with one per line. The *label* represented the
- relationship between the head and the modifier; e.g., if `purpose` is one
- your labels, you could possibly include `toothpastepurpose`.
-3. **Word Embeddings**. We used the
- [GloVe](https://nlp.stanford.edu/projects/glove/) word embeddings; in
- particular the 6B token, 300d variant. We'll assume you have this file as
- `${HOME}/data/glove.6B.300d.txt`.
-
-We first processed the embeddings from their text format into something that we
-can load a little bit more quickly:
-
- ./text_embeddings_to_binary.py \
- --input ${HOME}/data/glove.6B.300d.txt \
- --output_vocab ${HOME}/data/vocab.txt \
- --output_npy ${HOME}/data/glove.6B.300d.npy
-
-Next, we'll extract all the dependency parse paths connecting our labeled pairs
-from the corpus. This process takes a *looooong* time, but is trivially
-parallelized using map-reduce if you have access to that technology.
-
- ./extract_paths.py \
- --corpus ${HOME}/data/wiki.txt \
- --labeled_pairs ${HOME}/data/labeled-pairs.tsv \
- --output ${HOME}/data/paths.tsv
-
-The file it produces (`paths.tsv`) is a tab-separated file that contains the
-modifier, the head, the label, the encoded path, and the sentence from which the
-path was drawn. (This last is mostly for sanity checking.) A sample row might
-look something like this (where newlines would actually be tab characters):
-
- navy
- captain
- owner_emp_use
- /PROPN/dobj/>::enter/VERB/ROOT/^::follow/VERB/advcl/<::in/ADP/prep/<::footstep/NOUN/pobj/<::of/ADP/prep/<::father/NOUN/pobj/<::bover/PROPN/appos/<::/PROPN/compound/<
- He entered the Royal Navy following in the footsteps of his father Captain John Bover and two of his elder brothers as volunteer aboard HMS Perseus
-
-This file must be sorted as follows:
-
- sort -k1,3 -t$'\t' paths.tsv > sorted.paths.tsv
-
-In particular, rows with the same modifier, head, and label must appear
-contiguously.
-
-We next create a file that contains all the relation labels from our original
-labeled pairs:
-
- awk 'BEGIN {FS="\t"} {print $3}' < ${HOME}/data/labeled-pairs.tsv \
- | sort -u > ${HOME}/data/relations.txt
-
-With these in hand, we're ready to produce the train, validation, and test data:
-
- ./sorted_paths_to_examples.py \
- --input ${HOME}/data/sorted.paths.tsv \
- --vocab ${HOME}/data/vocab.txt \
- --relations ${HOME}/data/relations.txt \
- --splits ${HOME}/data/splits.txt \
- --output_dir ${HOME}/data
-
-Here, `splits.txt` is a file that indicates which "split" (train, test, or
-validation) you want the pair to appear in. It should be a tab-separate file
-which conatins the modifier, head, and the dataset ( `train`, `test`, or `val`)
-into which the pair should be placed; e.g.,:
-
- tooth paste train
- banana seat test
-
-The program will produce a separate file for each dataset split in the directory
-specified by `--output_dir`. Each file is contains `tf.train.Example` protocol
-buffers encoded using the `TFRecord` file format.
-
-## Create Path Embeddings
-
-Now we're ready to train the path embeddings using `learn_path_embeddings.py`:
-
- ./learn_path_embeddings.py \
- --train ${HOME}/data/train.tfrecs.gz \
- --val ${HOME}/data/val.tfrecs.gz \
- --text ${HOME}/data/test.tfrecs.gz \
- --embeddings ${HOME}/data/glove.6B.300d.npy
- --relations ${HOME}/data/relations.txt
- --output ${HOME}/data/path-embeddings \
- --logdir /tmp/learn_path_embeddings
-
-The path embeddings will be placed at the location specified by `--output`.
-
-## Train classifiers
-
-Train classifiers and evaluate on the validation and test data using
-`train_classifiers.py` script. This shell script fragment will iterate through
-each dataset, split, corpus, and model type to train and evaluate classifiers.
-
- LOGDIR=/tmp/learn_classifier
- for DATASET in tratz/fine_grained tratz/coarse_grained ; do
- for SPLIT in random lexical_head lexical_mod lexical_full ; do
- for CORPUS in wiki_gigiawords ; do
- for MODEL in dist dist-nc path integrated integrated-nc ; do
- # Filename for the log that will contain the classifier results.
- LOGFILE=$(echo "${DATASET}.${SPLIT}.${CORPUS}.${MODEL}.log" | sed -e "s,/,.,g")
- python learn_classifier.py \
- --dataset_dir ~/lexnet/datasets \
- --dataset "${DATASET}" \
- --corpus "${SPLIT}/${CORPUS}" \
- --embeddings_base_path ~/lexnet/embeddings \
- --logdir ${LOGDIR} \
- --input "${MODEL}" > "${LOGDIR}/${LOGFILE}"
- done
- done
- done
- done
-
-The log file will contain the final performance (precision, recall, F1) on the
-train, dev, and test sets, and will include a confusion matrix for each.
-
-# Contact
-
-If you have any questions, issues, or suggestions, feel free to contact either
-@vered1986 or @waterson.
-
-If you use this code for any published research, please include the following citation:
-
-Olive Oil Is Made of Olives, Baby Oil Is Made for Babies: Interpreting Noun Compounds Using Paraphrases in a Neural Model.
-Vered Shwartz and Chris Waterson. NAACL 2018. [link](https://arxiv.org/pdf/1803.08073.pdf).
diff --git a/research/lexnet_nc/extract_paths.py b/research/lexnet_nc/extract_paths.py
deleted file mode 100755
index 833eec2c1b8a176b487d4e663a737b9502b49eda..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/extract_paths.py
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import itertools
-import sys
-
-import spacy
-import tensorflow as tf
-
-tf.flags.DEFINE_string('corpus', '', 'Filename of corpus')
-tf.flags.DEFINE_string('labeled_pairs', '', 'Filename of labeled pairs')
-tf.flags.DEFINE_string('output', '', 'Filename of output file')
-FLAGS = tf.flags.FLAGS
-
-
-def get_path(mod_token, head_token):
- """Returns the path between a modifier token and a head token."""
- # Compute the path from the root to each token.
- mod_ancestors = list(reversed(list(mod_token.ancestors)))
- head_ancestors = list(reversed(list(head_token.ancestors)))
-
- # If the paths don't start at the same place (odd!) then there is no path at
- # all.
- if (not mod_ancestors or not head_ancestors
- or mod_ancestors[0] != head_ancestors[0]):
- return None
-
- # Eject elements from the common path until we reach the first differing
- # ancestor.
- ix = 1
- while (ix < len(mod_ancestors) and ix < len(head_ancestors)
- and mod_ancestors[ix] == head_ancestors[ix]):
- ix += 1
-
- # Construct the path. TODO: add "satellites", possibly honor sentence
- # ordering between modifier and head rather than just always traversing from
- # the modifier to the head?
- path = ['/'.join(('', mod_token.pos_, mod_token.dep_, '>'))]
-
- path += ['/'.join((tok.lemma_, tok.pos_, tok.dep_, '>'))
- for tok in reversed(mod_ancestors[ix:])]
-
- root_token = mod_ancestors[ix - 1]
- path += ['/'.join((root_token.lemma_, root_token.pos_, root_token.dep_, '^'))]
-
- path += ['/'.join((tok.lemma_, tok.pos_, tok.dep_, '<'))
- for tok in head_ancestors[ix:]]
-
- path += ['/'.join(('', head_token.pos_, head_token.dep_, '<'))]
-
- return '::'.join(path)
-
-
-def main(_):
- nlp = spacy.load('en_core_web_sm')
-
- # Grab the set of labeled pairs for which we wish to collect paths.
- with tf.gfile.GFile(FLAGS.labeled_pairs) as fh:
- parts = (l.decode('utf-8').split('\t') for l in fh.read().splitlines())
- labeled_pairs = {(mod, head): rel for mod, head, rel in parts}
-
- # Create a mapping from each head to the modifiers that are used with it.
- mods_for_head = {
- head: set(hm[1] for hm in head_mods)
- for head, head_mods in itertools.groupby(
- sorted((head, mod) for (mod, head) in labeled_pairs.iterkeys()),
- lambda (head, mod): head)}
-
- # Collect all the heads that we know about.
- heads = set(mods_for_head.keys())
-
- # For each sentence that contains a (head, modifier) pair that's in our set,
- # emit the dependency path that connects the pair.
- out_fh = sys.stdout if not FLAGS.output else tf.gfile.GFile(FLAGS.output, 'w')
- in_fh = sys.stdin if not FLAGS.corpus else tf.gfile.GFile(FLAGS.corpus)
-
- num_paths = 0
- for line, sen in enumerate(in_fh, start=1):
- if line % 100 == 0:
- print('\rProcessing line %d: %d paths' % (line, num_paths),
- end='', file=sys.stderr)
-
- sen = sen.decode('utf-8').strip()
- doc = nlp(sen)
-
- for head_token in doc:
- head_text = head_token.text.lower()
- if head_text in heads:
- mods = mods_for_head[head_text]
- for mod_token in doc:
- mod_text = mod_token.text.lower()
- if mod_text in mods:
- path = get_path(mod_token, head_token)
- if path:
- label = labeled_pairs[(mod_text, head_text)]
- line = '\t'.join((mod_text, head_text, label, path, sen))
- print(line.encode('utf-8'), file=out_fh)
- num_paths += 1
-
- out_fh.close()
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/lexnet_nc/get_indicative_paths.py b/research/lexnet_nc/get_indicative_paths.py
deleted file mode 100755
index f8b34cca221a07c0b633024b71f082b8f61b3a45..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/get_indicative_paths.py
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Extracts paths that are indicative of each relation."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import tensorflow as tf
-
-from . import path_model
-from . import lexnet_common
-
-tf.flags.DEFINE_string(
- 'dataset_dir', 'datasets',
- 'Dataset base directory')
-
-tf.flags.DEFINE_string(
- 'dataset',
- 'tratz/fine_grained',
- 'Subdirectory containing the corpus directories: '
- 'subdirectory of dataset_dir')
-
-tf.flags.DEFINE_string(
- 'corpus', 'random/wiki',
- 'Subdirectory containing the corpus and split: '
- 'subdirectory of dataset_dir/dataset')
-
-tf.flags.DEFINE_string(
- 'embeddings_base_path', 'embeddings',
- 'Embeddings base directory')
-
-tf.flags.DEFINE_string(
- 'logdir', 'logdir',
- 'Directory of model output files')
-
-tf.flags.DEFINE_integer(
- 'top_k', 20, 'Number of top paths to extract')
-
-tf.flags.DEFINE_float(
- 'threshold', 0.8, 'Threshold above which to consider paths as indicative')
-
-FLAGS = tf.flags.FLAGS
-
-
-def main(_):
- hparams = path_model.PathBasedModel.default_hparams()
-
- # First things first. Load the path data.
- path_embeddings_file = 'path_embeddings/{dataset}/{corpus}'.format(
- dataset=FLAGS.dataset,
- corpus=FLAGS.corpus)
-
- path_dim = (hparams.lemma_dim + hparams.pos_dim +
- hparams.dep_dim + hparams.dir_dim)
-
- path_embeddings, path_to_index = path_model.load_path_embeddings(
- os.path.join(FLAGS.embeddings_base_path, path_embeddings_file),
- path_dim)
-
- # Load and count the classes so we can correctly instantiate the model.
- classes_filename = os.path.join(
- FLAGS.dataset_dir, FLAGS.dataset, 'classes.txt')
-
- with open(classes_filename) as f_in:
- classes = f_in.read().splitlines()
-
- hparams.num_classes = len(classes)
-
- # We need the word embeddings to instantiate the model, too.
- print('Loading word embeddings...')
- lemma_embeddings = lexnet_common.load_word_embeddings(
- FLAGS.embeddings_base_path, hparams.lemma_embeddings_file)
-
- # Instantiate the model.
- with tf.Graph().as_default():
- with tf.variable_scope('lexnet'):
- instance = tf.placeholder(dtype=tf.string)
- model = path_model.PathBasedModel(
- hparams, lemma_embeddings, instance)
-
- with tf.Session() as session:
- model_dir = '{logdir}/results/{dataset}/path/{corpus}'.format(
- logdir=FLAGS.logdir,
- dataset=FLAGS.dataset,
- corpus=FLAGS.corpus)
-
- saver = tf.train.Saver()
- saver.restore(session, os.path.join(model_dir, 'best.ckpt'))
-
- path_model.get_indicative_paths(
- model, session, path_to_index, path_embeddings, classes,
- model_dir, FLAGS.top_k, FLAGS.threshold)
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/lexnet_nc/learn_classifier.py b/research/lexnet_nc/learn_classifier.py
deleted file mode 100755
index ec284029535609ffd2cc0f2f5cddb9b87954aa81..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/learn_classifier.py
+++ /dev/null
@@ -1,223 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Trains the integrated LexNET classifier."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import lexnet_common
-import lexnet_model
-import path_model
-from sklearn import metrics
-import tensorflow as tf
-
-tf.flags.DEFINE_string(
- 'dataset_dir', 'datasets',
- 'Dataset base directory')
-
-tf.flags.DEFINE_string(
- 'dataset', 'tratz/fine_grained',
- 'Subdirectory containing the corpus directories: '
- 'subdirectory of dataset_dir')
-
-tf.flags.DEFINE_string(
- 'corpus', 'wiki/random',
- 'Subdirectory containing the corpus and split: '
- 'subdirectory of dataset_dir/dataset')
-
-tf.flags.DEFINE_string(
- 'embeddings_base_path', 'embeddings',
- 'Embeddings base directory')
-
-tf.flags.DEFINE_string(
- 'logdir', 'logdir',
- 'Directory of model output files')
-
-tf.flags.DEFINE_string('hparams', '', 'Hyper-parameters')
-
-tf.flags.DEFINE_string(
- 'input', 'integrated',
- 'The model(dist/dist-nc/path/integrated/integrated-nc')
-
-FLAGS = tf.flags.FLAGS
-
-
-def main(_):
- # Pick up any one-off hyper-parameters.
- hparams = lexnet_model.LexNETModel.default_hparams()
- hparams.corpus = FLAGS.corpus
- hparams.input = FLAGS.input
- hparams.path_embeddings_file = 'path_embeddings/%s/%s' % (
- FLAGS.dataset, FLAGS.corpus)
-
- input_dir = hparams.input if hparams.input != 'path' else 'path_classifier'
-
- # Set the number of classes
- classes_filename = os.path.join(
- FLAGS.dataset_dir, FLAGS.dataset, 'classes.txt')
- with open(classes_filename) as f_in:
- classes = f_in.read().splitlines()
-
- hparams.num_classes = len(classes)
- print('Model will predict into %d classes' % hparams.num_classes)
-
- # Get the datasets
- train_set, val_set, test_set = (
- os.path.join(
- FLAGS.dataset_dir, FLAGS.dataset, FLAGS.corpus,
- filename + '.tfrecs.gz')
- for filename in ['train', 'val', 'test'])
-
- print('Running with hyper-parameters: {}'.format(hparams))
-
- # Load the instances
- print('Loading instances...')
- opts = tf.python_io.TFRecordOptions(
- compression_type=tf.python_io.TFRecordCompressionType.GZIP)
- train_instances = list(tf.python_io.tf_record_iterator(train_set, opts))
- val_instances = list(tf.python_io.tf_record_iterator(val_set, opts))
- test_instances = list(tf.python_io.tf_record_iterator(test_set, opts))
-
- # Load the word embeddings
- print('Loading word embeddings...')
- relata_embeddings, path_embeddings, nc_embeddings, path_to_index = (
- None, None, None, None)
- if hparams.input in ['dist', 'dist-nc', 'integrated', 'integrated-nc']:
- relata_embeddings = lexnet_common.load_word_embeddings(
- FLAGS.embeddings_base_path, hparams.relata_embeddings_file)
-
- if hparams.input in ['path', 'integrated', 'integrated-nc']:
- path_embeddings, path_to_index = path_model.load_path_embeddings(
- os.path.join(FLAGS.embeddings_base_path, hparams.path_embeddings_file),
- hparams.path_dim)
-
- if hparams.input in ['dist-nc', 'integrated-nc']:
- nc_embeddings = lexnet_common.load_word_embeddings(
- FLAGS.embeddings_base_path, hparams.nc_embeddings_file)
-
- # Define the graph and the model
- with tf.Graph().as_default():
- model = lexnet_model.LexNETModel(
- hparams, relata_embeddings, path_embeddings,
- nc_embeddings, path_to_index)
-
- # Initialize a session and start training
- session = tf.Session()
- session.run(tf.global_variables_initializer())
-
- # Initalize the path mapping
- if hparams.input in ['path', 'integrated', 'integrated-nc']:
- session.run(tf.tables_initializer())
- session.run(model.initialize_path_op, {
- model.path_initial_value_t: path_embeddings
- })
-
- # Initialize the NC embeddings
- if hparams.input in ['dist-nc', 'integrated-nc']:
- session.run(model.initialize_nc_op, {
- model.nc_initial_value_t: nc_embeddings
- })
-
- # Load the labels
- print('Loading labels...')
- train_labels = model.load_labels(session, train_instances)
- val_labels = model.load_labels(session, val_instances)
- test_labels = model.load_labels(session, test_instances)
-
- save_path = '{logdir}/results/{dataset}/{input}/{corpus}'.format(
- logdir=FLAGS.logdir, dataset=FLAGS.dataset,
- corpus=model.hparams.corpus, input=input_dir)
-
- if not os.path.exists(save_path):
- os.makedirs(save_path)
-
- # Train the model
- print('Training the model...')
- model.fit(session, train_instances, epoch_completed,
- val_instances, val_labels, save_path)
-
- # Print the best performance on the validation set
- print('Best performance on the validation set: F1=%.3f' %
- epoch_completed.best_f1)
-
- # Evaluate on the train and validation sets
- lexnet_common.full_evaluation(model, session, train_instances, train_labels,
- 'Train', classes)
- lexnet_common.full_evaluation(model, session, val_instances, val_labels,
- 'Validation', classes)
- test_predictions = lexnet_common.full_evaluation(
- model, session, test_instances, test_labels, 'Test', classes)
-
- # Write the test predictions to a file
- predictions_file = os.path.join(save_path, 'test_predictions.tsv')
- print('Saving test predictions to %s' % save_path)
- test_pairs = model.load_pairs(session, test_instances)
- lexnet_common.write_predictions(test_pairs, test_labels, test_predictions,
- classes, predictions_file)
-
-
-def epoch_completed(model, session, epoch, epoch_loss,
- val_instances, val_labels, save_path):
- """Runs every time an epoch completes.
-
- Print the performance on the validation set, and update the saved model if
- its performance is better on the previous ones. If the performance dropped,
- tell the training to stop.
-
- Args:
- model: The currently trained path-based model.
- session: The current TensorFlow session.
- epoch: The epoch number.
- epoch_loss: The current epoch loss.
- val_instances: The validation set instances (evaluation between epochs).
- val_labels: The validation set labels (for evaluation between epochs).
- save_path: Where to save the model.
-
- Returns:
- whether the training should stop.
- """
- stop_training = False
-
- # Evaluate on the validation set
- val_pred = model.predict(session, val_instances)
- precision, recall, f1, _ = metrics.precision_recall_fscore_support(
- val_labels, val_pred, average='weighted')
- print(
- 'Epoch: %d/%d, Loss: %f, validation set: P: %.3f, R: %.3f, F1: %.3f\n' % (
- epoch + 1, model.hparams.num_epochs, epoch_loss,
- precision, recall, f1))
-
- # If the F1 is much smaller than the previous one, stop training. Else, if
- # it's bigger, save the model.
- if f1 < epoch_completed.best_f1 - 0.08:
- stop_training = True
-
- if f1 > epoch_completed.best_f1:
- saver = tf.train.Saver()
- checkpoint_filename = os.path.join(save_path, 'best.ckpt')
- print('Saving model in: %s' % checkpoint_filename)
- saver.save(session, checkpoint_filename)
- print('Model saved in file: %s' % checkpoint_filename)
- epoch_completed.best_f1 = f1
-
- return stop_training
-
-epoch_completed.best_f1 = 0
-
-if __name__ == '__main__':
- tf.app.run(main)
diff --git a/research/lexnet_nc/learn_path_embeddings.py b/research/lexnet_nc/learn_path_embeddings.py
deleted file mode 100755
index 480378f4aa010ee27f0387685bac488cedbb2ab9..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/learn_path_embeddings.py
+++ /dev/null
@@ -1,186 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Trains the LexNET path-based model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import lexnet_common
-import path_model
-from sklearn import metrics
-import tensorflow as tf
-
-tf.flags.DEFINE_string('train', '', 'training dataset, tfrecs')
-tf.flags.DEFINE_string('val', '', 'validation dataset, tfrecs')
-tf.flags.DEFINE_string('test', '', 'test dataset, tfrecs')
-tf.flags.DEFINE_string('embeddings', '', 'embeddings, npy')
-tf.flags.DEFINE_string('relations', '', 'file containing relation labels')
-tf.flags.DEFINE_string('output_dir', '', 'output directory for path embeddings')
-tf.flags.DEFINE_string('logdir', '', 'directory for model training')
-FLAGS = tf.flags.FLAGS
-
-
-def main(_):
- # Pick up any one-off hyper-parameters.
- hparams = path_model.PathBasedModel.default_hparams()
-
- with open(FLAGS.relations) as fh:
- relations = fh.read().splitlines()
-
- hparams.num_classes = len(relations)
- print('Model will predict into %d classes' % hparams.num_classes)
-
- print('Running with hyper-parameters: {}'.format(hparams))
-
- # Load the instances
- print('Loading instances...')
- opts = tf.python_io.TFRecordOptions(
- compression_type=tf.python_io.TFRecordCompressionType.GZIP)
-
- train_instances = list(tf.python_io.tf_record_iterator(FLAGS.train, opts))
- val_instances = list(tf.python_io.tf_record_iterator(FLAGS.val, opts))
- test_instances = list(tf.python_io.tf_record_iterator(FLAGS.test, opts))
-
- # Load the word embeddings
- print('Loading word embeddings...')
- lemma_embeddings = lexnet_common.load_word_embeddings(FLAGS.embeddings)
-
- # Define the graph and the model
- with tf.Graph().as_default():
- with tf.variable_scope('lexnet'):
- options = tf.python_io.TFRecordOptions(
- compression_type=tf.python_io.TFRecordCompressionType.GZIP)
- reader = tf.TFRecordReader(options=options)
- _, train_instance = reader.read(
- tf.train.string_input_producer([FLAGS.train]))
- shuffled_train_instance = tf.train.shuffle_batch(
- [train_instance],
- batch_size=1,
- num_threads=1,
- capacity=len(train_instances),
- min_after_dequeue=100,
- )[0]
-
- train_model = path_model.PathBasedModel(
- hparams, lemma_embeddings, shuffled_train_instance)
-
- with tf.variable_scope('lexnet', reuse=True):
- val_instance = tf.placeholder(dtype=tf.string)
- val_model = path_model.PathBasedModel(
- hparams, lemma_embeddings, val_instance)
-
- # Initialize a session and start training
- best_model_saver = tf.train.Saver()
- f1_t = tf.placeholder(tf.float32)
- best_f1_t = tf.Variable(0.0, trainable=False, name='best_f1')
- assign_best_f1_op = tf.assign(best_f1_t, f1_t)
-
- supervisor = tf.train.Supervisor(
- logdir=FLAGS.logdir,
- global_step=train_model.global_step)
-
- with supervisor.managed_session() as session:
- # Load the labels
- print('Loading labels...')
- val_labels = train_model.load_labels(session, val_instances)
-
- # Train the model
- print('Training the model...')
-
- while True:
- step = session.run(train_model.global_step)
- epoch = (step + len(train_instances) - 1) // len(train_instances)
- if epoch > hparams.num_epochs:
- break
-
- print('Starting epoch %d (step %d)...' % (1 + epoch, step))
-
- epoch_loss = train_model.run_one_epoch(session, len(train_instances))
-
- best_f1 = session.run(best_f1_t)
- f1 = epoch_completed(val_model, session, epoch, epoch_loss,
- val_instances, val_labels, best_model_saver,
- FLAGS.logdir, best_f1)
-
- if f1 > best_f1:
- session.run(assign_best_f1_op, {f1_t: f1})
-
- if f1 < best_f1 - 0.08:
- tf.logging.info('Stopping training after %d epochs.\n' % epoch)
- break
-
- # Print the best performance on the validation set
- best_f1 = session.run(best_f1_t)
- print('Best performance on the validation set: F1=%.3f' % best_f1)
-
- # Save the path embeddings
- print('Computing the path embeddings...')
- instances = train_instances + val_instances + test_instances
- path_index, path_vectors = path_model.compute_path_embeddings(
- val_model, session, instances)
-
- if not os.path.exists(path_emb_dir):
- os.makedirs(path_emb_dir)
-
- path_model.save_path_embeddings(
- val_model, path_vectors, path_index, FLAGS.output_dir)
-
-
-def epoch_completed(model, session, epoch, epoch_loss,
- val_instances, val_labels, saver, save_path, best_f1):
- """Runs every time an epoch completes.
-
- Print the performance on the validation set, and update the saved model if
- its performance is better on the previous ones. If the performance dropped,
- tell the training to stop.
-
- Args:
- model: The currently trained path-based model.
- session: The current TensorFlow session.
- epoch: The epoch number.
- epoch_loss: The current epoch loss.
- val_instances: The validation set instances (evaluation between epochs).
- val_labels: The validation set labels (for evaluation between epochs).
- saver: tf.Saver object
- save_path: Where to save the model.
- best_f1: the best F1 achieved so far.
-
- Returns:
- The F1 achieved on the training set.
- """
- # Evaluate on the validation set
- val_pred = model.predict(session, val_instances)
- precision, recall, f1, _ = metrics.precision_recall_fscore_support(
- val_labels, val_pred, average='weighted')
- print(
- 'Epoch: %d/%d, Loss: %f, validation set: P: %.3f, R: %.3f, F1: %.3f\n' % (
- epoch + 1, model.hparams.num_epochs, epoch_loss,
- precision, recall, f1))
-
- if f1 > best_f1:
- save_filename = os.path.join(save_path, 'best.ckpt')
- print('Saving model in: %s' % save_filename)
- saver.save(session, save_filename)
- print('Model saved in file: %s' % save_filename)
-
- return f1
-
-
-if __name__ == '__main__':
- tf.app.run(main)
diff --git a/research/lexnet_nc/lexnet_common.py b/research/lexnet_nc/lexnet_common.py
deleted file mode 100644
index a2e8a104d00c1c2f90731f4045c3c8e69e370dbf..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/lexnet_common.py
+++ /dev/null
@@ -1,197 +0,0 @@
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Common stuff used with LexNET."""
-# pylint: disable=bad-whitespace
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import numpy as np
-from sklearn import metrics
-import tensorflow as tf
-
-# Part of speech tags used in the paths.
-POSTAGS = [
- 'PAD', 'VERB', 'CONJ', 'NOUN', 'PUNCT',
- 'ADP', 'ADJ', 'DET', 'ADV', 'PART',
- 'NUM', 'X', 'INTJ', 'SYM',
-]
-
-POSTAG_TO_ID = {tag: tid for tid, tag in enumerate(POSTAGS)}
-
-# Dependency labels used in the paths.
-DEPLABELS = [
- 'PAD', 'UNK', 'ROOT', 'abbrev', 'acomp', 'advcl',
- 'advmod', 'agent', 'amod', 'appos', 'attr', 'aux',
- 'auxpass', 'cc', 'ccomp', 'complm', 'conj', 'cop',
- 'csubj', 'csubjpass', 'dep', 'det', 'dobj', 'expl',
- 'infmod', 'iobj', 'mark', 'mwe', 'nc', 'neg',
- 'nn', 'npadvmod', 'nsubj', 'nsubjpass', 'num', 'number',
- 'p', 'parataxis', 'partmod', 'pcomp', 'pobj', 'poss',
- 'preconj', 'predet', 'prep', 'prepc', 'prt', 'ps',
- 'purpcl', 'quantmod', 'rcmod', 'ref', 'rel', 'suffix',
- 'title', 'tmod', 'xcomp', 'xsubj',
-]
-
-DEPLABEL_TO_ID = {label: lid for lid, label in enumerate(DEPLABELS)}
-
-# Direction codes used in the paths.
-DIRS = '_^V<>'
-DIR_TO_ID = {dir: did for did, dir in enumerate(DIRS)}
-
-
-def load_word_embeddings(embedding_filename):
- """Loads pretrained word embeddings from a binary file and returns the matrix.
-
- Adds the , , , and tokens to the beginning of the vocab.
-
- Args:
- embedding_filename: filename of the binary NPY data
-
- Returns:
- The word embeddings matrix
- """
- embeddings = np.load(embedding_filename)
- dim = embeddings.shape[1]
-
- # Four initially random vectors for the special tokens: , , ,
- special_embeddings = np.random.normal(0, 0.1, (4, dim))
- embeddings = np.vstack((special_embeddings, embeddings))
- embeddings = embeddings.astype(np.float32)
-
- return embeddings
-
-
-def full_evaluation(model, session, instances, labels, set_name, classes):
- """Prints a full evaluation on the current set.
-
- Performance (recall, precision and F1), classification report (per
- class performance), and confusion matrix).
-
- Args:
- model: The currently trained path-based model.
- session: The current TensorFlow session.
- instances: The current set instances.
- labels: The current set labels.
- set_name: The current set name (train/validation/test).
- classes: The class label names.
-
- Returns:
- The model's prediction for the given instances.
- """
-
- # Predict the labels
- pred = model.predict(session, instances)
-
- # Print the performance
- precision, recall, f1, _ = metrics.precision_recall_fscore_support(
- labels, pred, average='weighted')
-
- print('%s set: Precision: %.3f, Recall: %.3f, F1: %.3f' % (
- set_name, precision, recall, f1))
-
- # Print a classification report
- print('%s classification report:' % set_name)
- print(metrics.classification_report(labels, pred, target_names=classes))
-
- # Print the confusion matrix
- print('%s confusion matrix:' % set_name)
- cm = metrics.confusion_matrix(labels, pred, labels=range(len(classes)))
- cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] * 100
- print_cm(cm, labels=classes)
- return pred
-
-
-def print_cm(cm, labels):
- """Pretty print for confusion matrices.
-
- From: https://gist.github.com/zachguo/10296432.
-
- Args:
- cm: The confusion matrix.
- labels: The class names.
- """
- columnwidth = 10
- empty_cell = ' ' * columnwidth
- short_labels = [label[:12].rjust(10, ' ') for label in labels]
-
- # Print header
- header = empty_cell + ' '
- header += ''.join([' %{0}s '.format(columnwidth) % label
- for label in short_labels])
-
- print(header)
-
- # Print rows
- for i, label1 in enumerate(short_labels):
- row = '%{0}s '.format(columnwidth) % label1[:10]
- for j in range(len(short_labels)):
- value = int(cm[i, j]) if not np.isnan(cm[i, j]) else 0
- cell = ' %{0}d '.format(10) % value
- row += cell + ' '
- print(row)
-
-
-def load_all_labels(records):
- """Reads TensorFlow examples from a RecordReader and returns only the labels.
-
- Args:
- records: a record list with TensorFlow examples.
-
- Returns:
- The labels
- """
- curr_features = tf.parse_example(records, {
- 'rel_id': tf.FixedLenFeature([1], dtype=tf.int64),
- })
-
- labels = tf.squeeze(curr_features['rel_id'], [-1])
- return labels
-
-
-def load_all_pairs(records):
- """Reads TensorFlow examples from a RecordReader and returns the word pairs.
-
- Args:
- records: a record list with TensorFlow examples.
-
- Returns:
- The word pairs
- """
- curr_features = tf.parse_example(records, {
- 'pair': tf.FixedLenFeature([1], dtype=tf.string)
- })
-
- word_pairs = curr_features['pair']
- return word_pairs
-
-
-def write_predictions(pairs, labels, predictions, classes, predictions_file):
- """Write the predictions to a file.
-
- Args:
- pairs: the word pairs (list of tuple of two strings).
- labels: the gold-standard labels for these pairs (array of rel ID).
- predictions: the predicted labels for these pairs (array of rel ID).
- classes: a list of relation names.
- predictions_file: where to save the predictions.
- """
- with open(predictions_file, 'w') as f_out:
- for pair, label, pred in zip(pairs, labels, predictions):
- w1, w2 = pair
- f_out.write('\t'.join([w1, w2, classes[label], classes[pred]]) + '\n')
diff --git a/research/lexnet_nc/lexnet_model.py b/research/lexnet_nc/lexnet_model.py
deleted file mode 100644
index b0f16b030b3bb3fee68b91122bcd03226ffcfa4a..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/lexnet_model.py
+++ /dev/null
@@ -1,438 +0,0 @@
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""The integrated LexNET model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import lexnet_common
-import numpy as np
-import tensorflow as tf
-from six.moves import xrange
-
-
-class LexNETModel(object):
- """The LexNET model for classifying relationships between noun compounds."""
-
- @classmethod
- def default_hparams(cls):
- """Returns the default hyper-parameters."""
- return tf.contrib.training.HParams(
- batch_size=10,
- num_classes=37,
- num_epochs=30,
- input_keep_prob=0.9,
- input='integrated', # dist/ dist-nc/ path/ integrated/ integrated-nc
- learn_relata=False,
- corpus='wiki_gigawords',
- random_seed=133, # zero means no random seed
- relata_embeddings_file='glove/glove.6B.300d.bin',
- nc_embeddings_file='nc_glove/vecs.6B.300d.bin',
- path_embeddings_file='path_embeddings/tratz/fine_grained/wiki',
- hidden_layers=1,
- path_dim=60)
-
- def __init__(self, hparams, relata_embeddings, path_embeddings, nc_embeddings,
- path_to_index):
- """Initialize the LexNET classifier.
-
- Args:
- hparams: the hyper-parameters.
- relata_embeddings: word embeddings for the distributional component.
- path_embeddings: embeddings for the paths.
- nc_embeddings: noun compound embeddings.
- path_to_index: a mapping from string path to an index in the path
- embeddings matrix.
- """
- self.hparams = hparams
-
- self.path_embeddings = path_embeddings
- self.relata_embeddings = relata_embeddings
- self.nc_embeddings = nc_embeddings
-
- self.vocab_size, self.relata_dim = 0, 0
- self.path_to_index = None
- self.path_dim = 0
-
- # Set the random seed
- if hparams.random_seed > 0:
- tf.set_random_seed(hparams.random_seed)
-
- # Get the vocabulary size and relata dim
- if self.hparams.input in ['dist', 'dist-nc', 'integrated', 'integrated-nc']:
- self.vocab_size, self.relata_dim = self.relata_embeddings.shape
-
- # Create the mapping from string path to an index in the embeddings matrix
- if self.hparams.input in ['path', 'integrated', 'integrated-nc']:
- self.path_to_index = tf.contrib.lookup.HashTable(
- tf.contrib.lookup.KeyValueTensorInitializer(
- tf.constant(path_to_index.keys()),
- tf.constant(path_to_index.values()),
- key_dtype=tf.string, value_dtype=tf.int32), 0)
-
- self.path_dim = self.path_embeddings.shape[1]
-
- # Create the network
- self.__create_computation_graph__()
-
- def __create_computation_graph__(self):
- """Initialize the model and define the graph."""
- network_input = 0
-
- # Define the network inputs
- # Distributional x and y
- if self.hparams.input in ['dist', 'dist-nc', 'integrated', 'integrated-nc']:
- network_input += 2 * self.relata_dim
- self.relata_lookup = tf.get_variable(
- 'relata_lookup',
- initializer=self.relata_embeddings,
- dtype=tf.float32,
- trainable=self.hparams.learn_relata)
-
- # Path-based
- if self.hparams.input in ['path', 'integrated', 'integrated-nc']:
- network_input += self.path_dim
-
- self.path_initial_value_t = tf.placeholder(tf.float32, None)
-
- self.path_lookup = tf.get_variable(
- name='path_lookup',
- dtype=tf.float32,
- trainable=False,
- shape=self.path_embeddings.shape)
-
- self.initialize_path_op = tf.assign(
- self.path_lookup, self.path_initial_value_t, validate_shape=False)
-
- # Distributional noun compound
- if self.hparams.input in ['dist-nc', 'integrated-nc']:
- network_input += self.relata_dim
-
- self.nc_initial_value_t = tf.placeholder(tf.float32, None)
-
- self.nc_lookup = tf.get_variable(
- name='nc_lookup',
- dtype=tf.float32,
- trainable=False,
- shape=self.nc_embeddings.shape)
-
- self.initialize_nc_op = tf.assign(
- self.nc_lookup, self.nc_initial_value_t, validate_shape=False)
-
- hidden_dim = network_input // 2
-
- # Define the MLP
- if self.hparams.hidden_layers == 0:
- self.weights1 = tf.get_variable(
- 'W1',
- shape=[network_input, self.hparams.num_classes],
- dtype=tf.float32)
- self.bias1 = tf.get_variable(
- 'b1',
- shape=[self.hparams.num_classes],
- dtype=tf.float32)
-
- elif self.hparams.hidden_layers == 1:
-
- self.weights1 = tf.get_variable(
- 'W1',
- shape=[network_input, hidden_dim],
- dtype=tf.float32)
- self.bias1 = tf.get_variable(
- 'b1',
- shape=[hidden_dim],
- dtype=tf.float32)
-
- self.weights2 = tf.get_variable(
- 'W2',
- shape=[hidden_dim, self.hparams.num_classes],
- dtype=tf.float32)
- self.bias2 = tf.get_variable(
- 'b2',
- shape=[self.hparams.num_classes],
- dtype=tf.float32)
-
- else:
- raise ValueError('Only 0 or 1 hidden layers are supported')
-
- # Define the variables
- self.instances = tf.placeholder(dtype=tf.string,
- shape=[self.hparams.batch_size])
-
- (self.x_embedding_id,
- self.y_embedding_id,
- self.nc_embedding_id,
- self.path_embedding_id,
- self.path_counts,
- self.labels) = parse_tensorflow_examples(
- self.instances, self.hparams.batch_size, self.path_to_index)
-
- # Create the MLP
- self.__mlp__()
-
- self.instances_to_load = tf.placeholder(dtype=tf.string, shape=[None])
- self.labels_to_load = lexnet_common.load_all_labels(self.instances_to_load)
- self.pairs_to_load = lexnet_common.load_all_pairs(self.instances_to_load)
-
- def load_labels(self, session, instances):
- """Loads the labels for these instances.
-
- Args:
- session: The current TensorFlow session,
- instances: The instances for which to load the labels.
-
- Returns:
- the labels of these instances.
- """
- return session.run(self.labels_to_load,
- feed_dict={self.instances_to_load: instances})
-
- def load_pairs(self, session, instances):
- """Loads the word pairs for these instances.
-
- Args:
- session: The current TensorFlow session,
- instances: The instances for which to load the labels.
-
- Returns:
- the word pairs of these instances.
- """
- word_pairs = session.run(self.pairs_to_load,
- feed_dict={self.instances_to_load: instances})
- return [pair[0].split('::') for pair in word_pairs]
-
- def __train_single_batch__(self, session, batch_instances):
- """Train a single batch.
-
- Args:
- session: The current TensorFlow session.
- batch_instances: TensorFlow examples containing the training intances
-
- Returns:
- The cost for the current batch.
- """
- cost, _ = session.run([self.cost, self.train_op],
- feed_dict={self.instances: batch_instances})
-
- return cost
-
- def fit(self, session, inputs, on_epoch_completed, val_instances, val_labels,
- save_path):
- """Train the model.
-
- Args:
- session: The current TensorFlow session.
- inputs:
- on_epoch_completed: A method to call after each epoch.
- val_instances: The validation set instances (evaluation between epochs).
- val_labels: The validation set labels (for evaluation between epochs).
- save_path: Where to save the model.
- """
- for epoch in range(self.hparams.num_epochs):
-
- losses = []
- epoch_indices = list(np.random.permutation(len(inputs)))
-
- # If the number of instances doesn't divide by batch_size, enlarge it
- # by duplicating training examples
- mod = len(epoch_indices) % self.hparams.batch_size
- if mod > 0:
- epoch_indices.extend([np.random.randint(0, high=len(inputs))] * mod)
-
- # Define the batches
- n_batches = len(epoch_indices) // self.hparams.batch_size
-
- for minibatch in range(n_batches):
-
- batch_indices = epoch_indices[minibatch * self.hparams.batch_size:(
- minibatch + 1) * self.hparams.batch_size]
- batch_instances = [inputs[i] for i in batch_indices]
-
- loss = self.__train_single_batch__(session, batch_instances)
- losses.append(loss)
-
- epoch_loss = np.nanmean(losses)
-
- if on_epoch_completed:
- should_stop = on_epoch_completed(self, session, epoch, epoch_loss,
- val_instances, val_labels, save_path)
- if should_stop:
- print('Stopping training after %d epochs.' % epoch)
- return
-
- def predict(self, session, inputs):
- """Predict the classification of the test set.
-
- Args:
- session: The current TensorFlow session.
- inputs: the train paths, x, y and/or nc vectors
-
- Returns:
- The test predictions.
- """
- predictions, _ = zip(*self.predict_with_score(session, inputs))
- return np.array(predictions)
-
- def predict_with_score(self, session, inputs):
- """Predict the classification of the test set.
-
- Args:
- session: The current TensorFlow session.
- inputs: the test paths, x, y and/or nc vectors
-
- Returns:
- The test predictions along with their scores.
- """
- test_pred = [0] * len(inputs)
-
- for chunk in xrange(0, len(test_pred), self.hparams.batch_size):
-
- # Initialize the variables with the current batch data
- batch_indices = list(
- range(chunk, min(chunk + self.hparams.batch_size, len(test_pred))))
-
- # If the batch is too small, add a few other examples
- if len(batch_indices) < self.hparams.batch_size:
- batch_indices += [0] * (self.hparams.batch_size-len(batch_indices))
-
- batch_instances = [inputs[i] for i in batch_indices]
-
- predictions, scores = session.run(
- [self.predictions, self.scores],
- feed_dict={self.instances: batch_instances})
-
- for index_in_batch, index_in_dataset in enumerate(batch_indices):
- prediction = predictions[index_in_batch]
- score = scores[index_in_batch][prediction]
- test_pred[index_in_dataset] = (prediction, score)
-
- return test_pred
-
- def __mlp__(self):
- """Performs the MLP operations.
-
- Returns: the prediction object to be computed in a Session
- """
- # Define the operations
-
- # Network input
- vec_inputs = []
-
- # Distributional component
- if self.hparams.input in ['dist', 'dist-nc', 'integrated', 'integrated-nc']:
- for emb_id in [self.x_embedding_id, self.y_embedding_id]:
- vec_inputs.append(tf.nn.embedding_lookup(self.relata_lookup, emb_id))
-
- # Noun compound component
- if self.hparams.input in ['dist-nc', 'integrated-nc']:
- vec = tf.nn.embedding_lookup(self.nc_lookup, self.nc_embedding_id)
- vec_inputs.append(vec)
-
- # Path-based component
- if self.hparams.input in ['path', 'integrated', 'integrated-nc']:
-
- # Get the current paths for each batch instance
- self.path_embeddings = tf.nn.embedding_lookup(self.path_lookup,
- self.path_embedding_id)
-
- # self.path_embeddings is of shape
- # [batch_size, max_path_per_instance, output_dim]
- # We need to multiply it by path counts
- # ([batch_size, max_path_per_instance]).
- # Start by duplicating path_counts along the output_dim axis.
- self.path_freq = tf.tile(tf.expand_dims(self.path_counts, -1),
- [1, 1, self.path_dim])
-
- # Compute the averaged path vector for each instance.
- # First, multiply the path embeddings and frequencies element-wise.
- self.weighted = tf.multiply(self.path_freq, self.path_embeddings)
-
- # Second, take the sum to get a tensor of shape [batch_size, output_dim].
- self.pair_path_embeddings = tf.reduce_sum(self.weighted, 1)
-
- # Finally, divide by the total number of paths.
- # The number of paths for each pair has a shape [batch_size, 1],
- # We duplicate it output_dim times along the second axis.
- self.num_paths = tf.clip_by_value(
- tf.reduce_sum(self.path_counts, 1), 1, np.inf)
- self.num_paths = tf.tile(tf.expand_dims(self.num_paths, -1),
- [1, self.path_dim])
-
- # And finally, divide pair_path_embeddings by num_paths element-wise.
- self.pair_path_embeddings = tf.div(
- self.pair_path_embeddings, self.num_paths)
- vec_inputs.append(self.pair_path_embeddings)
-
- # Concatenate the inputs and feed to the MLP
- self.input_vec = tf.nn.dropout(
- tf.concat(vec_inputs, 1),
- keep_prob=self.hparams.input_keep_prob)
-
- h = tf.matmul(self.input_vec, self.weights1)
- self.output = h
-
- if self.hparams.hidden_layers == 1:
- self.output = tf.matmul(tf.nn.tanh(h), self.weights2)
-
- self.scores = self.output
- self.predictions = tf.argmax(self.scores, axis=1)
-
- # Define the loss function and the optimization algorithm
- self.cross_entropies = tf.nn.sparse_softmax_cross_entropy_with_logits(
- logits=self.scores, labels=self.labels)
- self.cost = tf.reduce_sum(self.cross_entropies, name='cost')
- self.global_step = tf.Variable(0, name='global_step', trainable=False)
- self.optimizer = tf.train.AdamOptimizer()
- self.train_op = self.optimizer.minimize(
- self.cost, global_step=self.global_step)
-
-
-def parse_tensorflow_examples(record, batch_size, path_to_index):
- """Reads TensorFlow examples from a RecordReader.
-
- Args:
- record: a record with TensorFlow examples.
- batch_size: the number of instances in a minibatch
- path_to_index: mapping from string path to index in the embeddings matrix.
-
- Returns:
- The word embeddings IDs, paths and counts
- """
- features = tf.parse_example(
- record, {
- 'x_embedding_id': tf.FixedLenFeature([1], dtype=tf.int64),
- 'y_embedding_id': tf.FixedLenFeature([1], dtype=tf.int64),
- 'nc_embedding_id': tf.FixedLenFeature([1], dtype=tf.int64),
- 'reprs': tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.string, allow_missing=True),
- 'counts': tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.int64, allow_missing=True),
- 'rel_id': tf.FixedLenFeature([1], dtype=tf.int64)
- })
-
- x_embedding_id = tf.squeeze(features['x_embedding_id'], [-1])
- y_embedding_id = tf.squeeze(features['y_embedding_id'], [-1])
- nc_embedding_id = tf.squeeze(features['nc_embedding_id'], [-1])
- labels = tf.squeeze(features['rel_id'], [-1])
- path_counts = tf.to_float(tf.reshape(features['counts'], [batch_size, -1]))
-
- path_embedding_id = None
- if path_to_index:
- path_embedding_id = path_to_index.lookup(features['reprs'])
-
- return (
- x_embedding_id, y_embedding_id, nc_embedding_id,
- path_embedding_id, path_counts, labels)
diff --git a/research/lexnet_nc/path_model.py b/research/lexnet_nc/path_model.py
deleted file mode 100644
index c283841775d673baa8a4bc8c438d65f288a2c555..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/path_model.py
+++ /dev/null
@@ -1,547 +0,0 @@
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""LexNET Path-based Model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import itertools
-import os
-
-import lexnet_common
-import numpy as np
-import tensorflow as tf
-
-
-class PathBasedModel(object):
- """The LexNET path-based model for classifying semantic relations."""
-
- @classmethod
- def default_hparams(cls):
- """Returns the default hyper-parameters."""
- return tf.contrib.training.HParams(
- max_path_len=8,
- num_classes=37,
- num_epochs=30,
- input_keep_prob=0.9,
- learning_rate=0.001,
- learn_lemmas=False,
- random_seed=133, # zero means no random seed
- lemma_embeddings_file='glove/glove.6B.50d.bin',
- num_pos=len(lexnet_common.POSTAGS),
- num_dep=len(lexnet_common.DEPLABELS),
- num_directions=len(lexnet_common.DIRS),
- lemma_dim=50,
- pos_dim=4,
- dep_dim=5,
- dir_dim=1)
-
- def __init__(self, hparams, lemma_embeddings, instance):
- """Initialize the LexNET classifier.
-
- Args:
- hparams: the hyper-parameters.
- lemma_embeddings: word embeddings for the path-based component.
- instance: string tensor containing the input instance
- """
- self.hparams = hparams
- self.lemma_embeddings = lemma_embeddings
- self.instance = instance
- self.vocab_size, self.lemma_dim = self.lemma_embeddings.shape
-
- # Set the random seed
- if hparams.random_seed > 0:
- tf.set_random_seed(hparams.random_seed)
-
- # Create the network
- self.__create_computation_graph__()
-
- def __create_computation_graph__(self):
- """Initialize the model and define the graph."""
- self.lstm_input_dim = sum([self.hparams.lemma_dim, self.hparams.pos_dim,
- self.hparams.dep_dim, self.hparams.dir_dim])
- self.lstm_output_dim = self.lstm_input_dim
-
- network_input = self.lstm_output_dim
- self.lemma_lookup = tf.get_variable(
- 'lemma_lookup',
- initializer=self.lemma_embeddings,
- dtype=tf.float32,
- trainable=self.hparams.learn_lemmas)
- self.pos_lookup = tf.get_variable(
- 'pos_lookup',
- shape=[self.hparams.num_pos, self.hparams.pos_dim],
- dtype=tf.float32)
- self.dep_lookup = tf.get_variable(
- 'dep_lookup',
- shape=[self.hparams.num_dep, self.hparams.dep_dim],
- dtype=tf.float32)
- self.dir_lookup = tf.get_variable(
- 'dir_lookup',
- shape=[self.hparams.num_directions, self.hparams.dir_dim],
- dtype=tf.float32)
-
- self.weights1 = tf.get_variable(
- 'W1',
- shape=[network_input, self.hparams.num_classes],
- dtype=tf.float32)
- self.bias1 = tf.get_variable(
- 'b1',
- shape=[self.hparams.num_classes],
- dtype=tf.float32)
-
- # Define the variables
- (self.batch_paths,
- self.path_counts,
- self.seq_lengths,
- self.path_strings,
- self.batch_labels) = _parse_tensorflow_example(
- self.instance, self.hparams.max_path_len, self.hparams.input_keep_prob)
-
- # Create the LSTM
- self.__lstm__()
-
- # Create the MLP
- self.__mlp__()
-
- self.instances_to_load = tf.placeholder(dtype=tf.string, shape=[None])
- self.labels_to_load = lexnet_common.load_all_labels(self.instances_to_load)
-
- def load_labels(self, session, batch_instances):
- """Loads the labels of the current instances.
-
- Args:
- session: the current TensorFlow session.
- batch_instances: the dataset instances.
-
- Returns:
- the labels.
- """
- return session.run(self.labels_to_load,
- feed_dict={self.instances_to_load: batch_instances})
-
- def run_one_epoch(self, session, num_steps):
- """Train the model.
-
- Args:
- session: The current TensorFlow session.
- num_steps: The number of steps in each epoch.
-
- Returns:
- The mean loss for the epoch.
-
- Raises:
- ArithmeticError: if the loss becomes non-finite.
- """
- losses = []
-
- for step in range(num_steps):
- curr_loss, _ = session.run([self.cost, self.train_op])
- if not np.isfinite(curr_loss):
- raise ArithmeticError('nan loss at step %d' % step)
-
- losses.append(curr_loss)
-
- return np.mean(losses)
-
- def predict(self, session, inputs):
- """Predict the classification of the test set.
-
- Args:
- session: The current TensorFlow session.
- inputs: the train paths, x, y and/or nc vectors
-
- Returns:
- The test predictions.
- """
- predictions, _ = zip(*self.predict_with_score(session, inputs))
- return np.array(predictions)
-
- def predict_with_score(self, session, inputs):
- """Predict the classification of the test set.
-
- Args:
- session: The current TensorFlow session.
- inputs: the test paths, x, y and/or nc vectors
-
- Returns:
- The test predictions along with their scores.
- """
- test_pred = [0] * len(inputs)
-
- for index, instance in enumerate(inputs):
-
- prediction, scores = session.run(
- [self.predictions, self.scores],
- feed_dict={self.instance: instance})
-
- test_pred[index] = (prediction, scores[prediction])
-
- return test_pred
-
- def __mlp__(self):
- """Performs the MLP operations.
-
- Returns: the prediction object to be computed in a Session
- """
- # Feed the paths to the MLP: path_embeddings is
- # [num_batch_paths, output_dim], and when we multiply it by W
- # ([output_dim, num_classes]), we get a matrix of class distributions:
- # [num_batch_paths, num_classes].
- self.distributions = tf.matmul(self.path_embeddings, self.weights1)
-
- # Now, compute weighted average on the class distributions, using the path
- # frequency as weights.
-
- # First, reshape path_freq to the same shape of distributions
- self.path_freq = tf.tile(tf.expand_dims(self.path_counts, -1),
- [1, self.hparams.num_classes])
-
- # Second, multiply the distributions and frequencies element-wise.
- self.weighted = tf.multiply(self.path_freq, self.distributions)
-
- # Finally, take the average to get a tensor of shape [1, num_classes].
- self.weighted_sum = tf.reduce_sum(self.weighted, 0)
- self.num_paths = tf.clip_by_value(tf.reduce_sum(self.path_counts),
- 1, np.inf)
- self.num_paths = tf.tile(tf.expand_dims(self.num_paths, -1),
- [self.hparams.num_classes])
- self.scores = tf.div(self.weighted_sum, self.num_paths)
- self.predictions = tf.argmax(self.scores)
-
- # Define the loss function and the optimization algorithm
- self.cross_entropies = tf.nn.sparse_softmax_cross_entropy_with_logits(
- logits=self.scores, labels=tf.reduce_mean(self.batch_labels))
- self.cost = tf.reduce_sum(self.cross_entropies, name='cost')
- self.global_step = tf.Variable(0, name='global_step', trainable=False)
- self.optimizer = tf.train.AdamOptimizer()
- self.train_op = self.optimizer.minimize(self.cost,
- global_step=self.global_step)
-
- def __lstm__(self):
- """Defines the LSTM operations.
-
- Returns:
- A matrix of path embeddings.
- """
- lookup_tables = [self.lemma_lookup, self.pos_lookup,
- self.dep_lookup, self.dir_lookup]
-
- # Split the edges to components: list of 4 tensors
- # [num_batch_paths, max_path_len, 1]
- self.edge_components = tf.split(self.batch_paths, 4, axis=2)
-
- # Look up the components embeddings and concatenate them back together
- self.path_matrix = tf.concat([
- tf.squeeze(tf.nn.embedding_lookup(lookup_table, component), 2)
- for lookup_table, component in
- zip(lookup_tables, self.edge_components)
- ], axis=2)
-
- self.sequence_lengths = tf.reshape(self.seq_lengths, [-1])
-
- # Define the LSTM.
- # The input is [num_batch_paths, max_path_len, input_dim].
- lstm_cell = tf.contrib.rnn.BasicLSTMCell(self.lstm_output_dim)
-
- # The output is [num_batch_paths, max_path_len, output_dim].
- self.lstm_outputs, _ = tf.nn.dynamic_rnn(
- lstm_cell, self.path_matrix, dtype=tf.float32,
- sequence_length=self.sequence_lengths)
-
- # Slice the last *relevant* output for each instance ->
- # [num_batch_paths, output_dim]
- self.path_embeddings = _extract_last_relevant(self.lstm_outputs,
- self.sequence_lengths)
-
-
-def _parse_tensorflow_example(record, max_path_len, input_keep_prob):
- """Reads TensorFlow examples from a RecordReader.
-
- Args:
- record: a record with TensorFlow example.
- max_path_len: the maximum path length.
- input_keep_prob: 1 - the word dropout probability
-
- Returns:
- The paths and counts
- """
- features = tf.parse_single_example(record, {
- 'lemmas':
- tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.int64, allow_missing=True),
- 'postags':
- tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.int64, allow_missing=True),
- 'deplabels':
- tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.int64, allow_missing=True),
- 'dirs':
- tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.int64, allow_missing=True),
- 'counts':
- tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.int64, allow_missing=True),
- 'pathlens':
- tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.int64, allow_missing=True),
- 'reprs':
- tf.FixedLenSequenceFeature(
- shape=(), dtype=tf.string, allow_missing=True),
- 'rel_id':
- tf.FixedLenFeature([], dtype=tf.int64)
- })
-
- path_counts = tf.to_float(features['counts'])
- seq_lengths = features['pathlens']
-
- # Concatenate the edge components to create a path tensor:
- # [max_paths_per_ins, max_path_length, 4]
- lemmas = _word_dropout(
- tf.reshape(features['lemmas'], [-1, max_path_len]), input_keep_prob)
-
- paths = tf.stack(
- [lemmas] + [
- tf.reshape(features[f], [-1, max_path_len])
- for f in ('postags', 'deplabels', 'dirs')
- ],
- axis=-1)
-
- path_strings = features['reprs']
-
- # Add an empty path to pairs with no paths
- paths = tf.cond(
- tf.shape(paths)[0] > 0,
- lambda: paths,
- lambda: tf.zeros([1, max_path_len, 4], dtype=tf.int64))
-
- # Paths are left-padded. We reverse them to make them right-padded.
- #paths = tf.reverse(paths, axis=[1])
-
- path_counts = tf.cond(
- tf.shape(path_counts)[0] > 0,
- lambda: path_counts,
- lambda: tf.constant([1.0], dtype=tf.float32))
-
- seq_lengths = tf.cond(
- tf.shape(seq_lengths)[0] > 0,
- lambda: seq_lengths,
- lambda: tf.constant([1], dtype=tf.int64))
-
- # Duplicate the label for each path
- labels = tf.ones_like(path_counts, dtype=tf.int64) * features['rel_id']
-
- return paths, path_counts, seq_lengths, path_strings, labels
-
-
-def _extract_last_relevant(output, seq_lengths):
- """Get the last relevant LSTM output cell for each batch instance.
-
- Args:
- output: the LSTM outputs - a tensor with shape
- [num_paths, output_dim, max_path_len]
- seq_lengths: the sequences length per instance
-
- Returns:
- The last relevant LSTM output cell for each batch instance.
- """
- max_length = int(output.get_shape()[1])
- path_lengths = tf.clip_by_value(seq_lengths - 1, 0, max_length)
- relevant = tf.reduce_sum(tf.multiply(output, tf.expand_dims(
- tf.one_hot(path_lengths, max_length), -1)), 1)
- return relevant
-
-
-def _word_dropout(words, input_keep_prob):
- """Drops words with probability 1 - input_keep_prob.
-
- Args:
- words: a list of lemmas from the paths.
- input_keep_prob: the probability to keep the word.
-
- Returns:
- The revised list where some of the words are ed.
- """
- # Create the mask: (-1) to drop, 1 to keep
- prob = tf.random_uniform(tf.shape(words), 0, 1)
- condition = tf.less(prob, (1 - input_keep_prob))
- mask = tf.where(condition,
- tf.negative(tf.ones_like(words)), tf.ones_like(words))
-
- # We need to keep zeros (), and change other numbers to 1 ()
- # if their mask is -1. First, we multiply the mask and the words.
- # Zeros will stay zeros, and words to drop will become negative.
- # Then, we change negative values to 1.
- masked_words = tf.multiply(mask, words)
- condition = tf.less(masked_words, 0)
- dropped_words = tf.where(condition, tf.ones_like(words), words)
- return dropped_words
-
-
-def compute_path_embeddings(model, session, instances):
- """Compute the path embeddings for all the distinct paths.
-
- Args:
- model: The trained path-based model.
- session: The current TensorFlow session.
- instances: All the train, test and validation instances.
-
- Returns:
- The path to ID index and the path embeddings.
- """
- # Get an index for each distinct path
- path_index = collections.defaultdict(itertools.count(0).next)
- path_vectors = {}
-
- for instance in instances:
- curr_path_embeddings, curr_path_strings = session.run(
- [model.path_embeddings, model.path_strings],
- feed_dict={model.instance: instance})
-
- for i, path in enumerate(curr_path_strings):
- if not path:
- continue
-
- # Set a new/existing index for the path
- index = path_index[path]
-
- # Save its vector
- path_vectors[index] = curr_path_embeddings[i, :]
-
- print('Number of distinct paths: %d' % len(path_index))
- return path_index, path_vectors
-
-
-def save_path_embeddings(model, path_vectors, path_index, embeddings_base_path):
- """Saves the path embeddings.
-
- Args:
- model: The trained path-based model.
- path_vectors: The path embeddings.
- path_index: A map from path to ID.
- embeddings_base_path: The base directory where the embeddings are.
- """
- index_range = range(max(path_index.values()) + 1)
- path_matrix = [path_vectors[i] for i in index_range]
- path_matrix = np.vstack(path_matrix)
-
- # Save the path embeddings
- path_vector_filename = os.path.join(
- embeddings_base_path, '%d_path_vectors' % model.lstm_output_dim)
- with open(path_vector_filename, 'w') as f_out:
- np.save(f_out, path_matrix)
-
- index_to_path = {i: p for p, i in path_index.iteritems()}
- path_vocab = [index_to_path[i] for i in index_range]
-
- # Save the path vocabulary
- path_vocab_filename = os.path.join(
- embeddings_base_path, '%d_path_vocab' % model.lstm_output_dim)
- with open(path_vocab_filename, 'w') as f_out:
- f_out.write('\n'.join(path_vocab))
- f_out.write('\n')
-
- print('Saved path embeddings.')
-
-
-def load_path_embeddings(path_embeddings_dir, path_dim):
- """Loads pretrained path embeddings from a binary file and returns the matrix.
-
- Args:
- path_embeddings_dir: The directory for the path embeddings.
- path_dim: The dimension of the path embeddings, used as prefix to the
- path_vocab and path_vectors files.
-
- Returns:
- The path embeddings matrix and the path_to_index dictionary.
- """
- prefix = path_embeddings_dir + '/%d' % path_dim + '_'
- with open(prefix + 'path_vocab') as f_in:
- vocab = f_in.read().splitlines()
-
- vocab_size = len(vocab)
- embedding_file = prefix + 'path_vectors'
-
- print('Embedding file "%s" has %d paths' % (embedding_file, vocab_size))
-
- with open(embedding_file) as f_in:
- embeddings = np.load(f_in)
-
- path_to_index = {p: i for i, p in enumerate(vocab)}
- return embeddings, path_to_index
-
-
-def get_indicative_paths(model, session, path_index, path_vectors, classes,
- save_dir, k=20, threshold=0.8):
- """Gets the most indicative paths for each class.
-
- Args:
- model: The trained path-based model.
- session: The current TensorFlow session.
- path_index: A map from path to ID.
- path_vectors: The path embeddings.
- classes: The class label names.
- save_dir: Where to save the paths.
- k: The k for top-k paths.
- threshold: The threshold above which to consider paths as indicative.
- """
- # Define graph variables for this operation
- p_path_embedding = tf.placeholder(dtype=tf.float32,
- shape=[1, model.lstm_output_dim])
- p_distributions = tf.nn.softmax(tf.matmul(p_path_embedding, model.weights1))
-
- # Treat each path as a pair instance with a single path, and get the
- # relation distribution for it. Then, take the top paths for each relation.
-
- # This dictionary contains a relation as a key, and the value is a list of
- # tuples of path index and score. A relation r will contain (p, s) if the
- # path p is classified to r with a confidence of s.
- prediction_per_relation = collections.defaultdict(list)
-
- index_to_path = {i: p for p, i in path_index.iteritems()}
-
- # Predict all the paths
- for index in range(len(path_index)):
- curr_path_vector = path_vectors[index]
-
- distribution = session.run(p_distributions,
- feed_dict={
- p_path_embedding: np.reshape(
- curr_path_vector,
- [1, model.lstm_output_dim])})
-
- distribution = distribution[0, :]
- prediction = np.argmax(distribution)
- prediction_per_relation[prediction].append(
- (index, distribution[prediction]))
-
- if index % 10000 == 0:
- print('Classified %d/%d (%3.2f%%) of the paths' % (
- index, len(path_index), 100 * index / len(path_index)))
-
- # Retrieve k-best scoring paths for each relation
- for relation_index, relation in enumerate(classes):
- curr_paths = sorted(prediction_per_relation[relation_index],
- key=lambda item: item[1], reverse=True)
- above_t = [(p, s) for (p, s) in curr_paths if s >= threshold]
- top_k = curr_paths[k+1]
- relation_paths = above_t if len(above_t) > len(top_k) else top_k
-
- paths_filename = os.path.join(save_dir, '%s.paths' % relation)
- with open(paths_filename, 'w') as f_out:
- for index, score in relation_paths:
- print('\t'.join([index_to_path[index], str(score)]), file=f_out)
diff --git a/research/lexnet_nc/sorted_paths_to_examples.py b/research/lexnet_nc/sorted_paths_to_examples.py
deleted file mode 100755
index c21d25d710ae793f6eefd889b98414c923e4fbe6..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/sorted_paths_to_examples.py
+++ /dev/null
@@ -1,202 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Takes as input a sorted, tab-separated of paths to produce tf.Examples."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import itertools
-import os
-import sys
-import tensorflow as tf
-
-import lexnet_common
-
-tf.flags.DEFINE_string('input', '', 'tab-separated input data')
-tf.flags.DEFINE_string('vocab', '', 'a text file containing lemma vocabulary')
-tf.flags.DEFINE_string('relations', '', 'a text file containing the relations')
-tf.flags.DEFINE_string('output_dir', '', 'output directory')
-tf.flags.DEFINE_string('splits', '', 'text file enumerating splits')
-tf.flags.DEFINE_string('default_split', '', 'default split for unlabeled pairs')
-tf.flags.DEFINE_string('compression', 'GZIP', 'compression for output records')
-tf.flags.DEFINE_integer('max_paths', 100, 'maximum number of paths per record')
-tf.flags.DEFINE_integer('max_pathlen', 8, 'maximum path length')
-FLAGS = tf.flags.FLAGS
-
-
-def _int64_features(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
-
-
-def _bytes_features(value):
- value = [v.encode('utf-8') if isinstance(v, unicode) else v for v in value]
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
-
-
-class CreateExampleFn(object):
-
- def __init__(self):
- # Read the vocabulary. N.B. that 0 = PAD, 1 = UNK, 2 = , 3 = , hence
- # the enumeration starting at 4.
- with tf.gfile.GFile(FLAGS.vocab) as fh:
- self.vocab = {w: ix for ix, w in enumerate(fh.read().splitlines(), start=4)}
-
- self.vocab.update({'': 0, '': 1, '': 2, '': 3})
-
- # Read the relations.
- with tf.gfile.GFile(FLAGS.relations) as fh:
- self.relations = {r: ix for ix, r in enumerate(fh.read().splitlines())}
-
- # Some hackery to map from SpaCy postags to Google's.
- lexnet_common.POSTAG_TO_ID['PROPN'] = lexnet_common.POSTAG_TO_ID['NOUN']
- lexnet_common.POSTAG_TO_ID['PRON'] = lexnet_common.POSTAG_TO_ID['NOUN']
- lexnet_common.POSTAG_TO_ID['CCONJ'] = lexnet_common.POSTAG_TO_ID['CONJ']
- #lexnet_common.DEPLABEL_TO_ID['relcl'] = lexnet_common.DEPLABEL_TO_ID['rel']
- #lexnet_common.DEPLABEL_TO_ID['compound'] = lexnet_common.DEPLABEL_TO_ID['xcomp']
- #lexnet_common.DEPLABEL_TO_ID['oprd'] = lexnet_common.DEPLABEL_TO_ID['UNK']
-
- def __call__(self, mod, head, rel, raw_paths):
- # Drop any really long paths.
- paths = []
- counts = []
- for raw, count in raw_paths.most_common(FLAGS.max_paths):
- path = raw.split('::')
- if len(path) <= FLAGS.max_pathlen:
- paths.append(path)
- counts.append(count)
-
- if not paths:
- return None
-
- # Compute the true length.
- pathlens = [len(path) for path in paths]
-
- # Pad each path out to max_pathlen so the LSTM can eat it.
- paths = (
- itertools.islice(
- itertools.chain(path, itertools.repeat('/PAD/PAD/_')),
- FLAGS.max_pathlen)
- for path in paths)
-
- # Split the lemma, POS, dependency label, and direction each into a
- # separate feature.
- lemmas, postags, deplabels, dirs = zip(
- *(part.split('/') for part in itertools.chain(*paths)))
-
- lemmas = [self.vocab.get(lemma, 1) for lemma in lemmas]
- postags = [lexnet_common.POSTAG_TO_ID[pos] for pos in postags]
- deplabels = [lexnet_common.DEPLABEL_TO_ID.get(dep, 1) for dep in deplabels]
- dirs = [lexnet_common.DIR_TO_ID.get(d, 0) for d in dirs]
-
- return tf.train.Example(features=tf.train.Features(feature={
- 'pair': _bytes_features(['::'.join((mod, head))]),
- 'rel': _bytes_features([rel]),
- 'rel_id': _int64_features([self.relations[rel]]),
- 'reprs': _bytes_features(raw_paths),
- 'pathlens': _int64_features(pathlens),
- 'counts': _int64_features(counts),
- 'lemmas': _int64_features(lemmas),
- 'dirs': _int64_features(dirs),
- 'deplabels': _int64_features(deplabels),
- 'postags': _int64_features(postags),
- 'x_embedding_id': _int64_features([self.vocab[mod]]),
- 'y_embedding_id': _int64_features([self.vocab[head]]),
- }))
-
-
-def main(_):
- # Read the splits file, if there is one.
- assignments = {}
- if FLAGS.splits:
- with tf.gfile.GFile(FLAGS.splits) as fh:
- parts = (line.split('\t') for line in fh.read().splitlines())
- assignments = {(mod, head): split for mod, head, split in parts}
-
- splits = set(assignments.itervalues())
- if FLAGS.default_split:
- default_split = FLAGS.default_split
- splits.add(FLAGS.default_split)
- elif splits:
- default_split = iter(splits).next()
- else:
- print('Please specify --splits, --default_split, or both', file=sys.stderr)
- return 1
-
- last_mod, last_head, last_label = None, None, None
- raw_paths = collections.Counter()
-
- # Keep track of pairs we've seen to ensure that we don't get unsorted data.
- seen_labeled_pairs = set()
-
- # Set up output compression
- compression_type = getattr(
- tf.python_io.TFRecordCompressionType, FLAGS.compression)
- options = tf.python_io.TFRecordOptions(compression_type=compression_type)
-
- writers = {
- split: tf.python_io.TFRecordWriter(
- os.path.join(FLAGS.output_dir, '%s.tfrecs.gz' % split),
- options=options)
- for split in splits}
-
- create_example = CreateExampleFn()
-
- in_fh = sys.stdin if not FLAGS.input else tf.gfile.GFile(FLAGS.input)
- for lineno, line in enumerate(in_fh, start=1):
- if lineno % 100 == 0:
- print('\rProcessed %d lines...' % lineno, end='', file=sys.stderr)
-
- parts = line.decode('utf-8').strip().split('\t')
- if len(parts) != 5:
- print('Skipping line %d: %d columns (expected 5)' % (
- lineno, len(parts)), file=sys.stderr)
-
- continue
-
- mod, head, label, raw_path, source = parts
- if mod == last_mod and head == last_head and label == last_label:
- raw_paths.update([raw_path])
- continue
-
- if last_mod and last_head and last_label and raw_paths:
- if (last_mod, last_head, last_label) in seen_labeled_pairs:
- print('It looks like the input data is not sorted; ignoring extra '
- 'record for (%s::%s, %s) at line %d' % (
- last_mod, last_head, last_label, lineno))
- else:
- ex = create_example(last_mod, last_head, last_label, raw_paths)
- if ex:
- split = assignments.get((last_mod, last_head), default_split)
- writers[split].write(ex.SerializeToString())
-
- seen_labeled_pairs.add((last_mod, last_head, last_label))
-
- last_mod, last_head, last_label = mod, head, label
- raw_paths = collections.Counter()
-
- if last_mod and last_head and last_label and raw_paths:
- ex = create_example(last_mod, last_head, last_label, raw_paths)
- if ex:
- split = assignments.get((last_mod, last_head), default_split)
- writers[split].write(ex.SerializeToString())
-
- for writer in writers.itervalues():
- writer.close()
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/lexnet_nc/text_embeddings_to_binary.py b/research/lexnet_nc/text_embeddings_to_binary.py
deleted file mode 100755
index 8226a7654e6da733ba1e8c46810a8ec8afd7a2c0..0000000000000000000000000000000000000000
--- a/research/lexnet_nc/text_embeddings_to_binary.py
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017, 2018 Google, Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Converts a text embedding file into a binary format for quicker loading."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-tf.flags.DEFINE_string('input', '', 'text file containing embeddings')
-tf.flags.DEFINE_string('output_vocab', '', 'output file for vocabulary')
-tf.flags.DEFINE_string('output_npy', '', 'output file for binary')
-FLAGS = tf.flags.FLAGS
-
-def main(_):
- vecs = []
- vocab = []
- with tf.gfile.GFile(FLAGS.input) as fh:
- for line in fh:
- parts = line.strip().split()
- vocab.append(parts[0])
- vecs.append([float(x) for x in parts[1:]])
-
- with tf.gfile.GFile(FLAGS.output_vocab, 'w') as fh:
- fh.write('\n'.join(vocab))
- fh.write('\n')
-
- vecs = np.array(vecs, dtype=np.float32)
- np.save(FLAGS.output_npy, vecs, allow_pickle=False)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/lm_1b/BUILD b/research/lm_1b/BUILD
deleted file mode 100644
index ca5bc1f6ce4347a3b5f18d1bb59284aa9d07a567..0000000000000000000000000000000000000000
--- a/research/lm_1b/BUILD
+++ /dev/null
@@ -1,27 +0,0 @@
-package(default_visibility = [":internal"])
-
-licenses(["notice"]) # Apache 2.0
-
-exports_files(["LICENSE"])
-
-package_group(
- name = "internal",
- packages = [
- "//lm_1b/...",
- ],
-)
-
-py_library(
- name = "data_utils",
- srcs = ["data_utils.py"],
-)
-
-py_binary(
- name = "lm_1b_eval",
- srcs = [
- "lm_1b_eval.py",
- ],
- deps = [
- ":data_utils",
- ],
-)
diff --git a/research/lm_1b/README.md b/research/lm_1b/README.md
deleted file mode 100644
index f48afbfe23aff6681e641296e73b2c6b0e5a9b48..0000000000000000000000000000000000000000
--- a/research/lm_1b/README.md
+++ /dev/null
@@ -1,198 +0,0 @@
-
-
-
-
-Language Model on One Billion Word Benchmark
-
-Authors:
-
-Oriol Vinyals (vinyals@google.com, github: OriolVinyals),
-Xin Pan
-
-Paper Authors:
-
-Rafal Jozefowicz, Oriol Vinyals, Mike Schuster, Noam Shazeer, Yonghui Wu
-
-TL;DR
-
-This is a pretrained model on One Billion Word Benchmark.
-If you use this model in your publication, please cite the original paper:
-
-@article{jozefowicz2016exploring,
- title={Exploring the Limits of Language Modeling},
- author={Jozefowicz, Rafal and Vinyals, Oriol and Schuster, Mike
- and Shazeer, Noam and Wu, Yonghui},
- journal={arXiv preprint arXiv:1602.02410},
- year={2016}
-}
-
-Introduction
-
-In this release, we open source a model trained on the One Billion Word
-Benchmark (http://arxiv.org/abs/1312.3005), a large language corpus in English
-which was released in 2013. This dataset contains about one billion words, and
-has a vocabulary size of about 800K words. It contains mostly news data. Since
-sentences in the training set are shuffled, models can ignore the context and
-focus on sentence level language modeling.
-
-In the original release and subsequent work, people have used the same test set
-to train models on this dataset as a standard benchmark for language modeling.
-Recently, we wrote an article (http://arxiv.org/abs/1602.02410) describing a
-model hybrid between character CNN, a large and deep LSTM, and a specific
-Softmax architecture which allowed us to train the best model on this dataset
-thus far, almost halving the best perplexity previously obtained by others.
-
-Code Release
-
-The open-sourced components include:
-
-* TensorFlow GraphDef proto buffer text file.
-* TensorFlow pre-trained checkpoint shards.
-* Code used to evaluate the pre-trained model.
-* Vocabulary file.
-* Test set from LM-1B evaluation.
-
-The code supports 4 evaluation modes:
-
-* Given provided dataset, calculate the model's perplexity.
-* Given a prefix sentence, predict the next words.
-* Dump the softmax embedding, character-level CNN word embeddings.
-* Give a sentence, dump the embedding from the LSTM state.
-
-Results
-
-Model | Test Perplexity | Number of Params [billions]
-------|-----------------|----------------------------
-Sigmoid-RNN-2048 [Blackout] | 68.3 | 4.1
-Interpolated KN 5-gram, 1.1B n-grams [chelba2013one] | 67.6 | 1.76
-Sparse Non-Negative Matrix LM [shazeer2015sparse] | 52.9 | 33
-RNN-1024 + MaxEnt 9-gram features [chelba2013one] | 51.3 | 20
-LSTM-512-512 | 54.1 | 0.82
-LSTM-1024-512 | 48.2 | 0.82
-LSTM-2048-512 | 43.7 | 0.83
-LSTM-8192-2048 (No Dropout) | 37.9 | 3.3
-LSTM-8192-2048 (50\% Dropout) | 32.2 | 3.3
-2-Layer LSTM-8192-1024 (BIG LSTM) | 30.6 | 1.8
-(THIS RELEASE) BIG LSTM+CNN Inputs | 30.0 | 1.04
-
-How To Run
-
-Prerequisites:
-
-* Install TensorFlow.
-* Install Bazel.
-* Download the data files:
- * Model GraphDef file:
- [link](http://download.tensorflow.org/models/LM_LSTM_CNN/graph-2016-09-10.pbtxt)
- * Model Checkpoint sharded file:
- [1](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-base)
- [2](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-char-embedding)
- [3](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-lstm)
- [4](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax0)
- [5](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax1)
- [6](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax2)
- [7](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax3)
- [8](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax4)
- [9](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax5)
- [10](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax6)
- [11](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax7)
- [12](http://download.tensorflow.org/models/LM_LSTM_CNN/all_shards-2016-09-10/ckpt-softmax8)
- * Vocabulary file:
- [link](http://download.tensorflow.org/models/LM_LSTM_CNN/vocab-2016-09-10.txt)
- * test dataset: link
- [link](http://download.tensorflow.org/models/LM_LSTM_CNN/test/news.en.heldout-00000-of-00050)
-* It is recommended to run on a modern desktop instead of a laptop.
-
-```shell
-# 1. Clone the code to your workspace.
-# 2. Download the data to your workspace.
-# 3. Create an empty WORKSPACE file in your workspace.
-# 4. Create an empty output directory in your workspace.
-# Example directory structure below:
-$ ls -R
-.:
-data lm_1b output WORKSPACE
-
-./data:
-ckpt-base ckpt-lstm ckpt-softmax1 ckpt-softmax3 ckpt-softmax5
-ckpt-softmax7 graph-2016-09-10.pbtxt vocab-2016-09-10.txt
-ckpt-char-embedding ckpt-softmax0 ckpt-softmax2 ckpt-softmax4 ckpt-softmax6
-ckpt-softmax8 news.en.heldout-00000-of-00050
-
-./lm_1b:
-BUILD data_utils.py lm_1b_eval.py README.md
-
-./output:
-
-# Build the codes.
-$ bazel build -c opt lm_1b/...
-# Run sample mode:
-$ bazel-bin/lm_1b/lm_1b_eval --mode sample \
- --prefix "I love that I" \
- --pbtxt data/graph-2016-09-10.pbtxt \
- --vocab_file data/vocab-2016-09-10.txt \
- --ckpt 'data/ckpt-*'
-...(omitted some TensorFlow output)
-I love
-I love that
-I love that I
-I love that I find
-I love that I find that
-I love that I find that amazing
-...(omitted)
-
-# Run eval mode:
-$ bazel-bin/lm_1b/lm_1b_eval --mode eval \
- --pbtxt data/graph-2016-09-10.pbtxt \
- --vocab_file data/vocab-2016-09-10.txt \
- --input_data data/news.en.heldout-00000-of-00050 \
- --ckpt 'data/ckpt-*'
-...(omitted some TensorFlow output)
-Loaded step 14108582.
-# perplexity is high initially because words without context are harder to
-# predict.
-Eval Step: 0, Average Perplexity: 2045.512297.
-Eval Step: 1, Average Perplexity: 229.478699.
-Eval Step: 2, Average Perplexity: 208.116787.
-Eval Step: 3, Average Perplexity: 338.870601.
-Eval Step: 4, Average Perplexity: 228.950107.
-Eval Step: 5, Average Perplexity: 197.685857.
-Eval Step: 6, Average Perplexity: 156.287063.
-Eval Step: 7, Average Perplexity: 124.866189.
-Eval Step: 8, Average Perplexity: 147.204975.
-Eval Step: 9, Average Perplexity: 90.124864.
-Eval Step: 10, Average Perplexity: 59.897914.
-Eval Step: 11, Average Perplexity: 42.591137.
-...(omitted)
-Eval Step: 4529, Average Perplexity: 29.243668.
-Eval Step: 4530, Average Perplexity: 29.302362.
-Eval Step: 4531, Average Perplexity: 29.285674.
-...(omitted. At convergence, it should be around 30.)
-
-# Run dump_emb mode:
-$ bazel-bin/lm_1b/lm_1b_eval --mode dump_emb \
- --pbtxt data/graph-2016-09-10.pbtxt \
- --vocab_file data/vocab-2016-09-10.txt \
- --ckpt 'data/ckpt-*' \
- --save_dir output
-...(omitted some TensorFlow output)
-Finished softmax weights
-Finished word embedding 0/793471
-Finished word embedding 1/793471
-Finished word embedding 2/793471
-...(omitted)
-$ ls output/
-embeddings_softmax.npy ...
-
-# Run dump_lstm_emb mode:
-$ bazel-bin/lm_1b/lm_1b_eval --mode dump_lstm_emb \
- --pbtxt data/graph-2016-09-10.pbtxt \
- --vocab_file data/vocab-2016-09-10.txt \
- --ckpt 'data/ckpt-*' \
- --sentence "I love who I am ." \
- --save_dir output
-$ ls output/
-lstm_emb_step_0.npy lstm_emb_step_2.npy lstm_emb_step_4.npy
-lstm_emb_step_6.npy lstm_emb_step_1.npy lstm_emb_step_3.npy
-lstm_emb_step_5.npy
-```
diff --git a/research/lm_1b/data_utils.py b/research/lm_1b/data_utils.py
deleted file mode 100644
index ad8d3391ef6db07c1d6c234450a6d23a8e19a178..0000000000000000000000000000000000000000
--- a/research/lm_1b/data_utils.py
+++ /dev/null
@@ -1,279 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""A library for loading 1B word benchmark dataset."""
-
-import random
-
-import numpy as np
-import tensorflow as tf
-
-
-class Vocabulary(object):
- """Class that holds a vocabulary for the dataset."""
-
- def __init__(self, filename):
- """Initialize vocabulary.
-
- Args:
- filename: Vocabulary file name.
- """
-
- self._id_to_word = []
- self._word_to_id = {}
- self._unk = -1
- self._bos = -1
- self._eos = -1
-
- with tf.gfile.Open(filename) as f:
- idx = 0
- for line in f:
- word_name = line.strip()
- if word_name == '':
- self._bos = idx
- elif word_name == '':
- self._eos = idx
- elif word_name == '':
- self._unk = idx
- if word_name == '!!!MAXTERMID':
- continue
-
- self._id_to_word.append(word_name)
- self._word_to_id[word_name] = idx
- idx += 1
-
- @property
- def bos(self):
- return self._bos
-
- @property
- def eos(self):
- return self._eos
-
- @property
- def unk(self):
- return self._unk
-
- @property
- def size(self):
- return len(self._id_to_word)
-
- def word_to_id(self, word):
- if word in self._word_to_id:
- return self._word_to_id[word]
- return self.unk
-
- def id_to_word(self, cur_id):
- if cur_id < self.size:
- return self._id_to_word[cur_id]
- return 'ERROR'
-
- def decode(self, cur_ids):
- """Convert a list of ids to a sentence, with space inserted."""
- return ' '.join([self.id_to_word(cur_id) for cur_id in cur_ids])
-
- def encode(self, sentence):
- """Convert a sentence to a list of ids, with special tokens added."""
- word_ids = [self.word_to_id(cur_word) for cur_word in sentence.split()]
- return np.array([self.bos] + word_ids + [self.eos], dtype=np.int32)
-
-
-class CharsVocabulary(Vocabulary):
- """Vocabulary containing character-level information."""
-
- def __init__(self, filename, max_word_length):
- super(CharsVocabulary, self).__init__(filename)
- self._max_word_length = max_word_length
- chars_set = set()
-
- for word in self._id_to_word:
- chars_set |= set(word)
-
- free_ids = []
- for i in range(256):
- if chr(i) in chars_set:
- continue
- free_ids.append(chr(i))
-
- if len(free_ids) < 5:
- raise ValueError('Not enough free char ids: %d' % len(free_ids))
-
- self.bos_char = free_ids[0] #
- self.eos_char = free_ids[1] #
- self.bow_char = free_ids[2] #
- self.eow_char = free_ids[3] #
- self.pad_char = free_ids[4] #
-
- chars_set |= {self.bos_char, self.eos_char, self.bow_char, self.eow_char,
- self.pad_char}
-
- self._char_set = chars_set
- num_words = len(self._id_to_word)
-
- self._word_char_ids = np.zeros([num_words, max_word_length], dtype=np.int32)
-
- self.bos_chars = self._convert_word_to_char_ids(self.bos_char)
- self.eos_chars = self._convert_word_to_char_ids(self.eos_char)
-
- for i, word in enumerate(self._id_to_word):
- self._word_char_ids[i] = self._convert_word_to_char_ids(word)
-
- @property
- def word_char_ids(self):
- return self._word_char_ids
-
- @property
- def max_word_length(self):
- return self._max_word_length
-
- def _convert_word_to_char_ids(self, word):
- code = np.zeros([self.max_word_length], dtype=np.int32)
- code[:] = ord(self.pad_char)
-
- if len(word) > self.max_word_length - 2:
- word = word[:self.max_word_length-2]
- cur_word = self.bow_char + word + self.eow_char
- for j in range(len(cur_word)):
- code[j] = ord(cur_word[j])
- return code
-
- def word_to_char_ids(self, word):
- if word in self._word_to_id:
- return self._word_char_ids[self._word_to_id[word]]
- else:
- return self._convert_word_to_char_ids(word)
-
- def encode_chars(self, sentence):
- chars_ids = [self.word_to_char_ids(cur_word)
- for cur_word in sentence.split()]
- return np.vstack([self.bos_chars] + chars_ids + [self.eos_chars])
-
-
-def get_batch(generator, batch_size, num_steps, max_word_length, pad=False):
- """Read batches of input."""
- cur_stream = [None] * batch_size
-
- inputs = np.zeros([batch_size, num_steps], np.int32)
- char_inputs = np.zeros([batch_size, num_steps, max_word_length], np.int32)
- global_word_ids = np.zeros([batch_size, num_steps], np.int32)
- targets = np.zeros([batch_size, num_steps], np.int32)
- weights = np.ones([batch_size, num_steps], np.float32)
-
- no_more_data = False
- while True:
- inputs[:] = 0
- char_inputs[:] = 0
- global_word_ids[:] = 0
- targets[:] = 0
- weights[:] = 0.0
-
- for i in range(batch_size):
- cur_pos = 0
-
- while cur_pos < num_steps:
- if cur_stream[i] is None or len(cur_stream[i][0]) <= 1:
- try:
- cur_stream[i] = list(generator.next())
- except StopIteration:
- # No more data, exhaust current streams and quit
- no_more_data = True
- break
-
- how_many = min(len(cur_stream[i][0]) - 1, num_steps - cur_pos)
- next_pos = cur_pos + how_many
-
- inputs[i, cur_pos:next_pos] = cur_stream[i][0][:how_many]
- char_inputs[i, cur_pos:next_pos] = cur_stream[i][1][:how_many]
- global_word_ids[i, cur_pos:next_pos] = cur_stream[i][2][:how_many]
- targets[i, cur_pos:next_pos] = cur_stream[i][0][1:how_many+1]
- weights[i, cur_pos:next_pos] = 1.0
-
- cur_pos = next_pos
- cur_stream[i][0] = cur_stream[i][0][how_many:]
- cur_stream[i][1] = cur_stream[i][1][how_many:]
- cur_stream[i][2] = cur_stream[i][2][how_many:]
-
- if pad:
- break
-
- if no_more_data and np.sum(weights) == 0:
- # There is no more data and this is an empty batch. Done!
- break
- yield inputs, char_inputs, global_word_ids, targets, weights
-
-
-class LM1BDataset(object):
- """Utility class for 1B word benchmark dataset.
-
- The current implementation reads the data from the tokenized text files.
- """
-
- def __init__(self, filepattern, vocab):
- """Initialize LM1BDataset reader.
-
- Args:
- filepattern: Dataset file pattern.
- vocab: Vocabulary.
- """
- self._vocab = vocab
- self._all_shards = tf.gfile.Glob(filepattern)
- tf.logging.info('Found %d shards at %s', len(self._all_shards), filepattern)
-
- def _load_random_shard(self):
- """Randomly select a file and read it."""
- return self._load_shard(random.choice(self._all_shards))
-
- def _load_shard(self, shard_name):
- """Read one file and convert to ids.
-
- Args:
- shard_name: file path.
-
- Returns:
- list of (id, char_id, global_word_id) tuples.
- """
- tf.logging.info('Loading data from: %s', shard_name)
- with tf.gfile.Open(shard_name) as f:
- sentences = f.readlines()
- chars_ids = [self.vocab.encode_chars(sentence) for sentence in sentences]
- ids = [self.vocab.encode(sentence) for sentence in sentences]
-
- global_word_ids = []
- current_idx = 0
- for word_ids in ids:
- current_size = len(word_ids) - 1 # without symbol
- cur_ids = np.arange(current_idx, current_idx + current_size)
- global_word_ids.append(cur_ids)
- current_idx += current_size
-
- tf.logging.info('Loaded %d words.', current_idx)
- tf.logging.info('Finished loading')
- return zip(ids, chars_ids, global_word_ids)
-
- def _get_sentence(self, forever=True):
- while True:
- ids = self._load_random_shard()
- for current_ids in ids:
- yield current_ids
- if not forever:
- break
-
- def get_batch(self, batch_size, num_steps, pad=False, forever=True):
- return get_batch(self._get_sentence(forever), batch_size, num_steps,
- self.vocab.max_word_length, pad=pad)
-
- @property
- def vocab(self):
- return self._vocab
diff --git a/research/lm_1b/lm_1b_eval.py b/research/lm_1b/lm_1b_eval.py
deleted file mode 100644
index ce8634757558c135ba137a9b9e09a733977adc3a..0000000000000000000000000000000000000000
--- a/research/lm_1b/lm_1b_eval.py
+++ /dev/null
@@ -1,308 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Eval pre-trained 1 billion word language model.
-"""
-import os
-import sys
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from google.protobuf import text_format
-import data_utils
-
-FLAGS = tf.flags.FLAGS
-# General flags.
-tf.flags.DEFINE_string('mode', 'eval',
- 'One of [sample, eval, dump_emb, dump_lstm_emb]. '
- '"sample" mode samples future word predictions, using '
- 'FLAGS.prefix as prefix (prefix could be left empty). '
- '"eval" mode calculates perplexity of the '
- 'FLAGS.input_data. '
- '"dump_emb" mode dumps word and softmax embeddings to '
- 'FLAGS.save_dir. embeddings are dumped in the same '
- 'order as words in vocabulary. All words in vocabulary '
- 'are dumped.'
- 'dump_lstm_emb dumps lstm embeddings of FLAGS.sentence '
- 'to FLAGS.save_dir.')
-tf.flags.DEFINE_string('pbtxt', '',
- 'GraphDef proto text file used to construct model '
- 'structure.')
-tf.flags.DEFINE_string('ckpt', '',
- 'Checkpoint directory used to fill model values.')
-tf.flags.DEFINE_string('vocab_file', '', 'Vocabulary file.')
-tf.flags.DEFINE_string('save_dir', '',
- 'Used for "dump_emb" mode to save word embeddings.')
-# sample mode flags.
-tf.flags.DEFINE_string('prefix', '',
- 'Used for "sample" mode to predict next words.')
-tf.flags.DEFINE_integer('max_sample_words', 100,
- 'Sampling stops either when is met or this number '
- 'of steps has passed.')
-tf.flags.DEFINE_integer('num_samples', 3,
- 'Number of samples to generate for the prefix.')
-# dump_lstm_emb mode flags.
-tf.flags.DEFINE_string('sentence', '',
- 'Used as input for "dump_lstm_emb" mode.')
-# eval mode flags.
-tf.flags.DEFINE_string('input_data', '',
- 'Input data files for eval model.')
-tf.flags.DEFINE_integer('max_eval_steps', 1000000,
- 'Maximum mumber of steps to run "eval" mode.')
-
-
-# For saving demo resources, use batch size 1 and step 1.
-BATCH_SIZE = 1
-NUM_TIMESTEPS = 1
-MAX_WORD_LEN = 50
-
-
-def _LoadModel(gd_file, ckpt_file):
- """Load the model from GraphDef and Checkpoint.
-
- Args:
- gd_file: GraphDef proto text file.
- ckpt_file: TensorFlow Checkpoint file.
-
- Returns:
- TensorFlow session and tensors dict.
- """
- with tf.Graph().as_default():
- sys.stderr.write('Recovering graph.\n')
- with tf.gfile.FastGFile(gd_file, 'r') as f:
- s = f.read().decode()
- gd = tf.GraphDef()
- text_format.Merge(s, gd)
-
- tf.logging.info('Recovering Graph %s', gd_file)
- t = {}
- [t['states_init'], t['lstm/lstm_0/control_dependency'],
- t['lstm/lstm_1/control_dependency'], t['softmax_out'], t['class_ids_out'],
- t['class_weights_out'], t['log_perplexity_out'], t['inputs_in'],
- t['targets_in'], t['target_weights_in'], t['char_inputs_in'],
- t['all_embs'], t['softmax_weights'], t['global_step']
- ] = tf.import_graph_def(gd, {}, ['states_init',
- 'lstm/lstm_0/control_dependency:0',
- 'lstm/lstm_1/control_dependency:0',
- 'softmax_out:0',
- 'class_ids_out:0',
- 'class_weights_out:0',
- 'log_perplexity_out:0',
- 'inputs_in:0',
- 'targets_in:0',
- 'target_weights_in:0',
- 'char_inputs_in:0',
- 'all_embs_out:0',
- 'Reshape_3:0',
- 'global_step:0'], name='')
-
- sys.stderr.write('Recovering checkpoint %s\n' % ckpt_file)
- sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
- sess.run('save/restore_all', {'save/Const:0': ckpt_file})
- sess.run(t['states_init'])
-
- return sess, t
-
-
-def _EvalModel(dataset):
- """Evaluate model perplexity using provided dataset.
-
- Args:
- dataset: LM1BDataset object.
- """
- sess, t = _LoadModel(FLAGS.pbtxt, FLAGS.ckpt)
-
- current_step = t['global_step'].eval(session=sess)
- sys.stderr.write('Loaded step %d.\n' % current_step)
-
- data_gen = dataset.get_batch(BATCH_SIZE, NUM_TIMESTEPS, forever=False)
- sum_num = 0.0
- sum_den = 0.0
- perplexity = 0.0
- for i, (inputs, char_inputs, _, targets, weights) in enumerate(data_gen):
- input_dict = {t['inputs_in']: inputs,
- t['targets_in']: targets,
- t['target_weights_in']: weights}
- if 'char_inputs_in' in t:
- input_dict[t['char_inputs_in']] = char_inputs
- log_perp = sess.run(t['log_perplexity_out'], feed_dict=input_dict)
-
- if np.isnan(log_perp):
- sys.stderr.error('log_perplexity is Nan.\n')
- else:
- sum_num += log_perp * weights.mean()
- sum_den += weights.mean()
- if sum_den > 0:
- perplexity = np.exp(sum_num / sum_den)
-
- sys.stderr.write('Eval Step: %d, Average Perplexity: %f.\n' %
- (i, perplexity))
-
- if i > FLAGS.max_eval_steps:
- break
-
-
-def _SampleSoftmax(softmax):
- return min(np.sum(np.cumsum(softmax) < np.random.rand()), len(softmax) - 1)
-
-
-def _SampleModel(prefix_words, vocab):
- """Predict next words using the given prefix words.
-
- Args:
- prefix_words: Prefix words.
- vocab: Vocabulary. Contains max word chard id length and converts between
- words and ids.
- """
- targets = np.zeros([BATCH_SIZE, NUM_TIMESTEPS], np.int32)
- weights = np.ones([BATCH_SIZE, NUM_TIMESTEPS], np.float32)
-
- sess, t = _LoadModel(FLAGS.pbtxt, FLAGS.ckpt)
-
- if prefix_words.find('') != 0:
- prefix_words = ' ' + prefix_words
-
- prefix = [vocab.word_to_id(w) for w in prefix_words.split()]
- prefix_char_ids = [vocab.word_to_char_ids(w) for w in prefix_words.split()]
- for _ in xrange(FLAGS.num_samples):
- inputs = np.zeros([BATCH_SIZE, NUM_TIMESTEPS], np.int32)
- char_ids_inputs = np.zeros(
- [BATCH_SIZE, NUM_TIMESTEPS, vocab.max_word_length], np.int32)
- samples = prefix[:]
- char_ids_samples = prefix_char_ids[:]
- sent = ''
- while True:
- inputs[0, 0] = samples[0]
- char_ids_inputs[0, 0, :] = char_ids_samples[0]
- samples = samples[1:]
- char_ids_samples = char_ids_samples[1:]
-
- softmax = sess.run(t['softmax_out'],
- feed_dict={t['char_inputs_in']: char_ids_inputs,
- t['inputs_in']: inputs,
- t['targets_in']: targets,
- t['target_weights_in']: weights})
-
- sample = _SampleSoftmax(softmax[0])
- sample_char_ids = vocab.word_to_char_ids(vocab.id_to_word(sample))
-
- if not samples:
- samples = [sample]
- char_ids_samples = [sample_char_ids]
- sent += vocab.id_to_word(samples[0]) + ' '
- sys.stderr.write('%s\n' % sent)
-
- if (vocab.id_to_word(samples[0]) == '' or
- len(sent) > FLAGS.max_sample_words):
- break
-
-
-def _DumpEmb(vocab):
- """Dump the softmax weights and word embeddings to files.
-
- Args:
- vocab: Vocabulary. Contains vocabulary size and converts word to ids.
- """
- assert FLAGS.save_dir, 'Must specify FLAGS.save_dir for dump_emb.'
- inputs = np.zeros([BATCH_SIZE, NUM_TIMESTEPS], np.int32)
- targets = np.zeros([BATCH_SIZE, NUM_TIMESTEPS], np.int32)
- weights = np.ones([BATCH_SIZE, NUM_TIMESTEPS], np.float32)
-
- sess, t = _LoadModel(FLAGS.pbtxt, FLAGS.ckpt)
-
- softmax_weights = sess.run(t['softmax_weights'])
- fname = FLAGS.save_dir + '/embeddings_softmax.npy'
- with tf.gfile.Open(fname, mode='w') as f:
- np.save(f, softmax_weights)
- sys.stderr.write('Finished softmax weights\n')
-
- all_embs = np.zeros([vocab.size, 1024])
- for i in xrange(vocab.size):
- input_dict = {t['inputs_in']: inputs,
- t['targets_in']: targets,
- t['target_weights_in']: weights}
- if 'char_inputs_in' in t:
- input_dict[t['char_inputs_in']] = (
- vocab.word_char_ids[i].reshape([-1, 1, MAX_WORD_LEN]))
- embs = sess.run(t['all_embs'], input_dict)
- all_embs[i, :] = embs
- sys.stderr.write('Finished word embedding %d/%d\n' % (i, vocab.size))
-
- fname = FLAGS.save_dir + '/embeddings_char_cnn.npy'
- with tf.gfile.Open(fname, mode='w') as f:
- np.save(f, all_embs)
- sys.stderr.write('Embedding file saved\n')
-
-
-def _DumpSentenceEmbedding(sentence, vocab):
- """Predict next words using the given prefix words.
-
- Args:
- sentence: Sentence words.
- vocab: Vocabulary. Contains max word chard id length and converts between
- words and ids.
- """
- targets = np.zeros([BATCH_SIZE, NUM_TIMESTEPS], np.int32)
- weights = np.ones([BATCH_SIZE, NUM_TIMESTEPS], np.float32)
-
- sess, t = _LoadModel(FLAGS.pbtxt, FLAGS.ckpt)
-
- if sentence.find('') != 0:
- sentence = ' ' + sentence
-
- word_ids = [vocab.word_to_id(w) for w in sentence.split()]
- char_ids = [vocab.word_to_char_ids(w) for w in sentence.split()]
-
- inputs = np.zeros([BATCH_SIZE, NUM_TIMESTEPS], np.int32)
- char_ids_inputs = np.zeros(
- [BATCH_SIZE, NUM_TIMESTEPS, vocab.max_word_length], np.int32)
- for i in xrange(len(word_ids)):
- inputs[0, 0] = word_ids[i]
- char_ids_inputs[0, 0, :] = char_ids[i]
-
- # Add 'lstm/lstm_0/control_dependency' if you want to dump previous layer
- # LSTM.
- lstm_emb = sess.run(t['lstm/lstm_1/control_dependency'],
- feed_dict={t['char_inputs_in']: char_ids_inputs,
- t['inputs_in']: inputs,
- t['targets_in']: targets,
- t['target_weights_in']: weights})
-
- fname = os.path.join(FLAGS.save_dir, 'lstm_emb_step_%d.npy' % i)
- with tf.gfile.Open(fname, mode='w') as f:
- np.save(f, lstm_emb)
- sys.stderr.write('LSTM embedding step %d file saved\n' % i)
-
-
-def main(unused_argv):
- vocab = data_utils.CharsVocabulary(FLAGS.vocab_file, MAX_WORD_LEN)
-
- if FLAGS.mode == 'eval':
- dataset = data_utils.LM1BDataset(FLAGS.input_data, vocab)
- _EvalModel(dataset)
- elif FLAGS.mode == 'sample':
- _SampleModel(FLAGS.prefix, vocab)
- elif FLAGS.mode == 'dump_emb':
- _DumpEmb(vocab)
- elif FLAGS.mode == 'dump_lstm_emb':
- _DumpSentenceEmbedding(FLAGS.sentence, vocab)
- else:
- raise Exception('Mode not supported.')
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/lm_commonsense/README.md b/research/lm_commonsense/README.md
deleted file mode 100644
index 78c8f53ca226f09c4b185490d6966f98bf584889..0000000000000000000000000000000000000000
--- a/research/lm_commonsense/README.md
+++ /dev/null
@@ -1,170 +0,0 @@
-
-
-
-
-# A Simple Method for Commonsense Reasoning
-
-This repository contains code to reproduce results from [*A Simple Method for Commonsense Reasoning*](https://arxiv.org/abs/1806.02847).
-
-Authors and contact:
-
-* Trieu H. Trinh (thtrieu@google.com, github: thtrieu)
-* Quoc V. Le (qvl@google.com)
-
-## TL;DR
-
-Commonsense reasoning is a long-standing challenge for deep learning. For example,
-it is difficult to use neural networks to tackle the Winograd Schema dataset - a difficult subset of Pronoun Disambiguation problems. In this work, we use language models to score substitued sentences to decide the correct reference of the ambiguous pronoun (see Figure below for an example).
-
-
-
-This simple unsupervised method achieves new state-of-the-art (*as of June 1st, 2018*) results on both benchmark PDP-60 and WSC-273 (See Table below), without using rule-based reasoning nor expensive annotated knowledge bases.
-
-| Commonsense-reasoning test | Previous best result | Ours |
-| ----------------------------|:----------------------:|:-----:|
-| Pronoun Disambiguation | 66.7% | 70% |
-| Winograd Schema Challenge | 52.8% | 63.7% |
-
-
-
-## Citation
-
-If you use our released models below in your publication, please cite the original paper:
-
-@article{TBD}
-
-
-## Requirements
-* Python >=2.6
-* Tensorflow >= v1.4
-* Numpy >= 1.12.1
-
-## Details of this release
-
-The open-sourced components include:
-
-* Test sets from Pronoun Disambiguation Problem (PDP-60) and Winograd Schema Challenges (WSC-273).
-* Tensorflow metagraph and checkpoints of 14 language models (See Appendix A in the paper).
-* A vocabulary file.
-* Code to reproduce results from the original paper.
-
-## How to run
-
-### 1. Download data files
-
-Download all files from the [Google Cloud Storage of this project](https://console.cloud.google.com/storage/browser/commonsense-reasoning/). The easiest way is to install and use `gsutil cp` command-line tool (See [install gsutil](https://cloud.google.com/storage/docs/gsutil_install)).
-
-
-```shell
-# Download everything from the project gs://commonsense-reasoning
-$ gsutil cp -R gs://commonsense-reasoning/* .
-Copying gs://commonsense-reasoning/reproduce/vocab.txt...
-Copying gs://commonsense-reasoning/reproduce/commonsense_test/pdp60.json...
-Copying gs://commonsense-reasoning/reproduce/commonsense_test/wsc273.json...
-
-...(omitted)
-```
-
-All downloaded content should be in `./reproduce/`. This includes two tests `pdp60.json` and `wsc273.json`, a vocabulary file `vocab.txt` and checkpoints for all 14 language models, each includes three files (`.data`, `.index` and `.meta`). All checkpoint names start with `ckpt-best` since they are saved at the best perplexity on a hold-out text corpus.
-
-```shell
-# Check for the content
-$ ls reproduce/*
-reproduce/vocab.txt
-
-reproduce/commonsense_test:
-pdp60.json wsc273.json
-
-reproduce/lm01:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm02:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm03:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm04:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm05:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm06:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm07:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm08:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm09:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm10:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm11:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm12:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm13:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-
-reproduce/lm14:
-ckpt-best.data-00000-of-00001 ckpt-best.index ckpt-best.meta
-```
-
-### 2. Run evaluation code
-
-To reproduce results from the paper, simply run `eval.py` script.
-
-```shell
-$ python eval.py --data_dir=reproduce
-
-Restored from ./reproduce/lm01
-Reset RNN states.
-Processing patch (1, 1) / (2, 4)
-Probs for
-[['Then' 'Dad' 'figured' ..., 'man' "'s" 'board-bill']
- ['Then' 'Dad' 'figured' ..., 'man' "'s" 'board-bill']
- ['Always' 'before' ',' ..., 'now' ',' 'for']
- ...,
- ['Mark' 'was' 'close' ..., 'promising' 'him' ',']
- ['Mark' 'was' 'close' ..., 'promising' 'him' ',']
- ['Mark' 'was' 'close' ..., 'promising' 'him' ',']]
-=
-[[ 1.64250596e-05 1.77780055e-06 4.14267970e-06 ..., 1.87315454e-03
- 1.57723188e-01 6.31845817e-02]
- [ 1.64250596e-05 1.77780055e-06 4.14267970e-06 ..., 1.87315454e-03
- 1.57723188e-01 6.31845817e-02]
- [ 1.28243030e-07 3.80435935e-03 1.12383246e-01 ..., 9.67682712e-03
- 2.17407525e-01 1.08243264e-01]
- ...,
- [ 1.15557734e-04 2.92792241e-03 3.46455898e-04 ..., 2.72328052e-05
- 3.37066874e-02 7.89367408e-02]
- [ 1.15557734e-04 2.92792241e-03 3.46455898e-04 ..., 2.72328052e-05
- 3.37066874e-02 7.89367408e-02]
- [ 1.15557734e-04 2.92792241e-03 3.46455898e-04 ..., 2.72328052e-05
- 3.37066874e-02 7.89367408e-02]]
-Processing patch (1, 2) / (2, 4)
-
-...(omitted)
-
-Accuracy of 1 LM(s) on pdp60 = 0.6
-
-...(omitted)
-
-Accuracy of 5 LM(s) on pdp60 = 0.7
-
-...(omitted)
-
-Accuracy of 10 LM(s) on wsc273 = 0.615
-
-...(omitted)
-
-Accuracy of 14 LM(s) on wsc273 = 0.637
-```
diff --git a/research/lm_commonsense/eval.py b/research/lm_commonsense/eval.py
deleted file mode 100644
index e5b7ff98b50a5af4e066d3d9f82c1acae81c3e93..0000000000000000000000000000000000000000
--- a/research/lm_commonsense/eval.py
+++ /dev/null
@@ -1,190 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import pickle as pkl
-import numpy as np
-import tensorflow as tf
-import utils
-
-tf.app.flags.DEFINE_string(
- 'data_dir', 'reproduce',
- 'Path to directory containing data and model checkpoints.')
-
-
-FLAGS = tf.app.flags.FLAGS
-
-
-class EnsembleLM(object):
- """Ensemble of language models."""
-
- def __init__(self, test_data_name='wsc273'):
- vocab_file = os.path.join(FLAGS.data_dir, 'vocab.txt')
- self.vocab = utils.CharsVocabulary(vocab_file, 50)
- assert test_data_name in ['pdp60', 'wsc273'], (
- 'Test data must be pdp60 or wsc273, got {}'.format(test_data_name))
- self.test_data_name = test_data_name
-
- test_data = utils.parse_commonsense_reasoning_test(test_data_name)
- self.question_ids, self.sentences, self.labels = test_data
- self.all_probs = [] # aggregate single-model prediction here.
-
- def add_single_model(self, model_name='lm1'):
- """Add a single model into the current ensemble."""
- # Create single LM
- single_lm = SingleRecurrentLanguageModel(self.vocab, model_name)
-
- # Add the single LM prediction.
- probs = single_lm.assign_probs(self.sentences, self.test_data_name)
- self.all_probs.append(probs)
- print('Done adding {}'.format(model_name))
-
- def evaluate(self):
- """Evaluate the current ensemble."""
- # Attach word probabilities and correctness label to each substitution
- ensembled_probs = sum(self.all_probs) / len(self.all_probs)
- scorings = []
- for i, sentence in enumerate(self.sentences):
- correctness = self.labels[i]
- word_probs = ensembled_probs[i, :len(sentence)]
- joint_prob = np.prod(word_probs, dtype=np.float64)
-
- scorings.append(dict(
- correctness=correctness,
- sentence=sentence,
- joint_prob=joint_prob,
- word_probs=word_probs))
- scoring_mode = 'full' if self.test_data_name == 'pdp60' else 'partial'
- return utils.compare_substitutions(
- self.question_ids, scorings, scoring_mode)
-
-
-class SingleRecurrentLanguageModel(object):
- """Single Recurrent Language Model."""
-
- def __init__(self, vocab, model_name='lm01'):
- self.vocab = vocab
- self.log_dir = os.path.join(FLAGS.data_dir, model_name)
-
- def reset(self):
- self.sess.run(self.tensors['states_init'])
-
- def _score(self, word_patch):
- """Score a matrix of shape (batch_size, num_timesteps+1) str tokens."""
- word_ids = np.array(
- [[self.vocab.word_to_id(word) for word in row]
- for row in word_patch])
- char_ids = np.array(
- [[self.vocab.word_to_char_ids(word) for word in row]
- for row in word_patch])
- print('Probs for \n{}\n='.format(np.array(word_patch)[:, 1:]))
-
- input_ids, target_ids = word_ids[:, :-1], word_ids[:, 1:]
- input_char_ids = char_ids[:, :-1, :]
-
- softmax = self.sess.run(self.tensors['softmax_out'], feed_dict={
- self.tensors['inputs_in']: input_ids,
- self.tensors['char_inputs_in']: input_char_ids
- })
-
- batch_size, num_timesteps = self.shape
- softmax = softmax.reshape((num_timesteps, batch_size, -1))
- softmax = np.transpose(softmax, [1, 0, 2])
- probs = np.array([[softmax[row, col, target_ids[row, col]]
- for col in range(num_timesteps)]
- for row in range(batch_size)])
- print(probs)
- return probs
-
- def _score_patches(self, word_patches):
- """Score a 2D matrix of word_patches and stitch results together."""
- batch_size, num_timesteps = self.shape
- nrow, ncol = len(word_patches), len(word_patches[0])
- max_len = num_timesteps * ncol
- probs = np.zeros([0, max_len]) # accumulate results into this.
-
- # Loop through the 2D matrix of word_patches and score each.
- for i, row in enumerate(word_patches):
- print('Reset RNN states.')
- self.reset() # reset states before processing each row.
- row_probs = np.zeros([batch_size, 0])
- for j, word_patch in enumerate(row):
- print('Processing patch '
- '({}, {}) / ({}, {})'.format(i+1, j+1, nrow, ncol))
- patch_probs = (self._score(word_patch) if word_patch else
- np.zeros([batch_size, num_timesteps]))
- row_probs = np.concatenate([row_probs, patch_probs], 1)
- probs = np.concatenate([probs, row_probs], 0)
- return probs
-
- def assign_probs(self, sentences, test_data_name='wsc273'):
- """Return prediction accuracy using this LM for a test."""
-
- probs_cache = os.path.join(self.log_dir, '{}.probs'.format(test_data_name))
- if os.path.exists(probs_cache):
- print('Reading cached result from {}'.format(probs_cache))
- with tf.gfile.Open(probs_cache, 'r') as f:
- probs = pkl.load(f)
- else:
- tf.reset_default_graph()
- self.sess = tf.Session()
- # Build the graph.
- saver = tf.train.import_meta_graph(
- os.path.join(self.log_dir, 'ckpt-best.meta'))
- saver.restore(self.sess, os.path.join(self.log_dir, 'ckpt-best'))
- print('Restored from {}'.format(self.log_dir))
- graph = tf.get_default_graph()
- self.tensors = dict(
- inputs_in=graph.get_tensor_by_name('test_inputs_in:0'),
- char_inputs_in=graph.get_tensor_by_name('test_char_inputs_in:0'),
- softmax_out=graph.get_tensor_by_name('SotaRNN_1/softmax_out:0'),
- states_init=graph.get_operation_by_name('SotaRNN_1/states_init'))
- self.shape = self.tensors['inputs_in'].shape.as_list()
-
- # Cut sentences into patches of shape processable by the LM.
- batch_size, num_timesteps = self.shape
- word_patches = utils.cut_to_patches(sentences, batch_size, num_timesteps)
- probs = self._score_patches(word_patches)
-
- # Cache the probs since they are expensive to evaluate
- with tf.gfile.Open(probs_cache, 'w') as f:
- pkl.dump(probs, f)
- return probs
-
-
-def evaluate_ensemble(test_data_name, number_of_lms):
- ensemble = EnsembleLM(test_data_name)
- model_list = ['lm{:02d}'.format(i+1) for i in range(number_of_lms)]
- for model_name in model_list:
- ensemble.add_single_model(model_name)
- accuracy = ensemble.evaluate()
- print('Accuracy of {} LM(s) on {} = {}'.format(
- number_of_lms, test_data_name, accuracy))
-
-
-def main(_):
- evaluate_ensemble('pdp60', 1) # 60%
- evaluate_ensemble('pdp60', 5) # 70%
- evaluate_ensemble('wsc273', 10) # 61.5%
- evaluate_ensemble('wsc273', 14) # 63.7%
-
-
-if __name__ == '__main__':
- tf.app.run(main)
diff --git a/research/lm_commonsense/method.jpg b/research/lm_commonsense/method.jpg
deleted file mode 100644
index ee8a5506fccca3cbb67f7bda0ccef78303cb228b..0000000000000000000000000000000000000000
Binary files a/research/lm_commonsense/method.jpg and /dev/null differ
diff --git a/research/lm_commonsense/utils.py b/research/lm_commonsense/utils.py
deleted file mode 100644
index d75f2b0fb72716860ea6d438e6b8ca2732d13c84..0000000000000000000000000000000000000000
--- a/research/lm_commonsense/utils.py
+++ /dev/null
@@ -1,368 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import json
-import os
-import numpy as np
-import tensorflow as tf
-
-FLAGS = tf.flags.FLAGS
-
-
-class Vocabulary(object):
- """Class that holds a vocabulary for the dataset."""
-
- def __init__(self, filename):
-
- self._id_to_word = []
- self._word_to_id = {}
- self._unk = -1
- self._bos = -1
- self._eos = -1
-
- with tf.gfile.Open(filename) as f:
- idx = 0
- for line in f:
- word_name = line.strip()
- if word_name == '':
- self._bos = idx
- elif word_name == '':
- self._eos = idx
- elif word_name == '':
- self._unk = idx
- if word_name == '!!!MAXTERMID':
- continue
-
- self._id_to_word.append(word_name)
- self._word_to_id[word_name] = idx
- idx += 1
-
- @property
- def bos(self):
- return self._bos
-
- @property
- def eos(self):
- return self._eos
-
- @property
- def unk(self):
- return self._unk
-
- @property
- def size(self):
- return len(self._id_to_word)
-
- def word_to_id(self, word):
- if word in self._word_to_id:
- return self._word_to_id[word]
- else:
- if word.lower() in self._word_to_id:
- return self._word_to_id[word.lower()]
- return self.unk
-
- def id_to_word(self, cur_id):
- if cur_id < self.size:
- return self._id_to_word[int(cur_id)]
- return ''
-
- def decode(self, cur_ids):
- return ' '.join([self.id_to_word(cur_id) for cur_id in cur_ids])
-
- def encode(self, sentence):
- word_ids = [self.word_to_id(cur_word) for cur_word in sentence.split()]
- return np.array([self.bos] + word_ids + [self.eos], dtype=np.int32)
-
-
-class CharsVocabulary(Vocabulary):
- """Vocabulary containing character-level information."""
-
- def __init__(self, filename, max_word_length):
- super(CharsVocabulary, self).__init__(filename)
-
- self._max_word_length = max_word_length
- chars_set = set()
-
- for word in self._id_to_word:
- chars_set |= set(word)
-
- free_ids = []
- for i in range(256):
- if chr(i) in chars_set:
- continue
- free_ids.append(chr(i))
-
- if len(free_ids) < 5:
- raise ValueError('Not enough free char ids: %d' % len(free_ids))
-
- self.bos_char = free_ids[0] #
- self.eos_char = free_ids[1] #
- self.bow_char = free_ids[2] #
- self.eow_char = free_ids[3] #
- self.pad_char = free_ids[4] #
-
- chars_set |= {self.bos_char, self.eos_char, self.bow_char, self.eow_char,
- self.pad_char}
-
- self._char_set = chars_set
- num_words = len(self._id_to_word)
-
- self._word_char_ids = np.zeros([num_words, max_word_length], dtype=np.int32)
-
- self.bos_chars = self._convert_word_to_char_ids(self.bos_char)
- self.eos_chars = self._convert_word_to_char_ids(self.eos_char)
-
- for i, word in enumerate(self._id_to_word):
- if i == self.bos:
- self._word_char_ids[i] = self.bos_chars
- elif i == self.eos:
- self._word_char_ids[i] = self.eos_chars
- else:
- self._word_char_ids[i] = self._convert_word_to_char_ids(word)
-
- @property
- def max_word_length(self):
- return self._max_word_length
-
- def _convert_word_to_char_ids(self, word):
- code = np.zeros([self.max_word_length], dtype=np.int32)
- code[:] = ord(self.pad_char)
-
- if len(word) > self.max_word_length - 2:
- word = word[:self.max_word_length-2]
- cur_word = self.bow_char + word + self.eow_char
- for j in range(len(cur_word)):
- code[j] = ord(cur_word[j])
- return code
-
- def word_to_char_ids(self, word):
- if word in self._word_to_id:
- return self._word_char_ids[self._word_to_id[word]]
- else:
- return self._convert_word_to_char_ids(word)
-
- def encode_chars(self, sentence):
- chars_ids = [self.word_to_char_ids(cur_word)
- for cur_word in sentence.split()]
- return np.vstack([self.bos_chars] + chars_ids + [self.eos_chars])
-
-
-_SPECIAL_CHAR_MAP = {
- '\xe2\x80\x98': '\'',
- '\xe2\x80\x99': '\'',
- '\xe2\x80\x9c': '"',
- '\xe2\x80\x9d': '"',
- '\xe2\x80\x93': '-',
- '\xe2\x80\x94': '-',
- '\xe2\x88\x92': '-',
- '\xce\x84': '\'',
- '\xc2\xb4': '\'',
- '`': '\''
-}
-
-_START_SPECIAL_CHARS = ['.', ',', '?', '!', ';', ':', '[', ']', '\'', '+', '/',
- '\xc2\xa3', '$', '~', '*', '%', '{', '}', '#', '&', '-',
- '"', '(', ')', '='] + list(_SPECIAL_CHAR_MAP.keys())
-_SPECIAL_CHARS = _START_SPECIAL_CHARS + [
- '\'s', '\'m', '\'t', '\'re', '\'d', '\'ve', '\'ll']
-
-
-def tokenize(sentence):
- """Tokenize a sentence."""
- sentence = str(sentence)
- words = sentence.strip().split()
- tokenized = [] # return this
-
- for word in words:
- if word.lower() in ['mr.', 'ms.']:
- tokenized.append(word)
- continue
-
- # Split special chars at the start of word
- will_split = True
- while will_split:
- will_split = False
- for char in _START_SPECIAL_CHARS:
- if word.startswith(char):
- tokenized.append(char)
- word = word[len(char):]
- will_split = True
-
- # Split special chars at the end of word
- special_end_tokens = []
- will_split = True
- while will_split:
- will_split = False
- for char in _SPECIAL_CHARS:
- if word.endswith(char):
- special_end_tokens = [char] + special_end_tokens
- word = word[:-len(char)]
- will_split = True
-
- if word:
- tokenized.append(word)
- tokenized += special_end_tokens
-
- # Add necessary end of sentence token.
- if tokenized[-1] not in ['.', '!', '?']:
- tokenized += ['.']
- return tokenized
-
-
-def parse_commonsense_reasoning_test(test_data_name):
- """Read JSON test data."""
- with tf.gfile.Open(os.path.join(
- FLAGS.data_dir, 'commonsense_test',
- '{}.json'.format(test_data_name)), 'r') as f:
- data = json.load(f)
-
- question_ids = [d['question_id'] for d in data]
- sentences = [tokenize(d['substitution']) for d in data]
- labels = [d['correctness'] for d in data]
-
- return question_ids, sentences, labels
-
-
-PAD = ''
-
-
-def cut_to_patches(sentences, batch_size, num_timesteps):
- """Cut sentences into patches of shape (batch_size, num_timesteps).
-
- Args:
- sentences: a list of sentences, each sentence is a list of str token.
- batch_size: batch size
- num_timesteps: number of backprop step
-
- Returns:
- patches: A 2D matrix,
- each entry is a matrix of shape (batch_size, num_timesteps).
- """
- preprocessed = [['']+sentence+[''] for sentence in sentences]
- max_len = max([len(sent) for sent in preprocessed])
-
- # Pad to shape [height, width]
- # where height is a multiple of batch_size
- # and width is a multiple of num_timesteps
- nrow = int(np.ceil(len(preprocessed) * 1.0 / batch_size))
- ncol = int(np.ceil(max_len * 1.0 / num_timesteps))
- height, width = nrow * batch_size, ncol * num_timesteps + 1
- preprocessed = [sent + [PAD] * (width - len(sent)) for sent in preprocessed]
- preprocessed += [[PAD] * width] * (height - len(preprocessed))
-
- # Cut preprocessed into patches of shape [batch_size, num_timesteps]
- patches = []
- for row in range(nrow):
- patches.append([])
- for col in range(ncol):
- patch = [sent[col * num_timesteps:
- (col+1) * num_timesteps + 1]
- for sent in preprocessed[row * batch_size:
- (row+1) * batch_size]]
- if np.all(np.array(patch)[:, 1:] == PAD):
- patch = None # no need to process this patch.
- patches[-1].append(patch)
- return patches
-
-
-def _substitution_mask(sent1, sent2):
- """Binary mask identifying substituted part in two sentences.
-
- Example sentence and their mask:
- First sentence = "I like the cat 's color"
- 0 0 0 1 0 0
- Second sentence = "I like the yellow dog 's color"
- 0 0 0 1 1 0 0
-
- Args:
- sent1: first sentence
- sent2: second sentence
-
- Returns:
- mask1: mask for first sentence
- mask2: mask for second sentence
- """
- mask1_start, mask2_start = [], []
- while sent1[0] == sent2[0]:
- sent1 = sent1[1:]
- sent2 = sent2[1:]
- mask1_start.append(0.)
- mask2_start.append(0.)
-
- mask1_end, mask2_end = [], []
- while sent1[-1] == sent2[-1]:
- if (len(sent1) == 1) or (len(sent2) == 1):
- break
- sent1 = sent1[:-1]
- sent2 = sent2[:-1]
- mask1_end = [0.] + mask1_end
- mask2_end = [0.] + mask2_end
-
- assert sent1 or sent2, 'Two sentences are identical.'
- return (mask1_start + [1.] * len(sent1) + mask1_end,
- mask2_start + [1.] * len(sent2) + mask2_end)
-
-
-def _convert_to_partial(scoring1, scoring2):
- """Convert full scoring into partial scoring."""
- mask1, mask2 = _substitution_mask(
- scoring1['sentence'], scoring2['sentence'])
-
- def _partial_score(scoring, mask):
- word_probs = [max(_) for _ in zip(scoring['word_probs'], mask)]
- scoring.update(word_probs=word_probs,
- joint_prob=np.prod(word_probs))
-
- _partial_score(scoring1, mask1)
- _partial_score(scoring2, mask2)
-
-
-def compare_substitutions(question_ids, scorings, mode='full'):
- """Return accuracy by comparing two consecutive scorings."""
- prediction_correctness = []
- # Compare two consecutive substitutions
- for i in range(len(scorings) // 2):
- scoring1, scoring2 = scorings[2*i: 2*i+2]
- if mode == 'partial': # fix joint prob into partial prob
- _convert_to_partial(scoring1, scoring2)
-
- prediction_correctness.append(
- (scoring2['joint_prob'] > scoring1['joint_prob']) ==
- scoring2['correctness'])
-
- # Two consecutive substitutions always belong to the same question
- question_ids = [qid for i, qid in enumerate(question_ids) if i % 2 == 0]
- assert len(question_ids) == len(prediction_correctness)
- num_questions = len(set(question_ids))
-
- # Question is correctly answered only if
- # all predictions of the same question_id is correct
- num_correct_answer = 0
- previous_qid = None
- correctly_answered = False
- for predict, qid in zip(prediction_correctness, question_ids):
- if qid != previous_qid:
- previous_qid = qid
- num_correct_answer += int(correctly_answered)
- correctly_answered = True
- correctly_answered = correctly_answered and predict
- num_correct_answer += int(correctly_answered)
-
- return num_correct_answer / num_questions
diff --git a/research/lstm_object_detection/tflite/protos/mobile_ssd_client_options.proto b/research/lstm_object_detection/tflite/protos/mobile_ssd_client_options.proto
index d501c213c11476675250f232b430e3ab1b62dac4..0fbd07574f74507cb128e4685e640c9497da5cb9 100644
--- a/research/lstm_object_detection/tflite/protos/mobile_ssd_client_options.proto
+++ b/research/lstm_object_detection/tflite/protos/mobile_ssd_client_options.proto
@@ -37,10 +37,10 @@ message ClientOptions {
// The threshold on intersection-over-union used by non-maxima suppression.
optional float iou_threshold = 5 [default = 0.3];
- // Optional whitelist of class names. If non-empty, detections whose class
+ // Optional allowlist of class names. If non-empty, detections whose class
// name is not in this set will be filtered out. Duplicate or unknown class
// names are ignored.
- repeated string class_name_whitelist = 6;
+ repeated string class_name_allowlist = 6;
// SSD in single class agnostic model.
optional bool agnostic_mode = 7 [default = false];
diff --git a/research/lstm_object_detection/tflite/utils/ssd_utils.h b/research/lstm_object_detection/tflite/utils/ssd_utils.h
index a9199e7fd5ca2c5826caf9ee6db9c1afc82a4534..a8efc00d3eafa00ee060348b798426f0ab0dad5e 100644
--- a/research/lstm_object_detection/tflite/utils/ssd_utils.h
+++ b/research/lstm_object_detection/tflite/utils/ssd_utils.h
@@ -63,7 +63,7 @@ void NonMaxSuppressionMultiClassFast(
// Similar to NonMaxSuppressionMultiClassFast, but restricts the results to
// the provided list of class indices. This effectively filters out any class
-// whose index is not in this whitelist.
+// whose index is not in this allowlist.
void NonMaxSuppressionMultiClassRestrict(
std::vector restricted_class_indices,
const protos::BoxCornerEncoding& boxes, const std::vector& scores,
diff --git a/research/maskgan/README.md b/research/maskgan/README.md
deleted file mode 100644
index 10ee8a4c4dd546983469b07e2fb8207fc200534d..0000000000000000000000000000000000000000
--- a/research/maskgan/README.md
+++ /dev/null
@@ -1,111 +0,0 @@
-
-
-
-
-# MaskGAN: Better Text Generation via Filling in the ______
-
-Code for [*MaskGAN: Better Text Generation via Filling in the
-______*](https://arxiv.org/abs/1801.07736) published at ICLR 2018.
-
-## Requirements
-
-* TensorFlow >= v1.5
-
-## Instructions
-
-Warning: The open-source version of this code is still in the process of being
-tested. Pretraining may not work correctly.
-
-For training on PTB:
-
-1. Follow instructions here ([Tensorflow RNN Language Model Tutorial](https://www.tensorflow.org/tutorials/sequences/recurrent)) to train a language model on PTB dataset.
-Copy PTB data downloaded from the above tensorflow RNN tutorial to folder "/tmp/ptb". It should contain following three files: ptb.train.txt, ptb.test.txt, ptb.valid.txt
-Make folder /tmp/pretrain-lm and copy checkpoints from above Tensorflow RNN tutorial under this folder.
-
-
-2. Run MaskGAN in MLE pretraining mode. If step 1 was not run*, set
-`language_model_ckpt_dir` to empty.
-
-```bash
-python train_mask_gan.py \
- --data_dir='/tmp/ptb' \
- --batch_size=20 \
- --sequence_length=20 \
- --base_directory='/tmp/maskGAN' \
- --hparams="gen_rnn_size=650,dis_rnn_size=650,gen_num_layers=2,dis_num_layers=2,gen_learning_rate=0.00074876,dis_learning_rate=5e-4,baseline_decay=0.99,dis_train_iterations=1,gen_learning_rate_decay=0.95" \
- --mode='TRAIN' \
- --max_steps=100000 \
- --language_model_ckpt_dir=/tmp/pretrain-lm/ \
- --generator_model='seq2seq_vd' \
- --discriminator_model='rnn_zaremba' \
- --is_present_rate=0.5 \
- --summaries_every=10 \
- --print_every=250 \
- --max_num_to_print=3 \
- --gen_training_strategy=cross_entropy \
- --seq2seq_share_embedding
-```
-
-3. Run MaskGAN in GAN mode. If step 2 was not run, set `maskgan_ckpt` to empty.
-```bash
-python train_mask_gan.py \
- --data_dir='/tmp/ptb' \
- --batch_size=128 \
- --sequence_length=20 \
- --base_directory='/tmp/maskGAN' \
- --mask_strategy=contiguous \
- --maskgan_ckpt='/tmp/maskGAN' \
- --hparams="gen_rnn_size=650,dis_rnn_size=650,gen_num_layers=2,dis_num_layers=2,gen_learning_rate=0.000038877,gen_learning_rate_decay=1.0,gen_full_learning_rate_steps=2000000,gen_vd_keep_prob=0.33971,rl_discount_rate=0.89072,dis_learning_rate=5e-4,baseline_decay=0.99,dis_train_iterations=2,dis_pretrain_learning_rate=0.005,critic_learning_rate=5.1761e-7,dis_vd_keep_prob=0.71940" \
- --mode='TRAIN' \
- --max_steps=100000 \
- --generator_model='seq2seq_vd' \
- --discriminator_model='seq2seq_vd' \
- --is_present_rate=0.5 \
- --summaries_every=250 \
- --print_every=250 \
- --max_num_to_print=3 \
- --gen_training_strategy='reinforce' \
- --seq2seq_share_embedding=true \
- --baseline_method=critic \
- --attention_option=luong
-```
-
-4. Generate samples:
-```bash
-python generate_samples.py \
- --data_dir /tmp/ptb/ \
- --data_set=ptb \
- --batch_size=256 \
- --sequence_length=20 \
- --base_directory /tmp/imdbsample/ \
- --hparams="gen_rnn_size=650,dis_rnn_size=650,gen_num_layers=2,gen_vd_keep_prob=0.33971" \
- --generator_model=seq2seq_vd \
- --discriminator_model=seq2seq_vd \
- --is_present_rate=0.0 \
- --maskgan_ckpt=/tmp/maskGAN \
- --seq2seq_share_embedding=True \
- --dis_share_embedding=True \
- --attention_option=luong \
- --mask_strategy=contiguous \
- --baseline_method=critic \
- --number_epochs=4
-```
-
-
-* While trying to run Step 2, the following error appears:
- NotFoundError (see above for traceback): Restoring from checkpoint failed. This is most likely due to a Variable name or other graph key that is missing from the checkpoint. Please ensure that you have not altered the graph expected based on the checkpoint. Original error:
-
- Key critic/rnn/biases not found in checkpoint
- [[node save/RestoreV2 (defined at train_mask_gan.py:431) ]]
-
- This is an issue with seq2seq model because it uses the attention mechanism.
- The issue arises if you saved the model with an earlier version (seq2seq is old) and restore with a recent one (saver.restore got updated).
- The naming convention for LSTM parameters changed, e.g. cell_0/basic_lstm_cell/weights became cell_0/basic_lstm_cell/kernel.
- Which is why you cannot restore them if you try to restore old checkpoints with recent TF.
- The below script will help rename the variables and everything will work as expected.
- https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/rnn/python/tools/checkpoint_convert.py
-
-## Contact for Issues
-
-* Liam Fedus, @liamb315
-* Andrew M. Dai, @a-dai
diff --git a/research/maskgan/data/__init__.py b/research/maskgan/data/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/maskgan/data/imdb_loader.py b/research/maskgan/data/imdb_loader.py
deleted file mode 100644
index 8169b3336b4ac0e1a36e35dbaed4c01f38f1ec02..0000000000000000000000000000000000000000
--- a/research/maskgan/data/imdb_loader.py
+++ /dev/null
@@ -1,136 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""IMDB data loader and helpers."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-# Dependency imports
-import numpy as np
-
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-tf.app.flags.DEFINE_boolean('prefix_label', True, 'Vocabulary file.')
-
-np.set_printoptions(precision=3)
-np.set_printoptions(suppress=True)
-
-EOS_INDEX = 88892
-
-
-def _read_words(filename, use_prefix=True):
- all_words = []
- sequence_example = tf.train.SequenceExample()
- for r in tf.python_io.tf_record_iterator(filename):
- sequence_example.ParseFromString(r)
-
- if FLAGS.prefix_label and use_prefix:
- label = sequence_example.context.feature['class'].int64_list.value[0]
- review_words = [EOS_INDEX + 1 + label]
- else:
- review_words = []
- review_words.extend([
- f.int64_list.value[0]
- for f in sequence_example.feature_lists.feature_list['token_id'].feature
- ])
- all_words.append(review_words)
- return all_words
-
-
-def build_vocab(vocab_file):
- word_to_id = {}
-
- with tf.gfile.GFile(vocab_file, 'r') as f:
- index = 0
- for word in f:
- word_to_id[word.strip()] = index
- index += 1
- word_to_id[''] = EOS_INDEX
-
- return word_to_id
-
-
-def imdb_raw_data(data_path=None):
- """Load IMDB raw data from data directory "data_path".
- Reads IMDB tf record files containing integer ids,
- and performs mini-batching of the inputs.
- Args:
- data_path: string path to the directory where simple-examples.tgz has
- been extracted.
- Returns:
- tuple (train_data, valid_data)
- where each of the data objects can be passed to IMDBIterator.
- """
-
- train_path = os.path.join(data_path, 'train_lm.tfrecords')
- valid_path = os.path.join(data_path, 'test_lm.tfrecords')
-
- train_data = _read_words(train_path)
- valid_data = _read_words(valid_path)
- return train_data, valid_data
-
-
-def imdb_iterator(raw_data, batch_size, num_steps, epoch_size_override=None):
- """Iterate on the raw IMDB data.
-
- This generates batch_size pointers into the raw IMDB data, and allows
- minibatch iteration along these pointers.
-
- Args:
- raw_data: one of the raw data outputs from imdb_raw_data.
- batch_size: int, the batch size.
- num_steps: int, the number of unrolls.
-
- Yields:
- Pairs of the batched data, each a matrix of shape [batch_size, num_steps].
- The second element of the tuple is the same data time-shifted to the
- right by one. The third is a set of weights with 1 indicating a word was
- present and 0 not.
-
- Raises:
- ValueError: if batch_size or num_steps are too high.
- """
- del epoch_size_override
- data_len = len(raw_data)
- num_batches = data_len // batch_size - 1
-
- for batch in range(num_batches):
- x = np.zeros([batch_size, num_steps], dtype=np.int32)
- y = np.zeros([batch_size, num_steps], dtype=np.int32)
- w = np.zeros([batch_size, num_steps], dtype=np.float)
-
- for i in range(batch_size):
- data_index = batch * batch_size + i
- example = raw_data[data_index]
-
- if len(example) > num_steps:
- final_x = example[:num_steps]
- final_y = example[1:(num_steps + 1)]
- w[i] = 1
-
- else:
- to_fill_in = num_steps - len(example)
- final_x = example + [EOS_INDEX] * to_fill_in
- final_y = final_x[1:] + [EOS_INDEX]
- w[i] = [1] * len(example) + [0] * to_fill_in
-
- x[i] = final_x
- y[i] = final_y
-
- yield (x, y, w)
diff --git a/research/maskgan/data/ptb_loader.py b/research/maskgan/data/ptb_loader.py
deleted file mode 100644
index 43105952a667f968faf12a4561f85964f0a123ae..0000000000000000000000000000000000000000
--- a/research/maskgan/data/ptb_loader.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""PTB data loader and helpers."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import os
-# Dependency imports
-import numpy as np
-
-import tensorflow as tf
-
-EOS_INDEX = 0
-
-
-def _read_words(filename):
- with tf.gfile.GFile(filename, "r") as f:
- return f.read().decode("utf-8").replace("\n", "").split()
-
-
-def build_vocab(filename):
- data = _read_words(filename)
-
- counter = collections.Counter(data)
- count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
-
- words, _ = list(zip(*count_pairs))
- word_to_id = dict(zip(words, range(len(words))))
- print(":", word_to_id[""])
- global EOS_INDEX
- EOS_INDEX = word_to_id[""]
-
- return word_to_id
-
-
-def _file_to_word_ids(filename, word_to_id):
- data = _read_words(filename)
- return [word_to_id[word] for word in data if word in word_to_id]
-
-
-def ptb_raw_data(data_path=None):
- """Load PTB raw data from data directory "data_path".
- Reads PTB text files, converts strings to integer ids,
- and performs mini-batching of the inputs.
- The PTB dataset comes from Tomas Mikolov's webpage:
- http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz
- Args:
- data_path: string path to the directory where simple-examples.tgz has
- been extracted.
- Returns:
- tuple (train_data, valid_data, test_data, vocabulary)
- where each of the data objects can be passed to PTBIterator.
- """
-
- train_path = os.path.join(data_path, "ptb.train.txt")
- valid_path = os.path.join(data_path, "ptb.valid.txt")
- test_path = os.path.join(data_path, "ptb.test.txt")
-
- word_to_id = build_vocab(train_path)
- train_data = _file_to_word_ids(train_path, word_to_id)
- valid_data = _file_to_word_ids(valid_path, word_to_id)
- test_data = _file_to_word_ids(test_path, word_to_id)
- vocabulary = len(word_to_id)
- return train_data, valid_data, test_data, vocabulary
-
-
-def ptb_iterator(raw_data, batch_size, num_steps, epoch_size_override=None):
- """Iterate on the raw PTB data.
-
- This generates batch_size pointers into the raw PTB data, and allows
- minibatch iteration along these pointers.
-
- Args:
- raw_data: one of the raw data outputs from ptb_raw_data.
- batch_size: int, the batch size.
- num_steps: int, the number of unrolls.
-
- Yields:
- Pairs of the batched data, each a matrix of shape [batch_size, num_steps].
- The second element of the tuple is the same data time-shifted to the
- right by one.
-
- Raises:
- ValueError: if batch_size or num_steps are too high.
- """
- raw_data = np.array(raw_data, dtype=np.int32)
-
- data_len = len(raw_data)
- batch_len = data_len // batch_size
- data = np.full([batch_size, batch_len], EOS_INDEX, dtype=np.int32)
- for i in range(batch_size):
- data[i] = raw_data[batch_len * i:batch_len * (i + 1)]
-
- if epoch_size_override:
- epoch_size = epoch_size_override
- else:
- epoch_size = (batch_len - 1) // num_steps
-
- if epoch_size == 0:
- raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
-
- # print("Number of batches per epoch: %d" % epoch_size)
- for i in range(epoch_size):
- x = data[:, i * num_steps:(i + 1) * num_steps]
- y = data[:, i * num_steps + 1:(i + 1) * num_steps + 1]
- w = np.ones_like(x)
- yield (x, y, w)
diff --git a/research/maskgan/generate_samples.py b/research/maskgan/generate_samples.py
deleted file mode 100644
index d4215ebc75a074b316010eb60189bf7428dfcfc5..0000000000000000000000000000000000000000
--- a/research/maskgan/generate_samples.py
+++ /dev/null
@@ -1,281 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Generate samples from the MaskGAN.
-
-Launch command:
- python generate_samples.py
- --data_dir=/tmp/data/imdb --data_set=imdb
- --batch_size=256 --sequence_length=20 --base_directory=/tmp/imdb
- --hparams="gen_rnn_size=650,dis_rnn_size=650,gen_num_layers=2,
- gen_vd_keep_prob=1.0" --generator_model=seq2seq_vd
- --discriminator_model=seq2seq_vd --is_present_rate=0.5
- --maskgan_ckpt=/tmp/model.ckpt-45494
- --seq2seq_share_embedding=True --dis_share_embedding=True
- --attention_option=luong --mask_strategy=contiguous --baseline_method=critic
- --number_epochs=4
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from functools import partial
-import os
-# Dependency imports
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import train_mask_gan
-from data import imdb_loader
-from data import ptb_loader
-
-# Data.
-from model_utils import helper
-from model_utils import model_utils
-
-SAMPLE_TRAIN = 'TRAIN'
-SAMPLE_VALIDATION = 'VALIDATION'
-
-## Sample Generation.
-## Binary and setup FLAGS.
-tf.app.flags.DEFINE_enum('sample_mode', 'TRAIN',
- [SAMPLE_TRAIN, SAMPLE_VALIDATION],
- 'Dataset to sample from.')
-tf.app.flags.DEFINE_string('output_path', '/tmp', 'Model output directory.')
-tf.app.flags.DEFINE_boolean(
- 'output_masked_logs', False,
- 'Whether to display for human evaluation (show masking).')
-tf.app.flags.DEFINE_integer('number_epochs', 1,
- 'The number of epochs to produce.')
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def get_iterator(data):
- """Return the data iterator."""
- if FLAGS.data_set == 'ptb':
- iterator = ptb_loader.ptb_iterator(data, FLAGS.batch_size,
- FLAGS.sequence_length,
- FLAGS.epoch_size_override)
- elif FLAGS.data_set == 'imdb':
- iterator = imdb_loader.imdb_iterator(data, FLAGS.batch_size,
- FLAGS.sequence_length)
- return iterator
-
-
-def convert_to_human_readable(id_to_word, arr, p, max_num_to_print):
- """Convert a np.array of indices into words using id_to_word dictionary.
- Return max_num_to_print results.
- """
-
- assert arr.ndim == 2
-
- samples = []
- for sequence_id in xrange(min(len(arr), max_num_to_print)):
- sample = []
- for i, index in enumerate(arr[sequence_id, :]):
- if p[sequence_id, i] == 1:
- sample.append(str(id_to_word[index]))
- else:
- sample.append('*' + str(id_to_word[index]))
- buffer_str = ' '.join(sample)
- samples.append(buffer_str)
- return samples
-
-
-def write_unmasked_log(log, id_to_word, sequence_eval):
- """Helper function for logging evaluated sequences without mask."""
- indices_arr = np.asarray(sequence_eval)
- samples = helper.convert_to_human_readable(id_to_word, indices_arr,
- FLAGS.batch_size)
- for sample in samples:
- log.write(sample + '\n')
- log.flush()
- return samples
-
-
-def write_masked_log(log, id_to_word, sequence_eval, present_eval):
- indices_arr = np.asarray(sequence_eval)
- samples = convert_to_human_readable(id_to_word, indices_arr, present_eval,
- FLAGS.batch_size)
- for sample in samples:
- log.write(sample + '\n')
- log.flush()
- return samples
-
-
-def generate_logs(sess, model, log, id_to_word, feed):
- """Impute Sequences using the model for a particular feed and send it to
- logs.
- """
- # Impute Sequences.
- [p, inputs_eval, sequence_eval] = sess.run(
- [model.present, model.inputs, model.fake_sequence], feed_dict=feed)
-
- # Add the 0th time-step for coherence.
- first_token = np.expand_dims(inputs_eval[:, 0], axis=1)
- sequence_eval = np.concatenate((first_token, sequence_eval), axis=1)
-
- # 0th token always present.
- p = np.concatenate((np.ones((FLAGS.batch_size, 1)), p), axis=1)
-
- if FLAGS.output_masked_logs:
- samples = write_masked_log(log, id_to_word, sequence_eval, p)
- else:
- samples = write_unmasked_log(log, id_to_word, sequence_eval)
- return samples
-
-
-def generate_samples(hparams, data, id_to_word, log_dir, output_file):
- """"Generate samples.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- data: Data to evaluate.
- id_to_word: Dictionary of indices to words.
- log_dir: Log directory.
- output_file: Output file for the samples.
- """
- # Boolean indicating operational mode.
- is_training = False
-
- # Set a random seed to keep fixed mask.
- np.random.seed(0)
-
- with tf.Graph().as_default():
- # Construct the model.
- model = train_mask_gan.create_MaskGAN(hparams, is_training)
-
- ## Retrieve the initial savers.
- init_savers = model_utils.retrieve_init_savers(hparams)
-
- ## Initial saver function to supervisor.
- init_fn = partial(model_utils.init_fn, init_savers)
-
- is_chief = FLAGS.task == 0
-
- # Create the supervisor. It will take care of initialization, summaries,
- # checkpoints, and recovery.
- sv = tf.Supervisor(
- logdir=log_dir,
- is_chief=is_chief,
- saver=model.saver,
- global_step=model.global_step,
- recovery_wait_secs=30,
- summary_op=None,
- init_fn=init_fn)
-
- # Get an initialized, and possibly recovered session. Launch the
- # services: Checkpointing, Summaries, step counting.
- #
- # When multiple replicas of this program are running the services are
- # only launched by the 'chief' replica.
- with sv.managed_session(
- FLAGS.master, start_standard_services=False) as sess:
-
- # Generator statefulness over the epoch.
- [gen_initial_state_eval, fake_gen_initial_state_eval] = sess.run(
- [model.eval_initial_state, model.fake_gen_initial_state])
-
- for n in xrange(FLAGS.number_epochs):
- print('Epoch number: %d' % n)
- # print('Percent done: %.2f' % float(n) / float(FLAGS.number_epochs))
- iterator = get_iterator(data)
- for x, y, _ in iterator:
- if FLAGS.eval_language_model:
- is_present_rate = 0.
- else:
- is_present_rate = FLAGS.is_present_rate
- tf.logging.info(
- 'Evaluating on is_present_rate=%.3f.' % is_present_rate)
-
- model_utils.assign_percent_real(sess, model.percent_real_update,
- model.new_rate, is_present_rate)
-
- # Randomly mask out tokens.
- p = model_utils.generate_mask()
-
- eval_feed = {model.inputs: x, model.targets: y, model.present: p}
-
- if FLAGS.data_set == 'ptb':
- # Statefulness for *evaluation* Generator.
- for i, (c, h) in enumerate(model.eval_initial_state):
- eval_feed[c] = gen_initial_state_eval[i].c
- eval_feed[h] = gen_initial_state_eval[i].h
-
- # Statefulness for the Generator.
- for i, (c, h) in enumerate(model.fake_gen_initial_state):
- eval_feed[c] = fake_gen_initial_state_eval[i].c
- eval_feed[h] = fake_gen_initial_state_eval[i].h
-
- [gen_initial_state_eval, fake_gen_initial_state_eval, _] = sess.run(
- [
- model.eval_final_state, model.fake_gen_final_state,
- model.global_step
- ],
- feed_dict=eval_feed)
-
- generate_logs(sess, model, output_file, id_to_word, eval_feed)
- output_file.close()
- print('Closing output_file.')
- return
-
-
-def main(_):
- hparams = train_mask_gan.create_hparams()
- log_dir = FLAGS.base_directory
-
- tf.gfile.MakeDirs(FLAGS.output_path)
- output_file = tf.gfile.GFile(
- os.path.join(FLAGS.output_path, 'reviews.txt'), mode='w')
-
- # Load data set.
- if FLAGS.data_set == 'ptb':
- raw_data = ptb_loader.ptb_raw_data(FLAGS.data_dir)
- train_data, valid_data, _, _ = raw_data
- elif FLAGS.data_set == 'imdb':
- raw_data = imdb_loader.imdb_raw_data(FLAGS.data_dir)
- train_data, valid_data = raw_data
- else:
- raise NotImplementedError
-
- # Generating more data on train set.
- if FLAGS.sample_mode == SAMPLE_TRAIN:
- data_set = train_data
- elif FLAGS.sample_mode == SAMPLE_VALIDATION:
- data_set = valid_data
- else:
- raise NotImplementedError
-
- # Dictionary and reverse dictionry.
- if FLAGS.data_set == 'ptb':
- word_to_id = ptb_loader.build_vocab(
- os.path.join(FLAGS.data_dir, 'ptb.train.txt'))
- elif FLAGS.data_set == 'imdb':
- word_to_id = imdb_loader.build_vocab(
- os.path.join(FLAGS.data_dir, 'vocab.txt'))
- id_to_word = {v: k for k, v in word_to_id.iteritems()}
-
- FLAGS.vocab_size = len(id_to_word)
- print('Vocab size: %d' % FLAGS.vocab_size)
-
- generate_samples(hparams, data_set, id_to_word, log_dir, output_file)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/maskgan/losses/__init__.py b/research/maskgan/losses/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/maskgan/losses/losses.py b/research/maskgan/losses/losses.py
deleted file mode 100644
index 38d0e7b4d13cfae9652d8c70f08bfba5c478e150..0000000000000000000000000000000000000000
--- a/research/maskgan/losses/losses.py
+++ /dev/null
@@ -1,186 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Losses for Generator and Discriminator."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-
-def discriminator_loss(predictions, labels, missing_tokens):
- """Discriminator loss based on predictions and labels.
-
- Args:
- predictions: Discriminator linear predictions Tensor of shape [batch_size,
- sequence_length]
- labels: Labels for predictions, Tensor of shape [batch_size,
- sequence_length]
- missing_tokens: Indicator for the missing tokens. Evaluate the loss only
- on the tokens that were missing.
-
- Returns:
- loss: Scalar tf.float32 loss.
-
- """
- loss = tf.losses.sigmoid_cross_entropy(labels,
- predictions,
- weights=missing_tokens)
- loss = tf.Print(
- loss, [loss, labels, missing_tokens],
- message='loss, labels, missing_tokens',
- summarize=25,
- first_n=25)
- return loss
-
-
-def cross_entropy_loss_matrix(gen_labels, gen_logits):
- """Computes the cross entropy loss for G.
-
- Args:
- gen_labels: Labels for the correct token.
- gen_logits: Generator logits.
-
- Returns:
- loss_matrix: Loss matrix of shape [batch_size, sequence_length].
- """
- cross_entropy_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
- labels=gen_labels, logits=gen_logits)
- return cross_entropy_loss
-
-
-def GAN_loss_matrix(dis_predictions):
- """Computes the cross entropy loss for G.
-
- Args:
- dis_predictions: Discriminator predictions.
-
- Returns:
- loss_matrix: Loss matrix of shape [batch_size, sequence_length].
- """
- eps = tf.constant(1e-7, tf.float32)
- gan_loss_matrix = -tf.log(dis_predictions + eps)
- return gan_loss_matrix
-
-
-def generator_GAN_loss(predictions):
- """Generator GAN loss based on Discriminator predictions."""
- return -tf.log(tf.reduce_mean(predictions))
-
-
-def generator_blended_forward_loss(gen_logits, gen_labels, dis_predictions,
- is_real_input):
- """Computes the masked-loss for G. This will be a blend of cross-entropy
- loss where the true label is known and GAN loss where the true label has been
- masked.
-
- Args:
- gen_logits: Generator logits.
- gen_labels: Labels for the correct token.
- dis_predictions: Discriminator predictions.
- is_real_input: Tensor indicating whether the label is present.
-
- Returns:
- loss: Scalar tf.float32 total loss.
- """
- cross_entropy_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
- labels=gen_labels, logits=gen_logits)
- gan_loss = -tf.log(dis_predictions)
- loss_matrix = tf.where(is_real_input, cross_entropy_loss, gan_loss)
- return tf.reduce_mean(loss_matrix)
-
-
-def wasserstein_generator_loss(gen_logits, gen_labels, dis_values,
- is_real_input):
- """Computes the masked-loss for G. This will be a blend of cross-entropy
- loss where the true label is known and GAN loss where the true label is
- missing.
-
- Args:
- gen_logits: Generator logits.
- gen_labels: Labels for the correct token.
- dis_values: Discriminator values Tensor of shape [batch_size,
- sequence_length].
- is_real_input: Tensor indicating whether the label is present.
-
- Returns:
- loss: Scalar tf.float32 total loss.
- """
- cross_entropy_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
- labels=gen_labels, logits=gen_logits)
- # Maximize the dis_values (minimize the negative)
- gan_loss = -dis_values
- loss_matrix = tf.where(is_real_input, cross_entropy_loss, gan_loss)
- loss = tf.reduce_mean(loss_matrix)
- return loss
-
-
-def wasserstein_discriminator_loss(real_values, fake_values):
- """Wasserstein discriminator loss.
-
- Args:
- real_values: Value given by the Wasserstein Discriminator to real data.
- fake_values: Value given by the Wasserstein Discriminator to fake data.
-
- Returns:
- loss: Scalar tf.float32 loss.
-
- """
- real_avg = tf.reduce_mean(real_values)
- fake_avg = tf.reduce_mean(fake_values)
-
- wasserstein_loss = real_avg - fake_avg
- return wasserstein_loss
-
-
-def wasserstein_discriminator_loss_intrabatch(values, is_real_input):
- """Wasserstein discriminator loss. This is an odd variant where the value
- difference is between the real tokens and the fake tokens within a single
- batch.
-
- Args:
- values: Value given by the Wasserstein Discriminator of shape [batch_size,
- sequence_length] to an imputed batch (real and fake).
- is_real_input: tf.bool Tensor of shape [batch_size, sequence_length]. If
- true, it indicates that the label is known.
-
- Returns:
- wasserstein_loss: Scalar tf.float32 loss.
-
- """
- zero_tensor = tf.constant(0., dtype=tf.float32, shape=[])
-
- present = tf.cast(is_real_input, tf.float32)
- missing = tf.cast(1 - present, tf.float32)
-
- # Counts for real and fake tokens.
- real_count = tf.reduce_sum(present)
- fake_count = tf.reduce_sum(missing)
-
- # Averages for real and fake token values.
- real = tf.mul(values, present)
- fake = tf.mul(values, missing)
- real_avg = tf.reduce_sum(real) / real_count
- fake_avg = tf.reduce_sum(fake) / fake_count
-
- # If there are no real or fake entries in the batch, we assign an average
- # value of zero.
- real_avg = tf.where(tf.equal(real_count, 0), zero_tensor, real_avg)
- fake_avg = tf.where(tf.equal(fake_count, 0), zero_tensor, fake_avg)
-
- wasserstein_loss = real_avg - fake_avg
- return wasserstein_loss
diff --git a/research/maskgan/model_utils/__init__.py b/research/maskgan/model_utils/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/maskgan/model_utils/helper.py b/research/maskgan/model_utils/helper.py
deleted file mode 100644
index 36115b484a007cda715b038e5cf52cbdd0b072ba..0000000000000000000000000000000000000000
--- a/research/maskgan/model_utils/helper.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Random helper functions for converting between indices and one-hot encodings
-as well as printing/logging helpers.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-
-def variable_summaries(var, name):
- """Attach a lot of summaries to a Tensor."""
- mean = tf.reduce_mean(var)
- tf.summary.scalar('mean/' + name, mean)
- with tf.name_scope('stddev'):
- stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean)))
- tf.summary.scalar('sttdev/' + name, stddev)
- tf.summary.scalar('max/' + name, tf.reduce_max(var))
- tf.summary.scalar('min/' + name, tf.reduce_min(var))
- tf.summary.histogram(name, var)
-
-
-def zip_seq_pred_crossent(id_to_word, sequences, predictions, cross_entropy):
- """Zip together the sequences, predictions, cross entropy."""
- indices = convert_to_indices(sequences)
-
- batch_of_metrics = []
-
- for ind_batch, pred_batch, crossent_batch in zip(indices, predictions,
- cross_entropy):
- metrics = []
-
- for index, pred, crossent in zip(ind_batch, pred_batch, crossent_batch):
- metrics.append([str(id_to_word[index]), pred, crossent])
-
- batch_of_metrics.append(metrics)
- return batch_of_metrics
-
-
-def print_and_log(log, id_to_word, sequence_eval, max_num_to_print=5):
- """Helper function for printing and logging evaluated sequences."""
- indices_eval = convert_to_indices(sequence_eval)
- indices_arr = np.asarray(indices_eval)
- samples = convert_to_human_readable(id_to_word, indices_arr, max_num_to_print)
-
- for i, sample in enumerate(samples):
- print('Sample', i, '. ', sample)
- log.write('\nSample ' + str(i) + '. ' + sample)
- log.write('\n')
- print('\n')
- log.flush()
-
-
-def convert_to_human_readable(id_to_word, arr, max_num_to_print):
- """Convert a np.array of indices into words using id_to_word dictionary.
- Return max_num_to_print results.
- """
- assert arr.ndim == 2
-
- samples = []
- for sequence_id in xrange(min(len(arr), max_num_to_print)):
- buffer_str = ' '.join(
- [str(id_to_word[index]) for index in arr[sequence_id, :]])
- samples.append(buffer_str)
- return samples
-
-
-def index_to_vocab_array(indices, vocab_size, sequence_length):
- """Convert the indices into an array with vocab_size one-hot encoding."""
-
- # Extract properties of the indices.
- num_batches = len(indices)
- shape = list(indices.shape)
- shape.append(vocab_size)
-
- # Construct the vocab_size array.
- new_arr = np.zeros(shape)
-
- for n in xrange(num_batches):
- indices_batch = indices[n]
- new_arr_batch = new_arr[n]
-
- # We map all indices greater than the vocabulary size to an unknown
- # character.
- indices_batch = np.where(indices_batch < vocab_size, indices_batch,
- vocab_size - 1)
-
- # Convert indices to vocab_size dimensions.
- new_arr_batch[np.arange(sequence_length), indices_batch] = 1
- return new_arr
-
-
-def convert_to_indices(sequences):
- """Convert a list of size [batch_size, sequence_length, vocab_size] to
- a list of size [batch_size, sequence_length] where the vocab element is
- denoted by the index.
- """
- batch_of_indices = []
-
- for sequence in sequences:
- indices = []
- for embedding in sequence:
- indices.append(np.argmax(embedding))
- batch_of_indices.append(indices)
- return batch_of_indices
-
-
-def convert_and_zip(id_to_word, sequences, predictions):
- """Helper function for printing or logging. Retrieves list of sequences
- and predictions and zips them together.
- """
- indices = convert_to_indices(sequences)
-
- batch_of_indices_predictions = []
-
- for index_batch, pred_batch in zip(indices, predictions):
- indices_predictions = []
-
- for index, pred in zip(index_batch, pred_batch):
- indices_predictions.append([str(id_to_word[index]), pred])
- batch_of_indices_predictions.append(indices_predictions)
- return batch_of_indices_predictions
-
-
-def recursive_length(item):
- """Recursively determine the total number of elements in nested list."""
- if type(item) == list:
- return sum(recursive_length(subitem) for subitem in item)
- else:
- return 1.
-
-
-def percent_correct(real_sequence, fake_sequences):
- """Determine the percent of tokens correctly generated within a batch."""
- identical = 0.
- for fake_sequence in fake_sequences:
- for real, fake in zip(real_sequence, fake_sequence):
- if real == fake:
- identical += 1.
- return identical / recursive_length(fake_sequences)
diff --git a/research/maskgan/model_utils/model_construction.py b/research/maskgan/model_utils/model_construction.py
deleted file mode 100644
index 8dfa1df343984d903ace5984a90c36cc0b67dbe3..0000000000000000000000000000000000000000
--- a/research/maskgan/model_utils/model_construction.py
+++ /dev/null
@@ -1,234 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Model construction."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-
-import tensorflow as tf
-from models import bidirectional
-from models import bidirectional_vd
-
-from models import bidirectional_zaremba
-from models import cnn
-from models import critic_vd
-from models import feedforward
-from models import rnn
-from models import rnn_nas
-from models import rnn_vd
-from models import rnn_zaremba
-from models import seq2seq
-from models import seq2seq_nas
-from models import seq2seq_vd
-from models import seq2seq_zaremba
-
-FLAGS = tf.app.flags.FLAGS
-
-
-# TODO(adai): IMDB labels placeholder to model.
-def create_generator(hparams,
- inputs,
- targets,
- present,
- is_training,
- is_validating,
- reuse=None):
- """Create the Generator model specified by the FLAGS and hparams.
-
- Args;
- hparams: Hyperparameters for the MaskGAN.
- inputs: tf.int32 Tensor of the sequence input of shape [batch_size,
- sequence_length].
- present: tf.bool Tensor indicating the presence or absence of the token
- of shape [batch_size, sequence_length].
- is_training: Whether the model is training.
- is_validating: Whether the model is being run in validation mode for
- calculating the perplexity.
- reuse (Optional): Whether to reuse the model.
-
- Returns:
- Tuple of the (sequence, logits, log_probs) of the Generator. Sequence
- and logits have shape [batch_size, sequence_length, vocab_size]. The
- log_probs will have shape [batch_size, sequence_length]. Log_probs
- corresponds to the log probability of selecting the words.
- """
- if FLAGS.generator_model == 'rnn':
- (sequence, logits, log_probs, initial_state, final_state) = rnn.generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- elif FLAGS.generator_model == 'rnn_zaremba':
- (sequence, logits, log_probs, initial_state,
- final_state) = rnn_zaremba.generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- elif FLAGS.generator_model == 'seq2seq':
- (sequence, logits, log_probs, initial_state,
- final_state) = seq2seq.generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- elif FLAGS.generator_model == 'seq2seq_zaremba':
- (sequence, logits, log_probs, initial_state,
- final_state) = seq2seq_zaremba.generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- elif FLAGS.generator_model == 'rnn_nas':
- (sequence, logits, log_probs, initial_state,
- final_state) = rnn_nas.generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- elif FLAGS.generator_model == 'seq2seq_nas':
- (sequence, logits, log_probs, initial_state,
- final_state) = seq2seq_nas.generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- elif FLAGS.generator_model == 'seq2seq_vd':
- (sequence, logits, log_probs, initial_state, final_state,
- encoder_states) = seq2seq_vd.generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- else:
- raise NotImplementedError
- return (sequence, logits, log_probs, initial_state, final_state,
- encoder_states)
-
-
-def create_discriminator(hparams,
- sequence,
- is_training,
- reuse=None,
- initial_state=None,
- inputs=None,
- present=None):
- """Create the Discriminator model specified by the FLAGS and hparams.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- sequence: tf.int32 Tensor sequence of shape [batch_size, sequence_length]
- is_training: Whether the model is training.
- reuse (Optional): Whether to reuse the model.
-
- Returns:
- predictions: tf.float32 Tensor of predictions of shape [batch_size,
- sequence_length]
- """
- if FLAGS.discriminator_model == 'cnn':
- predictions = cnn.discriminator(
- hparams, sequence, is_training=is_training, reuse=reuse)
- elif FLAGS.discriminator_model == 'fnn':
- predictions = feedforward.discriminator(
- hparams, sequence, is_training=is_training, reuse=reuse)
- elif FLAGS.discriminator_model == 'rnn':
- predictions = rnn.discriminator(
- hparams, sequence, is_training=is_training, reuse=reuse)
- elif FLAGS.discriminator_model == 'bidirectional':
- predictions = bidirectional.discriminator(
- hparams, sequence, is_training=is_training, reuse=reuse)
- elif FLAGS.discriminator_model == 'bidirectional_zaremba':
- predictions = bidirectional_zaremba.discriminator(
- hparams, sequence, is_training=is_training, reuse=reuse)
- elif FLAGS.discriminator_model == 'seq2seq_vd':
- predictions = seq2seq_vd.discriminator(
- hparams,
- inputs,
- present,
- sequence,
- is_training=is_training,
- reuse=reuse)
- elif FLAGS.discriminator_model == 'rnn_zaremba':
- predictions = rnn_zaremba.discriminator(
- hparams, sequence, is_training=is_training, reuse=reuse)
- elif FLAGS.discriminator_model == 'rnn_nas':
- predictions = rnn_nas.discriminator(
- hparams, sequence, is_training=is_training, reuse=reuse)
- elif FLAGS.discriminator_model == 'rnn_vd':
- predictions = rnn_vd.discriminator(
- hparams,
- sequence,
- is_training=is_training,
- reuse=reuse,
- initial_state=initial_state)
- elif FLAGS.discriminator_model == 'bidirectional_vd':
- predictions = bidirectional_vd.discriminator(
- hparams,
- sequence,
- is_training=is_training,
- reuse=reuse,
- initial_state=initial_state)
- else:
- raise NotImplementedError
- return predictions
-
-
-def create_critic(hparams, sequence, is_training, reuse=None):
- """Create the Critic model specified by the FLAGS and hparams.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- sequence: tf.int32 Tensor sequence of shape [batch_size, sequence_length]
- is_training: Whether the model is training.
- reuse (Optional): Whether to reuse the model.
-
- Returns:
- values: tf.float32 Tensor of predictions of shape [batch_size,
- sequence_length]
- """
- if FLAGS.baseline_method == 'critic':
- if FLAGS.discriminator_model == 'seq2seq_vd':
- values = critic_vd.critic_seq2seq_vd_derivative(
- hparams, sequence, is_training, reuse=reuse)
- else:
- raise NotImplementedError
- else:
- raise NotImplementedError
- return values
diff --git a/research/maskgan/model_utils/model_losses.py b/research/maskgan/model_utils/model_losses.py
deleted file mode 100644
index c8f337dc48b4f1efb1cf8604327376ddaa9994ea..0000000000000000000000000000000000000000
--- a/research/maskgan/model_utils/model_losses.py
+++ /dev/null
@@ -1,327 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Model loss construction."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-# Useful for REINFORCE baseline.
-from losses import losses
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def create_dis_loss(fake_predictions, real_predictions, targets_present):
- """Compute Discriminator loss across real/fake."""
-
- missing = tf.cast(targets_present, tf.int32)
- missing = 1 - missing
- missing = tf.cast(missing, tf.bool)
-
- real_labels = tf.ones([FLAGS.batch_size, FLAGS.sequence_length])
- dis_loss_real = tf.losses.sigmoid_cross_entropy(
- real_labels, real_predictions, weights=missing)
- dis_loss_fake = tf.losses.sigmoid_cross_entropy(
- targets_present, fake_predictions, weights=missing)
-
- dis_loss = (dis_loss_fake + dis_loss_real) / 2.
- return dis_loss, dis_loss_fake, dis_loss_real
-
-
-def create_critic_loss(cumulative_rewards, estimated_values, present):
- """Compute Critic loss in estimating the value function. This should be an
- estimate only for the missing elements."""
- missing = tf.cast(present, tf.int32)
- missing = 1 - missing
- missing = tf.cast(missing, tf.bool)
-
- loss = tf.losses.mean_squared_error(
- labels=cumulative_rewards, predictions=estimated_values, weights=missing)
- return loss
-
-
-def create_masked_cross_entropy_loss(targets, present, logits):
- """Calculate the cross entropy loss matrices for the masked tokens."""
- cross_entropy_losses = losses.cross_entropy_loss_matrix(targets, logits)
-
- # Zeros matrix.
- zeros_losses = tf.zeros(
- shape=[FLAGS.batch_size, FLAGS.sequence_length], dtype=tf.float32)
-
- missing_ce_loss = tf.where(present, zeros_losses, cross_entropy_losses)
-
- return missing_ce_loss
-
-
-def calculate_reinforce_objective(hparams,
- log_probs,
- dis_predictions,
- present,
- estimated_values=None):
- """Calculate the REINFORCE objectives. The REINFORCE objective should
- only be on the tokens that were missing. Specifically, the final Generator
- reward should be based on the Discriminator predictions on missing tokens.
- The log probaibilities should be only for missing tokens and the baseline
- should be calculated only on the missing tokens.
-
- For this model, we optimize the reward is the log of the *conditional*
- probability the Discriminator assigns to the distribution. Specifically, for
- a Discriminator D which outputs probability of real, given the past context,
-
- r_t = log D(x_t|x_0,x_1,...x_{t-1})
-
- And the policy for Generator G is the log-probability of taking action x2
- given the past context.
-
-
- Args:
- hparams: MaskGAN hyperparameters.
- log_probs: tf.float32 Tensor of log probailities of the tokens selected by
- the Generator. Shape [batch_size, sequence_length].
- dis_predictions: tf.float32 Tensor of the predictions from the
- Discriminator. Shape [batch_size, sequence_length].
- present: tf.bool Tensor indicating which tokens are present. Shape
- [batch_size, sequence_length].
- estimated_values: tf.float32 Tensor of estimated state values of tokens.
- Shape [batch_size, sequence_length]
-
- Returns:
- final_gen_objective: Final REINFORCE objective for the sequence.
- rewards: tf.float32 Tensor of rewards for sequence of shape [batch_size,
- sequence_length]
- advantages: tf.float32 Tensor of advantages for sequence of shape
- [batch_size, sequence_length]
- baselines: tf.float32 Tensor of baselines for sequence of shape
- [batch_size, sequence_length]
- maintain_averages_op: ExponentialMovingAverage apply average op to
- maintain the baseline.
- """
- # Final Generator objective.
- final_gen_objective = 0.
- gamma = hparams.rl_discount_rate
- eps = 1e-7
-
- # Generator rewards are log-probabilities.
- eps = tf.constant(1e-7, tf.float32)
- dis_predictions = tf.nn.sigmoid(dis_predictions)
- rewards = tf.log(dis_predictions + eps)
-
- # Apply only for missing elements.
- zeros = tf.zeros_like(present, dtype=tf.float32)
- log_probs = tf.where(present, zeros, log_probs)
- rewards = tf.where(present, zeros, rewards)
-
- # Unstack Tensors into lists.
- rewards_list = tf.unstack(rewards, axis=1)
- log_probs_list = tf.unstack(log_probs, axis=1)
- missing = 1. - tf.cast(present, tf.float32)
- missing_list = tf.unstack(missing, axis=1)
-
- # Cumulative Discounted Returns. The true value function V*(s).
- cumulative_rewards = []
- for t in xrange(FLAGS.sequence_length):
- cum_value = tf.zeros(shape=[FLAGS.batch_size])
- for s in xrange(t, FLAGS.sequence_length):
- cum_value += missing_list[s] * np.power(gamma, (s - t)) * rewards_list[s]
- cumulative_rewards.append(cum_value)
- cumulative_rewards = tf.stack(cumulative_rewards, axis=1)
-
- ## REINFORCE with different baselines.
- # We create a separate critic functionality for the Discriminator. This
- # will need to operate unidirectionally and it may take in the past context.
- if FLAGS.baseline_method == 'critic':
-
- # Critic loss calculated from the estimated value function \hat{V}(s)
- # versus the true value function V*(s).
- critic_loss = create_critic_loss(cumulative_rewards, estimated_values,
- present)
-
- # Baselines are coming from the critic's estimated state values.
- baselines = tf.unstack(estimated_values, axis=1)
-
- ## Calculate the Advantages, A(s,a) = Q(s,a) - \hat{V}(s).
- advantages = []
- for t in xrange(FLAGS.sequence_length):
- log_probability = log_probs_list[t]
- cum_advantage = tf.zeros(shape=[FLAGS.batch_size])
-
- for s in xrange(t, FLAGS.sequence_length):
- cum_advantage += missing_list[s] * np.power(gamma,
- (s - t)) * rewards_list[s]
- cum_advantage -= baselines[t]
- # Clip advantages.
- cum_advantage = tf.clip_by_value(cum_advantage, -FLAGS.advantage_clipping,
- FLAGS.advantage_clipping)
- advantages.append(missing_list[t] * cum_advantage)
- final_gen_objective += tf.multiply(
- log_probability, missing_list[t] * tf.stop_gradient(cum_advantage))
-
- maintain_averages_op = None
- baselines = tf.stack(baselines, axis=1)
- advantages = tf.stack(advantages, axis=1)
-
- # Split the batch into half. Use half for MC estimates for REINFORCE.
- # Use the other half to establish a baseline.
- elif FLAGS.baseline_method == 'dis_batch':
- # TODO(liamfedus): Recheck.
- [rewards_half, baseline_half] = tf.split(
- rewards, num_or_size_splits=2, axis=0)
- [log_probs_half, _] = tf.split(log_probs, num_or_size_splits=2, axis=0)
- [reward_present_half, baseline_present_half] = tf.split(
- present, num_or_size_splits=2, axis=0)
-
- # Unstack to lists.
- baseline_list = tf.unstack(baseline_half, axis=1)
- baseline_missing = 1. - tf.cast(baseline_present_half, tf.float32)
- baseline_missing_list = tf.unstack(baseline_missing, axis=1)
-
- baselines = []
- for t in xrange(FLAGS.sequence_length):
- # Calculate baseline only for missing tokens.
- num_missing = tf.reduce_sum(baseline_missing_list[t])
-
- avg_baseline = tf.reduce_sum(
- baseline_missing_list[t] * baseline_list[t], keep_dims=True) / (
- num_missing + eps)
- baseline = tf.tile(avg_baseline, multiples=[FLAGS.batch_size / 2])
- baselines.append(baseline)
-
- # Unstack to lists.
- rewards_list = tf.unstack(rewards_half, axis=1)
- log_probs_list = tf.unstack(log_probs_half, axis=1)
- reward_missing = 1. - tf.cast(reward_present_half, tf.float32)
- reward_missing_list = tf.unstack(reward_missing, axis=1)
-
- ## Calculate the Advantages, A(s,a) = Q(s,a) - \hat{V}(s).
- advantages = []
- for t in xrange(FLAGS.sequence_length):
- log_probability = log_probs_list[t]
- cum_advantage = tf.zeros(shape=[FLAGS.batch_size / 2])
-
- for s in xrange(t, FLAGS.sequence_length):
- cum_advantage += reward_missing_list[s] * np.power(gamma, (s - t)) * (
- rewards_list[s] - baselines[s])
- # Clip advantages.
- cum_advantage = tf.clip_by_value(cum_advantage, -FLAGS.advantage_clipping,
- FLAGS.advantage_clipping)
- advantages.append(reward_missing_list[t] * cum_advantage)
- final_gen_objective += tf.multiply(
- log_probability,
- reward_missing_list[t] * tf.stop_gradient(cum_advantage))
-
- # Cumulative Discounted Returns. The true value function V*(s).
- cumulative_rewards = []
- for t in xrange(FLAGS.sequence_length):
- cum_value = tf.zeros(shape=[FLAGS.batch_size / 2])
- for s in xrange(t, FLAGS.sequence_length):
- cum_value += reward_missing_list[s] * np.power(gamma, (
- s - t)) * rewards_list[s]
- cumulative_rewards.append(cum_value)
- cumulative_rewards = tf.stack(cumulative_rewards, axis=1)
-
- rewards = rewards_half
- critic_loss = None
- maintain_averages_op = None
- baselines = tf.stack(baselines, axis=1)
- advantages = tf.stack(advantages, axis=1)
-
- # Exponential Moving Average baseline.
- elif FLAGS.baseline_method == 'ema':
- # TODO(liamfedus): Recheck.
- # Lists of rewards and Log probabilities of the actions taken only for
- # missing tokens.
- ema = tf.train.ExponentialMovingAverage(decay=hparams.baseline_decay)
- maintain_averages_op = ema.apply(rewards_list)
-
- baselines = []
- for r in rewards_list:
- baselines.append(ema.average(r))
-
- ## Calculate the Advantages, A(s,a) = Q(s,a) - \hat{V}(s).
- advantages = []
- for t in xrange(FLAGS.sequence_length):
- log_probability = log_probs_list[t]
-
- # Calculate the forward advantage only on the missing tokens.
- cum_advantage = tf.zeros(shape=[FLAGS.batch_size])
- for s in xrange(t, FLAGS.sequence_length):
- cum_advantage += missing_list[s] * np.power(gamma, (s - t)) * (
- rewards_list[s] - baselines[s])
- # Clip advantages.
- cum_advantage = tf.clip_by_value(cum_advantage, -FLAGS.advantage_clipping,
- FLAGS.advantage_clipping)
- advantages.append(missing_list[t] * cum_advantage)
- final_gen_objective += tf.multiply(
- log_probability, missing_list[t] * tf.stop_gradient(cum_advantage))
-
- critic_loss = None
- baselines = tf.stack(baselines, axis=1)
- advantages = tf.stack(advantages, axis=1)
-
- elif FLAGS.baseline_method is None:
- num_missing = tf.reduce_sum(missing)
- final_gen_objective += tf.reduce_sum(rewards) / (num_missing + eps)
- baselines = tf.zeros_like(rewards)
- critic_loss = None
- maintain_averages_op = None
- advantages = cumulative_rewards
-
- else:
- raise NotImplementedError
-
- return [
- final_gen_objective, log_probs, rewards, advantages, baselines,
- maintain_averages_op, critic_loss, cumulative_rewards
- ]
-
-
-def calculate_log_perplexity(logits, targets, present):
- """Calculate the average log perplexity per *missing* token.
-
- Args:
- logits: tf.float32 Tensor of the logits of shape [batch_size,
- sequence_length, vocab_size].
- targets: tf.int32 Tensor of the sequence target of shape [batch_size,
- sequence_length].
- present: tf.bool Tensor indicating the presence or absence of the token
- of shape [batch_size, sequence_length].
-
- Returns:
- avg_log_perplexity: Scalar indicating the average log perplexity per
- missing token in the batch.
- """
- # logits = tf.Print(logits, [logits], message='logits:', summarize=50)
- # targets = tf.Print(targets, [targets], message='targets:', summarize=50)
- eps = 1e-12
- logits = tf.reshape(logits, [-1, FLAGS.vocab_size])
-
- # Only calculate log-perplexity on missing tokens.
- weights = tf.cast(present, tf.float32)
- weights = 1. - weights
- weights = tf.reshape(weights, [-1])
- num_missing = tf.reduce_sum(weights)
-
- log_perplexity = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
- [logits], [tf.reshape(targets, [-1])], [weights])
-
- avg_log_perplexity = tf.reduce_sum(log_perplexity) / (num_missing + eps)
- return avg_log_perplexity
diff --git a/research/maskgan/model_utils/model_optimization.py b/research/maskgan/model_utils/model_optimization.py
deleted file mode 100644
index caae271fe8bed390f032763972a43312f7a8ce9b..0000000000000000000000000000000000000000
--- a/research/maskgan/model_utils/model_optimization.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Model optimization."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def create_dis_pretrain_op(hparams, dis_loss, global_step):
- """Create a train op for pretraining."""
- with tf.name_scope('pretrain_generator'):
- optimizer = tf.train.AdamOptimizer(hparams.dis_pretrain_learning_rate)
- dis_vars = [
- v for v in tf.trainable_variables() if v.op.name.startswith('dis')
- ]
- if FLAGS.dis_update_share_embedding and FLAGS.dis_share_embedding:
- shared_embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == 'gen/decoder/rnn/embedding'
- ][0]
- dis_vars.append(shared_embedding)
- dis_grads = tf.gradients(dis_loss, dis_vars)
- dis_grads_clipped, _ = tf.clip_by_global_norm(dis_grads,
- FLAGS.grad_clipping)
- dis_pretrain_op = optimizer.apply_gradients(
- zip(dis_grads_clipped, dis_vars), global_step=global_step)
- return dis_pretrain_op
-
-
-def create_gen_pretrain_op(hparams, cross_entropy_loss, global_step):
- """Create a train op for pretraining."""
- with tf.name_scope('pretrain_generator'):
- optimizer = tf.train.AdamOptimizer(hparams.gen_pretrain_learning_rate)
- gen_vars = [
- v for v in tf.trainable_variables() if v.op.name.startswith('gen')
- ]
- gen_grads = tf.gradients(cross_entropy_loss, gen_vars)
- gen_grads_clipped, _ = tf.clip_by_global_norm(gen_grads,
- FLAGS.grad_clipping)
- gen_pretrain_op = optimizer.apply_gradients(
- zip(gen_grads_clipped, gen_vars), global_step=global_step)
- return gen_pretrain_op
-
-
-def create_gen_train_op(hparams, learning_rate, gen_loss, global_step, mode):
- """Create Generator train op."""
- del hparams
- with tf.name_scope('train_generator'):
- if FLAGS.generator_optimizer == 'sgd':
- gen_optimizer = tf.train.GradientDescentOptimizer(learning_rate)
- elif FLAGS.generator_optimizer == 'adam':
- gen_optimizer = tf.train.AdamOptimizer(learning_rate)
- else:
- raise NotImplementedError
- gen_vars = [
- v for v in tf.trainable_variables() if v.op.name.startswith('gen')
- ]
- print('Optimizing Generator vars.')
- for v in gen_vars:
- print(v)
- if mode == 'MINIMIZE':
- gen_grads = tf.gradients(gen_loss, gen_vars)
- elif mode == 'MAXIMIZE':
- gen_grads = tf.gradients(-gen_loss, gen_vars)
- else:
- raise ValueError("Must be one of 'MINIMIZE' or 'MAXIMIZE'")
- gen_grads_clipped, _ = tf.clip_by_global_norm(gen_grads,
- FLAGS.grad_clipping)
- gen_train_op = gen_optimizer.apply_gradients(
- zip(gen_grads_clipped, gen_vars), global_step=global_step)
- return gen_train_op, gen_grads_clipped, gen_vars
-
-
-def create_reinforce_gen_train_op(hparams, learning_rate, final_gen_reward,
- averages_op, global_step):
- """Create the Generator train_op when using REINFORCE.
-
- Args:
- hparams: MaskGAN hyperparameters.
- learning_rate: tf.Variable scalar learning rate.
- final_gen_objective: Scalar final REINFORCE objective for the sequence.
- averages_op: ExponentialMovingAverage apply average op to
- maintain the baseline.
- global_step: global_step tf.Variable.
-
- Returns:
- gen_train_op: Generator training op.
- """
- del hparams
- with tf.name_scope('train_generator'):
- if FLAGS.generator_optimizer == 'sgd':
- gen_optimizer = tf.train.GradientDescentOptimizer(learning_rate)
- elif FLAGS.generator_optimizer == 'adam':
- gen_optimizer = tf.train.AdamOptimizer(learning_rate)
- else:
- raise NotImplementedError
- gen_vars = [
- v for v in tf.trainable_variables() if v.op.name.startswith('gen')
- ]
- print('\nOptimizing Generator vars:')
- for v in gen_vars:
- print(v)
-
- # Maximize reward.
- gen_grads = tf.gradients(-final_gen_reward, gen_vars)
- gen_grads_clipped, _ = tf.clip_by_global_norm(gen_grads,
- FLAGS.grad_clipping)
- maximize_op = gen_optimizer.apply_gradients(
- zip(gen_grads_clipped, gen_vars), global_step=global_step)
-
- # Group maintain averages op.
- if averages_op:
- gen_train_op = tf.group(maximize_op, averages_op)
- else:
- gen_train_op = maximize_op
-
- return [gen_train_op, gen_grads, gen_vars]
-
-
-def create_dis_train_op(hparams, dis_loss, global_step):
- """Create Discriminator train op."""
- with tf.name_scope('train_discriminator'):
- dis_optimizer = tf.train.AdamOptimizer(hparams.dis_learning_rate)
- dis_vars = [
- v for v in tf.trainable_variables() if v.op.name.startswith('dis')
- ]
- if FLAGS.dis_update_share_embedding and FLAGS.dis_share_embedding:
- shared_embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == 'gen/decoder/rnn/embedding'
- ][0]
- dis_vars.append(shared_embedding)
- print('\nOptimizing Discriminator vars:')
- for v in dis_vars:
- print(v)
- dis_grads = tf.gradients(dis_loss, dis_vars)
- dis_grads_clipped, _ = tf.clip_by_global_norm(dis_grads,
- FLAGS.grad_clipping)
- dis_train_op = dis_optimizer.apply_gradients(
- zip(dis_grads_clipped, dis_vars), global_step=global_step)
- return dis_train_op, dis_grads_clipped, dis_vars
-
-
-def create_critic_train_op(hparams, critic_loss, global_step):
- """Create Discriminator train op."""
- with tf.name_scope('train_critic'):
- critic_optimizer = tf.train.AdamOptimizer(hparams.critic_learning_rate)
- output_vars = [
- v for v in tf.trainable_variables() if v.op.name.startswith('critic')
- ]
-
- if FLAGS.critic_update_dis_vars:
- if FLAGS.discriminator_model == 'bidirectional_vd':
- critic_vars = [
- v for v in tf.trainable_variables()
- if v.op.name.startswith('dis/rnn')
- ]
- elif FLAGS.discriminator_model == 'seq2seq_vd':
- critic_vars = [
- v for v in tf.trainable_variables()
- if v.op.name.startswith('dis/decoder/rnn/multi_rnn_cell')
- ]
- critic_vars.extend(output_vars)
- else:
- critic_vars = output_vars
- print('\nOptimizing Critic vars:')
- for v in critic_vars:
- print(v)
- critic_grads = tf.gradients(critic_loss, critic_vars)
- critic_grads_clipped, _ = tf.clip_by_global_norm(critic_grads,
- FLAGS.grad_clipping)
- critic_train_op = critic_optimizer.apply_gradients(
- zip(critic_grads_clipped, critic_vars), global_step=global_step)
- return critic_train_op, critic_grads_clipped, critic_vars
diff --git a/research/maskgan/model_utils/model_utils.py b/research/maskgan/model_utils/model_utils.py
deleted file mode 100644
index 0e3183582e0f17b7d4ca54450231ea9bad039e40..0000000000000000000000000000000000000000
--- a/research/maskgan/model_utils/model_utils.py
+++ /dev/null
@@ -1,291 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Model utilities."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-import numpy as np
-
-import tensorflow as tf
-from model_utils import variable_mapping
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def generate_mask():
- """Generate the mask to be fed into the model."""
- if FLAGS.mask_strategy == 'random':
- p = np.random.choice(
- [True, False],
- size=[FLAGS.batch_size, FLAGS.sequence_length],
- p=[FLAGS.is_present_rate, 1. - FLAGS.is_present_rate])
-
- elif FLAGS.mask_strategy == 'contiguous':
- masked_length = int((1 - FLAGS.is_present_rate) * FLAGS.sequence_length) - 1
- # Determine location to start masking.
- start_mask = np.random.randint(
- 1, FLAGS.sequence_length - masked_length + 1, size=FLAGS.batch_size)
- p = np.full([FLAGS.batch_size, FLAGS.sequence_length], True, dtype=bool)
-
- # Create contiguous masked section to be False.
- for i, index in enumerate(start_mask):
- p[i, index:index + masked_length] = False
-
- else:
- raise NotImplementedError
-
- return p
-
-
-def assign_percent_real(session, percent_real_update, new_rate, current_rate):
- """Run assign operation where the we load the current_rate of percent
- real into a Tensorflow variable.
-
- Args:
- session: Current tf.Session.
- percent_real_update: tf.assign operation.
- new_rate: tf.placeholder for the new rate.
- current_rate: Percent of tokens that are currently real. Fake tokens
- are the ones being imputed by the Generator.
- """
- session.run(percent_real_update, feed_dict={new_rate: current_rate})
-
-
-def assign_learning_rate(session, lr_update, lr_placeholder, new_lr):
- """Run assign operation where the we load the current_rate of percent
- real into a Tensorflow variable.
-
- Args:
- session: Current tf.Session.
- lr_update: tf.assign operation.
- lr_placeholder: tf.placeholder for the new learning rate.
- new_lr: New learning rate to use.
- """
- session.run(lr_update, feed_dict={lr_placeholder: new_lr})
-
-
-def clip_weights(variables, c_lower, c_upper):
- """Clip a list of weights to be within a certain range.
-
- Args:
- variables: List of tf.Variable weights.
- c_lower: Lower bound for weights.
- c_upper: Upper bound for weights.
- """
- clip_ops = []
-
- for var in variables:
- clipped_var = tf.clip_by_value(var, c_lower, c_upper)
-
- clip_ops.append(tf.assign(var, clipped_var))
- return tf.group(*clip_ops)
-
-
-def retrieve_init_savers(hparams):
- """Retrieve a dictionary of all the initial savers for the models.
-
- Args:
- hparams: MaskGAN hyperparameters.
- """
- ## Dictionary of init savers.
- init_savers = {}
-
- ## Load Generator weights from MaskGAN checkpoint.
- if FLAGS.maskgan_ckpt:
- gen_vars = [
- v for v in tf.trainable_variables() if v.op.name.startswith('gen')
- ]
- init_saver = tf.train.Saver(var_list=gen_vars)
- init_savers['init_saver'] = init_saver
-
- ## Load the Discriminator weights from the MaskGAN checkpoint if
- # the weights are compatible.
- if FLAGS.discriminator_model == 'seq2seq_vd':
- dis_variable_maps = variable_mapping.dis_seq2seq_vd(hparams)
- dis_init_saver = tf.train.Saver(var_list=dis_variable_maps)
- init_savers['dis_init_saver'] = dis_init_saver
-
- ## Load weights from language model checkpoint.
- if FLAGS.language_model_ckpt_dir:
- if FLAGS.maskgan_ckpt is None:
- ## Generator Variables/Savers.
- if FLAGS.generator_model == 'rnn_nas':
- gen_variable_maps = variable_mapping.rnn_nas(hparams, model='gen')
- gen_init_saver = tf.train.Saver(var_list=gen_variable_maps)
- init_savers['gen_init_saver'] = gen_init_saver
-
- elif FLAGS.generator_model == 'seq2seq_nas':
- # Encoder.
- gen_encoder_variable_maps = variable_mapping.gen_encoder_seq2seq_nas(
- hparams)
- gen_encoder_init_saver = tf.train.Saver(
- var_list=gen_encoder_variable_maps)
- # Decoder.
- gen_decoder_variable_maps = variable_mapping.gen_decoder_seq2seq_nas(
- hparams)
- gen_decoder_init_saver = tf.train.Saver(
- var_list=gen_decoder_variable_maps)
- init_savers['gen_encoder_init_saver'] = gen_encoder_init_saver
- init_savers['gen_decoder_init_saver'] = gen_decoder_init_saver
-
- # seq2seq_vd derived from the same code base as seq2seq_zaremba.
- elif (FLAGS.generator_model == 'seq2seq_zaremba' or
- FLAGS.generator_model == 'seq2seq_vd'):
- # Encoder.
- gen_encoder_variable_maps = variable_mapping.gen_encoder_seq2seq(
- hparams)
- gen_encoder_init_saver = tf.train.Saver(
- var_list=gen_encoder_variable_maps)
- # Decoder.
- gen_decoder_variable_maps = variable_mapping.gen_decoder_seq2seq(
- hparams)
- gen_decoder_init_saver = tf.train.Saver(
- var_list=gen_decoder_variable_maps)
- init_savers['gen_encoder_init_saver'] = gen_encoder_init_saver
- init_savers['gen_decoder_init_saver'] = gen_decoder_init_saver
-
- else:
- raise NotImplementedError
-
- ## Discriminator Variables/Savers.
- if FLAGS.discriminator_model == 'rnn_nas':
- dis_variable_maps = variable_mapping.rnn_nas(hparams, model='dis')
- dis_init_saver = tf.train.Saver(var_list=dis_variable_maps)
- init_savers['dis_init_saver'] = dis_init_saver
-
- # rnn_vd derived from the same code base as rnn_zaremba.
- elif (FLAGS.discriminator_model == 'rnn_zaremba' or
- FLAGS.discriminator_model == 'rnn_vd'):
- dis_variable_maps = variable_mapping.rnn_zaremba(hparams, model='dis')
- dis_init_saver = tf.train.Saver(var_list=dis_variable_maps)
- init_savers['dis_init_saver'] = dis_init_saver
-
- elif (FLAGS.discriminator_model == 'bidirectional_zaremba' or
- FLAGS.discriminator_model == 'bidirectional_vd'):
- dis_fwd_variable_maps = variable_mapping.dis_fwd_bidirectional(hparams)
- dis_bwd_variable_maps = variable_mapping.dis_bwd_bidirectional(hparams)
- # Savers for the forward/backward Discriminator components.
- dis_fwd_init_saver = tf.train.Saver(var_list=dis_fwd_variable_maps)
- dis_bwd_init_saver = tf.train.Saver(var_list=dis_bwd_variable_maps)
- init_savers['dis_fwd_init_saver'] = dis_fwd_init_saver
- init_savers['dis_bwd_init_saver'] = dis_bwd_init_saver
-
- elif FLAGS.discriminator_model == 'cnn':
- dis_variable_maps = variable_mapping.cnn()
- dis_init_saver = tf.train.Saver(var_list=dis_variable_maps)
- init_savers['dis_init_saver'] = dis_init_saver
-
- elif FLAGS.discriminator_model == 'seq2seq_vd':
- # Encoder.
- dis_encoder_variable_maps = variable_mapping.dis_encoder_seq2seq(hparams)
- dis_encoder_init_saver = tf.train.Saver(
- var_list=dis_encoder_variable_maps)
- # Decoder.
- dis_decoder_variable_maps = variable_mapping.dis_decoder_seq2seq(hparams)
- dis_decoder_init_saver = tf.train.Saver(
- var_list=dis_decoder_variable_maps)
- init_savers['dis_encoder_init_saver'] = dis_encoder_init_saver
- init_savers['dis_decoder_init_saver'] = dis_decoder_init_saver
-
- return init_savers
-
-
-def init_fn(init_savers, sess):
- """The init_fn to be passed to the Supervisor.
-
- Args:
- init_savers: Dictionary of init_savers. 'init_saver_name': init_saver.
- sess: tf.Session.
- """
- ## Load Generator weights from MaskGAN checkpoint.
- if FLAGS.maskgan_ckpt:
- print('Restoring Generator from %s.' % FLAGS.maskgan_ckpt)
- tf.logging.info('Restoring Generator from %s.' % FLAGS.maskgan_ckpt)
- print('Asserting Generator is a seq2seq-variant.')
- tf.logging.info('Asserting Generator is a seq2seq-variant.')
- assert FLAGS.generator_model.startswith('seq2seq')
- init_saver = init_savers['init_saver']
- init_saver.restore(sess, FLAGS.maskgan_ckpt)
-
- ## Load the Discriminator weights from the MaskGAN checkpoint if
- # the weights are compatible.
- if FLAGS.discriminator_model == 'seq2seq_vd':
- print('Restoring Discriminator from %s.' % FLAGS.maskgan_ckpt)
- tf.logging.info('Restoring Discriminator from %s.' % FLAGS.maskgan_ckpt)
- dis_init_saver = init_savers['dis_init_saver']
- dis_init_saver.restore(sess, FLAGS.maskgan_ckpt)
-
- ## Load weights from language model checkpoint.
- if FLAGS.language_model_ckpt_dir:
- if FLAGS.maskgan_ckpt is None:
- ## Generator Models.
- if FLAGS.generator_model == 'rnn_nas':
- load_ckpt = tf.train.latest_checkpoint(FLAGS.language_model_ckpt_dir)
- print('Restoring Generator from %s.' % load_ckpt)
- tf.logging.info('Restoring Generator from %s.' % load_ckpt)
- gen_init_saver = init_savers['gen_init_saver']
- gen_init_saver.restore(sess, load_ckpt)
-
- elif FLAGS.generator_model.startswith('seq2seq'):
- load_ckpt = tf.train.latest_checkpoint(FLAGS.language_model_ckpt_dir)
- print('Restoring Generator from %s.' % load_ckpt)
- tf.logging.info('Restoring Generator from %s.' % load_ckpt)
- gen_encoder_init_saver = init_savers['gen_encoder_init_saver']
- gen_decoder_init_saver = init_savers['gen_decoder_init_saver']
- gen_encoder_init_saver.restore(sess, load_ckpt)
- gen_decoder_init_saver.restore(sess, load_ckpt)
-
- ## Discriminator Models.
- if (FLAGS.discriminator_model == 'rnn_nas' or
- FLAGS.discriminator_model == 'rnn_zaremba' or
- FLAGS.discriminator_model == 'rnn_vd' or
- FLAGS.discriminator_model == 'cnn'):
- load_ckpt = tf.train.latest_checkpoint(FLAGS.language_model_ckpt_dir)
- print('Restoring Discriminator from %s.' % load_ckpt)
- tf.logging.info('Restoring Discriminator from %s.' % load_ckpt)
- dis_init_saver = init_savers['dis_init_saver']
- dis_init_saver.restore(sess, load_ckpt)
-
- elif (FLAGS.discriminator_model == 'bidirectional_zaremba' or
- FLAGS.discriminator_model == 'bidirectional_vd'):
- assert FLAGS.language_model_ckpt_dir_reversed is not None, (
- 'Need a reversed directory to fill in the backward components.')
- load_fwd_ckpt = tf.train.latest_checkpoint(FLAGS.language_model_ckpt_dir)
- load_bwd_ckpt = tf.train.latest_checkpoint(
- FLAGS.language_model_ckpt_dir_reversed)
- print('Restoring Discriminator from %s and %s.' % (load_fwd_ckpt,
- load_bwd_ckpt))
- tf.logging.info('Restoring Discriminator from %s and %s.' %
- (load_fwd_ckpt, load_bwd_ckpt))
- dis_fwd_init_saver = init_savers['dis_fwd_init_saver']
- dis_bwd_init_saver = init_savers['dis_bwd_init_saver']
- dis_fwd_init_saver.restore(sess, load_fwd_ckpt)
- dis_bwd_init_saver.restore(sess, load_bwd_ckpt)
-
- elif FLAGS.discriminator_model == 'seq2seq_vd':
- load_ckpt = tf.train.latest_checkpoint(FLAGS.language_model_ckpt_dir)
- print('Restoring Discriminator from %s.' % load_ckpt)
- tf.logging.info('Restoring Discriminator from %s.' % load_ckpt)
- dis_encoder_init_saver = init_savers['dis_encoder_init_saver']
- dis_decoder_init_saver = init_savers['dis_decoder_init_saver']
- dis_encoder_init_saver.restore(sess, load_ckpt)
- dis_decoder_init_saver.restore(sess, load_ckpt)
-
- else:
- return
diff --git a/research/maskgan/model_utils/n_gram.py b/research/maskgan/model_utils/n_gram.py
deleted file mode 100644
index b889dde849a60d95aa38c57cd8c864249233514f..0000000000000000000000000000000000000000
--- a/research/maskgan/model_utils/n_gram.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""We calculate n-Grams from the training text. We will use this as an
-evaluation metric."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from six.moves import xrange
-
-
-def hash_function(input_tuple):
- """Hash function for a tuple."""
- return hash(input_tuple)
-
-
-def find_all_ngrams(dataset, n):
- """Generate a list of all ngrams."""
- return zip(*[dataset[i:] for i in xrange(n)])
-
-
-def construct_ngrams_dict(ngrams_list):
- """Construct a ngram dictionary which maps an ngram tuple to the number
- of times it appears in the text."""
- counts = {}
-
- for t in ngrams_list:
- key = hash_function(t)
- if key in counts:
- counts[key] += 1
- else:
- counts[key] = 1
- return counts
-
-
-def percent_unique_ngrams_in_train(train_ngrams_dict, gen_ngrams_dict):
- """Compute the percent of ngrams generated by the model that are
- present in the training text and are unique."""
-
- # *Total* number of n-grams produced by the generator.
- total_ngrams_produced = 0
-
- for _, value in gen_ngrams_dict.iteritems():
- total_ngrams_produced += value
-
- # The unique ngrams in the training set.
- unique_ngrams_in_train = 0.
-
- for key, _ in gen_ngrams_dict.iteritems():
- if key in train_ngrams_dict:
- unique_ngrams_in_train += 1
- return float(unique_ngrams_in_train) / float(total_ngrams_produced)
diff --git a/research/maskgan/model_utils/variable_mapping.py b/research/maskgan/model_utils/variable_mapping.py
deleted file mode 100644
index 0301b969716fe473ac98c2e3bba5c04662461954..0000000000000000000000000000000000000000
--- a/research/maskgan/model_utils/variable_mapping.py
+++ /dev/null
@@ -1,745 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def rnn_nas(hparams, model):
- assert model == 'gen' or model == 'dis'
-
- # This logic is only valid for rnn_zaremba
- if model == 'gen':
- assert FLAGS.generator_model == 'rnn_nas'
- assert hparams.gen_num_layers == 2
-
- if model == 'dis':
- assert FLAGS.discriminator_model == 'rnn_nas'
- assert hparams.dis_num_layers == 2
-
- # Output variables only for the Generator. Discriminator output biases
- # will begin randomly initialized.
- if model == 'gen':
- softmax_b = [
- v for v in tf.trainable_variables() if v.op.name == 'gen/rnn/softmax_b'
- ][0]
-
- # Common elements to Generator and Discriminator.
- embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == str(model) + '/rnn/embedding'
- ][0]
- lstm_w_0 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- str(model) + '/rnn/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_h_mat'
- ][0]
- lstm_b_0 = [
- v for v in tf.trainable_variables()
- if v.op.name == str(model) +
- '/rnn/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_inputs_mat'
- ][0]
- lstm_w_1 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- str(model) + '/rnn/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_h_mat'
- ][0]
- lstm_b_1 = [
- v for v in tf.trainable_variables()
- if v.op.name == str(model) +
- '/rnn/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_inputs_mat'
- ][0]
-
- # Dictionary mapping.
- if model == 'gen':
- variable_mapping = {
- 'Model/embeddings/input_embedding':
- embedding,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_h_mat':
- lstm_w_0,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_inputs_mat':
- lstm_b_0,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_h_mat':
- lstm_w_1,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_inputs_mat':
- lstm_b_1,
- 'Model/softmax_b':
- softmax_b
- }
- else:
- variable_mapping = {
- 'Model/embeddings/input_embedding':
- embedding,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_h_mat':
- lstm_w_0,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_inputs_mat':
- lstm_b_0,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_h_mat':
- lstm_w_1,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_inputs_mat':
- lstm_b_1
- }
-
- return variable_mapping
-
-
-def cnn():
- """Variable mapping for the CNN embedding.
-
- Returns:
- variable_mapping: Dictionary with Key: ckpt_name, Value: model_var.
- """
- # This logic is only valid for cnn
- assert FLAGS.discriminator_model == 'cnn'
-
- # Retrieve CNN embedding.
- embedding = [
- v for v in tf.trainable_variables() if v.op.name == 'dis/embedding'
- ][0]
-
- # Variable mapping.
- variable_mapping = {'Model/embedding': embedding}
-
- return variable_mapping
-
-
-def rnn_zaremba(hparams, model):
- """Returns the PTB Variable name to MaskGAN Variable dictionary mapping. This
- is a highly restrictive function just for testing. This will need to be
- generalized.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- model: Model type, one of ['gen', 'dis'].
-
- Returns:
- variable_mapping: Dictionary with Key: ckpt_name, Value: model_var.
- """
- assert model == 'gen' or model == 'dis'
-
- # This logic is only valid for rnn_zaremba
- if model == 'gen':
- assert FLAGS.generator_model == 'rnn_zaremba'
- assert hparams.gen_num_layers == 2
-
- if model == 'dis':
- assert (FLAGS.discriminator_model == 'rnn_zaremba' or
- FLAGS.discriminator_model == 'rnn_vd')
- assert hparams.dis_num_layers == 2
-
- # Output variables only for the Generator. Discriminator output weights
- # and biases will begin randomly initialized.
- if model == 'gen':
- softmax_w = [
- v for v in tf.trainable_variables() if v.op.name == 'gen/rnn/softmax_w'
- ][0]
- softmax_b = [
- v for v in tf.trainable_variables() if v.op.name == 'gen/rnn/softmax_b'
- ][0]
-
- # Common elements to Generator and Discriminator.
- if not FLAGS.dis_share_embedding or model != 'dis':
- embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == str(model) + '/rnn/embedding'
- ][0]
- lstm_w_0 = [
- v for v in tf.trainable_variables() if v.op.name == str(model) +
- '/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- lstm_b_0 = [
- v for v in tf.trainable_variables() if v.op.name == str(model) +
- '/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- lstm_w_1 = [
- v for v in tf.trainable_variables() if v.op.name == str(model) +
- '/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- lstm_b_1 = [
- v for v in tf.trainable_variables() if v.op.name == str(model) +
- '/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
-
- # Dictionary mapping.
- if model == 'gen':
- variable_mapping = {
- 'Model/embedding': embedding,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel': lstm_w_0,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias': lstm_b_0,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel': lstm_w_1,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias': lstm_b_1,
- 'Model/softmax_w': softmax_w,
- 'Model/softmax_b': softmax_b
- }
- else:
- if FLAGS.dis_share_embedding:
- variable_mapping = {
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel': lstm_w_0,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias': lstm_b_0,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel': lstm_w_1,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias': lstm_b_1
- }
- else:
- variable_mapping = {
- 'Model/embedding': embedding,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel': lstm_w_0,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias': lstm_b_0,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel': lstm_w_1,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias': lstm_b_1
- }
-
- return variable_mapping
-
-
-def gen_encoder_seq2seq_nas(hparams):
- """Returns the NAS Variable name to MaskGAN Variable
- dictionary mapping. This is a highly restrictive function just for testing.
- This is for the *unidirecitional* seq2seq_nas encoder.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
-
- Returns:
- variable_mapping: Dictionary with Key: ckpt_name, Value: model_varself.
- """
- assert FLAGS.generator_model == 'seq2seq_nas'
- assert hparams.gen_num_layers == 2
- ## Encoder forward variables.
-
- if not FLAGS.seq2seq_share_embedding:
- encoder_embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == 'gen/encoder/rnn/embedding'
- ][0]
- encoder_lstm_w_0 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- 'gen/encoder/rnn/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_h_mat'
- ][0]
- encoder_lstm_b_0 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- 'gen/encoder/rnn/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_inputs_mat'
- ][0]
- encoder_lstm_w_1 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- 'gen/encoder/rnn/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_h_mat'
- ][0]
- encoder_lstm_b_1 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- 'gen/encoder/rnn/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_inputs_mat'
- ][0]
-
- if not FLAGS.seq2seq_share_embedding:
- variable_mapping = {
- 'Model/embeddings/input_embedding':
- encoder_embedding,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_h_mat':
- encoder_lstm_w_0,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_inputs_mat':
- encoder_lstm_b_0,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_h_mat':
- encoder_lstm_w_1,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_inputs_mat':
- encoder_lstm_b_1
- }
- else:
- variable_mapping = {
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_h_mat':
- encoder_lstm_w_0,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_inputs_mat':
- encoder_lstm_b_0,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_h_mat':
- encoder_lstm_w_1,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_inputs_mat':
- encoder_lstm_b_1
- }
- return variable_mapping
-
-
-def gen_decoder_seq2seq_nas(hparams):
- assert FLAGS.generator_model == 'seq2seq_nas'
- assert hparams.gen_num_layers == 2
-
- decoder_embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == 'gen/decoder/rnn/embedding'
- ][0]
- decoder_lstm_w_0 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- 'gen/decoder/rnn/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_h_mat'
- ][0]
- decoder_lstm_b_0 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- 'gen/decoder/rnn/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_inputs_mat'
- ][0]
- decoder_lstm_w_1 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- 'gen/decoder/rnn/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_h_mat'
- ][0]
- decoder_lstm_b_1 = [
- v for v in tf.trainable_variables()
- if v.op.name ==
- 'gen/decoder/rnn/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_inputs_mat'
- ][0]
-
- decoder_softmax_b = [
- v for v in tf.trainable_variables()
- if v.op.name == 'gen/decoder/rnn/softmax_b'
- ][0]
-
- variable_mapping = {
- 'Model/embeddings/input_embedding':
- decoder_embedding,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_h_mat':
- decoder_lstm_w_0,
- 'Model/RNN/GenericMultiRNNCell/Cell0/Alien/rnn_builder/big_inputs_mat':
- decoder_lstm_b_0,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_h_mat':
- decoder_lstm_w_1,
- 'Model/RNN/GenericMultiRNNCell/Cell1/Alien/rnn_builder/big_inputs_mat':
- decoder_lstm_b_1,
- 'Model/softmax_b':
- decoder_softmax_b
- }
-
- return variable_mapping
-
-
-def gen_encoder_seq2seq(hparams):
- """Returns the PTB Variable name to MaskGAN Variable
- dictionary mapping. This is a highly restrictive function just for testing.
- This is foe the *unidirecitional* seq2seq_zaremba encoder.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
-
- Returns:
- variable_mapping: Dictionary with Key: ckpt_name, Value: model_varself.
- """
- assert (FLAGS.generator_model == 'seq2seq_zaremba' or
- FLAGS.generator_model == 'seq2seq_vd')
- assert hparams.gen_num_layers == 2
-
- ## Encoder forward variables.
- if not FLAGS.seq2seq_share_embedding:
- encoder_embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == 'gen/encoder/rnn/embedding'
- ][0]
- encoder_lstm_w_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'gen/encoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- encoder_lstm_b_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'gen/encoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- encoder_lstm_w_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'gen/encoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- encoder_lstm_b_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'gen/encoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
-
- if FLAGS.data_set == 'ptb':
- model_str = 'Model'
- else:
- model_str = 'model'
-
- if not FLAGS.seq2seq_share_embedding:
- variable_mapping = {
- str(model_str) + '/embedding':
- encoder_embedding,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel':
- encoder_lstm_w_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias':
- encoder_lstm_b_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel':
- encoder_lstm_w_1,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias':
- encoder_lstm_b_1
- }
- else:
- variable_mapping = {
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel':
- encoder_lstm_w_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias':
- encoder_lstm_b_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel':
- encoder_lstm_w_1,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias':
- encoder_lstm_b_1
- }
- return variable_mapping
-
-
-def gen_decoder_seq2seq(hparams):
- assert (FLAGS.generator_model == 'seq2seq_zaremba' or
- FLAGS.generator_model == 'seq2seq_vd')
- assert hparams.gen_num_layers == 2
-
- decoder_embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == 'gen/decoder/rnn/embedding'
- ][0]
- decoder_lstm_w_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'gen/decoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- decoder_lstm_b_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'gen/decoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- decoder_lstm_w_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'gen/decoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- decoder_lstm_b_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'gen/decoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
- decoder_softmax_b = [
- v for v in tf.trainable_variables()
- if v.op.name == 'gen/decoder/rnn/softmax_b'
- ][0]
-
- if FLAGS.data_set == 'ptb':
- model_str = 'Model'
- else:
- model_str = 'model'
-
- variable_mapping = {
- str(model_str) + '/embedding':
- decoder_embedding,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel':
- decoder_lstm_w_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias':
- decoder_lstm_b_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel':
- decoder_lstm_w_1,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias':
- decoder_lstm_b_1,
- str(model_str) + '/softmax_b':
- decoder_softmax_b
- }
- return variable_mapping
-
-
-def dis_fwd_bidirectional(hparams):
- """Returns the *forward* PTB Variable name to MaskGAN Variable dictionary
- mapping. This is a highly restrictive function just for testing. This is for
- the bidirectional_zaremba discriminator.
-
- Args:
- FLAGS: Flags for the model.
- hparams: Hyperparameters for the MaskGAN.
-
- Returns:
- variable_mapping: Dictionary with Key: ckpt_name, Value: model_varself.
- """
- assert (FLAGS.discriminator_model == 'bidirectional_zaremba' or
- FLAGS.discriminator_model == 'bidirectional_vd')
- assert hparams.dis_num_layers == 2
-
- # Forward Discriminator Elements.
- if not FLAGS.dis_share_embedding:
- embedding = [
- v for v in tf.trainable_variables() if v.op.name == 'dis/embedding'
- ][0]
- fw_lstm_w_0 = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/rnn/fw/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- fw_lstm_b_0 = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/rnn/fw/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- fw_lstm_w_1 = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/rnn/fw/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- fw_lstm_b_1 = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/rnn/fw/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
- if FLAGS.dis_share_embedding:
- variable_mapping = {
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel': fw_lstm_w_0,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias': fw_lstm_b_0,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel': fw_lstm_w_1,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias': fw_lstm_b_1
- }
- else:
- variable_mapping = {
- 'Model/embedding': embedding,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel': fw_lstm_w_0,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias': fw_lstm_b_0,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel': fw_lstm_w_1,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias': fw_lstm_b_1
- }
- return variable_mapping
-
-
-def dis_bwd_bidirectional(hparams):
- """Returns the *backward* PTB Variable name to MaskGAN Variable dictionary
- mapping. This is a highly restrictive function just for testing. This is for
- the bidirectional_zaremba discriminator.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
-
- Returns:
- variable_mapping: Dictionary with Key: ckpt_name, Value: model_varself.
- """
- assert (FLAGS.discriminator_model == 'bidirectional_zaremba' or
- FLAGS.discriminator_model == 'bidirectional_vd')
- assert hparams.dis_num_layers == 2
-
- # Backward Discriminator Elements.
- bw_lstm_w_0 = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/rnn/bw/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- bw_lstm_b_0 = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/rnn/bw/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- bw_lstm_w_1 = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/rnn/bw/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- bw_lstm_b_1 = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/rnn/bw/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
-
- variable_mapping = {
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel': bw_lstm_w_0,
- 'Model/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias': bw_lstm_b_0,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel': bw_lstm_w_1,
- 'Model/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias': bw_lstm_b_1
- }
- return variable_mapping
-
-
-def dis_encoder_seq2seq(hparams):
- """Returns the PTB Variable name to MaskGAN Variable
- dictionary mapping.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
-
- Returns:
- variable_mapping: Dictionary with Key: ckpt_name, Value: model_varself.
- """
- assert FLAGS.discriminator_model == 'seq2seq_vd'
- assert hparams.dis_num_layers == 2
-
- ## Encoder forward variables.
- encoder_lstm_w_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/encoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- encoder_lstm_b_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/encoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- encoder_lstm_w_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/encoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- encoder_lstm_b_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/encoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
-
- if FLAGS.data_set == 'ptb':
- model_str = 'Model'
- else:
- model_str = 'model'
-
- variable_mapping = {
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel':
- encoder_lstm_w_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias':
- encoder_lstm_b_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel':
- encoder_lstm_w_1,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias':
- encoder_lstm_b_1
- }
- return variable_mapping
-
-
-def dis_decoder_seq2seq(hparams):
- assert FLAGS.discriminator_model == 'seq2seq_vd'
- assert hparams.dis_num_layers == 2
-
- if not FLAGS.dis_share_embedding:
- decoder_embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/decoder/rnn/embedding'
- ][0]
- decoder_lstm_w_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/decoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- decoder_lstm_b_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/decoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- decoder_lstm_w_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/decoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- decoder_lstm_b_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/decoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
-
- if FLAGS.data_set == 'ptb':
- model_str = 'Model'
- else:
- model_str = 'model'
-
- if not FLAGS.dis_share_embedding:
- variable_mapping = {
- str(model_str) + '/embedding':
- decoder_embedding,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel':
- decoder_lstm_w_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias':
- decoder_lstm_b_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel':
- decoder_lstm_w_1,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias':
- decoder_lstm_b_1
- }
- else:
- variable_mapping = {
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/kernel':
- decoder_lstm_w_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_0/basic_lstm_cell/bias':
- decoder_lstm_b_0,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/kernel':
- decoder_lstm_w_1,
- str(model_str) + '/RNN/multi_rnn_cell/cell_1/basic_lstm_cell/bias':
- decoder_lstm_b_1,
- }
- return variable_mapping
-
-
-def dis_seq2seq_vd(hparams):
- assert FLAGS.discriminator_model == 'seq2seq_vd'
- assert hparams.dis_num_layers == 2
-
- if not FLAGS.dis_share_embedding:
- decoder_embedding = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/decoder/rnn/embedding'
- ][0]
-
- ## Encoder variables.
- encoder_lstm_w_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/encoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- encoder_lstm_b_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/encoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- encoder_lstm_w_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/encoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- encoder_lstm_b_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/encoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
-
- ## Attention.
- if FLAGS.attention_option is not None:
- decoder_attention_keys = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/decoder/attention_keys/weights'
- ][0]
- decoder_attention_construct_weights = [
- v for v in tf.trainable_variables()
- if v.op.name == 'dis/decoder/rnn/attention_construct/weights'
- ][0]
-
- ## Decoder.
- decoder_lstm_w_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/decoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel'
- ][0]
- decoder_lstm_b_0 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/decoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias'
- ][0]
- decoder_lstm_w_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/decoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel'
- ][0]
- decoder_lstm_b_1 = [
- v for v in tf.trainable_variables() if v.op.name ==
- 'dis/decoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias'
- ][0]
-
- # Standard variable mappings.
- variable_mapping = {
- 'gen/encoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel':
- encoder_lstm_w_0,
- 'gen/encoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias':
- encoder_lstm_b_0,
- 'gen/encoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel':
- encoder_lstm_w_1,
- 'gen/encoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias':
- encoder_lstm_b_1,
- 'gen/decoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel':
- decoder_lstm_w_0,
- 'gen/decoder/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias':
- decoder_lstm_b_0,
- 'gen/decoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel':
- decoder_lstm_w_1,
- 'gen/decoder/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias':
- decoder_lstm_b_1
- }
-
- # Optional variable mappings.
- if not FLAGS.dis_share_embedding:
- variable_mapping['gen/decoder/rnn/embedding'] = decoder_embedding
- if FLAGS.attention_option is not None:
- variable_mapping[
- 'gen/decoder/attention_keys/weights'] = decoder_attention_keys
- variable_mapping[
- 'gen/decoder/rnn/attention_construct/weights'] = decoder_attention_construct_weights
-
- return variable_mapping
diff --git a/research/maskgan/models/__init__.py b/research/maskgan/models/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/maskgan/models/attention_utils.py b/research/maskgan/models/attention_utils.py
deleted file mode 100644
index 4bd9e41dd3178d6210e8f81d628b7d92004a6601..0000000000000000000000000000000000000000
--- a/research/maskgan/models/attention_utils.py
+++ /dev/null
@@ -1,477 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Attention-based decoder functions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-from tensorflow.python.framework import function
-
-__all__ = [
- "prepare_attention", "attention_decoder_fn_train",
- "attention_decoder_fn_inference"
-]
-
-
-def attention_decoder_fn_train(encoder_state,
- attention_keys,
- attention_values,
- attention_score_fn,
- attention_construct_fn,
- name=None):
- """Attentional decoder function for `dynamic_rnn_decoder` during training.
-
- The `attention_decoder_fn_train` is a training function for an
- attention-based sequence-to-sequence model. It should be used when
- `dynamic_rnn_decoder` is in the training mode.
-
- The `attention_decoder_fn_train` is called with a set of the user arguments
- and returns the `decoder_fn`, which can be passed to the
- `dynamic_rnn_decoder`, such that
-
- ```
- dynamic_fn_train = attention_decoder_fn_train(encoder_state)
- outputs_train, state_train = dynamic_rnn_decoder(
- decoder_fn=dynamic_fn_train, ...)
- ```
-
- Further usage can be found in the `kernel_tests/seq2seq_test.py`.
-
- Args:
- encoder_state: The encoded state to initialize the `dynamic_rnn_decoder`.
- attention_keys: to be compared with target states.
- attention_values: to be used to construct context vectors.
- attention_score_fn: to compute similarity between key and target states.
- attention_construct_fn: to build attention states.
- name: (default: `None`) NameScope for the decoder function;
- defaults to "simple_decoder_fn_train"
-
- Returns:
- A decoder function with the required interface of `dynamic_rnn_decoder`
- intended for training.
- """
- with tf.name_scope(name, "attention_decoder_fn_train", [
- encoder_state, attention_keys, attention_values, attention_score_fn,
- attention_construct_fn
- ]):
- pass
-
- def decoder_fn(time, cell_state, cell_input, cell_output, context_state):
- """Decoder function used in the `dynamic_rnn_decoder` for training.
-
- Args:
- time: positive integer constant reflecting the current timestep.
- cell_state: state of RNNCell.
- cell_input: input provided by `dynamic_rnn_decoder`.
- cell_output: output of RNNCell.
- context_state: context state provided by `dynamic_rnn_decoder`.
-
- Returns:
- A tuple (done, next state, next input, emit output, next context state)
- where:
-
- done: `None`, which is used by the `dynamic_rnn_decoder` to indicate
- that `sequence_lengths` in `dynamic_rnn_decoder` should be used.
-
- next state: `cell_state`, this decoder function does not modify the
- given state.
-
- next input: `cell_input`, this decoder function does not modify the
- given input. The input could be modified when applying e.g. attention.
-
- emit output: `cell_output`, this decoder function does not modify the
- given output.
-
- next context state: `context_state`, this decoder function does not
- modify the given context state. The context state could be modified when
- applying e.g. beam search.
- """
- with tf.name_scope(
- name, "attention_decoder_fn_train",
- [time, cell_state, cell_input, cell_output, context_state]):
- if cell_state is None: # first call, return encoder_state
- cell_state = encoder_state
-
- # init attention
- attention = _init_attention(encoder_state)
- else:
- # construct attention
- attention = attention_construct_fn(cell_output, attention_keys,
- attention_values)
- cell_output = attention
-
- # combine cell_input and attention
- next_input = tf.concat([cell_input, attention], 1)
-
- return (None, cell_state, next_input, cell_output, context_state)
-
- return decoder_fn
-
-
-def attention_decoder_fn_inference(output_fn,
- encoder_state,
- attention_keys,
- attention_values,
- attention_score_fn,
- attention_construct_fn,
- embeddings,
- start_of_sequence_id,
- end_of_sequence_id,
- maximum_length,
- num_decoder_symbols,
- dtype=tf.int32,
- name=None):
- """Attentional decoder function for `dynamic_rnn_decoder` during inference.
-
- The `attention_decoder_fn_inference` is a simple inference function for a
- sequence-to-sequence model. It should be used when `dynamic_rnn_decoder` is
- in the inference mode.
-
- The `attention_decoder_fn_inference` is called with user arguments
- and returns the `decoder_fn`, which can be passed to the
- `dynamic_rnn_decoder`, such that
-
- ```
- dynamic_fn_inference = attention_decoder_fn_inference(...)
- outputs_inference, state_inference = dynamic_rnn_decoder(
- decoder_fn=dynamic_fn_inference, ...)
- ```
-
- Further usage can be found in the `kernel_tests/seq2seq_test.py`.
-
- Args:
- output_fn: An output function to project your `cell_output` onto class
- logits.
-
- An example of an output function;
-
- ```
- tf.variable_scope("decoder") as varscope
- output_fn = lambda x: tf.contrib.layers.linear(x, num_decoder_symbols,
- scope=varscope)
-
- outputs_train, state_train = seq2seq.dynamic_rnn_decoder(...)
- logits_train = output_fn(outputs_train)
-
- varscope.reuse_variables()
- logits_inference, state_inference = seq2seq.dynamic_rnn_decoder(
- output_fn=output_fn, ...)
- ```
-
- If `None` is supplied it will act as an identity function, which
- might be wanted when using the RNNCell `OutputProjectionWrapper`.
-
- encoder_state: The encoded state to initialize the `dynamic_rnn_decoder`.
- attention_keys: to be compared with target states.
- attention_values: to be used to construct context vectors.
- attention_score_fn: to compute similarity between key and target states.
- attention_construct_fn: to build attention states.
- embeddings: The embeddings matrix used for the decoder sized
- `[num_decoder_symbols, embedding_size]`.
- start_of_sequence_id: The start of sequence ID in the decoder embeddings.
- end_of_sequence_id: The end of sequence ID in the decoder embeddings.
- maximum_length: The maximum allowed of time steps to decode.
- num_decoder_symbols: The number of classes to decode at each time step.
- dtype: (default: `tf.int32`) The default data type to use when
- handling integer objects.
- name: (default: `None`) NameScope for the decoder function;
- defaults to "attention_decoder_fn_inference"
-
- Returns:
- A decoder function with the required interface of `dynamic_rnn_decoder`
- intended for inference.
- """
- with tf.name_scope(name, "attention_decoder_fn_inference", [
- output_fn, encoder_state, attention_keys, attention_values,
- attention_score_fn, attention_construct_fn, embeddings,
- start_of_sequence_id, end_of_sequence_id, maximum_length,
- num_decoder_symbols, dtype
- ]):
- start_of_sequence_id = tf.convert_to_tensor(start_of_sequence_id, dtype)
- end_of_sequence_id = tf.convert_to_tensor(end_of_sequence_id, dtype)
- maximum_length = tf.convert_to_tensor(maximum_length, dtype)
- num_decoder_symbols = tf.convert_to_tensor(num_decoder_symbols, dtype)
- encoder_info = tf.contrib.framework.nest.flatten(encoder_state)[0]
- batch_size = encoder_info.get_shape()[0].value
- if output_fn is None:
- output_fn = lambda x: x
- if batch_size is None:
- batch_size = tf.shape(encoder_info)[0]
-
- def decoder_fn(time, cell_state, cell_input, cell_output, context_state):
- """Decoder function used in the `dynamic_rnn_decoder` for inference.
-
- The main difference between this decoder function and the `decoder_fn` in
- `attention_decoder_fn_train` is how `next_cell_input` is calculated. In
- decoder function we calculate the next input by applying an argmax across
- the feature dimension of the output from the decoder. This is a
- greedy-search approach. (Bahdanau et al., 2014) & (Sutskever et al., 2014)
- use beam-search instead.
-
- Args:
- time: positive integer constant reflecting the current timestep.
- cell_state: state of RNNCell.
- cell_input: input provided by `dynamic_rnn_decoder`.
- cell_output: output of RNNCell.
- context_state: context state provided by `dynamic_rnn_decoder`.
-
- Returns:
- A tuple (done, next state, next input, emit output, next context state)
- where:
-
- done: A boolean vector to indicate which sentences has reached a
- `end_of_sequence_id`. This is used for early stopping by the
- `dynamic_rnn_decoder`. When `time>=maximum_length` a boolean vector with
- all elements as `true` is returned.
-
- next state: `cell_state`, this decoder function does not modify the
- given state.
-
- next input: The embedding from argmax of the `cell_output` is used as
- `next_input`.
-
- emit output: If `output_fn is None` the supplied `cell_output` is
- returned, else the `output_fn` is used to update the `cell_output`
- before calculating `next_input` and returning `cell_output`.
-
- next context state: `context_state`, this decoder function does not
- modify the given context state. The context state could be modified when
- applying e.g. beam search.
-
- Raises:
- ValueError: if cell_input is not None.
-
- """
- with tf.name_scope(
- name, "attention_decoder_fn_inference",
- [time, cell_state, cell_input, cell_output, context_state]):
- if cell_input is not None:
- raise ValueError(
- "Expected cell_input to be None, but saw: %s" % cell_input)
- if cell_output is None:
- # invariant that this is time == 0
- next_input_id = tf.ones(
- [
- batch_size,
- ], dtype=dtype) * (
- start_of_sequence_id)
- done = tf.zeros(
- [
- batch_size,
- ], dtype=tf.bool)
- cell_state = encoder_state
- cell_output = tf.zeros([num_decoder_symbols], dtype=tf.float32)
- cell_input = tf.gather(embeddings, next_input_id)
-
- # init attention
- attention = _init_attention(encoder_state)
- else:
- # construct attention
- attention = attention_construct_fn(cell_output, attention_keys,
- attention_values)
- cell_output = attention
-
- # argmax decoder
- cell_output = output_fn(cell_output) # logits
- next_input_id = tf.cast(tf.argmax(cell_output, 1), dtype=dtype)
- done = tf.equal(next_input_id, end_of_sequence_id)
- cell_input = tf.gather(embeddings, next_input_id)
-
- # combine cell_input and attention
- next_input = tf.concat([cell_input, attention], 1)
-
- # if time > maxlen, return all true vector
- done = tf.cond(
- tf.greater(time, maximum_length),
- lambda: tf.ones([
- batch_size,], dtype=tf.bool), lambda: done)
- return (done, cell_state, next_input, cell_output, context_state)
-
- return decoder_fn
-
-
-## Helper functions ##
-def prepare_attention(attention_states, attention_option, num_units,
- reuse=None):
- """Prepare keys/values/functions for attention.
-
- Args:
- attention_states: hidden states to attend over.
- attention_option: how to compute attention, either "luong" or "bahdanau".
- num_units: hidden state dimension.
- reuse: whether to reuse variable scope.
-
- Returns:
- attention_keys: to be compared with target states.
- attention_values: to be used to construct context vectors.
- attention_score_fn: to compute similarity between key and target states.
- attention_construct_fn: to build attention states.
- """
- # Prepare attention keys / values from attention_states
- with tf.variable_scope("attention_keys", reuse=reuse) as scope:
- attention_keys = tf.contrib.layers.linear(
- attention_states, num_units, biases_initializer=None, scope=scope)
- attention_values = attention_states
-
- # Attention score function
- attention_score_fn = _create_attention_score_fn("attention_score", num_units,
- attention_option, reuse)
- # Attention construction function
- attention_construct_fn = _create_attention_construct_fn(
- "attention_construct", num_units, attention_score_fn, reuse)
-
- return (attention_keys, attention_values, attention_score_fn,
- attention_construct_fn)
-
-
-def _init_attention(encoder_state):
- """Initialize attention. Handling both LSTM and GRU.
-
- Args:
- encoder_state: The encoded state to initialize the `dynamic_rnn_decoder`.
-
- Returns:
- attn: initial zero attention vector.
- """
-
- # Multi- vs single-layer
- # TODO(thangluong): is this the best way to check?
- if isinstance(encoder_state, tuple):
- top_state = encoder_state[-1]
- else:
- top_state = encoder_state
-
- # LSTM vs GRU
- if isinstance(top_state, tf.contrib.rnn.LSTMStateTuple):
- attn = tf.zeros_like(top_state.h)
- else:
- attn = tf.zeros_like(top_state)
-
- return attn
-
-
-def _create_attention_construct_fn(name, num_units, attention_score_fn, reuse):
- """Function to compute attention vectors.
-
- Args:
- name: to label variables.
- num_units: hidden state dimension.
- attention_score_fn: to compute similarity between key and target states.
- reuse: whether to reuse variable scope.
-
- Returns:
- attention_construct_fn: to build attention states.
- """
-
- def construct_fn(attention_query, attention_keys, attention_values):
- with tf.variable_scope(name, reuse=reuse) as scope:
- context = attention_score_fn(attention_query, attention_keys,
- attention_values)
- concat_input = tf.concat([attention_query, context], 1)
- attention = tf.contrib.layers.linear(
- concat_input, num_units, biases_initializer=None, scope=scope)
- return attention
-
- return construct_fn
-
-
-# keys: [batch_size, attention_length, attn_size]
-# query: [batch_size, 1, attn_size]
-# return weights [batch_size, attention_length]
-@function.Defun(func_name="attn_add_fun", noinline=True)
-def _attn_add_fun(v, keys, query):
- return tf.reduce_sum(v * tf.tanh(keys + query), [2])
-
-
-@function.Defun(func_name="attn_mul_fun", noinline=True)
-def _attn_mul_fun(keys, query):
- return tf.reduce_sum(keys * query, [2])
-
-
-def _create_attention_score_fn(name,
- num_units,
- attention_option,
- reuse,
- dtype=tf.float32):
- """Different ways to compute attention scores.
-
- Args:
- name: to label variables.
- num_units: hidden state dimension.
- attention_option: how to compute attention, either "luong" or "bahdanau".
- "bahdanau": additive (Bahdanau et al., ICLR'2015)
- "luong": multiplicative (Luong et al., EMNLP'2015)
- reuse: whether to reuse variable scope.
- dtype: (default: `tf.float32`) data type to use.
-
- Returns:
- attention_score_fn: to compute similarity between key and target states.
- """
- with tf.variable_scope(name, reuse=reuse):
- if attention_option == "bahdanau":
- query_w = tf.get_variable("attnW", [num_units, num_units], dtype=dtype)
- score_v = tf.get_variable("attnV", [num_units], dtype=dtype)
-
- def attention_score_fn(query, keys, values):
- """Put attention masks on attention_values using attention_keys and query.
-
- Args:
- query: A Tensor of shape [batch_size, num_units].
- keys: A Tensor of shape [batch_size, attention_length, num_units].
- values: A Tensor of shape [batch_size, attention_length, num_units].
-
- Returns:
- context_vector: A Tensor of shape [batch_size, num_units].
-
- Raises:
- ValueError: if attention_option is neither "luong" or "bahdanau".
-
-
- """
- if attention_option == "bahdanau":
- # transform query
- query = tf.matmul(query, query_w)
-
- # reshape query: [batch_size, 1, num_units]
- query = tf.reshape(query, [-1, 1, num_units])
-
- # attn_fun
- scores = _attn_add_fun(score_v, keys, query)
- elif attention_option == "luong":
- # reshape query: [batch_size, 1, num_units]
- query = tf.reshape(query, [-1, 1, num_units])
-
- # attn_fun
- scores = _attn_mul_fun(keys, query)
- else:
- raise ValueError("Unknown attention option %s!" % attention_option)
-
- # Compute alignment weights
- # scores: [batch_size, length]
- # alignments: [batch_size, length]
- # TODO(thangluong): not normalize over padding positions.
- alignments = tf.nn.softmax(scores)
-
- # Now calculate the attention-weighted vector.
- alignments = tf.expand_dims(alignments, 2)
- context_vector = tf.reduce_sum(alignments * values, [1])
- context_vector.set_shape([None, num_units])
-
- return context_vector
-
- return attention_score_fn
diff --git a/research/maskgan/models/bidirectional.py b/research/maskgan/models/bidirectional.py
deleted file mode 100644
index 1e6b3fe45f9ffe7dffdeb5c0d571de7e68227498..0000000000000000000000000000000000000000
--- a/research/maskgan/models/bidirectional.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple bidirectional model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-# ZoneoutWrapper.
-from regularization import zoneout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def discriminator(hparams, sequence, is_training, reuse=None):
- """Define the bidirectional Discriminator graph."""
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- 'If you wish to share Discriminator/Generator embeddings, they must be'
- ' same dimension.')
- with tf.variable_scope('gen/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('dis', reuse=reuse):
- cell_fwd = tf.contrib.rnn.LayerNormBasicLSTMCell(
- hparams.dis_rnn_size, forget_bias=1.0, reuse=reuse)
- cell_bwd = tf.contrib.rnn.LayerNormBasicLSTMCell(
- hparams.dis_rnn_size, forget_bias=1.0, reuse=reuse)
- if FLAGS.zoneout_drop_prob > 0.0:
- cell_fwd = zoneout.ZoneoutWrapper(
- cell_fwd,
- zoneout_drop_prob=FLAGS.zoneout_drop_prob,
- is_training=is_training)
- cell_bwd = zoneout.ZoneoutWrapper(
- cell_bwd,
- zoneout_drop_prob=FLAGS.zoneout_drop_prob,
- is_training=is_training)
-
- state_fwd = cell_fwd.zero_state(FLAGS.batch_size, tf.float32)
- state_bwd = cell_bwd.zero_state(FLAGS.batch_size, tf.float32)
-
- if not FLAGS.dis_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
- rnn_inputs = tf.unstack(rnn_inputs, axis=1)
-
- with tf.variable_scope('rnn') as vs:
- outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn(
- cell_fwd, cell_bwd, rnn_inputs, state_fwd, state_bwd, scope=vs)
-
- # Prediction is linear output for Discriminator.
- predictions = tf.contrib.layers.linear(outputs, 1, scope=vs)
-
- predictions = tf.transpose(predictions, [1, 0, 2])
- return tf.squeeze(predictions, axis=2)
diff --git a/research/maskgan/models/bidirectional_vd.py b/research/maskgan/models/bidirectional_vd.py
deleted file mode 100644
index 469af9da57a8a0dbf280327308a17fa6e0277a86..0000000000000000000000000000000000000000
--- a/research/maskgan/models/bidirectional_vd.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple bidirectional model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-from regularization import variational_dropout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def discriminator(hparams,
- sequence,
- is_training,
- reuse=None,
- initial_state=None):
- """Define the Discriminator graph."""
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- 'If you wish to share Discriminator/Generator embeddings, they must be'
- ' same dimension.')
- with tf.variable_scope('gen/decoder/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('dis', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(
- hparams.dis_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and hparams.dis_vd_keep_prob < 1:
-
- def attn_cell():
- return variational_dropout.VariationalDropoutWrapper(
- lstm_cell(), FLAGS.batch_size, hparams.dis_rnn_size,
- hparams.dis_vd_keep_prob, hparams.dis_vd_keep_prob)
-
- cell_fwd = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- cell_bwd = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- # print initial_state
- # print cell_fwd.zero_state(FLAGS.batch_size, tf.float32)
- if initial_state:
- state_fwd = [[tf.identity(x) for x in inner_initial_state]
- for inner_initial_state in initial_state]
- state_bwd = cell_bwd.zero_state(FLAGS.batch_size, tf.float32)
- else:
- state_fwd = cell_fwd.zero_state(FLAGS.batch_size, tf.float32)
- state_bwd = cell_bwd.zero_state(FLAGS.batch_size, tf.float32)
-
- def make_mask(keep_prob, units):
- random_tensor = keep_prob
- # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
- random_tensor += tf.random_uniform(tf.stack([FLAGS.batch_size, units]))
- return tf.floor(random_tensor) / keep_prob
-
- if is_training:
- output_mask = make_mask(hparams.dis_vd_keep_prob,
- 2 * hparams.dis_rnn_size)
-
- if not FLAGS.dis_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- rnn_inputs = tf.unstack(rnn_inputs, axis=1)
-
- with tf.variable_scope('rnn') as vs:
- outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn(
- cell_fwd, cell_bwd, rnn_inputs, state_fwd, state_bwd, scope=vs)
-
- if is_training:
- outputs *= output_mask
-
- # Prediction is linear output for Discriminator.
- predictions = tf.contrib.layers.linear(outputs, 1, scope=vs)
- predictions = tf.transpose(predictions, [1, 0, 2])
-
- if FLAGS.baseline_method == 'critic':
- with tf.variable_scope('critic', reuse=reuse) as critic_scope:
- values = tf.contrib.layers.linear(outputs, 1, scope=critic_scope)
- values = tf.transpose(values, [1, 0, 2])
-
- return tf.squeeze(predictions, axis=2), tf.squeeze(values, axis=2)
-
- else:
- return tf.squeeze(predictions, axis=2), None
diff --git a/research/maskgan/models/bidirectional_zaremba.py b/research/maskgan/models/bidirectional_zaremba.py
deleted file mode 100644
index b0683d7cc1493a8aa0298b7dc91020a152a9da36..0000000000000000000000000000000000000000
--- a/research/maskgan/models/bidirectional_zaremba.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple bidirectional model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def discriminator(hparams, sequence, is_training, reuse=None):
- """Define the bidirectional Discriminator graph."""
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- 'If you wish to share Discriminator/Generator embeddings, they must be'
- ' same dimension.')
- with tf.variable_scope('gen/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('dis', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(
- hparams.dis_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and FLAGS.keep_prob < 1:
-
- def attn_cell():
- return tf.contrib.rnn.DropoutWrapper(
- lstm_cell(), output_keep_prob=FLAGS.keep_prob)
-
- cell_fwd = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- cell_bwd = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- state_fwd = cell_fwd.zero_state(FLAGS.batch_size, tf.float32)
- state_bwd = cell_bwd.zero_state(FLAGS.batch_size, tf.float32)
-
- if not FLAGS.dis_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
- if is_training and FLAGS.keep_prob < 1:
- rnn_inputs = tf.nn.dropout(rnn_inputs, FLAGS.keep_prob)
- rnn_inputs = tf.unstack(rnn_inputs, axis=1)
-
- with tf.variable_scope('rnn') as vs:
- outputs, _, _ = tf.contrib.rnn.static_bidirectional_rnn(
- cell_fwd, cell_bwd, rnn_inputs, state_fwd, state_bwd, scope=vs)
-
- # Prediction is linear output for Discriminator.
- predictions = tf.contrib.layers.linear(outputs, 1, scope=vs)
-
- predictions = tf.transpose(predictions, [1, 0, 2])
- return tf.squeeze(predictions, axis=2)
diff --git a/research/maskgan/models/cnn.py b/research/maskgan/models/cnn.py
deleted file mode 100644
index ca682debf1630f5773cef48b874334d28d1fc6fc..0000000000000000000000000000000000000000
--- a/research/maskgan/models/cnn.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple CNN model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def discriminator(hparams, sequence, is_training, reuse=None):
- """Define the Discriminator graph."""
- del is_training
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- "If you wish to share Discriminator/Generator embeddings, they must be"
- " same dimension.")
- with tf.variable_scope("gen/rnn", reuse=True):
- embedding = tf.get_variable("embedding",
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- dis_filter_sizes = [3, 4, 5, 6, 7, 8, 9, 10, 15, 20]
-
- with tf.variable_scope("dis", reuse=reuse):
- if not FLAGS.dis_share_embedding:
- embedding = tf.get_variable("embedding",
- [FLAGS.vocab_size, hparams.dis_rnn_size])
- cnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- # Create a convolution layer for each filter size
- conv_outputs = []
- for filter_size in dis_filter_sizes:
- with tf.variable_scope("conv-%s" % filter_size):
- # Convolution Layer
- filter_shape = [
- filter_size, hparams.dis_rnn_size, hparams.dis_num_filters
- ]
- W = tf.get_variable(
- name="W", initializer=tf.truncated_normal(filter_shape, stddev=0.1))
- b = tf.get_variable(
- name="b",
- initializer=tf.constant(0.1, shape=[hparams.dis_num_filters]))
- conv = tf.nn.conv1d(
- cnn_inputs, W, stride=1, padding="SAME", name="conv")
-
- # Apply nonlinearity
- h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
-
- conv_outputs.append(h)
-
- # Combine all the pooled features
- dis_num_filters_total = hparams.dis_num_filters * len(dis_filter_sizes)
-
- h_conv = tf.concat(conv_outputs, axis=2)
- h_conv_flat = tf.reshape(h_conv, [-1, dis_num_filters_total])
-
- # Add dropout
- with tf.variable_scope("dropout"):
- h_drop = tf.nn.dropout(h_conv_flat, FLAGS.keep_prob)
-
- with tf.variable_scope("fully_connected"):
- fc = tf.contrib.layers.fully_connected(
- h_drop, num_outputs=dis_num_filters_total / 2)
-
- # Final (unnormalized) scores and predictions
- with tf.variable_scope("output"):
- W = tf.get_variable(
- "W",
- shape=[dis_num_filters_total / 2, 1],
- initializer=tf.contrib.layers.xavier_initializer())
- b = tf.get_variable(name="b", initializer=tf.constant(0.1, shape=[1]))
- predictions = tf.nn.xw_plus_b(fc, W, b, name="predictions")
- predictions = tf.reshape(
- predictions, shape=[FLAGS.batch_size, FLAGS.sequence_length])
- return predictions
diff --git a/research/maskgan/models/critic_vd.py b/research/maskgan/models/critic_vd.py
deleted file mode 100644
index ede8b7bb77af28f562c2e3942728899fe9b16422..0000000000000000000000000000000000000000
--- a/research/maskgan/models/critic_vd.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Critic model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from six.moves import xrange
-import tensorflow as tf
-from regularization import variational_dropout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def critic_seq2seq_vd_derivative(hparams, sequence, is_training, reuse=None):
- """Define the Critic graph which is derived from the seq2seq_vd
- Discriminator. This will be initialized with the same parameters as the
- language model and will share the forward RNN components with the
- Discriminator. This estimates the V(s_t), where the state
- s_t = x_0,...,x_t-1.
- """
- assert FLAGS.discriminator_model == 'seq2seq_vd'
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- 'If you wish to share Discriminator/Generator embeddings, they must be'
- ' same dimension.')
- with tf.variable_scope('gen/decoder/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- else:
- with tf.variable_scope('dis/decoder/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- with tf.variable_scope(
- 'dis/decoder/rnn/multi_rnn_cell', reuse=True) as dis_scope:
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(
- hparams.dis_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=True)
-
- attn_cell = lstm_cell
- if is_training and hparams.dis_vd_keep_prob < 1:
-
- def attn_cell():
- return variational_dropout.VariationalDropoutWrapper(
- lstm_cell(), FLAGS.batch_size, hparams.dis_rnn_size,
- hparams.dis_vd_keep_prob, hparams.dis_vd_keep_prob)
-
- cell_critic = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- with tf.variable_scope('critic', reuse=reuse):
- state_dis = cell_critic.zero_state(FLAGS.batch_size, tf.float32)
-
- def make_mask(keep_prob, units):
- random_tensor = keep_prob
- # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
- random_tensor += tf.random_uniform(tf.stack([FLAGS.batch_size, units]))
- return tf.floor(random_tensor) / keep_prob
-
- if is_training:
- output_mask = make_mask(hparams.dis_vd_keep_prob, hparams.dis_rnn_size)
-
- with tf.variable_scope('rnn') as vs:
- values = []
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- if t == 0:
- rnn_in = tf.zeros_like(rnn_inputs[:, 0])
- else:
- rnn_in = rnn_inputs[:, t - 1]
- rnn_out, state_dis = cell_critic(rnn_in, state_dis, scope=dis_scope)
-
- if is_training:
- rnn_out *= output_mask
-
- # Prediction is linear output for Discriminator.
- value = tf.contrib.layers.linear(rnn_out, 1, scope=vs)
-
- values.append(value)
- values = tf.stack(values, axis=1)
- return tf.squeeze(values, axis=2)
diff --git a/research/maskgan/models/evaluation_utils.py b/research/maskgan/models/evaluation_utils.py
deleted file mode 100644
index fc2a3a16f0b2c03736bfaa881c5c14546240d283..0000000000000000000000000000000000000000
--- a/research/maskgan/models/evaluation_utils.py
+++ /dev/null
@@ -1,280 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Evaluation utilities."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import Counter
-# Dependency imports
-import numpy as np
-from scipy.special import expit
-
-import tensorflow as tf
-
-from model_utils import helper
-from model_utils import n_gram
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def print_and_log_losses(log, step, is_present_rate, avg_dis_loss,
- avg_gen_loss):
- """Prints and logs losses to the log file.
-
- Args:
- log: GFile for logs.
- step: Global step.
- is_present_rate: Current masking rate.
- avg_dis_loss: List of Discriminator losses.
- avg_gen_loss: List of Generator losses.
- """
- print('global_step: %d' % step)
- print(' is_present_rate: %.3f' % is_present_rate)
- print(' D train loss: %.5f' % np.mean(avg_dis_loss))
- print(' G train loss: %.5f' % np.mean(avg_gen_loss))
- log.write('\nglobal_step: %d\n' % step)
- log.write((' is_present_rate: %.3f\n' % is_present_rate))
- log.write(' D train loss: %.5f\n' % np.mean(avg_dis_loss))
- log.write(' G train loss: %.5f\n' % np.mean(avg_gen_loss))
-
-
-def print_and_log(log, id_to_word, sequence_eval, max_num_to_print=5):
- """Helper function for printing and logging evaluated sequences."""
- indices_arr = np.asarray(sequence_eval)
- samples = helper.convert_to_human_readable(id_to_word, indices_arr,
- max_num_to_print)
-
- for i, sample in enumerate(samples):
- print('Sample', i, '. ', sample)
- log.write('\nSample ' + str(i) + '. ' + sample)
- log.write('\n')
- print('\n')
- log.flush()
- return samples
-
-
-def zip_seq_pred_crossent(id_to_word, sequences, predictions, cross_entropy):
- """Zip together the sequences, predictions, cross entropy."""
- indices = np.asarray(sequences)
-
- batch_of_metrics = []
-
- for ind_batch, pred_batch, crossent_batch in zip(indices, predictions,
- cross_entropy):
- metrics = []
-
- for index, pred, crossent in zip(ind_batch, pred_batch, crossent_batch):
- metrics.append([str(id_to_word[index]), pred, crossent])
-
- batch_of_metrics.append(metrics)
- return batch_of_metrics
-
-
-def zip_metrics(indices, *args):
- """Zip together the indices matrices with the provided metrics matrices."""
- batch_of_metrics = []
- for metrics_batch in zip(indices, *args):
-
- metrics = []
- for m in zip(*metrics_batch):
- metrics.append(m)
- batch_of_metrics.append(metrics)
- return batch_of_metrics
-
-
-def print_formatted(present, id_to_word, log, batch_of_tuples):
- """Print and log metrics."""
- num_cols = len(batch_of_tuples[0][0])
- repeat_float_format = '{:<12.3f} '
- repeat_str_format = '{:<13}'
-
- format_str = ''.join(
- ['[{:<1}] {:<20}',
- str(repeat_float_format * (num_cols - 1))])
-
- # TODO(liamfedus): Generalize the logging. This is sloppy.
- header_format_str = ''.join(
- ['[{:<1}] {:<20}',
- str(repeat_str_format * (num_cols - 1))])
- header_str = header_format_str.format('p', 'Word', 'p(real)', 'log-perp',
- 'log(p(a))', 'r', 'R=V*(s)', 'b=V(s)',
- 'A(a,s)')
-
- for i, batch in enumerate(batch_of_tuples):
- print(' Sample: %d' % i)
- log.write(' Sample %d.\n' % i)
- print(' ', header_str)
- log.write(' ' + str(header_str) + '\n')
-
- for j, t in enumerate(batch):
- t = list(t)
- t[0] = id_to_word[t[0]]
- buffer_str = format_str.format(int(present[i][j]), *t)
- print(' ', buffer_str)
- log.write(' ' + str(buffer_str) + '\n')
- log.flush()
-
-
-def generate_RL_logs(sess, model, log, id_to_word, feed):
- """Generate complete logs while running with REINFORCE."""
- # Impute Sequences.
- [
- p,
- fake_sequence_eval,
- fake_predictions_eval,
- _,
- fake_cross_entropy_losses_eval,
- _,
- fake_log_probs_eval,
- fake_rewards_eval,
- fake_baselines_eval,
- cumulative_rewards_eval,
- fake_advantages_eval,
- ] = sess.run(
- [
- model.present,
- model.fake_sequence,
- model.fake_predictions,
- model.real_predictions,
- model.fake_cross_entropy_losses,
- model.fake_logits,
- model.fake_log_probs,
- model.fake_rewards,
- model.fake_baselines,
- model.cumulative_rewards,
- model.fake_advantages,
- ],
- feed_dict=feed)
-
- indices = np.asarray(fake_sequence_eval)
-
- # Convert Discriminator linear layer to probability.
- fake_prob_eval = expit(fake_predictions_eval)
-
- # Add metrics.
- fake_tuples = zip_metrics(indices, fake_prob_eval,
- fake_cross_entropy_losses_eval, fake_log_probs_eval,
- fake_rewards_eval, cumulative_rewards_eval,
- fake_baselines_eval, fake_advantages_eval)
-
- # real_tuples = zip_metrics(indices, )
-
- # Print forward sequences.
- tuples_to_print = fake_tuples[:FLAGS.max_num_to_print]
- print_formatted(p, id_to_word, log, tuples_to_print)
-
- print('Samples')
- log.write('Samples\n')
- samples = print_and_log(log, id_to_word, fake_sequence_eval,
- FLAGS.max_num_to_print)
- return samples
-
-
-def generate_logs(sess, model, log, id_to_word, feed):
- """Impute Sequences using the model for a particular feed and send it to
- logs."""
- # Impute Sequences.
- [
- p, sequence_eval, fake_predictions_eval, fake_cross_entropy_losses_eval,
- fake_logits_eval
- ] = sess.run(
- [
- model.present, model.fake_sequence, model.fake_predictions,
- model.fake_cross_entropy_losses, model.fake_logits
- ],
- feed_dict=feed)
-
- # Convert Discriminator linear layer to probability.
- fake_prob_eval = expit(fake_predictions_eval)
-
- # Forward Masked Tuples.
- fake_tuples = zip_seq_pred_crossent(id_to_word, sequence_eval, fake_prob_eval,
- fake_cross_entropy_losses_eval)
-
- tuples_to_print = fake_tuples[:FLAGS.max_num_to_print]
-
- if FLAGS.print_verbose:
- print('fake_logits_eval')
- print(fake_logits_eval)
-
- for i, batch in enumerate(tuples_to_print):
- print(' Sample %d.' % i)
- log.write(' Sample %d.\n' % i)
- for j, pred in enumerate(batch):
- buffer_str = ('[{:<1}] {:<20} {:<7.3f} {:<7.3f}').format(
- int(p[i][j]), pred[0], pred[1], pred[2])
- print(' ', buffer_str)
- log.write(' ' + str(buffer_str) + '\n')
- log.flush()
-
- print('Samples')
- log.write('Samples\n')
- samples = print_and_log(log, id_to_word, sequence_eval,
- FLAGS.max_num_to_print)
- return samples
-
-
-def create_merged_ngram_dictionaries(indices, n):
- """Generate a single dictionary for the full batch.
-
- Args:
- indices: List of lists of indices.
- n: Degree of n-grams.
-
- Returns:
- Dictionary of hashed(n-gram tuples) to counts in the batch of indices.
- """
- ngram_dicts = []
-
- for ind in indices:
- ngrams = n_gram.find_all_ngrams(ind, n=n)
- ngram_counts = n_gram.construct_ngrams_dict(ngrams)
- ngram_dicts.append(ngram_counts)
-
- merged_gen_dict = Counter()
- for ngram_dict in ngram_dicts:
- merged_gen_dict += Counter(ngram_dict)
- return merged_gen_dict
-
-
-def sequence_ngram_evaluation(sess, sequence, log, feed, data_ngram_count, n):
- """Calculates the percent of ngrams produced in the sequence is present in
- data_ngram_count.
-
- Args:
- sess: tf.Session.
- sequence: Sequence Tensor from the MaskGAN model.
- log: gFile log.
- feed: Feed to evaluate.
- data_ngram_count: Dictionary of hashed(n-gram tuples) to counts in the
- data_set.
-
- Returns:
- avg_percent_captured: Percent of produced ngrams that appear in the
- data_ngram_count.
- """
- del log
- # Impute sequence.
- [sequence_eval] = sess.run([sequence], feed_dict=feed)
- indices = sequence_eval
-
- # Retrieve the counts across the batch of indices.
- gen_ngram_counts = create_merged_ngram_dictionaries(
- indices, n=n)
- return n_gram.percent_unique_ngrams_in_train(data_ngram_count,
- gen_ngram_counts)
diff --git a/research/maskgan/models/feedforward.py b/research/maskgan/models/feedforward.py
deleted file mode 100644
index d48a517d6bea65477b8a940ed770f92203da6dfd..0000000000000000000000000000000000000000
--- a/research/maskgan/models/feedforward.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple FNN model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from six.moves import xrange
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def discriminator(hparams, sequence, is_training, reuse=None):
- """Define the Discriminator graph."""
- del is_training
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- "If you wish to share Discriminator/Generator embeddings, they must be"
- " same dimension.")
- with tf.variable_scope("gen/rnn", reuse=True):
- embedding = tf.get_variable("embedding",
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope("dis", reuse=reuse):
- if not FLAGS.dis_share_embedding:
- embedding = tf.get_variable("embedding",
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- embeddings = tf.nn.embedding_lookup(embedding, sequence)
-
- # Input matrices.
- W = tf.get_variable(
- "W",
- initializer=tf.truncated_normal(
- shape=[3 * hparams.dis_embedding_dim, hparams.dis_hidden_dim],
- stddev=0.1))
- b = tf.get_variable(
- "b", initializer=tf.constant(0.1, shape=[hparams.dis_hidden_dim]))
-
- # Output matrices.
- W_out = tf.get_variable(
- "W_out",
- initializer=tf.truncated_normal(
- shape=[hparams.dis_hidden_dim, 1], stddev=0.1))
- b_out = tf.get_variable("b_out", initializer=tf.constant(0.1, shape=[1]))
-
- predictions = []
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- inp = embeddings[:, t]
-
- if t > 0:
- past_inp = tf.unstack(embeddings[:, 0:t], axis=1)
- avg_past_inp = tf.add_n(past_inp) / len(past_inp)
- else:
- avg_past_inp = tf.zeros_like(inp)
-
- if t < FLAGS.sequence_length:
- future_inp = tf.unstack(embeddings[:, t:], axis=1)
- avg_future_inp = tf.add_n(future_inp) / len(future_inp)
- else:
- avg_future_inp = tf.zeros_like(inp)
-
- # Cumulative input.
- concat_inp = tf.concat([avg_past_inp, inp, avg_future_inp], axis=1)
-
- # Hidden activations.
- hidden = tf.nn.relu(tf.nn.xw_plus_b(concat_inp, W, b, name="scores"))
-
- # Add dropout
- with tf.variable_scope("dropout"):
- hidden = tf.nn.dropout(hidden, FLAGS.keep_prob)
-
- # Output.
- output = tf.nn.xw_plus_b(hidden, W_out, b_out, name="output")
-
- predictions.append(output)
- predictions = tf.stack(predictions, axis=1)
- return tf.squeeze(predictions, axis=2)
diff --git a/research/maskgan/models/rnn.py b/research/maskgan/models/rnn.py
deleted file mode 100644
index 40b3a7aa3b85ddfd3002d845416b5004088620fc..0000000000000000000000000000000000000000
--- a/research/maskgan/models/rnn.py
+++ /dev/null
@@ -1,211 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple RNN model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from six.moves import xrange
-import tensorflow as tf
-
-# ZoneoutWrapper.
-from regularization import zoneout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def generator(hparams,
- inputs,
- targets,
- targets_present,
- is_training,
- is_validating,
- reuse=None):
- """Define the Generator graph.
-
- G will now impute tokens that have been masked from the input seqeunce.
- """
- tf.logging.warning(
- 'Undirectional generative model is not a useful model for this MaskGAN '
- 'because future context is needed. Use only for debugging purposes.')
- init_scale = 0.05
- initializer = tf.random_uniform_initializer(-init_scale, init_scale)
-
- with tf.variable_scope('gen', reuse=reuse, initializer=initializer):
-
- def lstm_cell():
- return tf.contrib.rnn.LayerNormBasicLSTMCell(
- hparams.gen_rnn_size, reuse=reuse)
-
- attn_cell = lstm_cell
- if FLAGS.zoneout_drop_prob > 0.0:
-
- def attn_cell():
- return zoneout.ZoneoutWrapper(
- lstm_cell(),
- zoneout_drop_prob=FLAGS.zoneout_drop_prob,
- is_training=is_training)
-
- cell_gen = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.gen_num_layers)],
- state_is_tuple=True)
-
- initial_state = cell_gen.zero_state(FLAGS.batch_size, tf.float32)
-
- with tf.variable_scope('rnn'):
- sequence, logits, log_probs = [], [], []
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- softmax_w = tf.get_variable('softmax_w',
- [hparams.gen_rnn_size, FLAGS.vocab_size])
- softmax_b = tf.get_variable('softmax_b', [FLAGS.vocab_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, inputs)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- # Input to the model is the first token to provide context. The
- # model will then predict token t > 0.
- if t == 0:
- # Always provide the real input at t = 0.
- state_gen = initial_state
- rnn_inp = rnn_inputs[:, t]
-
- # If the target at the last time-step was present, read in the real.
- # If the target at the last time-step was not present, read in the fake.
- else:
- real_rnn_inp = rnn_inputs[:, t]
- fake_rnn_inp = tf.nn.embedding_lookup(embedding, fake)
-
- # Use teacher forcing.
- if (is_training and
- FLAGS.gen_training_strategy == 'cross_entropy') or is_validating:
- rnn_inp = real_rnn_inp
- else:
- # Note that targets_t-1 == inputs_(t)
- rnn_inp = tf.where(targets_present[:, t - 1], real_rnn_inp,
- fake_rnn_inp)
-
- # RNN.
- rnn_out, state_gen = cell_gen(rnn_inp, state_gen)
- logit = tf.matmul(rnn_out, softmax_w) + softmax_b
-
- # Real sample.
- real = targets[:, t]
-
- # Fake sample.
- categorical = tf.contrib.distributions.Categorical(logits=logit)
- fake = categorical.sample()
- log_prob = categorical.log_prob(fake)
-
- # Output for Generator will either be generated or the target.
- # If present: Return real.
- # If not present: Return fake.
- output = tf.where(targets_present[:, t], real, fake)
-
- # Append to lists.
- sequence.append(output)
- logits.append(logit)
- log_probs.append(log_prob)
-
- # Produce the RNN state had the model operated only
- # over real data.
- real_state_gen = initial_state
- for t in xrange(FLAGS.sequence_length):
- tf.get_variable_scope().reuse_variables()
-
- rnn_inp = rnn_inputs[:, t]
-
- # RNN.
- rnn_out, real_state_gen = cell_gen(rnn_inp, real_state_gen)
-
- final_state = real_state_gen
-
- return (tf.stack(sequence, axis=1), tf.stack(logits, axis=1), tf.stack(
- log_probs, axis=1), initial_state, final_state)
-
-
-def discriminator(hparams, sequence, is_training, reuse=None):
- """Define the Discriminator graph.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- FLAGS: Current flags.
- sequence: [FLAGS.batch_size, FLAGS.sequence_length]
- is_training:
- reuse
-
- Returns:
- predictions:
- """
- tf.logging.warning(
- 'Undirectional Discriminative model is not a useful model for this '
- 'MaskGAN because future context is needed. Use only for debugging '
- 'purposes.')
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- 'If you wish to share Discriminator/Generator embeddings, they must be'
- ' same dimension.')
- with tf.variable_scope('gen/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('dis', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.LayerNormBasicLSTMCell(
- hparams.dis_rnn_size, reuse=reuse)
-
- attn_cell = lstm_cell
- if FLAGS.zoneout_drop_prob > 0.0:
-
- def attn_cell():
- return zoneout.ZoneoutWrapper(
- lstm_cell(),
- zoneout_drop_prob=FLAGS.zoneout_drop_prob,
- is_training=is_training)
-
- cell_dis = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
- state_dis = cell_dis.zero_state(FLAGS.batch_size, tf.float32)
-
- with tf.variable_scope('rnn') as vs:
- predictions = []
- if not FLAGS.dis_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_in = rnn_inputs[:, t]
- rnn_out, state_dis = cell_dis(rnn_in, state_dis)
-
- # Prediction is linear output for Discriminator.
- pred = tf.contrib.layers.linear(rnn_out, 1, scope=vs)
-
- predictions.append(pred)
- predictions = tf.stack(predictions, axis=1)
- return tf.squeeze(predictions, axis=2)
diff --git a/research/maskgan/models/rnn_nas.py b/research/maskgan/models/rnn_nas.py
deleted file mode 100644
index 618ace2f8196fb4718ae01bc406f114523fd44cc..0000000000000000000000000000000000000000
--- a/research/maskgan/models/rnn_nas.py
+++ /dev/null
@@ -1,234 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple RNN model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-from six.moves import xrange
-import tensorflow as tf
-
-# NAS Code..
-from nas_utils import configs
-from nas_utils import custom_cell
-from nas_utils import variational_dropout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def get_config():
- return configs.AlienConfig2()
-
-
-LSTMTuple = collections.namedtuple('LSTMTuple', ['c', 'h'])
-
-
-def generator(hparams,
- inputs,
- targets,
- targets_present,
- is_training,
- is_validating,
- reuse=None):
- """Define the Generator graph.
-
- G will now impute tokens that have been masked from the input seqeunce.
- """
- tf.logging.info(
- 'Undirectional generative model is not a useful model for this MaskGAN '
- 'because future context is needed. Use only for debugging purposes.')
- config = get_config()
- config.keep_prob = [hparams.gen_nas_keep_prob_0, hparams.gen_nas_keep_prob_1]
- configs.print_config(config)
-
- init_scale = config.init_scale
- initializer = tf.random_uniform_initializer(-init_scale, init_scale)
-
- with tf.variable_scope('gen', reuse=reuse, initializer=initializer):
- # Neural architecture search cell.
- cell = custom_cell.Alien(config.hidden_size)
-
- if is_training:
- [h2h_masks, _, _,
- output_mask] = variational_dropout.generate_variational_dropout_masks(
- hparams, config.keep_prob)
- else:
- output_mask = None
-
- cell_gen = custom_cell.GenericMultiRNNCell([cell] * config.num_layers)
- initial_state = cell_gen.zero_state(FLAGS.batch_size, tf.float32)
-
- with tf.variable_scope('rnn'):
- sequence, logits, log_probs = [], [], []
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- softmax_w = tf.matrix_transpose(embedding)
- softmax_b = tf.get_variable('softmax_b', [FLAGS.vocab_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, inputs)
-
- if is_training and FLAGS.keep_prob < 1:
- rnn_inputs = tf.nn.dropout(rnn_inputs, FLAGS.keep_prob)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- # Input to the model is the first token to provide context. The
- # model will then predict token t > 0.
- if t == 0:
- # Always provide the real input at t = 0.
- state_gen = initial_state
- rnn_inp = rnn_inputs[:, t]
-
- # If the input is present, read in the input at t.
- # If the input is not present, read in the previously generated.
- else:
- real_rnn_inp = rnn_inputs[:, t]
- fake_rnn_inp = tf.nn.embedding_lookup(embedding, fake)
-
- # While validating, the decoder should be operating in teacher
- # forcing regime. Also, if we're just training with cross_entropy
- # use teacher forcing.
- if is_validating or (is_training and
- FLAGS.gen_training_strategy == 'cross_entropy'):
- rnn_inp = real_rnn_inp
- else:
- rnn_inp = tf.where(targets_present[:, t - 1], real_rnn_inp,
- fake_rnn_inp)
-
- if is_training:
- state_gen = list(state_gen)
- for layer_num, per_layer_state in enumerate(state_gen):
- per_layer_state = LSTMTuple(
- per_layer_state[0], per_layer_state[1] * h2h_masks[layer_num])
- state_gen[layer_num] = per_layer_state
-
- # RNN.
- rnn_out, state_gen = cell_gen(rnn_inp, state_gen)
-
- if is_training:
- rnn_out = output_mask * rnn_out
-
- logit = tf.matmul(rnn_out, softmax_w) + softmax_b
-
- # Real sample.
- real = targets[:, t]
-
- categorical = tf.contrib.distributions.Categorical(logits=logit)
- fake = categorical.sample()
- log_prob = categorical.log_prob(fake)
-
- # Output for Generator will either be generated or the input.
- #
- # If present: Return real.
- # If not present: Return fake.
- output = tf.where(targets_present[:, t], real, fake)
-
- # Add to lists.
- sequence.append(output)
- log_probs.append(log_prob)
- logits.append(logit)
-
- # Produce the RNN state had the model operated only
- # over real data.
- real_state_gen = initial_state
- for t in xrange(FLAGS.sequence_length):
- tf.get_variable_scope().reuse_variables()
-
- rnn_inp = rnn_inputs[:, t]
-
- # RNN.
- rnn_out, real_state_gen = cell_gen(rnn_inp, real_state_gen)
-
- final_state = real_state_gen
-
- return (tf.stack(sequence, axis=1), tf.stack(logits, axis=1), tf.stack(
- log_probs, axis=1), initial_state, final_state)
-
-
-def discriminator(hparams, sequence, is_training, reuse=None):
- """Define the Discriminator graph."""
- tf.logging.info(
- 'Undirectional Discriminative model is not a useful model for this '
- 'MaskGAN because future context is needed. Use only for debugging '
- 'purposes.')
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- 'If you wish to share Discriminator/Generator embeddings, they must be'
- ' same dimension.')
- with tf.variable_scope('gen/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- config = get_config()
- config.keep_prob = [hparams.dis_nas_keep_prob_0, hparams.dis_nas_keep_prob_1]
- configs.print_config(config)
-
- with tf.variable_scope('dis', reuse=reuse):
- # Neural architecture search cell.
- cell = custom_cell.Alien(config.hidden_size)
-
- if is_training:
- [h2h_masks, _, _,
- output_mask] = variational_dropout.generate_variational_dropout_masks(
- hparams, config.keep_prob)
- else:
- output_mask = None
-
- cell_dis = custom_cell.GenericMultiRNNCell([cell] * config.num_layers)
- state_dis = cell_dis.zero_state(FLAGS.batch_size, tf.float32)
-
- with tf.variable_scope('rnn') as vs:
- predictions = []
- if not FLAGS.dis_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- if is_training and FLAGS.keep_prob < 1:
- rnn_inputs = tf.nn.dropout(rnn_inputs, FLAGS.keep_prob)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_in = rnn_inputs[:, t]
-
- if is_training:
- state_dis = list(state_dis)
- for layer_num, per_layer_state in enumerate(state_dis):
- per_layer_state = LSTMTuple(
- per_layer_state[0], per_layer_state[1] * h2h_masks[layer_num])
- state_dis[layer_num] = per_layer_state
-
- # RNN.
- rnn_out, state_dis = cell_dis(rnn_in, state_dis)
-
- if is_training:
- rnn_out = output_mask * rnn_out
-
- # Prediction is linear output for Discriminator.
- pred = tf.contrib.layers.linear(rnn_out, 1, scope=vs)
-
- predictions.append(pred)
- predictions = tf.stack(predictions, axis=1)
- return tf.squeeze(predictions, axis=2)
diff --git a/research/maskgan/models/rnn_vd.py b/research/maskgan/models/rnn_vd.py
deleted file mode 100644
index 428f1a54bda7d6e5f9dd55061149664b1b3e751d..0000000000000000000000000000000000000000
--- a/research/maskgan/models/rnn_vd.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple RNN model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from six.moves import xrange
-import tensorflow as tf
-from regularization import variational_dropout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def discriminator(hparams,
- sequence,
- is_training,
- reuse=None,
- initial_state=None):
- """Define the Discriminator graph."""
- tf.logging.info(
- 'Undirectional Discriminative model is not a useful model for this '
- 'MaskGAN because future context is needed. Use only for debugging '
- 'purposes.')
- sequence = tf.cast(sequence, tf.int32)
-
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- 'If you wish to share Discriminator/Generator embeddings, they must be'
- ' same dimension.')
- with tf.variable_scope('gen/decoder/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('dis', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(
- hparams.dis_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and hparams.dis_vd_keep_prob < 1:
-
- def attn_cell():
- return variational_dropout.VariationalDropoutWrapper(
- lstm_cell(), FLAGS.batch_size, hparams.dis_rnn_size,
- hparams.dis_vd_keep_prob, hparams.dis_vd_keep_prob)
-
- cell_dis = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- if initial_state:
- state_dis = [[tf.identity(x) for x in inner_initial_state]
- for inner_initial_state in initial_state]
- else:
- state_dis = cell_dis.zero_state(FLAGS.batch_size, tf.float32)
-
- def make_mask(keep_prob, units):
- random_tensor = keep_prob
- # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
- random_tensor += tf.random_uniform(tf.stack([FLAGS.batch_size, units]))
- return tf.floor(random_tensor) / keep_prob
-
- if is_training:
- output_mask = make_mask(hparams.dis_vd_keep_prob, hparams.dis_rnn_size)
-
- with tf.variable_scope('rnn') as vs:
- predictions, rnn_outs = [], []
-
- if not FLAGS.dis_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_in = rnn_inputs[:, t]
- rnn_out, state_dis = cell_dis(rnn_in, state_dis)
-
- if is_training:
- rnn_out *= output_mask
-
- # Prediction is linear output for Discriminator.
- pred = tf.contrib.layers.linear(rnn_out, 1, scope=vs)
- predictions.append(pred)
- rnn_outs.append(rnn_out)
-
- predictions = tf.stack(predictions, axis=1)
-
- if FLAGS.baseline_method == 'critic':
- with tf.variable_scope('critic', reuse=reuse) as critic_scope:
- rnn_outs = tf.stack(rnn_outs, axis=1)
- values = tf.contrib.layers.linear(rnn_outs, 1, scope=critic_scope)
- return tf.squeeze(predictions, axis=2), tf.squeeze(values, axis=2)
-
- else:
- return tf.squeeze(predictions, axis=2), None
diff --git a/research/maskgan/models/rnn_zaremba.py b/research/maskgan/models/rnn_zaremba.py
deleted file mode 100644
index 9369c77fbb849551721b46321e6868a7aeaceea6..0000000000000000000000000000000000000000
--- a/research/maskgan/models/rnn_zaremba.py
+++ /dev/null
@@ -1,196 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple RNN model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from six.moves import xrange
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def generator(hparams,
- inputs,
- targets,
- targets_present,
- is_training,
- is_validating,
- reuse=None):
- """Define the Generator graph.
-
- G will now impute tokens that have been masked from the input seqeunce.
- """
- tf.logging.warning(
- 'Undirectional generative model is not a useful model for this MaskGAN '
- 'because future context is needed. Use only for debugging purposes.')
- init_scale = 0.05
- initializer = tf.random_uniform_initializer(-init_scale, init_scale)
- with tf.variable_scope('gen', reuse=reuse, initializer=initializer):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(hparams.gen_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and FLAGS.keep_prob < 1:
-
- def attn_cell():
- return tf.contrib.rnn.DropoutWrapper(
- lstm_cell(), output_keep_prob=FLAGS.keep_prob)
-
- cell_gen = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.gen_num_layers)],
- state_is_tuple=True)
-
- initial_state = cell_gen.zero_state(FLAGS.batch_size, tf.float32)
-
- with tf.variable_scope('rnn'):
- sequence, logits, log_probs = [], [], []
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- softmax_w = tf.get_variable('softmax_w',
- [hparams.gen_rnn_size, FLAGS.vocab_size])
- softmax_b = tf.get_variable('softmax_b', [FLAGS.vocab_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, inputs)
-
- if is_training and FLAGS.keep_prob < 1:
- rnn_inputs = tf.nn.dropout(rnn_inputs, FLAGS.keep_prob)
-
- fake = None
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- # Input to the model is the first token to provide context. The
- # model will then predict token t > 0.
- if t == 0:
- # Always provide the real input at t = 0.
- state_gen = initial_state
- rnn_inp = rnn_inputs[:, t]
-
- # If the input is present, read in the input at t.
- # If the input is not present, read in the previously generated.
- else:
- real_rnn_inp = rnn_inputs[:, t]
- fake_rnn_inp = tf.nn.embedding_lookup(embedding, fake)
-
- # While validating, the decoder should be operating in teacher
- # forcing regime. Also, if we're just training with cross_entropy
- # use teacher forcing.
- if is_validating or (is_training and
- FLAGS.gen_training_strategy == 'cross_entropy'):
- rnn_inp = real_rnn_inp
- else:
- rnn_inp = tf.where(targets_present[:, t - 1], real_rnn_inp,
- fake_rnn_inp)
-
- # RNN.
- rnn_out, state_gen = cell_gen(rnn_inp, state_gen)
- logit = tf.matmul(rnn_out, softmax_w) + softmax_b
-
- # Real sample.
- real = targets[:, t]
-
- categorical = tf.contrib.distributions.Categorical(logits=logit)
- fake = categorical.sample()
- log_prob = categorical.log_prob(fake)
-
- # Output for Generator will either be generated or the input.
- #
- # If present: Return real.
- # If not present: Return fake.
- output = tf.where(targets_present[:, t], real, fake)
-
- # Add to lists.
- sequence.append(output)
- log_probs.append(log_prob)
- logits.append(logit)
-
- # Produce the RNN state had the model operated only
- # over real data.
- real_state_gen = initial_state
- for t in xrange(FLAGS.sequence_length):
- tf.get_variable_scope().reuse_variables()
-
- rnn_inp = rnn_inputs[:, t]
-
- # RNN.
- rnn_out, real_state_gen = cell_gen(rnn_inp, real_state_gen)
-
- final_state = real_state_gen
-
- return (tf.stack(sequence, axis=1), tf.stack(logits, axis=1), tf.stack(
- log_probs, axis=1), initial_state, final_state)
-
-
-def discriminator(hparams, sequence, is_training, reuse=None):
- """Define the Discriminator graph."""
- tf.logging.warning(
- 'Undirectional Discriminative model is not a useful model for this '
- 'MaskGAN because future context is needed. Use only for debugging '
- 'purposes.')
- sequence = tf.cast(sequence, tf.int32)
-
- with tf.variable_scope('dis', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(hparams.dis_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and FLAGS.keep_prob < 1:
-
- def attn_cell():
- return tf.contrib.rnn.DropoutWrapper(
- lstm_cell(), output_keep_prob=FLAGS.keep_prob)
-
- cell_dis = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- state_dis = cell_dis.zero_state(FLAGS.batch_size, tf.float32)
-
- with tf.variable_scope('rnn') as vs:
- predictions = []
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- if is_training and FLAGS.keep_prob < 1:
- rnn_inputs = tf.nn.dropout(rnn_inputs, FLAGS.keep_prob)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_in = rnn_inputs[:, t]
- rnn_out, state_dis = cell_dis(rnn_in, state_dis)
-
- # Prediction is linear output for Discriminator.
- pred = tf.contrib.layers.linear(rnn_out, 1, scope=vs)
-
- predictions.append(pred)
- predictions = tf.stack(predictions, axis=1)
- return tf.squeeze(predictions, axis=2)
diff --git a/research/maskgan/models/rollout.py b/research/maskgan/models/rollout.py
deleted file mode 100644
index 6919af2e31fa362f702e96e135d4a2bc06e063a2..0000000000000000000000000000000000000000
--- a/research/maskgan/models/rollout.py
+++ /dev/null
@@ -1,384 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Rollout RNN model definitions which call rnn_zaremba code."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-
-from six.moves import xrange
-import tensorflow as tf
-
-from losses import losses
-from model_utils import helper
-from model_utils import model_construction
-from model_utils import model_losses
-from model_utils import model_optimization
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def create_rollout_MaskGAN(hparams, is_training):
- """Create the MaskGAN model.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- is_training: Boolean indicating operational mode (train/inference).
- evaluated with a teacher forcing regime.
-
- Return:
- model: Namedtuple for specifying the MaskGAN."""
- global_step = tf.Variable(0, name='global_step', trainable=False)
-
- new_learning_rate = tf.placeholder(tf.float32, [], name='new_learning_rate')
- learning_rate = tf.Variable(0.0, name='learning_rate', trainable=False)
- learning_rate_update = tf.assign(learning_rate, new_learning_rate)
-
- new_rate = tf.placeholder(tf.float32, [], name='new_rate')
- percent_real_var = tf.Variable(0.0, trainable=False)
- percent_real_update = tf.assign(percent_real_var, new_rate)
-
- ## Placeholders.
- inputs = tf.placeholder(
- tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length])
- present = tf.placeholder(
- tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length])
- inv_present = tf.placeholder(
- tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length])
-
- ## Rollout Generator.
- fwd_gen_rollouts = rollout_generator(
- hparams, inputs, present, is_training=is_training, is_validating=False)
- inv_gen_rollouts = rollout_generator(
- hparams,
- inputs,
- inv_present,
- is_training=is_training,
- is_validating=False,
- reuse=True)
-
- ## Rollout Discriminator.
- fwd_dis_rollouts = rollout_discriminator(
- hparams, fwd_gen_rollouts, is_training=is_training)
- inv_dis_rollouts = rollout_discriminator(
- hparams, inv_gen_rollouts, is_training=is_training, reuse=True)
-
- ## Discriminator Loss.
- [dis_loss, dis_loss_pred, dis_loss_inv_pred] = rollout_discriminator_loss(
- fwd_dis_rollouts, present, inv_dis_rollouts, inv_present)
-
- ## Average log-perplexity for only missing words. However, to do this,
- # the logits are still computed using teacher forcing, that is, the ground
- # truth tokens are fed in at each time point to be valid.
- # TODO(liamfedus): Fix the naming convention.
- with tf.variable_scope('gen_rollout'):
- _, fwd_eval_logits, _ = model_construction.create_generator(
- hparams,
- inputs,
- present,
- is_training=False,
- is_validating=True,
- reuse=True)
-
- avg_log_perplexity = model_losses.calculate_log_perplexity(
- fwd_eval_logits, inputs, present)
-
- ## Generator Loss.
- # 1. Cross Entropy losses on missing tokens.
- [fwd_cross_entropy_losses,
- inv_cross_entropy_losses] = rollout_masked_cross_entropy_loss(
- inputs, present, inv_present, fwd_gen_rollouts, inv_gen_rollouts)
-
- # 2. GAN losses on missing tokens.
- [fwd_RL_loss,
- fwd_RL_statistics, fwd_averages_op] = rollout_reinforce_objective(
- hparams, fwd_gen_rollouts, fwd_dis_rollouts, present)
- [inv_RL_loss,
- inv_RL_statistics, inv_averages_op] = rollout_reinforce_objective(
- hparams, inv_gen_rollouts, inv_dis_rollouts, inv_present)
-
- # TODO(liamfedus): Generalize this to use all logs.
- [fwd_sequence, fwd_logits, fwd_log_probs] = fwd_gen_rollouts[-1]
- [inv_sequence, inv_logits, inv_log_probs] = inv_gen_rollouts[-1]
-
- # TODO(liamfedus): Generalize this to use all logs.
- fwd_predictions = fwd_dis_rollouts[-1]
- inv_predictions = inv_dis_rollouts[-1]
-
- # TODO(liamfedus): Generalize this to use all logs.
- [fwd_log_probs, fwd_rewards, fwd_advantages,
- fwd_baselines] = fwd_RL_statistics[-1]
- [inv_log_probs, inv_rewards, inv_advantages,
- inv_baselines] = inv_RL_statistics[-1]
-
- ## Pre-training.
- if FLAGS.gen_pretrain_steps:
- # TODO(liamfedus): Rewrite this.
- fwd_cross_entropy_loss = tf.reduce_mean(fwd_cross_entropy_losses)
- gen_pretrain_op = model_optimization.create_gen_pretrain_op(
- hparams, fwd_cross_entropy_loss, global_step)
- else:
- gen_pretrain_op = tf.no_op('gen_pretrain_no_op')
- if FLAGS.dis_pretrain_steps:
- dis_pretrain_op = model_optimization.create_dis_pretrain_op(
- hparams, dis_loss, global_step)
- else:
- dis_pretrain_op = tf.no_op('dis_pretrain_no_op')
-
- ## Generator Train Op.
- # 1. Cross-Entropy.
- if FLAGS.gen_training_strategy == 'cross_entropy':
- gen_loss = tf.reduce_mean(
- fwd_cross_entropy_losses + inv_cross_entropy_losses) / 2.
- [gen_train_op, gen_grads,
- gen_vars] = model_optimization.create_gen_train_op(
- hparams, learning_rate, gen_loss, global_step, mode='MINIMIZE')
-
- # 2. GAN (REINFORCE)
- elif FLAGS.gen_training_strategy == 'reinforce':
- gen_loss = (fwd_RL_loss + inv_RL_loss) / 2.
- [gen_train_op, gen_grads,
- gen_vars] = model_optimization.create_reinforce_gen_train_op(
- hparams, learning_rate, gen_loss, fwd_averages_op, inv_averages_op,
- global_step)
-
- else:
- raise NotImplementedError
-
- ## Discriminator Train Op.
- dis_train_op, dis_grads, dis_vars = model_optimization.create_dis_train_op(
- hparams, dis_loss, global_step)
-
- ## Summaries.
- with tf.name_scope('general'):
- tf.summary.scalar('percent_real', percent_real_var)
- tf.summary.scalar('learning_rate', learning_rate)
-
- with tf.name_scope('generator_losses'):
- tf.summary.scalar('gen_loss', tf.reduce_mean(gen_loss))
- tf.summary.scalar('gen_loss_fwd_cross_entropy',
- tf.reduce_mean(fwd_cross_entropy_losses))
- tf.summary.scalar('gen_loss_inv_cross_entropy',
- tf.reduce_mean(inv_cross_entropy_losses))
-
- with tf.name_scope('REINFORCE'):
- with tf.name_scope('objective'):
- tf.summary.scalar('fwd_RL_loss', tf.reduce_mean(fwd_RL_loss))
- tf.summary.scalar('inv_RL_loss', tf.reduce_mean(inv_RL_loss))
-
- with tf.name_scope('rewards'):
- helper.variable_summaries(fwd_rewards, 'fwd_rewards')
- helper.variable_summaries(inv_rewards, 'inv_rewards')
-
- with tf.name_scope('advantages'):
- helper.variable_summaries(fwd_advantages, 'fwd_advantages')
- helper.variable_summaries(inv_advantages, 'inv_advantages')
-
- with tf.name_scope('baselines'):
- helper.variable_summaries(fwd_baselines, 'fwd_baselines')
- helper.variable_summaries(inv_baselines, 'inv_baselines')
-
- with tf.name_scope('log_probs'):
- helper.variable_summaries(fwd_log_probs, 'fwd_log_probs')
- helper.variable_summaries(inv_log_probs, 'inv_log_probs')
-
- with tf.name_scope('discriminator_losses'):
- tf.summary.scalar('dis_loss', dis_loss)
- tf.summary.scalar('dis_loss_fwd_sequence', dis_loss_pred)
- tf.summary.scalar('dis_loss_inv_sequence', dis_loss_inv_pred)
-
- with tf.name_scope('logits'):
- helper.variable_summaries(fwd_logits, 'fwd_logits')
- helper.variable_summaries(inv_logits, 'inv_logits')
-
- for v, g in zip(gen_vars, gen_grads):
- helper.variable_summaries(v, v.op.name)
- helper.variable_summaries(g, 'grad/' + v.op.name)
-
- for v, g in zip(dis_vars, dis_grads):
- helper.variable_summaries(v, v.op.name)
- helper.variable_summaries(g, 'grad/' + v.op.name)
-
- merge_summaries_op = tf.summary.merge_all()
-
- # Model saver.
- saver = tf.train.Saver(keep_checkpoint_every_n_hours=1, max_to_keep=5)
-
- # Named tuple that captures elements of the MaskGAN model.
- Model = collections.namedtuple('Model', [
- 'inputs', 'present', 'inv_present', 'percent_real_update', 'new_rate',
- 'fwd_sequence', 'fwd_logits', 'fwd_rewards', 'fwd_advantages',
- 'fwd_log_probs', 'fwd_predictions', 'fwd_cross_entropy_losses',
- 'inv_sequence', 'inv_logits', 'inv_rewards', 'inv_advantages',
- 'inv_log_probs', 'inv_predictions', 'inv_cross_entropy_losses',
- 'avg_log_perplexity', 'dis_loss', 'gen_loss', 'dis_train_op',
- 'gen_train_op', 'gen_pretrain_op', 'dis_pretrain_op',
- 'merge_summaries_op', 'global_step', 'new_learning_rate',
- 'learning_rate_update', 'saver'
- ])
-
- model = Model(
- inputs, present, inv_present, percent_real_update, new_rate, fwd_sequence,
- fwd_logits, fwd_rewards, fwd_advantages, fwd_log_probs, fwd_predictions,
- fwd_cross_entropy_losses, inv_sequence, inv_logits, inv_rewards,
- inv_advantages, inv_log_probs, inv_predictions, inv_cross_entropy_losses,
- avg_log_perplexity, dis_loss, gen_loss, dis_train_op, gen_train_op,
- gen_pretrain_op, dis_pretrain_op, merge_summaries_op, global_step,
- new_learning_rate, learning_rate_update, saver)
- return model
-
-
-def rollout_generator(hparams,
- inputs,
- input_present,
- is_training,
- is_validating,
- reuse=None):
- """Define the Generator graph which does rollouts.
-
- G will now impute tokens that have been masked from the input seqeunce.
- """
- rollouts = []
-
- with tf.variable_scope('gen_rollout'):
- for n in xrange(FLAGS.num_rollouts):
- if n > 0:
- # TODO(liamfedus): Why is it necessary here to manually set reuse?
- reuse = True
- tf.get_variable_scope().reuse_variables()
-
- [sequence, logits, log_probs] = model_construction.create_generator(
- hparams,
- inputs,
- input_present,
- is_training,
- is_validating,
- reuse=reuse)
-
- rollouts.append([sequence, logits, log_probs])
-
- # Length assertion.
- assert len(rollouts) == FLAGS.num_rollouts
-
- return rollouts
-
-
-def rollout_discriminator(hparams, gen_rollouts, is_training, reuse=None):
- """Define the Discriminator graph which does rollouts.
-
- G will now impute tokens that have been masked from the input seqeunce.
- """
- rollout_predictions = []
-
- with tf.variable_scope('dis_rollout'):
- for n, rollout in enumerate(gen_rollouts):
- if n > 0:
- # TODO(liamfedus): Why is it necessary here to manually set reuse?
- reuse = True
- tf.get_variable_scope().reuse_variables()
-
- [sequence, _, _] = rollout
-
- predictions = model_construction.create_discriminator(
- hparams, sequence, is_training=is_training, reuse=reuse)
-
- # Predictions for each rollout.
- rollout_predictions.append(predictions)
-
- # Length assertion.
- assert len(rollout_predictions) == FLAGS.num_rollouts
-
- return rollout_predictions
-
-
-def rollout_reinforce_objective(hparams, gen_rollouts, dis_rollouts, present):
- cumulative_gen_objective = 0.
- cumulative_averages_op = []
- cumulative_statistics = []
-
- assert len(gen_rollouts) == len(dis_rollouts)
-
- for gen_rollout, dis_rollout in zip(gen_rollouts, dis_rollouts):
- [_, _, log_probs] = gen_rollout
- dis_predictions = dis_rollout
-
- [
- final_gen_objective, log_probs, rewards, advantages, baselines,
- maintain_averages_op
- ] = model_losses.calculate_reinforce_objective(hparams, log_probs,
- dis_predictions, present)
-
- # Accumulate results.
- cumulative_gen_objective += final_gen_objective
- cumulative_averages_op.append(maintain_averages_op)
- cumulative_statistics.append([log_probs, rewards, advantages, baselines])
-
- # Group all the averaging operations.
- cumulative_averages_op = tf.group(*cumulative_averages_op)
- cumulative_gen_objective /= FLAGS.num_rollouts
- [log_probs, rewards, advantages, baselines] = cumulative_statistics[-1]
-
- # Length assertion.
- assert len(cumulative_statistics) == FLAGS.num_rollouts
-
- return [
- cumulative_gen_objective, cumulative_statistics, cumulative_averages_op
- ]
-
-
-def rollout_masked_cross_entropy_loss(inputs, present, inv_present,
- fwd_rollouts, inv_rollouts):
- cumulative_fwd_cross_entropy_losses = tf.zeros(
- shape=[FLAGS.batch_size, FLAGS.sequence_length])
- cumulative_inv_cross_entropy_losses = tf.zeros(
- shape=[FLAGS.batch_size, FLAGS.sequence_length])
-
- for fwd_rollout, inv_rollout in zip(fwd_rollouts, inv_rollouts):
- [_, fwd_logits, _] = fwd_rollout
- [_, inv_logits, _] = inv_rollout
-
- [fwd_cross_entropy_losses,
- inv_cross_entropy_losses] = model_losses.create_masked_cross_entropy_loss(
- inputs, present, inv_present, fwd_logits, inv_logits)
-
- cumulative_fwd_cross_entropy_losses = tf.add(
- cumulative_fwd_cross_entropy_losses, fwd_cross_entropy_losses)
- cumulative_inv_cross_entropy_losses = tf.add(
- cumulative_inv_cross_entropy_losses, inv_cross_entropy_losses)
-
- return [
- cumulative_fwd_cross_entropy_losses, cumulative_inv_cross_entropy_losses
- ]
-
-
-def rollout_discriminator_loss(fwd_rollouts, present, inv_rollouts,
- inv_present):
-
- dis_loss = 0
- dis_loss_pred = 0
- dis_loss_inv_pred = 0
-
- for fwd_predictions, inv_predictions in zip(fwd_rollouts, inv_rollouts):
- dis_loss_pred += losses.discriminator_loss(fwd_predictions, present)
- dis_loss_inv_pred += losses.discriminator_loss(inv_predictions, inv_present)
-
- dis_loss_pred /= FLAGS.num_rollouts
- dis_loss_inv_pred /= FLAGS.num_rollouts
-
- dis_loss = (dis_loss_pred + dis_loss_inv_pred) / 2.
- return [dis_loss, dis_loss_pred, dis_loss_inv_pred]
diff --git a/research/maskgan/models/seq2seq.py b/research/maskgan/models/seq2seq.py
deleted file mode 100644
index fac397c98381309f6c7c6d428fcec3c665bcff98..0000000000000000000000000000000000000000
--- a/research/maskgan/models/seq2seq.py
+++ /dev/null
@@ -1,277 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple seq2seq model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-from six.moves import xrange
-from models import attention_utils
-
-# ZoneoutWrapper.
-from regularization import zoneout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def transform_input_with_is_missing_token(inputs, targets_present):
- """Transforms the inputs to have missing tokens when it's masked out. The
- mask is for the targets, so therefore, to determine if an input at time t is
- masked, we have to check if the target at time t - 1 is masked out.
-
- e.g.
- inputs = [a, b, c, d]
- targets = [b, c, d, e]
- targets_present = [1, 0, 1, 0]
-
- then,
- transformed_input = [a, b, , d]
-
- Args:
- inputs: tf.int32 Tensor of shape [batch_size, sequence_length] with tokens
- up to, but not including, vocab_size.
- targets_present: tf.bool Tensor of shape [batch_size, sequence_length] with
- True representing the presence of the word.
-
- Returns:
- transformed_input: tf.int32 Tensor of shape [batch_size, sequence_length]
- which takes on value of inputs when the input is present and takes on
- value=vocab_size to indicate a missing token.
- """
- # To fill in if the input is missing.
- input_missing = tf.constant(
- FLAGS.vocab_size,
- dtype=tf.int32,
- shape=[FLAGS.batch_size, FLAGS.sequence_length])
-
- # The 0th input will always be present to MaskGAN.
- zeroth_input_present = tf.constant(True, tf.bool, shape=[FLAGS.batch_size, 1])
-
- # Input present mask.
- inputs_present = tf.concat(
- [zeroth_input_present, targets_present[:, :-1]], axis=1)
-
- transformed_input = tf.where(inputs_present, inputs, input_missing)
- return transformed_input
-
-
-def gen_encoder(hparams, inputs, targets_present, is_training, reuse=None):
- """Define the Encoder graph."""
- # We will use the same variable from the decoder.
- if FLAGS.seq2seq_share_embedding:
- with tf.variable_scope('decoder/rnn'):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('encoder', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.LayerNormBasicLSTMCell(
- hparams.gen_rnn_size, reuse=reuse)
-
- attn_cell = lstm_cell
- if FLAGS.zoneout_drop_prob > 0.0:
-
- def attn_cell():
- return zoneout.ZoneoutWrapper(
- lstm_cell(),
- zoneout_drop_prob=FLAGS.zoneout_drop_prob,
- is_training=is_training)
-
- cell = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.gen_num_layers)],
- state_is_tuple=True)
-
- initial_state = cell.zero_state(FLAGS.batch_size, tf.float32)
-
- # Add a missing token for inputs not present.
- real_inputs = inputs
- masked_inputs = transform_input_with_is_missing_token(
- inputs, targets_present)
-
- with tf.variable_scope('rnn'):
- hidden_states = []
-
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size + 1, hparams.gen_rnn_size])
-
- real_rnn_inputs = tf.nn.embedding_lookup(embedding, real_inputs)
- masked_rnn_inputs = tf.nn.embedding_lookup(embedding, masked_inputs)
-
- state = initial_state
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_inp = masked_rnn_inputs[:, t]
- rnn_out, state = cell(rnn_inp, state)
- hidden_states.append(rnn_out)
- final_masked_state = state
- hidden_states = tf.stack(hidden_states, axis=1)
-
- # Produce the RNN state had the model operated only
- # over real data.
- real_state = initial_state
- for t in xrange(FLAGS.sequence_length):
- tf.get_variable_scope().reuse_variables()
-
- # RNN.
- rnn_inp = real_rnn_inputs[:, t]
- rnn_out, real_state = cell(rnn_inp, real_state)
- final_state = real_state
-
- return (hidden_states, final_masked_state), initial_state, final_state
-
-
-def gen_decoder(hparams,
- inputs,
- targets,
- targets_present,
- encoding_state,
- is_training,
- is_validating,
- reuse=None):
- """Define the Decoder graph. The Decoder will now impute tokens that
- have been masked from the input seqeunce.
- """
- gen_decoder_rnn_size = hparams.gen_rnn_size
-
- with tf.variable_scope('decoder', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.LayerNormBasicLSTMCell(
- gen_decoder_rnn_size, reuse=reuse)
-
- attn_cell = lstm_cell
- if FLAGS.zoneout_drop_prob > 0.0:
-
- def attn_cell():
- return zoneout.ZoneoutWrapper(
- lstm_cell(),
- zoneout_drop_prob=FLAGS.zoneout_drop_prob,
- is_training=is_training)
-
- cell_gen = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.gen_num_layers)],
- state_is_tuple=True)
-
- # Hidden encoder states.
- hidden_vector_encodings = encoding_state[0]
-
- # Carry forward the final state tuple from the encoder.
- # State tuples.
- state_gen = encoding_state[1]
-
- if FLAGS.attention_option is not None:
- (attention_keys, attention_values, _,
- attention_construct_fn) = attention_utils.prepare_attention(
- hidden_vector_encodings,
- FLAGS.attention_option,
- num_units=gen_decoder_rnn_size,
- reuse=reuse)
-
- with tf.variable_scope('rnn'):
- sequence, logits, log_probs = [], [], []
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, gen_decoder_rnn_size])
- softmax_w = tf.get_variable('softmax_w',
- [gen_decoder_rnn_size, FLAGS.vocab_size])
- softmax_b = tf.get_variable('softmax_b', [FLAGS.vocab_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, inputs)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- # Input to the Decoder.
- if t == 0:
- # Always provide the real input at t = 0.
- rnn_inp = rnn_inputs[:, t]
-
- # If the input is present, read in the input at t.
- # If the input is not present, read in the previously generated.
- else:
- real_rnn_inp = rnn_inputs[:, t]
- fake_rnn_inp = tf.nn.embedding_lookup(embedding, fake)
-
- # While validating, the decoder should be operating in teacher
- # forcing regime. Also, if we're just training with cross_entropy
- # use teacher forcing.
- if is_validating or (is_training and
- FLAGS.gen_training_strategy == 'cross_entropy'):
- rnn_inp = real_rnn_inp
- else:
- rnn_inp = tf.where(targets_present[:, t - 1], real_rnn_inp,
- fake_rnn_inp)
-
- # RNN.
- rnn_out, state_gen = cell_gen(rnn_inp, state_gen)
-
- if FLAGS.attention_option is not None:
- rnn_out = attention_construct_fn(rnn_out, attention_keys,
- attention_values)
- # # TODO(liamfedus): Assert not "monotonic" attention_type.
- # # TODO(liamfedus): FLAGS.attention_type.
- # context_state = revised_attention_utils._empty_state()
- # rnn_out, context_state = attention_construct_fn(
- # rnn_out, attention_keys, attention_values, context_state, t)
- logit = tf.matmul(rnn_out, softmax_w) + softmax_b
-
- # Output for Decoder.
- # If input is present: Return real at t+1.
- # If input is not present: Return fake for t+1.
- real = targets[:, t]
-
- categorical = tf.contrib.distributions.Categorical(logits=logit)
- fake = categorical.sample()
- log_prob = categorical.log_prob(fake)
-
- output = tf.where(targets_present[:, t], real, fake)
-
- # Add to lists.
- sequence.append(output)
- log_probs.append(log_prob)
- logits.append(logit)
-
- return (tf.stack(sequence, axis=1), tf.stack(logits, axis=1), tf.stack(
- log_probs, axis=1))
-
-
-def generator(hparams,
- inputs,
- targets,
- targets_present,
- is_training,
- is_validating,
- reuse=None):
- """Define the Generator graph."""
- with tf.variable_scope('gen', reuse=reuse):
- encoder_states, initial_state, final_state = gen_encoder(
- hparams, inputs, targets_present, is_training=is_training, reuse=reuse)
- stacked_sequence, stacked_logits, stacked_log_probs = gen_decoder(
- hparams,
- inputs,
- targets,
- targets_present,
- encoder_states,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- return (stacked_sequence, stacked_logits, stacked_log_probs, initial_state,
- final_state)
diff --git a/research/maskgan/models/seq2seq_nas.py b/research/maskgan/models/seq2seq_nas.py
deleted file mode 100644
index cede90f5625c6e46740ad7601681712e73f07450..0000000000000000000000000000000000000000
--- a/research/maskgan/models/seq2seq_nas.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple seq2seq model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-from six.moves import xrange
-import tensorflow as tf
-
-from models import attention_utils
-
-# NAS Code..
-from nas_utils import configs
-from nas_utils import custom_cell
-from nas_utils import variational_dropout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def get_config():
- return configs.AlienConfig2()
-
-
-LSTMTuple = collections.namedtuple('LSTMTuple', ['c', 'h'])
-
-
-def transform_input_with_is_missing_token(inputs, targets_present):
- """Transforms the inputs to have missing tokens when it's masked out. The
- mask is for the targets, so therefore, to determine if an input at time t is
- masked, we have to check if the target at time t - 1 is masked out.
-
- e.g.
- inputs = [a, b, c, d]
- targets = [b, c, d, e]
- targets_present = [1, 0, 1, 0]
-
- then,
- transformed_input = [a, b, , d]
-
- Args:
- inputs: tf.int32 Tensor of shape [batch_size, sequence_length] with tokens
- up to, but not including, vocab_size.
- targets_present: tf.bool Tensor of shape [batch_size, sequence_length] with
- True representing the presence of the word.
-
- Returns:
- transformed_input: tf.int32 Tensor of shape [batch_size, sequence_length]
- which takes on value of inputs when the input is present and takes on
- value=vocab_size to indicate a missing token.
- """
- # To fill in if the input is missing.
- input_missing = tf.constant(
- FLAGS.vocab_size,
- dtype=tf.int32,
- shape=[FLAGS.batch_size, FLAGS.sequence_length])
-
- # The 0th input will always be present to MaskGAN.
- zeroth_input_present = tf.constant(True, tf.bool, shape=[FLAGS.batch_size, 1])
-
- # Input present mask.
- inputs_present = tf.concat(
- [zeroth_input_present, targets_present[:, :-1]], axis=1)
-
- transformed_input = tf.where(inputs_present, inputs, input_missing)
- return transformed_input
-
-
-def gen_encoder(hparams, inputs, targets_present, is_training, reuse=None):
- """Define the Encoder graph.
-
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- inputs: tf.int32 Tensor of shape [batch_size, sequence_length] with tokens
- up to, but not including, vocab_size.
- targets_present: tf.bool Tensor of shape [batch_size, sequence_length] with
- True representing the presence of the target.
- is_training: Boolean indicating operational mode (train/inference).
- reuse (Optional): Whether to reuse the variables.
-
- Returns:
- Tuple of (hidden_states, final_state).
- """
- config = get_config()
- configs.print_config(config)
- # We will use the same variable from the decoder.
- if FLAGS.seq2seq_share_embedding:
- with tf.variable_scope('decoder/rnn'):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('encoder', reuse=reuse):
- # Neural architecture search cell.
- cell = custom_cell.Alien(config.hidden_size)
-
- if is_training:
- [h2h_masks, h2i_masks, _,
- output_mask] = variational_dropout.generate_variational_dropout_masks(
- hparams, config.keep_prob)
- else:
- h2i_masks, output_mask = None, None
-
- cell = custom_cell.GenericMultiRNNCell([cell] * config.num_layers)
-
- initial_state = cell.zero_state(FLAGS.batch_size, tf.float32)
-
- # Add a missing token for inputs not present.
- real_inputs = inputs
- masked_inputs = transform_input_with_is_missing_token(
- inputs, targets_present)
-
- with tf.variable_scope('rnn'):
- hidden_states = []
-
- # Split the embedding into two parts so that we can load the PTB
- # weights into one part of the Variable.
- if not FLAGS.seq2seq_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- missing_embedding = tf.get_variable('missing_embedding',
- [1, hparams.gen_rnn_size])
- embedding = tf.concat([embedding, missing_embedding], axis=0)
-
- real_rnn_inputs = tf.nn.embedding_lookup(embedding, real_inputs)
- masked_rnn_inputs = tf.nn.embedding_lookup(embedding, masked_inputs)
-
- if is_training and FLAGS.keep_prob < 1:
- masked_rnn_inputs = tf.nn.dropout(masked_rnn_inputs, FLAGS.keep_prob)
-
- state = initial_state
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_inp = masked_rnn_inputs[:, t]
-
- if is_training:
- state = list(state)
- for layer_num, per_layer_state in enumerate(state):
- per_layer_state = LSTMTuple(
- per_layer_state[0], per_layer_state[1] * h2h_masks[layer_num])
- state[layer_num] = per_layer_state
-
- rnn_out, state = cell(rnn_inp, state, h2i_masks)
-
- if is_training:
- rnn_out = output_mask * rnn_out
-
- hidden_states.append(rnn_out)
- final_masked_state = state
- hidden_states = tf.stack(hidden_states, axis=1)
-
- # Produce the RNN state had the model operated only
- # over real data.
- real_state = initial_state
- for t in xrange(FLAGS.sequence_length):
- tf.get_variable_scope().reuse_variables()
-
- # RNN.
- rnn_inp = real_rnn_inputs[:, t]
- rnn_out, real_state = cell(rnn_inp, real_state)
- final_state = real_state
-
- return (hidden_states, final_masked_state), initial_state, final_state
-
-
-def gen_decoder(hparams,
- inputs,
- targets,
- targets_present,
- encoding_state,
- is_training,
- is_validating,
- reuse=None):
- """Define the Decoder graph. The Decoder will now impute tokens that
- have been masked from the input seqeunce.
- """
- config = get_config()
- gen_decoder_rnn_size = hparams.gen_rnn_size
-
- if FLAGS.seq2seq_share_embedding:
- with tf.variable_scope('decoder/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, gen_decoder_rnn_size])
-
- with tf.variable_scope('decoder', reuse=reuse):
- # Neural architecture search cell.
- cell = custom_cell.Alien(config.hidden_size)
-
- if is_training:
- [h2h_masks, _, _,
- output_mask] = variational_dropout.generate_variational_dropout_masks(
- hparams, config.keep_prob)
- else:
- output_mask = None
-
- cell_gen = custom_cell.GenericMultiRNNCell([cell] * config.num_layers)
-
- # Hidden encoder states.
- hidden_vector_encodings = encoding_state[0]
-
- # Carry forward the final state tuple from the encoder.
- # State tuples.
- state_gen = encoding_state[1]
-
- if FLAGS.attention_option is not None:
- (attention_keys, attention_values, _,
- attention_construct_fn) = attention_utils.prepare_attention(
- hidden_vector_encodings,
- FLAGS.attention_option,
- num_units=gen_decoder_rnn_size,
- reuse=reuse)
-
- with tf.variable_scope('rnn'):
- sequence, logits, log_probs = [], [], []
-
- if not FLAGS.seq2seq_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, gen_decoder_rnn_size])
- softmax_w = tf.matrix_transpose(embedding)
- softmax_b = tf.get_variable('softmax_b', [FLAGS.vocab_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, inputs)
-
- if is_training and FLAGS.keep_prob < 1:
- rnn_inputs = tf.nn.dropout(rnn_inputs, FLAGS.keep_prob)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- # Input to the Decoder.
- if t == 0:
- # Always provide the real input at t = 0.
- rnn_inp = rnn_inputs[:, t]
-
- # If the input is present, read in the input at t.
- # If the input is not present, read in the previously generated.
- else:
- real_rnn_inp = rnn_inputs[:, t]
- fake_rnn_inp = tf.nn.embedding_lookup(embedding, fake)
-
- # While validating, the decoder should be operating in teacher
- # forcing regime. Also, if we're just training with cross_entropy
- # use teacher forcing.
- if is_validating or (is_training and
- FLAGS.gen_training_strategy == 'cross_entropy'):
- rnn_inp = real_rnn_inp
- else:
- rnn_inp = tf.where(targets_present[:, t - 1], real_rnn_inp,
- fake_rnn_inp)
-
- if is_training:
- state_gen = list(state_gen)
- for layer_num, per_layer_state in enumerate(state_gen):
- per_layer_state = LSTMTuple(
- per_layer_state[0], per_layer_state[1] * h2h_masks[layer_num])
- state_gen[layer_num] = per_layer_state
-
- # RNN.
- rnn_out, state_gen = cell_gen(rnn_inp, state_gen)
-
- if is_training:
- rnn_out = output_mask * rnn_out
-
- if FLAGS.attention_option is not None:
- rnn_out = attention_construct_fn(rnn_out, attention_keys,
- attention_values)
- # # TODO(liamfedus): Assert not "monotonic" attention_type.
- # # TODO(liamfedus): FLAGS.attention_type.
- # context_state = revised_attention_utils._empty_state()
- # rnn_out, context_state = attention_construct_fn(
- # rnn_out, attention_keys, attention_values, context_state, t)
- logit = tf.matmul(rnn_out, softmax_w) + softmax_b
-
- # Output for Decoder.
- # If input is present: Return real at t+1.
- # If input is not present: Return fake for t+1.
- real = targets[:, t]
-
- categorical = tf.contrib.distributions.Categorical(logits=logit)
- fake = categorical.sample()
- log_prob = categorical.log_prob(fake)
-
- output = tf.where(targets_present[:, t], real, fake)
-
- # Add to lists.
- sequence.append(output)
- log_probs.append(log_prob)
- logits.append(logit)
-
- return (tf.stack(sequence, axis=1), tf.stack(logits, axis=1), tf.stack(
- log_probs, axis=1))
-
-
-def generator(hparams,
- inputs,
- targets,
- targets_present,
- is_training,
- is_validating,
- reuse=None):
- """Define the Generator graph."""
- with tf.variable_scope('gen', reuse=reuse):
- encoder_states, initial_state, final_state = gen_encoder(
- hparams, inputs, targets_present, is_training=is_training, reuse=reuse)
- stacked_sequence, stacked_logits, stacked_log_probs = gen_decoder(
- hparams,
- inputs,
- targets,
- targets_present,
- encoder_states,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- return (stacked_sequence, stacked_logits, stacked_log_probs, initial_state,
- final_state)
diff --git a/research/maskgan/models/seq2seq_vd.py b/research/maskgan/models/seq2seq_vd.py
deleted file mode 100644
index 850eda435c48c73d574a06b1b65a12f71a18f276..0000000000000000000000000000000000000000
--- a/research/maskgan/models/seq2seq_vd.py
+++ /dev/null
@@ -1,609 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple seq2seq model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from six.moves import xrange
-import tensorflow as tf
-
-from models import attention_utils
-from regularization import variational_dropout
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def transform_input_with_is_missing_token(inputs, targets_present):
- """Transforms the inputs to have missing tokens when it's masked out. The
- mask is for the targets, so therefore, to determine if an input at time t is
- masked, we have to check if the target at time t - 1 is masked out.
-
- e.g.
- inputs = [a, b, c, d]
- targets = [b, c, d, e]
- targets_present = [1, 0, 1, 0]
-
- which computes,
- inputs_present = [1, 1, 0, 1]
-
- and outputs,
- transformed_input = [a, b, , d]
-
- Args:
- inputs: tf.int32 Tensor of shape [batch_size, sequence_length] with tokens
- up to, but not including, vocab_size.
- targets_present: tf.bool Tensor of shape [batch_size, sequence_length] with
- True representing the presence of the word.
-
- Returns:
- transformed_input: tf.int32 Tensor of shape [batch_size, sequence_length]
- which takes on value of inputs when the input is present and takes on
- value=vocab_size to indicate a missing token.
- """
- # To fill in if the input is missing.
- input_missing = tf.constant(
- FLAGS.vocab_size,
- dtype=tf.int32,
- shape=[FLAGS.batch_size, FLAGS.sequence_length])
-
- # The 0th input will always be present to MaskGAN.
- zeroth_input_present = tf.constant(True, tf.bool, shape=[FLAGS.batch_size, 1])
-
- # Input present mask.
- inputs_present = tf.concat(
- [zeroth_input_present, targets_present[:, :-1]], axis=1)
-
- transformed_input = tf.where(inputs_present, inputs, input_missing)
- return transformed_input
-
-
-# TODO(adai): IMDB labels placeholder to encoder.
-def gen_encoder(hparams, inputs, targets_present, is_training, reuse=None):
- """Define the Encoder graph.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- inputs: tf.int32 Tensor of shape [batch_size, sequence_length] with tokens
- up to, but not including, vocab_size.
- targets_present: tf.bool Tensor of shape [batch_size, sequence_length] with
- True representing the presence of the target.
- is_training: Boolean indicating operational mode (train/inference).
- reuse (Optional): Whether to reuse the variables.
-
- Returns:
- Tuple of (hidden_states, final_state).
- """
- # We will use the same variable from the decoder.
- if FLAGS.seq2seq_share_embedding:
- with tf.variable_scope('decoder/rnn'):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('encoder', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(
- hparams.gen_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and hparams.gen_vd_keep_prob < 1:
-
- def attn_cell():
- return variational_dropout.VariationalDropoutWrapper(
- lstm_cell(), FLAGS.batch_size, hparams.gen_rnn_size,
- hparams.gen_vd_keep_prob, hparams.gen_vd_keep_prob)
-
- cell = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.gen_num_layers)],
- state_is_tuple=True)
-
- initial_state = cell.zero_state(FLAGS.batch_size, tf.float32)
-
- # Add a missing token for inputs not present.
- real_inputs = inputs
- masked_inputs = transform_input_with_is_missing_token(
- inputs, targets_present)
-
- with tf.variable_scope('rnn') as scope:
- hidden_states = []
-
- # Split the embedding into two parts so that we can load the PTB
- # weights into one part of the Variable.
- if not FLAGS.seq2seq_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- missing_embedding = tf.get_variable('missing_embedding',
- [1, hparams.gen_rnn_size])
- embedding = tf.concat([embedding, missing_embedding], axis=0)
-
- # TODO(adai): Perhaps append IMDB labels placeholder to input at
- # each time point.
- real_rnn_inputs = tf.nn.embedding_lookup(embedding, real_inputs)
- masked_rnn_inputs = tf.nn.embedding_lookup(embedding, masked_inputs)
-
- state = initial_state
-
- def make_mask(keep_prob, units):
- random_tensor = keep_prob
- # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
- random_tensor += tf.random_uniform(
- tf.stack([FLAGS.batch_size, 1, units]))
- return tf.floor(random_tensor) / keep_prob
-
- if is_training:
- output_mask = make_mask(hparams.gen_vd_keep_prob, hparams.gen_rnn_size)
-
- hidden_states, state = tf.nn.dynamic_rnn(
- cell, masked_rnn_inputs, initial_state=state, scope=scope)
- if is_training:
- hidden_states *= output_mask
-
- final_masked_state = state
-
- # Produce the RNN state had the model operated only
- # over real data.
- real_state = initial_state
- _, real_state = tf.nn.dynamic_rnn(
- cell, real_rnn_inputs, initial_state=real_state, scope=scope)
- final_state = real_state
-
- return (hidden_states, final_masked_state), initial_state, final_state
-
-
-# TODO(adai): IMDB labels placeholder to encoder.
-def gen_encoder_cnn(hparams, inputs, targets_present, is_training, reuse=None):
- """Define the CNN Encoder graph."""
- del reuse
- sequence = transform_input_with_is_missing_token(inputs, targets_present)
-
- # TODO(liamfedus): Make this a hyperparameter.
- dis_filter_sizes = [3, 4, 5, 6, 7, 8, 9, 10, 15, 20]
-
- # Keeping track of l2 regularization loss (optional)
- # l2_loss = tf.constant(0.0)
-
- with tf.variable_scope('encoder', reuse=True):
- with tf.variable_scope('rnn'):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- cnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- # Create a convolution layer for each filter size
- conv_outputs = []
- for filter_size in dis_filter_sizes:
- with tf.variable_scope('conv-%s' % filter_size):
- # Convolution Layer
- filter_shape = [
- filter_size, hparams.gen_rnn_size, hparams.dis_num_filters
- ]
- W = tf.get_variable(
- name='W', initializer=tf.truncated_normal(filter_shape, stddev=0.1))
- b = tf.get_variable(
- name='b',
- initializer=tf.constant(0.1, shape=[hparams.dis_num_filters]))
- conv = tf.nn.conv1d(cnn_inputs, W, stride=1, padding='SAME', name='conv')
-
- # Apply nonlinearity
- h = tf.nn.relu(tf.nn.bias_add(conv, b), name='relu')
-
- conv_outputs.append(h)
-
- # Combine all the pooled features
- dis_num_filters_total = hparams.dis_num_filters * len(dis_filter_sizes)
-
- h_conv = tf.concat(conv_outputs, axis=2)
- h_conv_flat = tf.reshape(h_conv, [-1, dis_num_filters_total])
-
- # Add dropout
- if is_training:
- with tf.variable_scope('dropout'):
- h_conv_flat = tf.nn.dropout(h_conv_flat, hparams.gen_vd_keep_prob)
-
- # Final (unnormalized) scores and predictions
- with tf.variable_scope('output'):
- W = tf.get_variable(
- 'W',
- shape=[dis_num_filters_total, hparams.gen_rnn_size],
- initializer=tf.contrib.layers.xavier_initializer())
- b = tf.get_variable(
- name='b', initializer=tf.constant(0.1, shape=[hparams.gen_rnn_size]))
- # l2_loss += tf.nn.l2_loss(W)
- # l2_loss += tf.nn.l2_loss(b)
- predictions = tf.nn.xw_plus_b(h_conv_flat, W, b, name='predictions')
- predictions = tf.reshape(
- predictions,
- shape=[FLAGS.batch_size, FLAGS.sequence_length, hparams.gen_rnn_size])
- final_state = tf.reduce_mean(predictions, 1)
- return predictions, (final_state, final_state)
-
-
-# TODO(adai): IMDB labels placeholder to decoder.
-def gen_decoder(hparams,
- inputs,
- targets,
- targets_present,
- encoding_state,
- is_training,
- is_validating,
- reuse=None):
- """Define the Decoder graph. The Decoder will now impute tokens that
- have been masked from the input seqeunce.
- """
- gen_decoder_rnn_size = hparams.gen_rnn_size
-
- targets = tf.Print(targets, [targets], message='targets', summarize=50)
- if FLAGS.seq2seq_share_embedding:
- with tf.variable_scope('decoder/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
-
- with tf.variable_scope('decoder', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(
- gen_decoder_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and hparams.gen_vd_keep_prob < 1:
-
- def attn_cell():
- return variational_dropout.VariationalDropoutWrapper(
- lstm_cell(), FLAGS.batch_size, hparams.gen_rnn_size,
- hparams.gen_vd_keep_prob, hparams.gen_vd_keep_prob)
-
- cell_gen = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.gen_num_layers)],
- state_is_tuple=True)
-
- # Hidden encoder states.
- hidden_vector_encodings = encoding_state[0]
-
- # Carry forward the final state tuple from the encoder.
- # State tuples.
- state_gen = encoding_state[1]
-
- if FLAGS.attention_option is not None:
- (attention_keys, attention_values, _,
- attention_construct_fn) = attention_utils.prepare_attention(
- hidden_vector_encodings,
- FLAGS.attention_option,
- num_units=gen_decoder_rnn_size,
- reuse=reuse)
-
- def make_mask(keep_prob, units):
- random_tensor = keep_prob
- # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
- random_tensor += tf.random_uniform(tf.stack([FLAGS.batch_size, units]))
- return tf.floor(random_tensor) / keep_prob
-
- if is_training:
- output_mask = make_mask(hparams.gen_vd_keep_prob, hparams.gen_rnn_size)
-
- with tf.variable_scope('rnn'):
- sequence, logits, log_probs = [], [], []
-
- if not FLAGS.seq2seq_share_embedding:
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- softmax_w = tf.matrix_transpose(embedding)
- softmax_b = tf.get_variable('softmax_b', [FLAGS.vocab_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, inputs)
- # TODO(adai): Perhaps append IMDB labels placeholder to input at
- # each time point.
-
- rnn_outs = []
-
- fake = None
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- # Input to the Decoder.
- if t == 0:
- # Always provide the real input at t = 0.
- rnn_inp = rnn_inputs[:, t]
-
- # If the input is present, read in the input at t.
- # If the input is not present, read in the previously generated.
- else:
- real_rnn_inp = rnn_inputs[:, t]
-
- # While validating, the decoder should be operating in teacher
- # forcing regime. Also, if we're just training with cross_entropy
- # use teacher forcing.
- if is_validating or FLAGS.gen_training_strategy == 'cross_entropy':
- rnn_inp = real_rnn_inp
- else:
- fake_rnn_inp = tf.nn.embedding_lookup(embedding, fake)
- rnn_inp = tf.where(targets_present[:, t - 1], real_rnn_inp,
- fake_rnn_inp)
-
- # RNN.
- rnn_out, state_gen = cell_gen(rnn_inp, state_gen)
-
- if FLAGS.attention_option is not None:
- rnn_out = attention_construct_fn(rnn_out, attention_keys,
- attention_values)
- if is_training:
- rnn_out *= output_mask
-
- rnn_outs.append(rnn_out)
- if FLAGS.gen_training_strategy != 'cross_entropy':
- logit = tf.nn.bias_add(tf.matmul(rnn_out, softmax_w), softmax_b)
-
- # Output for Decoder.
- # If input is present: Return real at t+1.
- # If input is not present: Return fake for t+1.
- real = targets[:, t]
-
- categorical = tf.contrib.distributions.Categorical(logits=logit)
- if FLAGS.use_gen_mode:
- fake = categorical.mode()
- else:
- fake = categorical.sample()
- log_prob = categorical.log_prob(fake)
- output = tf.where(targets_present[:, t], real, fake)
-
- else:
- real = targets[:, t]
- logit = tf.zeros(tf.stack([FLAGS.batch_size, FLAGS.vocab_size]))
- log_prob = tf.zeros(tf.stack([FLAGS.batch_size]))
- output = real
-
- # Add to lists.
- sequence.append(output)
- log_probs.append(log_prob)
- logits.append(logit)
-
- if FLAGS.gen_training_strategy == 'cross_entropy':
- logits = tf.nn.bias_add(
- tf.matmul(
- tf.reshape(tf.stack(rnn_outs, 1), [-1, gen_decoder_rnn_size]),
- softmax_w), softmax_b)
- logits = tf.reshape(logits,
- [-1, FLAGS.sequence_length, FLAGS.vocab_size])
- else:
- logits = tf.stack(logits, axis=1)
-
- return (tf.stack(sequence, axis=1), logits, tf.stack(log_probs, axis=1))
-
-
-def dis_encoder(hparams, masked_inputs, is_training, reuse=None,
- embedding=None):
- """Define the Discriminator encoder. Reads in the masked inputs for context
- and produces the hidden states of the encoder."""
- with tf.variable_scope('encoder', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(
- hparams.dis_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and hparams.dis_vd_keep_prob < 1:
-
- def attn_cell():
- return variational_dropout.VariationalDropoutWrapper(
- lstm_cell(), FLAGS.batch_size, hparams.dis_rnn_size,
- hparams.dis_vd_keep_prob, hparams.dis_vd_keep_prob)
-
- cell_dis = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- state_dis = cell_dis.zero_state(FLAGS.batch_size, tf.float32)
-
- with tf.variable_scope('rnn'):
- hidden_states = []
-
- missing_embedding = tf.get_variable('missing_embedding',
- [1, hparams.dis_rnn_size])
- embedding = tf.concat([embedding, missing_embedding], axis=0)
- masked_rnn_inputs = tf.nn.embedding_lookup(embedding, masked_inputs)
-
- def make_mask(keep_prob, units):
- random_tensor = keep_prob
- # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
- random_tensor += tf.random_uniform(tf.stack([FLAGS.batch_size, units]))
- return tf.floor(random_tensor) / keep_prob
-
- if is_training:
- output_mask = make_mask(hparams.dis_vd_keep_prob, hparams.dis_rnn_size)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_in = masked_rnn_inputs[:, t]
- rnn_out, state_dis = cell_dis(rnn_in, state_dis)
- if is_training:
- rnn_out *= output_mask
- hidden_states.append(rnn_out)
- final_state = state_dis
-
- return (tf.stack(hidden_states, axis=1), final_state)
-
-
-def dis_decoder(hparams,
- sequence,
- encoding_state,
- is_training,
- reuse=None,
- embedding=None):
- """Define the Discriminator decoder. Read in the sequence and predict
- at each time point."""
- sequence = tf.cast(sequence, tf.int32)
-
- with tf.variable_scope('decoder', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(
- hparams.dis_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and hparams.dis_vd_keep_prob < 1:
-
- def attn_cell():
- return variational_dropout.VariationalDropoutWrapper(
- lstm_cell(), FLAGS.batch_size, hparams.dis_rnn_size,
- hparams.dis_vd_keep_prob, hparams.dis_vd_keep_prob)
-
- cell_dis = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.dis_num_layers)],
- state_is_tuple=True)
-
- # Hidden encoder states.
- hidden_vector_encodings = encoding_state[0]
-
- # Carry forward the final state tuple from the encoder.
- # State tuples.
- state = encoding_state[1]
-
- if FLAGS.attention_option is not None:
- (attention_keys, attention_values, _,
- attention_construct_fn) = attention_utils.prepare_attention(
- hidden_vector_encodings,
- FLAGS.attention_option,
- num_units=hparams.dis_rnn_size,
- reuse=reuse)
-
- def make_mask(keep_prob, units):
- random_tensor = keep_prob
- # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
- random_tensor += tf.random_uniform(tf.stack([FLAGS.batch_size, units]))
- return tf.floor(random_tensor) / keep_prob
-
- if is_training:
- output_mask = make_mask(hparams.dis_vd_keep_prob, hparams.dis_rnn_size)
-
- with tf.variable_scope('rnn') as vs:
- predictions = []
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, sequence)
-
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_in = rnn_inputs[:, t]
- rnn_out, state = cell_dis(rnn_in, state)
-
- if FLAGS.attention_option is not None:
- rnn_out = attention_construct_fn(rnn_out, attention_keys,
- attention_values)
- if is_training:
- rnn_out *= output_mask
-
- # Prediction is linear output for Discriminator.
- pred = tf.contrib.layers.linear(rnn_out, 1, scope=vs)
- predictions.append(pred)
-
- predictions = tf.stack(predictions, axis=1)
- return tf.squeeze(predictions, axis=2)
-
-
-def discriminator(hparams,
- inputs,
- targets_present,
- sequence,
- is_training,
- reuse=None):
- """Define the Discriminator graph."""
- if FLAGS.dis_share_embedding:
- assert hparams.dis_rnn_size == hparams.gen_rnn_size, (
- 'If you wish to share Discriminator/Generator embeddings, they must be'
- ' same dimension.')
- with tf.variable_scope('gen/decoder/rnn', reuse=True):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- else:
- # Explicitly share the embedding.
- with tf.variable_scope('dis/decoder/rnn', reuse=reuse):
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.dis_rnn_size])
-
- # Mask the input sequence.
- masked_inputs = transform_input_with_is_missing_token(inputs, targets_present)
-
- # Confirm masking.
- masked_inputs = tf.Print(
- masked_inputs, [inputs, targets_present, masked_inputs, sequence],
- message='inputs, targets_present, masked_inputs, sequence',
- summarize=10)
-
- with tf.variable_scope('dis', reuse=reuse):
- encoder_states = dis_encoder(
- hparams,
- masked_inputs,
- is_training=is_training,
- reuse=reuse,
- embedding=embedding)
- predictions = dis_decoder(
- hparams,
- sequence,
- encoder_states,
- is_training=is_training,
- reuse=reuse,
- embedding=embedding)
-
- # if FLAGS.baseline_method == 'critic':
- # with tf.variable_scope('critic', reuse=reuse) as critic_scope:
- # values = tf.contrib.layers.linear(rnn_outs, 1, scope=critic_scope)
- # values = tf.squeeze(values, axis=2)
- # else:
- # values = None
-
- return predictions
-
-
-# TODO(adai): IMDB labels placeholder to encoder/decoder.
-def generator(hparams,
- inputs,
- targets,
- targets_present,
- is_training,
- is_validating,
- reuse=None):
- """Define the Generator graph."""
- with tf.variable_scope('gen', reuse=reuse):
- encoder_states, initial_state, final_state = gen_encoder(
- hparams, inputs, targets_present, is_training=is_training, reuse=reuse)
- stacked_sequence, stacked_logits, stacked_log_probs = gen_decoder(
- hparams,
- inputs,
- targets,
- targets_present,
- encoder_states,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- return (stacked_sequence, stacked_logits, stacked_log_probs, initial_state,
- final_state, encoder_states)
diff --git a/research/maskgan/models/seq2seq_zaremba.py b/research/maskgan/models/seq2seq_zaremba.py
deleted file mode 100644
index 25f6ce44f0cb2fe650e23b332ace014ab7cdf469..0000000000000000000000000000000000000000
--- a/research/maskgan/models/seq2seq_zaremba.py
+++ /dev/null
@@ -1,305 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Simple seq2seq model definitions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-from six.moves import xrange
-from models import attention_utils
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def transform_input_with_is_missing_token(inputs, targets_present):
- """Transforms the inputs to have missing tokens when it's masked out. The
- mask is for the targets, so therefore, to determine if an input at time t is
- masked, we have to check if the target at time t - 1 is masked out.
-
- e.g.
- inputs = [a, b, c, d]
- targets = [b, c, d, e]
- targets_present = [1, 0, 1, 0]
-
- then,
- transformed_input = [a, b, , d]
-
- Args:
- inputs: tf.int32 Tensor of shape [batch_size, sequence_length] with tokens
- up to, but not including, vocab_size.
- targets_present: tf.bool Tensor of shape [batch_size, sequence_length] with
- True representing the presence of the word.
-
- Returns:
- transformed_input: tf.int32 Tensor of shape [batch_size, sequence_length]
- which takes on value of inputs when the input is present and takes on
- value=vocab_size to indicate a missing token.
- """
- # To fill in if the input is missing.
- input_missing = tf.constant(FLAGS.vocab_size,
- dtype=tf.int32,
- shape=[FLAGS.batch_size, FLAGS.sequence_length])
-
- # The 0th input will always be present to MaskGAN.
- zeroth_input_present = tf.constant(True, tf.bool, shape=[FLAGS.batch_size, 1])
-
- # Input present mask.
- inputs_present = tf.concat(
- [zeroth_input_present, targets_present[:, :-1]], axis=1)
-
- transformed_input = tf.where(inputs_present, inputs, input_missing)
- return transformed_input
-
-
-def gen_encoder(hparams, inputs, targets_present, is_training, reuse=None):
- """Define the Encoder graph.
-
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- inputs: tf.int32 Tensor of shape [batch_size, sequence_length] with tokens
- up to, but not including, vocab_size.
- targets_present: tf.bool Tensor of shape [batch_size, sequence_length] with
- True representing the presence of the target.
- is_training: Boolean indicating operational mode (train/inference).
- reuse (Optional): Whether to reuse the variables.
-
- Returns:
- Tuple of (hidden_states, final_state).
- """
- with tf.variable_scope('encoder', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(hparams.gen_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and FLAGS.keep_prob < 1:
-
- def attn_cell():
- return tf.contrib.rnn.DropoutWrapper(
- lstm_cell(), output_keep_prob=FLAGS.keep_prob)
-
- cell = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.gen_num_layers)],
- state_is_tuple=True)
-
- initial_state = cell.zero_state(FLAGS.batch_size, tf.float32)
-
- # Add a missing token for inputs not present.
- real_inputs = inputs
- masked_inputs = transform_input_with_is_missing_token(inputs,
- targets_present)
-
- with tf.variable_scope('rnn'):
- hidden_states = []
-
- # Split the embedding into two parts so that we can load the PTB
- # weights into one part of the Variable.
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- missing_embedding = tf.get_variable('missing_embedding',
- [1, hparams.gen_rnn_size])
- embedding = tf.concat([embedding, missing_embedding], axis=0)
-
- real_rnn_inputs = tf.nn.embedding_lookup(embedding, real_inputs)
- masked_rnn_inputs = tf.nn.embedding_lookup(embedding, masked_inputs)
-
- if is_training and FLAGS.keep_prob < 1:
- masked_rnn_inputs = tf.nn.dropout(masked_rnn_inputs, FLAGS.keep_prob)
-
- state = initial_state
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- rnn_inp = masked_rnn_inputs[:, t]
- rnn_out, state = cell(rnn_inp, state)
- hidden_states.append(rnn_out)
- final_masked_state = state
- hidden_states = tf.stack(hidden_states, axis=1)
-
- # Produce the RNN state had the model operated only
- # over real data.
- real_state = initial_state
- for t in xrange(FLAGS.sequence_length):
- tf.get_variable_scope().reuse_variables()
-
- # RNN.
- rnn_inp = real_rnn_inputs[:, t]
- rnn_out, real_state = cell(rnn_inp, real_state)
- final_state = real_state
-
- return (hidden_states, final_masked_state), initial_state, final_state
-
-
-def gen_decoder(hparams,
- inputs,
- targets,
- targets_present,
- encoding_state,
- is_training,
- is_validating,
- reuse=None):
- """Define the Decoder graph. The Decoder will now impute tokens that
- have been masked from the input seqeunce.
- """
- gen_decoder_rnn_size = hparams.gen_rnn_size
-
- with tf.variable_scope('decoder', reuse=reuse):
-
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(gen_decoder_rnn_size,
- forget_bias=0.0,
- state_is_tuple=True,
- reuse=reuse)
-
- attn_cell = lstm_cell
- if is_training and FLAGS.keep_prob < 1:
-
- def attn_cell():
- return tf.contrib.rnn.DropoutWrapper(
- lstm_cell(), output_keep_prob=FLAGS.keep_prob)
-
- cell_gen = tf.contrib.rnn.MultiRNNCell(
- [attn_cell() for _ in range(hparams.gen_num_layers)],
- state_is_tuple=True)
-
- # Hidden encoder states.
- hidden_vector_encodings = encoding_state[0]
-
- # Carry forward the final state tuple from the encoder.
- # State tuples.
- state_gen = encoding_state[1]
-
- if FLAGS.attention_option is not None:
- (attention_keys, attention_values, _,
- attention_construct_fn) = attention_utils.prepare_attention(
- hidden_vector_encodings,
- FLAGS.attention_option,
- num_units=gen_decoder_rnn_size,
- reuse=reuse)
-
- with tf.variable_scope('rnn'):
- sequence, logits, log_probs = [], [], []
-
- embedding = tf.get_variable('embedding',
- [FLAGS.vocab_size, hparams.gen_rnn_size])
- softmax_w = tf.matrix_transpose(embedding)
- softmax_b = tf.get_variable('softmax_b', [FLAGS.vocab_size])
-
- rnn_inputs = tf.nn.embedding_lookup(embedding, inputs)
-
- if is_training and FLAGS.keep_prob < 1:
- rnn_inputs = tf.nn.dropout(rnn_inputs, FLAGS.keep_prob)
-
- rnn_outs = []
-
- fake = None
- for t in xrange(FLAGS.sequence_length):
- if t > 0:
- tf.get_variable_scope().reuse_variables()
-
- # Input to the Decoder.
- if t == 0:
- # Always provide the real input at t = 0.
- rnn_inp = rnn_inputs[:, t]
-
- # If the input is present, read in the input at t.
- # If the input is not present, read in the previously generated.
- else:
- real_rnn_inp = rnn_inputs[:, t]
-
- # While validating, the decoder should be operating in teacher
- # forcing regime. Also, if we're just training with cross_entropy
- # use teacher forcing.
- if is_validating or FLAGS.gen_training_strategy == 'cross_entropy':
- rnn_inp = real_rnn_inp
- else:
- fake_rnn_inp = tf.nn.embedding_lookup(embedding, fake)
- rnn_inp = tf.where(targets_present[:, t - 1], real_rnn_inp,
- fake_rnn_inp)
-
- # RNN.
- rnn_out, state_gen = cell_gen(rnn_inp, state_gen)
-
- if FLAGS.attention_option is not None:
- rnn_out = attention_construct_fn(rnn_out, attention_keys,
- attention_values)
- rnn_outs.append(rnn_out)
- if FLAGS.gen_training_strategy != 'cross_entropy':
- logit = tf.nn.bias_add(tf.matmul(rnn_out, softmax_w), softmax_b)
-
- # Output for Decoder.
- # If input is present: Return real at t+1.
- # If input is not present: Return fake for t+1.
- real = targets[:, t]
-
- categorical = tf.contrib.distributions.Categorical(logits=logit)
- fake = categorical.sample()
- log_prob = categorical.log_prob(fake)
-
- output = tf.where(targets_present[:, t], real, fake)
-
- else:
- batch_size = tf.shape(rnn_out)[0]
- logit = tf.zeros(tf.stack([batch_size, FLAGS.vocab_size]))
- log_prob = tf.zeros(tf.stack([batch_size]))
- output = targets[:, t]
-
- # Add to lists.
- sequence.append(output)
- log_probs.append(log_prob)
- logits.append(logit)
- if FLAGS.gen_training_strategy == 'cross_entropy':
- logits = tf.nn.bias_add(
- tf.matmul(
- tf.reshape(tf.stack(rnn_outs, 1), [-1, gen_decoder_rnn_size]),
- softmax_w), softmax_b)
- logits = tf.reshape(logits,
- [-1, FLAGS.sequence_length, FLAGS.vocab_size])
- else:
- logits = tf.stack(logits, axis=1)
-
- return (tf.stack(sequence, axis=1), logits, tf.stack(log_probs, axis=1))
-
-
-def generator(hparams,
- inputs,
- targets,
- targets_present,
- is_training,
- is_validating,
- reuse=None):
- """Define the Generator graph."""
- with tf.variable_scope('gen', reuse=reuse):
- encoder_states, initial_state, final_state = gen_encoder(
- hparams, inputs, targets_present, is_training=is_training, reuse=reuse)
- stacked_sequence, stacked_logits, stacked_log_probs = gen_decoder(
- hparams,
- inputs,
- targets,
- targets_present,
- encoder_states,
- is_training=is_training,
- is_validating=is_validating,
- reuse=reuse)
- return (stacked_sequence, stacked_logits, stacked_log_probs, initial_state,
- final_state)
diff --git a/research/maskgan/nas_utils/__init__.py b/research/maskgan/nas_utils/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/maskgan/nas_utils/configs.py b/research/maskgan/nas_utils/configs.py
deleted file mode 100644
index 80d867c36d1de07663d59d6c161aaf9cbe241d95..0000000000000000000000000000000000000000
--- a/research/maskgan/nas_utils/configs.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-def print_config(config):
- print("-" * 10, "Configuration Specs", "-" * 10)
- for item in dir(config):
- if list(item)[0] != "_":
- print(item, getattr(config, item))
- print("-" * 29)
-
-
-class AlienConfig2(object):
- """Base 8 740 shared embeddings, gets 64.0 (mean: std: min: max: )."""
- init_scale = 0.05
- learning_rate = 1.0
- max_grad_norm = 10
- num_layers = 2
- num_steps = 25
- hidden_size = 740
- max_epoch = 70
- max_max_epoch = 250
- keep_prob = [1 - 0.15, 1 - 0.45]
- lr_decay = 0.95
- batch_size = 20
- vocab_size = 10000
- weight_decay = 1e-4
- share_embeddings = True
- cell = "alien"
- dropout_type = "variational"
diff --git a/research/maskgan/nas_utils/custom_cell.py b/research/maskgan/nas_utils/custom_cell.py
deleted file mode 100644
index 6add7ffa4e0d69da56d2bba7d9da3875b5c4dd3b..0000000000000000000000000000000000000000
--- a/research/maskgan/nas_utils/custom_cell.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import numpy as np
-import tensorflow as tf
-
-flags = tf.flags
-FLAGS = tf.app.flags.FLAGS
-LSTMTuple = collections.namedtuple('LSTMTuple', ['c', 'h'])
-
-
-def cell_depth(num):
- num /= 2
- val = np.log2(1 + num)
- assert abs(val - int(val)) == 0
- return int(val)
-
-
-class GenericMultiRNNCell(tf.contrib.rnn.RNNCell):
- """More generic version of MultiRNNCell that allows you to pass in a dropout mask"""
-
- def __init__(self, cells):
- """Create a RNN cell composed sequentially of a number of RNNCells.
-
- Args:
- cells: list of RNNCells that will be composed in this order.
- state_is_tuple: If True, accepted and returned states are n-tuples, where
- `n = len(cells)`. If False, the states are all
- concatenated along the column axis. This latter behavior will soon be
- deprecated.
-
- Raises:
- ValueError: if cells is empty (not allowed), or at least one of the cells
- returns a state tuple but the flag `state_is_tuple` is `False`.
- """
- self._cells = cells
-
- @property
- def state_size(self):
- return tuple(cell.state_size for cell in self._cells)
-
- @property
- def output_size(self):
- return self._cells[-1].output_size
-
- def __call__(self, inputs, state, input_masks=None, scope=None):
- """Run this multi-layer cell on inputs, starting from state."""
- with tf.variable_scope(scope or type(self).__name__):
- cur_inp = inputs
- new_states = []
- for i, cell in enumerate(self._cells):
- with tf.variable_scope('Cell%d' % i):
- cur_state = state[i]
- if input_masks is not None:
- cur_inp *= input_masks[i]
- cur_inp, new_state = cell(cur_inp, cur_state)
- new_states.append(new_state)
- new_states = tuple(new_states)
- return cur_inp, new_states
-
-
-class AlienRNNBuilder(tf.contrib.rnn.RNNCell):
-
- def __init__(self, num_units, params, additional_params, base_size):
- self.num_units = num_units
- self.cell_create_index = additional_params[0]
- self.cell_inject_index = additional_params[1]
- self.base_size = base_size
- self.cell_params = params[
- -2:] # Cell injection parameters are always the last two
- params = params[:-2]
- self.depth = cell_depth(len(params))
- self.params = params
- self.units_per_layer = [2**i for i in range(self.depth)
- ][::-1] # start with the biggest layer
-
- def __call__(self, inputs, state, scope=None):
- with tf.variable_scope(scope or type(self).__name__):
- definition1 = ['add', 'elem_mult', 'max']
- definition2 = [tf.identity, tf.tanh, tf.sigmoid, tf.nn.relu, tf.sin]
- layer_outputs = [[] for _ in range(self.depth)]
- with tf.variable_scope('rnn_builder'):
- curr_index = 0
- c, h = state
-
- # Run all dense matrix multiplications at once
- big_h_mat = tf.get_variable(
- 'big_h_mat', [self.num_units,
- self.base_size * self.num_units], tf.float32)
- big_inputs_mat = tf.get_variable(
- 'big_inputs_mat', [self.num_units,
- self.base_size * self.num_units], tf.float32)
- big_h_output = tf.matmul(h, big_h_mat)
- big_inputs_output = tf.matmul(inputs, big_inputs_mat)
- h_splits = tf.split(big_h_output, self.base_size, axis=1)
- inputs_splits = tf.split(big_inputs_output, self.base_size, axis=1)
-
- for layer_num, units in enumerate(self.units_per_layer):
- for unit_num in range(units):
- with tf.variable_scope(
- 'layer_{}_unit_{}'.format(layer_num, unit_num)):
- if layer_num == 0:
- prev1_mat = h_splits[unit_num]
- prev2_mat = inputs_splits[unit_num]
- else:
- prev1_mat = layer_outputs[layer_num - 1][2 * unit_num]
- prev2_mat = layer_outputs[layer_num - 1][2 * unit_num + 1]
- if definition1[self.params[curr_index]] == 'add':
- output = prev1_mat + prev2_mat
- elif definition1[self.params[curr_index]] == 'elem_mult':
- output = prev1_mat * prev2_mat
- elif definition1[self.params[curr_index]] == 'max':
- output = tf.maximum(prev1_mat, prev2_mat)
- if curr_index / 2 == self.cell_create_index: # Take the new cell before the activation
- new_c = tf.identity(output)
- output = definition2[self.params[curr_index + 1]](output)
- if curr_index / 2 == self.cell_inject_index:
- if definition1[self.cell_params[0]] == 'add':
- output += c
- elif definition1[self.cell_params[0]] == 'elem_mult':
- output *= c
- elif definition1[self.cell_params[0]] == 'max':
- output = tf.maximum(output, c)
- output = definition2[self.cell_params[1]](output)
- layer_outputs[layer_num].append(output)
- curr_index += 2
- new_h = layer_outputs[-1][-1]
- return new_h, LSTMTuple(new_c, new_h)
-
- @property
- def state_size(self):
- return LSTMTuple(self.num_units, self.num_units)
-
- @property
- def output_size(self):
- return self.num_units
-
-
-class Alien(AlienRNNBuilder):
- """Base 8 Cell."""
-
- def __init__(self, num_units):
- params = [
- 0, 2, 0, 3, 0, 2, 1, 3, 0, 1, 0, 2, 0, 1, 0, 2, 1, 1, 0, 1, 1, 1, 0, 2,
- 1, 0, 0, 1, 1, 1, 0, 1
- ]
- additional_params = [12, 8]
- base_size = 8
- super(Alien, self).__init__(num_units, params, additional_params, base_size)
diff --git a/research/maskgan/nas_utils/variational_dropout.py b/research/maskgan/nas_utils/variational_dropout.py
deleted file mode 100644
index 49cc29f0cd77f7bef9e3c47e7d7dae73fa877ecd..0000000000000000000000000000000000000000
--- a/research/maskgan/nas_utils/variational_dropout.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Variational Dropout."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def generate_dropout_masks(keep_prob, shape, amount):
- masks = []
- for _ in range(amount):
- dropout_mask = tf.random_uniform(shape) + (keep_prob)
- dropout_mask = tf.floor(dropout_mask) / (keep_prob)
- masks.append(dropout_mask)
- return masks
-
-
-def generate_variational_dropout_masks(hparams, keep_prob):
- [batch_size, num_steps, size, num_layers] = [
- FLAGS.batch_size, FLAGS.sequence_length, hparams.gen_rnn_size,
- hparams.gen_num_layers
- ]
- if len(keep_prob) == 2:
- emb_keep_prob = keep_prob[0] # keep prob for embedding matrix
- h2h_keep_prob = emb_keep_prob # keep prob for hidden to hidden connections
- h2i_keep_prob = keep_prob[1] # keep prob for hidden to input connections
- out_keep_prob = h2i_keep_prob # keep probability for output state
- else:
- emb_keep_prob = keep_prob[0] # keep prob for embedding matrix
- h2h_keep_prob = keep_prob[1] # keep prob for hidden to hidden connections
- h2i_keep_prob = keep_prob[2] # keep prob for hidden to input connections
- out_keep_prob = keep_prob[3] # keep probability for output state
- h2i_masks = [] # Masks for input to recurrent connections
- h2h_masks = [] # Masks for recurrent to recurrent connections
-
- # Input word dropout mask
- emb_masks = generate_dropout_masks(emb_keep_prob, [num_steps, 1], batch_size)
- output_mask = generate_dropout_masks(out_keep_prob, [batch_size, size], 1)[0]
- h2i_masks = generate_dropout_masks(h2i_keep_prob, [batch_size, size],
- num_layers)
- h2h_masks = generate_dropout_masks(h2h_keep_prob, [batch_size, size],
- num_layers)
- return h2h_masks, h2i_masks, emb_masks, output_mask
diff --git a/research/maskgan/pretrain_mask_gan.py b/research/maskgan/pretrain_mask_gan.py
deleted file mode 100644
index 1a9d8ee947deaa3e31cc4c332969ed529e60305e..0000000000000000000000000000000000000000
--- a/research/maskgan/pretrain_mask_gan.py
+++ /dev/null
@@ -1,231 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Pretraining functions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# Dependency imports
-
-import numpy as np
-
-import tensorflow as tf
-
-from data import imdb_loader
-from data import ptb_loader
-
-# Data.
-from model_utils import model_utils
-from models import evaluation_utils
-
-tf.app.flags.DEFINE_integer(
- 'gen_pretrain_steps', None,
- 'The number of steps to pretrain the generator with cross entropy loss.')
-tf.app.flags.DEFINE_integer(
- 'dis_pretrain_steps', None,
- 'The number of steps to pretrain the discriminator.')
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def pretrain_generator(sv, sess, model, data, log, id_to_word,
- data_ngram_counts, is_chief):
- """Pretrain the generator with classic language modeling training."""
- print('\nPretraining generator for %d steps.' % FLAGS.gen_pretrain_steps)
- log.write(
- '\nPretraining generator for %d steps.\n' % FLAGS.gen_pretrain_steps)
-
- is_pretraining = True
-
- while is_pretraining:
-
- costs = 0.
- iters = 0
- if FLAGS.data_set == 'ptb':
- iterator = ptb_loader.ptb_iterator(data, FLAGS.batch_size,
- FLAGS.sequence_length,
- FLAGS.epoch_size_override)
- elif FLAGS.data_set == 'imdb':
- iterator = imdb_loader.imdb_iterator(data, FLAGS.batch_size,
- FLAGS.sequence_length)
-
- for x, y, _ in iterator:
-
- # For pretraining with cross entropy loss, we have all tokens in the
- # forward sequence present (all True).
- model_utils.assign_percent_real(sess, model.percent_real_update,
- model.new_rate, 1.0)
- p = np.ones(shape=[FLAGS.batch_size, FLAGS.sequence_length], dtype=bool)
-
- pretrain_feed = {model.inputs: x, model.targets: y, model.present: p}
-
- [losses, cost_eval, _, step] = sess.run(
- [
- model.fake_cross_entropy_losses, model.avg_log_perplexity,
- model.gen_pretrain_op, model.global_step
- ],
- feed_dict=pretrain_feed)
-
- costs += cost_eval
- iters += FLAGS.sequence_length
-
- # Calulate rolling perplexity.
- perplexity = np.exp(costs / iters)
-
- # Summaries.
- if is_chief and step % FLAGS.summaries_every == 0:
- # Graph summaries.
- summary_str = sess.run(
- model.merge_summaries_op, feed_dict=pretrain_feed)
- sv.SummaryComputed(sess, summary_str)
-
- # Additional summary.
- for n, data_ngram_count in data_ngram_counts.iteritems():
- avg_percent_captured = evaluation_utils.sequence_ngram_evaluation(
- sess, model.fake_sequence, log, pretrain_feed, data_ngram_count,
- int(n))
- summary_percent_str = tf.Summary(value=[
- tf.Summary.Value(
- tag='general/%s-grams_percent_correct' % n,
- simple_value=avg_percent_captured)
- ])
- sv.SummaryComputed(sess, summary_percent_str, global_step=step)
-
- summary_perplexity_str = tf.Summary(value=[
- tf.Summary.Value(tag='general/perplexity', simple_value=perplexity)
- ])
- sv.SummaryComputed(sess, summary_perplexity_str, global_step=step)
-
- # Printing and logging
- if is_chief and step % FLAGS.print_every == 0:
- print('global_step: %d' % step)
- print(' generator loss: %.3f' % np.mean(losses))
- print(' perplexity: %.3f' % perplexity)
- log.write('global_step: %d\n' % step)
- log.write(' generator loss: %.3f\n' % np.mean(losses))
- log.write(' perplexity: %.3f\n' % perplexity)
-
- for n, data_ngram_count in data_ngram_counts.iteritems():
- avg_percent_captured = evaluation_utils.sequence_ngram_evaluation(
- sess, model.fake_sequence, log, pretrain_feed, data_ngram_count,
- int(n))
- print(' percent of %s-grams captured: %.3f.\n' %
- (n, avg_percent_captured))
- log.write(' percent of %s-grams captured: %.3f.\n\n' %
- (n, avg_percent_captured))
-
- evaluation_utils.generate_logs(sess, model, log, id_to_word,
- pretrain_feed)
-
- if step >= FLAGS.gen_pretrain_steps:
- is_pretraining = False
- break
- return
-
-
-def pretrain_discriminator(sv, sess, model, data, log, id_to_word,
- data_ngram_counts, is_chief):
- print('\nPretraining discriminator for %d steps.' % FLAGS.dis_pretrain_steps)
- log.write(
- '\nPretraining discriminator for %d steps.\n' % FLAGS.dis_pretrain_steps)
-
- is_pretraining = True
-
- while is_pretraining:
-
- cumulative_costs = 0.
- iters = 0
- if FLAGS.data_set == 'ptb':
- iterator = ptb_loader.ptb_iterator(data, FLAGS.batch_size,
- FLAGS.sequence_length,
- FLAGS.epoch_size_override)
- elif FLAGS.data_set == 'imdb':
- iterator = imdb_loader.imdb_iterator(data, FLAGS.batch_size,
- FLAGS.sequence_length)
-
- for x, y, _ in iterator:
- is_present_rate = FLAGS.is_present_rate
- # is_present_rate = np.random.uniform(low=0.0, high=1.0)
- model_utils.assign_percent_real(sess, model.percent_real_update,
- model.new_rate, is_present_rate)
- # Randomly mask out tokens.
- p = model_utils.generate_mask()
-
- pretrain_feed = {model.inputs: x, model.targets: y, model.present: p}
-
- [_, dis_loss_eval, gen_log_perplexity_eval, step] = sess.run(
- [
- model.dis_pretrain_op, model.dis_loss, model.avg_log_perplexity,
- model.global_step
- ],
- feed_dict=pretrain_feed)
-
- cumulative_costs += gen_log_perplexity_eval
- iters += 1
-
- # Calulate rolling perplexity.
- perplexity = np.exp(cumulative_costs / iters)
-
- # Summaries.
- if is_chief and step % FLAGS.summaries_every == 0:
- # Graph summaries.
- summary_str = sess.run(
- model.merge_summaries_op, feed_dict=pretrain_feed)
- sv.SummaryComputed(sess, summary_str)
-
- # Additional summary.
- for n, data_ngram_count in data_ngram_counts.iteritems():
- avg_percent_captured = evaluation_utils.sequence_ngram_evaluation(
- sess, model.fake_sequence, log, pretrain_feed, data_ngram_count,
- int(n))
- summary_percent_str = tf.Summary(value=[
- tf.Summary.Value(
- tag='general/%s-grams_percent_correct' % n,
- simple_value=avg_percent_captured)
- ])
- sv.SummaryComputed(sess, summary_percent_str, global_step=step)
-
- summary_perplexity_str = tf.Summary(value=[
- tf.Summary.Value(tag='general/perplexity', simple_value=perplexity)
- ])
- sv.SummaryComputed(sess, summary_perplexity_str, global_step=step)
-
- # Printing and logging
- if is_chief and step % FLAGS.print_every == 0:
- print('global_step: %d' % step)
- print(' discriminator loss: %.3f' % dis_loss_eval)
- print(' perplexity: %.3f' % perplexity)
- log.write('global_step: %d\n' % step)
- log.write(' discriminator loss: %.3f\n' % dis_loss_eval)
- log.write(' perplexity: %.3f\n' % perplexity)
-
- for n, data_ngram_count in data_ngram_counts.iteritems():
- avg_percent_captured = evaluation_utils.sequence_ngram_evaluation(
- sess, model.fake_sequence, log, pretrain_feed, data_ngram_count,
- int(n))
- print(' percent of %s-grams captured: %.3f.\n' %
- (n, avg_percent_captured))
- log.write(' percent of %s-grams captured: %.3f.\n\n' %
- (n, avg_percent_captured))
-
- evaluation_utils.generate_logs(sess, model, log, id_to_word,
- pretrain_feed)
-
- if step >= FLAGS.dis_pretrain_steps + int(FLAGS.gen_pretrain_steps or 0):
- is_pretraining = False
- break
- return
diff --git a/research/maskgan/regularization/__init__.py b/research/maskgan/regularization/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/maskgan/regularization/variational_dropout.py b/research/maskgan/regularization/variational_dropout.py
deleted file mode 100644
index d67fe52eee45c31012fe50e5de662d27565befae..0000000000000000000000000000000000000000
--- a/research/maskgan/regularization/variational_dropout.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Variational Dropout Wrapper."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-
-class VariationalDropoutWrapper(tf.contrib.rnn.RNNCell):
- """Add variational dropout to a RNN cell."""
-
- def __init__(self, cell, batch_size, input_size, recurrent_keep_prob,
- input_keep_prob):
- self._cell = cell
- self._recurrent_keep_prob = recurrent_keep_prob
- self._input_keep_prob = input_keep_prob
-
- def make_mask(keep_prob, units):
- random_tensor = keep_prob
- # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
- random_tensor += tf.random_uniform(tf.stack([batch_size, units]))
- return tf.floor(random_tensor) / keep_prob
-
- self._recurrent_mask = make_mask(recurrent_keep_prob,
- self._cell.state_size[0])
- self._input_mask = self._recurrent_mask
-
- @property
- def state_size(self):
- return self._cell.state_size
-
- @property
- def output_size(self):
- return self._cell.output_size
-
- def __call__(self, inputs, state, scope=None):
- dropped_inputs = inputs * self._input_mask
- dropped_state = (state[0], state[1] * self._recurrent_mask)
- new_h, new_state = self._cell(dropped_inputs, dropped_state, scope)
- return new_h, new_state
diff --git a/research/maskgan/regularization/zoneout.py b/research/maskgan/regularization/zoneout.py
deleted file mode 100644
index 5f9ef3e3014ae6f2e7eea1a2937c5f1e2c356411..0000000000000000000000000000000000000000
--- a/research/maskgan/regularization/zoneout.py
+++ /dev/null
@@ -1,64 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Zoneout Wrapper"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-
-class ZoneoutWrapper(tf.contrib.rnn.RNNCell):
- """Add Zoneout to a RNN cell."""
-
- def __init__(self, cell, zoneout_drop_prob, is_training=True):
- self._cell = cell
- self._zoneout_prob = zoneout_drop_prob
- self._is_training = is_training
-
- @property
- def state_size(self):
- return self._cell.state_size
-
- @property
- def output_size(self):
- return self._cell.output_size
-
- def __call__(self, inputs, state, scope=None):
- output, new_state = self._cell(inputs, state, scope)
- if not isinstance(self._cell.state_size, tuple):
- new_state = tf.split(value=new_state, num_or_size_splits=2, axis=1)
- state = tf.split(value=state, num_or_size_splits=2, axis=1)
- final_new_state = [new_state[0], new_state[1]]
- if self._is_training:
- for i, state_element in enumerate(state):
- random_tensor = 1 - self._zoneout_prob # keep probability
- random_tensor += tf.random_uniform(tf.shape(state_element))
- # 0. if [zoneout_prob, 1.0) and 1. if [1.0, 1.0 + zoneout_prob)
- binary_tensor = tf.floor(random_tensor)
- final_new_state[
- i] = (new_state[i] - state_element) * binary_tensor + state_element
- else:
- for i, state_element in enumerate(state):
- final_new_state[
- i] = state_element * self._zoneout_prob + new_state[i] * (
- 1 - self._zoneout_prob)
- if isinstance(self._cell.state_size, tuple):
- return output, tf.contrib.rnn.LSTMStateTuple(
- final_new_state[0], final_new_state[1])
-
- return output, tf.concat([final_new_state[0], final_new_state[1]], 1)
diff --git a/research/maskgan/sample_shuffler.py b/research/maskgan/sample_shuffler.py
deleted file mode 100644
index 58c31fb573a864b33f3d6e2f17b42e42f1d0ea4d..0000000000000000000000000000000000000000
--- a/research/maskgan/sample_shuffler.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Shuffle samples for human evaluation.
-
-Local launch command:
- python sample_shuffler.py
- --input_ml_path=/tmp/ptb/seq2seq_vd_shareemb_forreal_55_3
- --input_gan_path=/tmp/ptb/MaskGAN_PTB_ari_avg_56.29_v2.0.0
- --output_file_name=/tmp/ptb/shuffled_output.txt
-
- python sample_shuffler.py
- --input_ml_path=/tmp/generate_samples/MaskGAN_IMDB_Benchmark_87.1_v0.3.0
- --input_gan_path=/tmp/generate_samples/MaskGAN_IMDB_v1.0.1
- --output_file_name=/tmp/imdb/shuffled_output.txt
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-# Dependency imports
-import numpy as np
-
-import tensorflow as tf
-
-tf.app.flags.DEFINE_string('input_ml_path', '/tmp', 'Model output directory.')
-tf.app.flags.DEFINE_string('input_gan_path', '/tmp', 'Model output directory.')
-tf.app.flags.DEFINE_string('output_file_name', '/tmp/ptb/shuffled_output.txt',
- 'Model output file.')
-tf.app.flags.DEFINE_boolean(
- 'output_masked_logs', False,
- 'Whether to display for human evaluation (show masking).')
-tf.app.flags.DEFINE_integer('number_epochs', 1,
- 'The number of epochs to produce.')
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def shuffle_samples(input_file_1, input_file_2):
- """Shuffle the examples."""
- shuffled = []
-
- # Set a random seed to keep fixed mask.
- np.random.seed(0)
-
- for line_1, line_2 in zip(input_file_1, input_file_2):
- rand = np.random.randint(1, 3)
- if rand == 1:
- shuffled.append((rand, line_1, line_2))
- else:
- shuffled.append((rand, line_2, line_1))
- input_file_1.close()
- input_file_2.close()
- return shuffled
-
-
-def generate_output(shuffled_tuples, output_file_name):
- output_file = tf.gfile.GFile(output_file_name, mode='w')
-
- for tup in shuffled_tuples:
- formatted_tuple = ('\n{:<1}, {:<1}, {:<1}').format(tup[0], tup[1].rstrip(),
- tup[2].rstrip())
- output_file.write(formatted_tuple)
- output_file.close()
-
-
-def main(_):
- ml_samples_file = tf.gfile.GFile(
- os.path.join(FLAGS.input_ml_path, 'reviews.txt'), mode='r')
- gan_samples_file = tf.gfile.GFile(
- os.path.join(FLAGS.input_gan_path, 'reviews.txt'), mode='r')
-
- # Generate shuffled tuples.
- shuffled_tuples = shuffle_samples(ml_samples_file, gan_samples_file)
-
- # Output to file.
- generate_output(shuffled_tuples, FLAGS.output_file_name)
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/maskgan/train_mask_gan.py b/research/maskgan/train_mask_gan.py
deleted file mode 100644
index 1e70c2284a8704b1c92dcdec850ac29fc9625667..0000000000000000000000000000000000000000
--- a/research/maskgan/train_mask_gan.py
+++ /dev/null
@@ -1,1167 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Launch example:
-
-[IMDB]
-python train_mask_gan.py --data_dir
-/tmp/imdb --data_set imdb --batch_size 128
---sequence_length 20 --base_directory /tmp/maskGAN_v0.01
---hparams="gen_rnn_size=650,gen_num_layers=2,dis_rnn_size=650,dis_num_layers=2
-,critic_learning_rate=0.0009756,dis_learning_rate=0.0000585,
-dis_train_iterations=8,gen_learning_rate=0.0016624,
-gen_full_learning_rate_steps=1e9,gen_learning_rate_decay=0.999999,
-rl_discount_rate=0.8835659" --mode TRAIN --max_steps 1000000
---generator_model seq2seq_vd --discriminator_model seq2seq_vd
---is_present_rate 0.5 --summaries_every 25 --print_every 25
- --max_num_to_print=3 --generator_optimizer=adam
- --seq2seq_share_embedding=True --baseline_method=critic
- --attention_option=luong --n_gram_eval=4 --mask_strategy=contiguous
- --gen_training_strategy=reinforce --dis_pretrain_steps=100
- --perplexity_threshold=1000000
- --dis_share_embedding=True --maskgan_ckpt
- /tmp/model.ckpt-171091
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-
-from functools import partial
-import os
-import time
-# Dependency imports
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import pretrain_mask_gan
-from data import imdb_loader
-from data import ptb_loader
-from model_utils import helper
-from model_utils import model_construction
-from model_utils import model_losses
-from model_utils import model_optimization
-
-# Data.
-from model_utils import model_utils
-
-from model_utils import n_gram
-from models import evaluation_utils
-
-from models import rollout
-
-np.set_printoptions(precision=3)
-np.set_printoptions(suppress=True)
-
-MODE_TRAIN = 'TRAIN'
-MODE_TRAIN_EVAL = 'TRAIN_EVAL'
-MODE_VALIDATION = 'VALIDATION'
-MODE_TEST = 'TEST'
-
-## Binary and setup FLAGS.
-tf.app.flags.DEFINE_enum(
- 'mode', 'TRAIN', [MODE_TRAIN, MODE_VALIDATION, MODE_TEST, MODE_TRAIN_EVAL],
- 'What this binary will do.')
-tf.app.flags.DEFINE_string('master', '',
- """Name of the TensorFlow master to use.""")
-tf.app.flags.DEFINE_string('eval_master', '',
- """Name prefix of the Tensorflow eval master.""")
-tf.app.flags.DEFINE_integer('task', 0,
- """Task id of the replica running the training.""")
-tf.app.flags.DEFINE_integer('ps_tasks', 0, """Number of tasks in the ps job.
- If 0 no ps job is used.""")
-
-## General FLAGS.
-tf.app.flags.DEFINE_string(
- 'hparams', '', 'Comma separated list of name=value hyperparameter pairs.')
-tf.app.flags.DEFINE_integer('batch_size', 20, 'The batch size.')
-tf.app.flags.DEFINE_integer('vocab_size', 10000, 'The vocabulary size.')
-tf.app.flags.DEFINE_integer('sequence_length', 20, 'The sequence length.')
-tf.app.flags.DEFINE_integer('max_steps', 1000000,
- 'Maximum number of steps to run.')
-tf.app.flags.DEFINE_string(
- 'mask_strategy', 'random', 'Strategy for masking the words. Determine the '
- 'characterisitics of how the words are dropped out. One of '
- "['contiguous', 'random'].")
-tf.app.flags.DEFINE_float('is_present_rate', 0.5,
- 'Percent of tokens present in the forward sequence.')
-tf.app.flags.DEFINE_float('is_present_rate_decay', None, 'Decay rate for the '
- 'percent of words that are real (are present).')
-tf.app.flags.DEFINE_string(
- 'generator_model', 'seq2seq',
- "Type of Generator model. One of ['rnn', 'seq2seq', 'seq2seq_zaremba',"
- "'rnn_zaremba', 'rnn_nas', 'seq2seq_nas']")
-tf.app.flags.DEFINE_string(
- 'attention_option', None,
- "Attention mechanism. One of [None, 'luong', 'bahdanau']")
-tf.app.flags.DEFINE_string(
- 'discriminator_model', 'bidirectional',
- "Type of Discriminator model. One of ['cnn', 'rnn', 'bidirectional', "
- "'rnn_zaremba', 'bidirectional_zaremba', 'rnn_nas', 'rnn_vd', 'seq2seq_vd']"
-)
-tf.app.flags.DEFINE_boolean('seq2seq_share_embedding', False,
- 'Whether to share the '
- 'embeddings between the encoder and decoder.')
-tf.app.flags.DEFINE_boolean(
- 'dis_share_embedding', False, 'Whether to share the '
- 'embeddings between the generator and discriminator.')
-tf.app.flags.DEFINE_boolean('dis_update_share_embedding', False, 'Whether the '
- 'discriminator should update the shared embedding.')
-tf.app.flags.DEFINE_boolean('use_gen_mode', False,
- 'Use the mode of the generator '
- 'to produce samples.')
-tf.app.flags.DEFINE_boolean('critic_update_dis_vars', False,
- 'Whether the critic '
- 'updates the discriminator variables.')
-
-## Training FLAGS.
-tf.app.flags.DEFINE_string(
- 'gen_training_strategy', 'reinforce',
- "Method for training the Generator. One of ['cross_entropy', 'reinforce']")
-tf.app.flags.DEFINE_string(
- 'generator_optimizer', 'adam',
- "Type of Generator optimizer. One of ['sgd', 'adam']")
-tf.app.flags.DEFINE_float('grad_clipping', 10., 'Norm for gradient clipping.')
-tf.app.flags.DEFINE_float('advantage_clipping', 5., 'Clipping for advantages.')
-tf.app.flags.DEFINE_string(
- 'baseline_method', None,
- "Approach for baseline. One of ['critic', 'dis_batch', 'ema', None]")
-tf.app.flags.DEFINE_float('perplexity_threshold', 15000,
- 'Limit for perplexity before terminating job.')
-tf.app.flags.DEFINE_float('zoneout_drop_prob', 0.1,
- 'Probability for dropping parameter for zoneout.')
-tf.app.flags.DEFINE_float('keep_prob', 0.5,
- 'Probability for keeping parameter for dropout.')
-
-## Logging and evaluation FLAGS.
-tf.app.flags.DEFINE_integer('print_every', 250,
- 'Frequency to print and log the '
- 'outputs of the model.')
-tf.app.flags.DEFINE_integer('max_num_to_print', 5,
- 'Number of samples to log/print.')
-tf.app.flags.DEFINE_boolean('print_verbose', False, 'Whether to print in full.')
-tf.app.flags.DEFINE_integer('summaries_every', 100,
- 'Frequency to compute summaries.')
-tf.app.flags.DEFINE_boolean('eval_language_model', False,
- 'Whether to evaluate on '
- 'all words as in language modeling.')
-tf.app.flags.DEFINE_float('eval_interval_secs', 60,
- 'Delay for evaluating model.')
-tf.app.flags.DEFINE_integer(
- 'n_gram_eval', 4, """The degree of the n-grams to use for evaluation.""")
-tf.app.flags.DEFINE_integer(
- 'epoch_size_override', None,
- 'If an integer, this dictates the size of the epochs and will potentially '
- 'not iterate over all the data.')
-tf.app.flags.DEFINE_integer('eval_epoch_size_override', None,
- 'Number of evaluation steps.')
-
-## Directories and checkpoints.
-tf.app.flags.DEFINE_string('base_directory', '/tmp/maskGAN_v0.00',
- 'Base directory for the logging, events and graph.')
-tf.app.flags.DEFINE_string('data_set', 'ptb', 'Data set to operate on. One of'
- "['ptb', 'imdb']")
-tf.app.flags.DEFINE_string('data_dir', '/tmp/data/ptb',
- 'Directory for the training data.')
-tf.app.flags.DEFINE_string(
- 'language_model_ckpt_dir', None,
- 'Directory storing checkpoints to initialize the model. Pretrained models'
- 'are stored at /tmp/maskGAN/pretrained/')
-tf.app.flags.DEFINE_string(
- 'language_model_ckpt_dir_reversed', None,
- 'Directory storing checkpoints of reversed models to initialize the model.'
- 'Pretrained models stored at'
- 'are stored at /tmp/PTB/pretrained_reversed')
-tf.app.flags.DEFINE_string(
- 'maskgan_ckpt', None,
- 'Override which checkpoint file to use to restore the '
- 'model. A pretrained seq2seq_zaremba model is stored at '
- '/tmp/maskGAN/pretrain/seq2seq_zaremba/train/model.ckpt-64912')
-
-tf.app.flags.DEFINE_boolean('wasserstein_objective', False,
- '(DEPRECATED) Whether to use the WGAN training.')
-tf.app.flags.DEFINE_integer('num_rollouts', 1,
- 'The number of rolled out predictions to make.')
-tf.app.flags.DEFINE_float('c_lower', -0.01, 'Lower bound for weights.')
-tf.app.flags.DEFINE_float('c_upper', 0.01, 'Upper bound for weights.')
-
-FLAGS = tf.app.flags.FLAGS
-
-
-def create_hparams():
- """Create the hparams object for generic training hyperparameters."""
- hparams = tf.contrib.training.HParams(
- gen_num_layers=2,
- dis_num_layers=2,
- gen_rnn_size=740,
- dis_rnn_size=740,
- gen_learning_rate=5e-4,
- dis_learning_rate=5e-3,
- critic_learning_rate=5e-3,
- dis_train_iterations=1,
- gen_learning_rate_decay=1.0,
- gen_full_learning_rate_steps=1e7,
- baseline_decay=0.999999,
- rl_discount_rate=0.9,
- gen_vd_keep_prob=0.5,
- dis_vd_keep_prob=0.5,
- dis_pretrain_learning_rate=5e-3,
- dis_num_filters=128,
- dis_hidden_dim=128,
- gen_nas_keep_prob_0=0.85,
- gen_nas_keep_prob_1=0.55,
- dis_nas_keep_prob_0=0.85,
- dis_nas_keep_prob_1=0.55)
- # Command line flags override any of the preceding hyperparameter values.
- if FLAGS.hparams:
- hparams = hparams.parse(FLAGS.hparams)
- return hparams
-
-
-def create_MaskGAN(hparams, is_training):
- """Create the MaskGAN model.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- is_training: Boolean indicating operational mode (train/inference).
- evaluated with a teacher forcing regime.
-
- Return:
- model: Namedtuple for specifying the MaskGAN.
- """
- global_step = tf.Variable(0, name='global_step', trainable=False)
-
- new_learning_rate = tf.placeholder(tf.float32, [], name='new_learning_rate')
- learning_rate = tf.Variable(0.0, name='learning_rate', trainable=False)
- learning_rate_update = tf.assign(learning_rate, new_learning_rate)
-
- new_rate = tf.placeholder(tf.float32, [], name='new_rate')
- percent_real_var = tf.Variable(0.0, trainable=False)
- percent_real_update = tf.assign(percent_real_var, new_rate)
-
- ## Placeholders.
- inputs = tf.placeholder(
- tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length])
- targets = tf.placeholder(
- tf.int32, shape=[FLAGS.batch_size, FLAGS.sequence_length])
- present = tf.placeholder(
- tf.bool, shape=[FLAGS.batch_size, FLAGS.sequence_length])
- # TODO(adai): Placeholder for IMDB label.
-
- ## Real Sequence is the targets.
- real_sequence = targets
-
- ## Fakse Sequence from the Generator.
- # TODO(adai): Generator must have IMDB labels placeholder.
- (fake_sequence, fake_logits, fake_log_probs, fake_gen_initial_state,
- fake_gen_final_state, _) = model_construction.create_generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=is_training,
- is_validating=False)
- (_, eval_logits, _, eval_initial_state, eval_final_state,
- _) = model_construction.create_generator(
- hparams,
- inputs,
- targets,
- present,
- is_training=False,
- is_validating=True,
- reuse=True)
-
- ## Discriminator.
- fake_predictions = model_construction.create_discriminator(
- hparams,
- fake_sequence,
- is_training=is_training,
- inputs=inputs,
- present=present)
- real_predictions = model_construction.create_discriminator(
- hparams,
- real_sequence,
- is_training=is_training,
- reuse=True,
- inputs=inputs,
- present=present)
-
- ## Critic.
- # The critic will be used to estimate the forward rewards to the Generator.
- if FLAGS.baseline_method == 'critic':
- est_state_values = model_construction.create_critic(
- hparams, fake_sequence, is_training=is_training)
- else:
- est_state_values = None
-
- ## Discriminator Loss.
- [dis_loss, dis_loss_fake, dis_loss_real] = model_losses.create_dis_loss(
- fake_predictions, real_predictions, present)
-
- ## Average log-perplexity for only missing words. However, to do this,
- # the logits are still computed using teacher forcing, that is, the ground
- # truth tokens are fed in at each time point to be valid.
- avg_log_perplexity = model_losses.calculate_log_perplexity(
- eval_logits, targets, present)
-
- ## Generator Objective.
- # 1. Cross Entropy losses on missing tokens.
- fake_cross_entropy_losses = model_losses.create_masked_cross_entropy_loss(
- targets, present, fake_logits)
-
- # 2. GAN REINFORCE losses.
- [
- fake_RL_loss, fake_log_probs, fake_rewards, fake_advantages,
- fake_baselines, fake_averages_op, critic_loss, cumulative_rewards
- ] = model_losses.calculate_reinforce_objective(
- hparams, fake_log_probs, fake_predictions, present, est_state_values)
-
- ## Pre-training.
- if FLAGS.gen_pretrain_steps:
- raise NotImplementedError
- # # TODO(liamfedus): Rewrite this.
- # fwd_cross_entropy_loss = tf.reduce_mean(fwd_cross_entropy_losses)
- # gen_pretrain_op = model_optimization.create_gen_pretrain_op(
- # hparams, fwd_cross_entropy_loss, global_step)
- else:
- gen_pretrain_op = None
- if FLAGS.dis_pretrain_steps:
- dis_pretrain_op = model_optimization.create_dis_pretrain_op(
- hparams, dis_loss, global_step)
- else:
- dis_pretrain_op = None
-
- ## Generator Train Op.
- # 1. Cross-Entropy.
- if FLAGS.gen_training_strategy == 'cross_entropy':
- gen_loss = tf.reduce_mean(fake_cross_entropy_losses)
- [gen_train_op, gen_grads,
- gen_vars] = model_optimization.create_gen_train_op(
- hparams, learning_rate, gen_loss, global_step, mode='MINIMIZE')
-
- # 2. GAN (REINFORCE)
- elif FLAGS.gen_training_strategy == 'reinforce':
- gen_loss = fake_RL_loss
- [gen_train_op, gen_grads,
- gen_vars] = model_optimization.create_reinforce_gen_train_op(
- hparams, learning_rate, gen_loss, fake_averages_op, global_step)
-
- else:
- raise NotImplementedError
-
- ## Discriminator Train Op.
- dis_train_op, dis_grads, dis_vars = model_optimization.create_dis_train_op(
- hparams, dis_loss, global_step)
-
- ## Critic Train Op.
- if critic_loss is not None:
- [critic_train_op, _, _] = model_optimization.create_critic_train_op(
- hparams, critic_loss, global_step)
- dis_train_op = tf.group(dis_train_op, critic_train_op)
-
- ## Summaries.
- with tf.name_scope('general'):
- tf.summary.scalar('percent_real', percent_real_var)
- tf.summary.scalar('learning_rate', learning_rate)
-
- with tf.name_scope('generator_objectives'):
- tf.summary.scalar('gen_objective', tf.reduce_mean(gen_loss))
- tf.summary.scalar('gen_loss_cross_entropy',
- tf.reduce_mean(fake_cross_entropy_losses))
-
- with tf.name_scope('REINFORCE'):
- with tf.name_scope('objective'):
- tf.summary.scalar('fake_RL_loss', tf.reduce_mean(fake_RL_loss))
-
- with tf.name_scope('rewards'):
- helper.variable_summaries(cumulative_rewards, 'rewards')
-
- with tf.name_scope('advantages'):
- helper.variable_summaries(fake_advantages, 'advantages')
-
- with tf.name_scope('baselines'):
- helper.variable_summaries(fake_baselines, 'baselines')
-
- with tf.name_scope('log_probs'):
- helper.variable_summaries(fake_log_probs, 'log_probs')
-
- with tf.name_scope('discriminator_losses'):
- tf.summary.scalar('dis_loss', dis_loss)
- tf.summary.scalar('dis_loss_fake_sequence', dis_loss_fake)
- tf.summary.scalar('dis_loss_prob_fake_sequence', tf.exp(-dis_loss_fake))
- tf.summary.scalar('dis_loss_real_sequence', dis_loss_real)
- tf.summary.scalar('dis_loss_prob_real_sequence', tf.exp(-dis_loss_real))
-
- if critic_loss is not None:
- with tf.name_scope('critic_losses'):
- tf.summary.scalar('critic_loss', critic_loss)
-
- with tf.name_scope('logits'):
- helper.variable_summaries(fake_logits, 'fake_logits')
-
- for v, g in zip(gen_vars, gen_grads):
- helper.variable_summaries(v, v.op.name)
- helper.variable_summaries(g, 'grad/' + v.op.name)
-
- for v, g in zip(dis_vars, dis_grads):
- helper.variable_summaries(v, v.op.name)
- helper.variable_summaries(g, 'grad/' + v.op.name)
-
- merge_summaries_op = tf.summary.merge_all()
- text_summary_placeholder = tf.placeholder(tf.string)
- text_summary_op = tf.summary.text('Samples', text_summary_placeholder)
-
- # Model saver.
- saver = tf.train.Saver(keep_checkpoint_every_n_hours=1, max_to_keep=5)
-
- # Named tuple that captures elements of the MaskGAN model.
- Model = collections.namedtuple('Model', [
- 'inputs', 'targets', 'present', 'percent_real_update', 'new_rate',
- 'fake_sequence', 'fake_logits', 'fake_rewards', 'fake_baselines',
- 'fake_advantages', 'fake_log_probs', 'fake_predictions',
- 'real_predictions', 'fake_cross_entropy_losses', 'fake_gen_initial_state',
- 'fake_gen_final_state', 'eval_initial_state', 'eval_final_state',
- 'avg_log_perplexity', 'dis_loss', 'gen_loss', 'critic_loss',
- 'cumulative_rewards', 'dis_train_op', 'gen_train_op', 'gen_pretrain_op',
- 'dis_pretrain_op', 'merge_summaries_op', 'global_step',
- 'new_learning_rate', 'learning_rate_update', 'saver', 'text_summary_op',
- 'text_summary_placeholder'
- ])
-
- model = Model(
- inputs, targets, present, percent_real_update, new_rate, fake_sequence,
- fake_logits, fake_rewards, fake_baselines, fake_advantages,
- fake_log_probs, fake_predictions, real_predictions,
- fake_cross_entropy_losses, fake_gen_initial_state, fake_gen_final_state,
- eval_initial_state, eval_final_state, avg_log_perplexity, dis_loss,
- gen_loss, critic_loss, cumulative_rewards, dis_train_op, gen_train_op,
- gen_pretrain_op, dis_pretrain_op, merge_summaries_op, global_step,
- new_learning_rate, learning_rate_update, saver, text_summary_op,
- text_summary_placeholder)
- return model
-
-
-def compute_geometric_average(percent_captured):
- """Compute the geometric average of the n-gram metrics."""
-
- res = 1.
- for _, n_gram_percent in percent_captured.iteritems():
- res *= n_gram_percent
-
- return np.power(res, 1. / float(len(percent_captured)))
-
-
-def compute_arithmetic_average(percent_captured):
- """Compute the arithmetic average of the n-gram metrics."""
- N = len(percent_captured)
-
- res = 0.
- for _, n_gram_percent in percent_captured.iteritems():
- res += n_gram_percent
-
- return res / float(N)
-
-
-def get_iterator(data):
- """Return the data iterator."""
- if FLAGS.data_set == 'ptb':
- iterator = ptb_loader.ptb_iterator(data, FLAGS.batch_size,
- FLAGS.sequence_length,
- FLAGS.epoch_size_override)
- elif FLAGS.data_set == 'imdb':
- iterator = imdb_loader.imdb_iterator(data, FLAGS.batch_size,
- FLAGS.sequence_length)
- return iterator
-
-
-def train_model(hparams, data, log_dir, log, id_to_word, data_ngram_counts):
- """Train model.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- data: Data to evaluate.
- log_dir: Directory to save checkpoints.
- log: Readable log for the experiment.
- id_to_word: Dictionary of indices to words.
- data_ngram_counts: Dictionary of hashed(n-gram tuples) to counts in the
- data_set.
- """
- print('Training model.')
- tf.logging.info('Training model.')
-
- # Boolean indicating operational mode.
- is_training = True
-
- # Write all the information to the logs.
- log.write('hparams\n')
- log.write(str(hparams))
- log.flush()
-
- is_chief = FLAGS.task == 0
-
- with tf.Graph().as_default():
- with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
- container_name = ''
- with tf.container(container_name):
- # Construct the model.
- if FLAGS.num_rollouts == 1:
- model = create_MaskGAN(hparams, is_training)
- elif FLAGS.num_rollouts > 1:
- model = rollout.create_rollout_MaskGAN(hparams, is_training)
- else:
- raise ValueError
-
- print('\nTrainable Variables in Graph:')
- for v in tf.trainable_variables():
- print(v)
-
- ## Retrieve the initial savers.
- init_savers = model_utils.retrieve_init_savers(hparams)
-
- ## Initial saver function to supervisor.
- init_fn = partial(model_utils.init_fn, init_savers)
-
- # Create the supervisor. It will take care of initialization,
- # summaries, checkpoints, and recovery.
- sv = tf.train.Supervisor(
- logdir=log_dir,
- is_chief=is_chief,
- saver=model.saver,
- global_step=model.global_step,
- save_model_secs=60,
- recovery_wait_secs=30,
- summary_op=None,
- init_fn=init_fn)
-
- # Get an initialized, and possibly recovered session. Launch the
- # services: Checkpointing, Summaries, step counting.
- #
- # When multiple replicas of this program are running the services are
- # only launched by the 'chief' replica.
- with sv.managed_session(FLAGS.master) as sess:
-
- ## Pretrain the generator.
- if FLAGS.gen_pretrain_steps:
- pretrain_mask_gan.pretrain_generator(sv, sess, model, data, log,
- id_to_word, data_ngram_counts,
- is_chief)
-
- ## Pretrain the discriminator.
- if FLAGS.dis_pretrain_steps:
- pretrain_mask_gan.pretrain_discriminator(
- sv, sess, model, data, log, id_to_word, data_ngram_counts,
- is_chief)
-
- # Initial indicators for printing and summarizing.
- print_step_division = -1
- summary_step_division = -1
-
- # Run iterative computation in a loop.
- while not sv.ShouldStop():
- is_present_rate = FLAGS.is_present_rate
-
- if FLAGS.is_present_rate_decay is not None:
- is_present_rate *= (1. - FLAGS.is_present_rate_decay)
-
- model_utils.assign_percent_real(sess, model.percent_real_update,
- model.new_rate, is_present_rate)
-
- # GAN training.
- avg_epoch_gen_loss, avg_epoch_dis_loss = [], []
- cumulative_costs = 0.
- gen_iters = 0
-
- # Generator and Discriminator statefulness initial evaluation.
- # TODO(liamfedus): Throughout the code I am implicitly assuming
- # that the Generator and Discriminator are equal sized.
- [gen_initial_state_eval, fake_gen_initial_state_eval] = sess.run(
- [model.eval_initial_state, model.fake_gen_initial_state])
- dis_initial_state_eval = fake_gen_initial_state_eval
-
- # Save zeros state to reset later.
- zeros_state = fake_gen_initial_state_eval
-
- ## Offset Discriminator.
- if FLAGS.ps_tasks == 0:
- dis_offset = 1
- else:
- dis_offset = FLAGS.task * 1000 + 1
- dis_iterator = get_iterator(data)
-
- for i in range(dis_offset):
- try:
- dis_x, dis_y, _ = next(dis_iterator)
- except StopIteration:
- dis_iterator = get_iterator(data)
- dis_initial_state_eval = zeros_state
- dis_x, dis_y, _ = next(dis_iterator)
-
- p = model_utils.generate_mask()
-
- # Construct the train feed.
- train_feed = {
- model.inputs: dis_x,
- model.targets: dis_y,
- model.present: p
- }
-
- if FLAGS.data_set == 'ptb':
- # Statefulness of the Generator being used for Discriminator.
- for i, (c, h) in enumerate(model.fake_gen_initial_state):
- train_feed[c] = dis_initial_state_eval[i].c
- train_feed[h] = dis_initial_state_eval[i].h
-
- # Determine the state had the Generator run over real data. We
- # use this state for the Discriminator.
- [dis_initial_state_eval] = sess.run(
- [model.fake_gen_final_state], train_feed)
-
- ## Training loop.
- iterator = get_iterator(data)
- gen_initial_state_eval = zeros_state
-
- if FLAGS.ps_tasks > 0:
- gen_offset = FLAGS.task * 1000 + 1
- for i in range(gen_offset):
- try:
- next(iterator)
- except StopIteration:
- dis_iterator = get_iterator(data)
- dis_initial_state_eval = zeros_state
- next(dis_iterator)
-
- for x, y, _ in iterator:
- for _ in xrange(hparams.dis_train_iterations):
- try:
- dis_x, dis_y, _ = next(dis_iterator)
- except StopIteration:
- dis_iterator = get_iterator(data)
- dis_initial_state_eval = zeros_state
- dis_x, dis_y, _ = next(dis_iterator)
-
- if FLAGS.data_set == 'ptb':
- [dis_initial_state_eval] = sess.run(
- [model.fake_gen_initial_state])
-
- p = model_utils.generate_mask()
-
- # Construct the train feed.
- train_feed = {
- model.inputs: dis_x,
- model.targets: dis_y,
- model.present: p
- }
-
- # Statefulness for the Discriminator.
- if FLAGS.data_set == 'ptb':
- for i, (c, h) in enumerate(model.fake_gen_initial_state):
- train_feed[c] = dis_initial_state_eval[i].c
- train_feed[h] = dis_initial_state_eval[i].h
-
- _, dis_loss_eval, step = sess.run(
- [model.dis_train_op, model.dis_loss, model.global_step],
- feed_dict=train_feed)
-
- # Determine the state had the Generator run over real data.
- # Use this state for the Discriminator.
- [dis_initial_state_eval] = sess.run(
- [model.fake_gen_final_state], train_feed)
-
- # Randomly mask out tokens.
- p = model_utils.generate_mask()
-
- # Construct the train feed.
- train_feed = {model.inputs: x, model.targets: y, model.present: p}
-
- # Statefulness for Generator.
- if FLAGS.data_set == 'ptb':
- tf.logging.info('Generator is stateful.')
- print('Generator is stateful.')
- # Statefulness for *evaluation* Generator.
- for i, (c, h) in enumerate(model.eval_initial_state):
- train_feed[c] = gen_initial_state_eval[i].c
- train_feed[h] = gen_initial_state_eval[i].h
-
- # Statefulness for Generator.
- for i, (c, h) in enumerate(model.fake_gen_initial_state):
- train_feed[c] = fake_gen_initial_state_eval[i].c
- train_feed[h] = fake_gen_initial_state_eval[i].h
-
- # Determine whether to decay learning rate.
- lr_decay = hparams.gen_learning_rate_decay**max(
- step + 1 - hparams.gen_full_learning_rate_steps, 0.0)
-
- # Assign learning rate.
- gen_learning_rate = hparams.gen_learning_rate * lr_decay
- model_utils.assign_learning_rate(sess, model.learning_rate_update,
- model.new_learning_rate,
- gen_learning_rate)
-
- [_, gen_loss_eval, gen_log_perplexity_eval, step] = sess.run(
- [
- model.gen_train_op, model.gen_loss,
- model.avg_log_perplexity, model.global_step
- ],
- feed_dict=train_feed)
-
- cumulative_costs += gen_log_perplexity_eval
- gen_iters += 1
-
- # Determine the state had the Generator run over real data.
- [gen_initial_state_eval, fake_gen_initial_state_eval] = sess.run(
- [model.eval_final_state,
- model.fake_gen_final_state], train_feed)
-
- avg_epoch_dis_loss.append(dis_loss_eval)
- avg_epoch_gen_loss.append(gen_loss_eval)
-
- ## Summaries.
- # Calulate rolling perplexity.
- perplexity = np.exp(cumulative_costs / gen_iters)
-
- if is_chief and (step / FLAGS.summaries_every >
- summary_step_division):
- summary_step_division = step / FLAGS.summaries_every
-
- # Confirm perplexity is not infinite.
- if (not np.isfinite(perplexity) or
- perplexity >= FLAGS.perplexity_threshold):
- print('Training raising FloatingPoinError.')
- raise FloatingPointError(
- 'Training infinite perplexity: %.3f' % perplexity)
-
- # Graph summaries.
- summary_str = sess.run(
- model.merge_summaries_op, feed_dict=train_feed)
- sv.SummaryComputed(sess, summary_str)
-
- # Summary: n-gram
- avg_percent_captured = {'2': 0., '3': 0., '4': 0.}
- for n, data_ngram_count in data_ngram_counts.iteritems():
- batch_percent_captured = evaluation_utils.sequence_ngram_evaluation(
- sess, model.fake_sequence, log, train_feed,
- data_ngram_count, int(n))
- summary_percent_str = tf.Summary(value=[
- tf.Summary.Value(
- tag='general/%s-grams_percent_correct' % n,
- simple_value=batch_percent_captured)
- ])
- sv.SummaryComputed(
- sess, summary_percent_str, global_step=step)
-
- # Summary: geometric_avg
- geometric_avg = compute_geometric_average(avg_percent_captured)
- summary_geometric_avg_str = tf.Summary(value=[
- tf.Summary.Value(
- tag='general/geometric_avg', simple_value=geometric_avg)
- ])
- sv.SummaryComputed(
- sess, summary_geometric_avg_str, global_step=step)
-
- # Summary: arithmetic_avg
- arithmetic_avg = compute_arithmetic_average(
- avg_percent_captured)
- summary_arithmetic_avg_str = tf.Summary(value=[
- tf.Summary.Value(
- tag='general/arithmetic_avg',
- simple_value=arithmetic_avg)
- ])
- sv.SummaryComputed(
- sess, summary_arithmetic_avg_str, global_step=step)
-
- # Summary: perplexity
- summary_perplexity_str = tf.Summary(value=[
- tf.Summary.Value(
- tag='general/perplexity', simple_value=perplexity)
- ])
- sv.SummaryComputed(
- sess, summary_perplexity_str, global_step=step)
-
- ## Printing and logging
- if is_chief and (step / FLAGS.print_every > print_step_division):
- print_step_division = (step / FLAGS.print_every)
- print('global_step: %d' % step)
- print(' perplexity: %.3f' % perplexity)
- print(' gen_learning_rate: %.6f' % gen_learning_rate)
- log.write('global_step: %d\n' % step)
- log.write(' perplexity: %.3f\n' % perplexity)
- log.write(' gen_learning_rate: %.6f' % gen_learning_rate)
-
- # Average percent captured for each of the n-grams.
- avg_percent_captured = {'2': 0., '3': 0., '4': 0.}
- for n, data_ngram_count in data_ngram_counts.iteritems():
- batch_percent_captured = evaluation_utils.sequence_ngram_evaluation(
- sess, model.fake_sequence, log, train_feed,
- data_ngram_count, int(n))
- avg_percent_captured[n] = batch_percent_captured
- print(' percent of %s-grams captured: %.3f.' %
- (n, batch_percent_captured))
- log.write(' percent of %s-grams captured: %.3f.\n' %
- (n, batch_percent_captured))
- geometric_avg = compute_geometric_average(avg_percent_captured)
- print(' geometric_avg: %.3f.' % geometric_avg)
- log.write(' geometric_avg: %.3f.' % geometric_avg)
- arithmetic_avg = compute_arithmetic_average(
- avg_percent_captured)
- print(' arithmetic_avg: %.3f.' % arithmetic_avg)
- log.write(' arithmetic_avg: %.3f.' % arithmetic_avg)
-
- evaluation_utils.print_and_log_losses(
- log, step, is_present_rate, avg_epoch_dis_loss,
- avg_epoch_gen_loss)
-
- if FLAGS.gen_training_strategy == 'reinforce':
- evaluation_utils.generate_RL_logs(sess, model, log,
- id_to_word, train_feed)
- else:
- evaluation_utils.generate_logs(sess, model, log, id_to_word,
- train_feed)
- log.flush()
-
- log.close()
-
-
-def evaluate_once(data, sv, model, sess, train_dir, log, id_to_word,
- data_ngram_counts, eval_saver):
- """Evaluate model for a number of steps.
-
- Args:
- data: Dataset.
- sv: Supervisor.
- model: The GAN model we have just built.
- sess: A session to use.
- train_dir: Path to a directory containing checkpoints.
- log: Evaluation log for evaluation.
- id_to_word: Dictionary of indices to words.
- data_ngram_counts: Dictionary of hashed(n-gram tuples) to counts in the
- data_set.
- eval_saver: Evaluation saver.r.
- """
- tf.logging.info('Evaluate Once.')
- # Load the last model checkpoint, or initialize the graph.
- model_save_path = tf.latest_checkpoint(train_dir)
- if not model_save_path:
- tf.logging.warning('No checkpoint yet in: %s', train_dir)
- return
-
- tf.logging.info('Starting eval of: %s' % model_save_path)
- tf.logging.info('Only restoring trainable variables.')
- eval_saver.restore(sess, model_save_path)
-
- # Run the requested number of evaluation steps
- avg_epoch_gen_loss, avg_epoch_dis_loss = [], []
- cumulative_costs = 0.
-
- # Average percent captured for each of the n-grams.
- avg_percent_captured = {'2': 0., '3': 0., '4': 0.}
-
- # Set a random seed to keep fixed mask.
- np.random.seed(0)
- gen_iters = 0
-
- # Generator statefulness over the epoch.
- # TODO(liamfedus): Check this.
- [gen_initial_state_eval, fake_gen_initial_state_eval] = sess.run(
- [model.eval_initial_state, model.fake_gen_initial_state])
-
- if FLAGS.eval_language_model:
- is_present_rate = 0.
- tf.logging.info('Overriding is_present_rate=0. for evaluation.')
- print('Overriding is_present_rate=0. for evaluation.')
-
- iterator = get_iterator(data)
-
- for x, y, _ in iterator:
- if FLAGS.eval_language_model:
- is_present_rate = 0.
- else:
- is_present_rate = FLAGS.is_present_rate
- tf.logging.info('Evaluating on is_present_rate=%.3f.' % is_present_rate)
-
- model_utils.assign_percent_real(sess, model.percent_real_update,
- model.new_rate, is_present_rate)
-
- # Randomly mask out tokens.
- p = model_utils.generate_mask()
-
- eval_feed = {model.inputs: x, model.targets: y, model.present: p}
-
- if FLAGS.data_set == 'ptb':
- # Statefulness for *evaluation* Generator.
- for i, (c, h) in enumerate(model.eval_initial_state):
- eval_feed[c] = gen_initial_state_eval[i].c
- eval_feed[h] = gen_initial_state_eval[i].h
-
- # Statefulness for the Generator.
- for i, (c, h) in enumerate(model.fake_gen_initial_state):
- eval_feed[c] = fake_gen_initial_state_eval[i].c
- eval_feed[h] = fake_gen_initial_state_eval[i].h
-
- [
- gen_log_perplexity_eval, dis_loss_eval, gen_loss_eval,
- gen_initial_state_eval, fake_gen_initial_state_eval, step
- ] = sess.run(
- [
- model.avg_log_perplexity, model.dis_loss, model.gen_loss,
- model.eval_final_state, model.fake_gen_final_state,
- model.global_step
- ],
- feed_dict=eval_feed)
-
- for n, data_ngram_count in data_ngram_counts.iteritems():
- batch_percent_captured = evaluation_utils.sequence_ngram_evaluation(
- sess, model.fake_sequence, log, eval_feed, data_ngram_count, int(n))
- avg_percent_captured[n] += batch_percent_captured
-
- cumulative_costs += gen_log_perplexity_eval
-
- avg_epoch_dis_loss.append(dis_loss_eval)
- avg_epoch_gen_loss.append(gen_loss_eval)
-
- gen_iters += 1
-
- # Calulate rolling metrics.
- perplexity = np.exp(cumulative_costs / gen_iters)
- for n, _ in avg_percent_captured.iteritems():
- avg_percent_captured[n] /= gen_iters
-
- # Confirm perplexity is not infinite.
- if not np.isfinite(perplexity) or perplexity >= FLAGS.perplexity_threshold:
- print('Evaluation raising FloatingPointError.')
- raise FloatingPointError(
- 'Evaluation infinite perplexity: %.3f' % perplexity)
-
- ## Printing and logging.
- evaluation_utils.print_and_log_losses(log, step, is_present_rate,
- avg_epoch_dis_loss, avg_epoch_gen_loss)
- print(' perplexity: %.3f' % perplexity)
- log.write(' perplexity: %.3f\n' % perplexity)
-
- for n, n_gram_percent in avg_percent_captured.iteritems():
- n = int(n)
- print(' percent of %d-grams captured: %.3f.' % (n, n_gram_percent))
- log.write(' percent of %d-grams captured: %.3f.\n' % (n, n_gram_percent))
-
- samples = evaluation_utils.generate_logs(sess, model, log, id_to_word,
- eval_feed)
-
- ## Summaries.
- summary_str = sess.run(model.merge_summaries_op, feed_dict=eval_feed)
- sv.SummaryComputed(sess, summary_str)
-
- # Summary: text
- summary_str = sess.run(model.text_summary_op,
- {model.text_summary_placeholder: '\n\n'.join(samples)})
- sv.SummaryComputed(sess, summary_str, global_step=step)
-
- # Summary: n-gram
- for n, n_gram_percent in avg_percent_captured.iteritems():
- n = int(n)
- summary_percent_str = tf.Summary(value=[
- tf.Summary.Value(
- tag='general/%d-grams_percent_correct' % n,
- simple_value=n_gram_percent)
- ])
- sv.SummaryComputed(sess, summary_percent_str, global_step=step)
-
- # Summary: geometric_avg
- geometric_avg = compute_geometric_average(avg_percent_captured)
- summary_geometric_avg_str = tf.Summary(value=[
- tf.Summary.Value(tag='general/geometric_avg', simple_value=geometric_avg)
- ])
- sv.SummaryComputed(sess, summary_geometric_avg_str, global_step=step)
-
- # Summary: arithmetic_avg
- arithmetic_avg = compute_arithmetic_average(avg_percent_captured)
- summary_arithmetic_avg_str = tf.Summary(value=[
- tf.Summary.Value(
- tag='general/arithmetic_avg', simple_value=arithmetic_avg)
- ])
- sv.SummaryComputed(sess, summary_arithmetic_avg_str, global_step=step)
-
- # Summary: perplexity
- summary_perplexity_str = tf.Summary(value=[
- tf.Summary.Value(tag='general/perplexity', simple_value=perplexity)
- ])
- sv.SummaryComputed(sess, summary_perplexity_str, global_step=step)
-
-
-def evaluate_model(hparams, data, train_dir, log, id_to_word,
- data_ngram_counts):
- """Evaluate MaskGAN model.
-
- Args:
- hparams: Hyperparameters for the MaskGAN.
- data: Data to evaluate.
- train_dir: Path to a directory containing checkpoints.
- id_to_word: Dictionary of indices to words.
- data_ngram_counts: Dictionary of hashed(n-gram tuples) to counts in the
- data_set.
- """
- tf.logging.error('Evaluate model.')
-
- # Boolean indicating operational mode.
- is_training = False
-
- if FLAGS.mode == MODE_VALIDATION:
- logdir = FLAGS.base_directory + '/validation'
- elif FLAGS.mode == MODE_TRAIN_EVAL:
- logdir = FLAGS.base_directory + '/train_eval'
- elif FLAGS.mode == MODE_TEST:
- logdir = FLAGS.base_directory + '/test'
- else:
- raise NotImplementedError
-
- # Wait for a checkpoint to exist.
- print(train_dir)
- print(tf.train.latest_checkpoint(train_dir))
- while not tf.train.latest_checkpoint(train_dir):
- tf.logging.error('Waiting for checkpoint...')
- print('Waiting for checkpoint...')
- time.sleep(10)
-
- with tf.Graph().as_default():
- # Use a separate container for each trial
- container_name = ''
- with tf.container(container_name):
-
- # Construct the model.
- if FLAGS.num_rollouts == 1:
- model = create_MaskGAN(hparams, is_training)
- elif FLAGS.num_rollouts > 1:
- model = rollout.create_rollout_MaskGAN(hparams, is_training)
- else:
- raise ValueError
-
- # Create the supervisor. It will take care of initialization, summaries,
- # checkpoints, and recovery. We only pass the trainable variables
- # to load since things like baselines keep batch_size which may not
- # match between training and evaluation.
- evaluation_variables = tf.trainable_variables()
- evaluation_variables.append(model.global_step)
- eval_saver = tf.train.Saver(var_list=evaluation_variables)
- sv = tf.Supervisor(logdir=logdir)
- sess = sv.PrepareSession(FLAGS.eval_master, start_standard_services=False)
-
- tf.logging.info('Before sv.Loop.')
- sv.Loop(FLAGS.eval_interval_secs, evaluate_once,
- (data, sv, model, sess, train_dir, log, id_to_word,
- data_ngram_counts, eval_saver))
-
- sv.WaitForStop()
- tf.logging.info('sv.Stop().')
- sv.Stop()
-
-
-def main(_):
- hparams = create_hparams()
- train_dir = FLAGS.base_directory + '/train'
-
- # Load data set.
- if FLAGS.data_set == 'ptb':
- raw_data = ptb_loader.ptb_raw_data(FLAGS.data_dir)
- train_data, valid_data, test_data, _ = raw_data
- valid_data_flat = valid_data
- elif FLAGS.data_set == 'imdb':
- raw_data = imdb_loader.imdb_raw_data(FLAGS.data_dir)
- # TODO(liamfedus): Get an IMDB test partition.
- train_data, valid_data = raw_data
- valid_data_flat = [word for review in valid_data for word in review]
- else:
- raise NotImplementedError
-
- if FLAGS.mode == MODE_TRAIN or FLAGS.mode == MODE_TRAIN_EVAL:
- data_set = train_data
- elif FLAGS.mode == MODE_VALIDATION:
- data_set = valid_data
- elif FLAGS.mode == MODE_TEST:
- data_set = test_data
- else:
- raise NotImplementedError
-
- # Dictionary and reverse dictionry.
- if FLAGS.data_set == 'ptb':
- word_to_id = ptb_loader.build_vocab(
- os.path.join(FLAGS.data_dir, 'ptb.train.txt'))
- elif FLAGS.data_set == 'imdb':
- word_to_id = imdb_loader.build_vocab(
- os.path.join(FLAGS.data_dir, 'vocab.txt'))
- id_to_word = {v: k for k, v in word_to_id.iteritems()}
-
- # Dictionary of Training Set n-gram counts.
- bigram_tuples = n_gram.find_all_ngrams(valid_data_flat, n=2)
- trigram_tuples = n_gram.find_all_ngrams(valid_data_flat, n=3)
- fourgram_tuples = n_gram.find_all_ngrams(valid_data_flat, n=4)
-
- bigram_counts = n_gram.construct_ngrams_dict(bigram_tuples)
- trigram_counts = n_gram.construct_ngrams_dict(trigram_tuples)
- fourgram_counts = n_gram.construct_ngrams_dict(fourgram_tuples)
- print('Unique %d-grams: %d' % (2, len(bigram_counts)))
- print('Unique %d-grams: %d' % (3, len(trigram_counts)))
- print('Unique %d-grams: %d' % (4, len(fourgram_counts)))
-
- data_ngram_counts = {
- '2': bigram_counts,
- '3': trigram_counts,
- '4': fourgram_counts
- }
-
- # TODO(liamfedus): This was necessary because there was a problem with our
- # originally trained IMDB models. The EOS_INDEX was off by one, which means,
- # two words were mapping to index 86933. The presence of '' is going
- # to throw and out of vocabulary error.
- FLAGS.vocab_size = len(id_to_word)
- print('Vocab size: %d' % FLAGS.vocab_size)
-
- tf.gfile.MakeDirs(FLAGS.base_directory)
-
- if FLAGS.mode == MODE_TRAIN:
- log = tf.gfile.GFile(
- os.path.join(FLAGS.base_directory, 'train-log.txt'), mode='w')
- elif FLAGS.mode == MODE_VALIDATION:
- log = tf.gfile.GFile(
- os.path.join(FLAGS.base_directory, 'validation-log.txt'), mode='w')
- elif FLAGS.mode == MODE_TRAIN_EVAL:
- log = tf.gfile.GFile(
- os.path.join(FLAGS.base_directory, 'train_eval-log.txt'), mode='w')
- else:
- log = tf.gfile.GFile(
- os.path.join(FLAGS.base_directory, 'test-log.txt'), mode='w')
-
- if FLAGS.mode == MODE_TRAIN:
- train_model(hparams, data_set, train_dir, log, id_to_word,
- data_ngram_counts)
-
- elif FLAGS.mode == MODE_VALIDATION:
- evaluate_model(hparams, data_set, train_dir, log, id_to_word,
- data_ngram_counts)
- elif FLAGS.mode == MODE_TRAIN_EVAL:
- evaluate_model(hparams, data_set, train_dir, log, id_to_word,
- data_ngram_counts)
-
- elif FLAGS.mode == MODE_TEST:
- evaluate_model(hparams, data_set, train_dir, log, id_to_word,
- data_ngram_counts)
-
- else:
- raise NotImplementedError
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/namignizer/.gitignore b/research/namignizer/.gitignore
deleted file mode 100644
index 2dae8043534bc7a079f36caa6c673f74c39e5dfa..0000000000000000000000000000000000000000
--- a/research/namignizer/.gitignore
+++ /dev/null
@@ -1,6 +0,0 @@
-# Remove the pyc files
-*.pyc
-
-# Ignore the model and the data
-model/
-data/
diff --git a/research/namignizer/README.md b/research/namignizer/README.md
deleted file mode 100644
index 475a087541913aaa3fca9d2094b4c23de52dbb41..0000000000000000000000000000000000000000
--- a/research/namignizer/README.md
+++ /dev/null
@@ -1,86 +0,0 @@
-
-
-
-
-# Namignizer
-
-Use a variation of the [PTB](https://www.tensorflow.org/versions/r0.8/tutorials/recurrent/index.html#recurrent-neural-networks) model to recognize and generate names using the [Kaggle Baby Name Database](https://www.kaggle.com/kaggle/us-baby-names).
-
-### API
-Namignizer is implemented in Tensorflow 0.8r and uses the python package `pandas` for some data processing.
-
-#### How to use
-Download the data from Kaggle and place it in your data directory (or use the small training data provided). The example data looks like so:
-
-```
-Id,Name,Year,Gender,Count
-1,Mary,1880,F,7065
-2,Anna,1880,F,2604
-3,Emma,1880,F,2003
-4,Elizabeth,1880,F,1939
-5,Minnie,1880,F,1746
-6,Margaret,1880,F,1578
-7,Ida,1880,F,1472
-8,Alice,1880,F,1414
-9,Bertha,1880,F,1320
-```
-
-But any data with the two columns: `Name` and `Count` will work.
-
-With the data, we can then train the model:
-
-```python
-train("data/SmallNames.txt", "model/namignizer", SmallConfig)
-```
-
-And you will get the output:
-
-```
-Reading Name data in data/SmallNames.txt
-Epoch: 1 Learning rate: 1.000
-0.090 perplexity: 18.539 speed: 282 lps
-...
-0.890 perplexity: 1.478 speed: 285 lps
-0.990 perplexity: 1.477 speed: 284 lps
-Epoch: 13 Train Perplexity: 1.477
-```
-
-This will as a side effect write model checkpoints to the `model` directory. With this you will be able to determine the perplexity your model will give you for any arbitrary set of names like so:
-
-```python
-namignize(["mary", "ida", "gazorpazorp", "houyhnhnms", "bob"],
- tf.train.latest_checkpoint("model"), SmallConfig)
-```
-You will provide the same config and the same checkpoint directory. This will allow you to use a the model you just trained. You will then get a perplexity output for each name like so:
-
-```
-Name mary gives us a perplexity of 1.03105580807
-Name ida gives us a perplexity of 1.07770049572
-Name gazorpazorp gives us a perplexity of 175.940353394
-Name houyhnhnms gives us a perplexity of 9.53870773315
-Name bob gives us a perplexity of 6.03938627243
-```
-
-Finally, you will also be able generate names using the model like so:
-
-```python
-namignator(tf.train.latest_checkpoint("model"), SmallConfig)
-```
-
-Again, you will need to provide the same config and the same checkpoint directory. This will allow you to use a the model you just trained. You will then get a single generated name. Examples of output that I got when using the provided data are:
-
-```
-['b', 'e', 'r', 't', 'h', 'a', '`']
-['m', 'a', 'r', 'y', '`']
-['a', 'n', 'n', 'a', '`']
-['m', 'a', 'r', 'y', '`']
-['b', 'e', 'r', 't', 'h', 'a', '`']
-['a', 'n', 'n', 'a', '`']
-['e', 'l', 'i', 'z', 'a', 'b', 'e', 't', 'h', '`']
-```
-
-Notice that each name ends with a backtick. This marks the end of the name.
-
-### Contact Info
-
-Feel free to reach out to me at knt(at google) or k.nathaniel.tucker(at gmail)
diff --git a/research/namignizer/data_utils.py b/research/namignizer/data_utils.py
deleted file mode 100644
index 4320215026ccf7a2b31ffd476c25a153ecd92b86..0000000000000000000000000000000000000000
--- a/research/namignizer/data_utils.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Utilities for parsing Kaggle baby names files."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import os
-
-import numpy as np
-import tensorflow as tf
-import pandas as pd
-
-# the default end of name rep will be zero
-_EON = 0
-
-
-def read_names(names_path):
- """read data from downloaded file. See SmallNames.txt for example format
- or go to https://www.kaggle.com/kaggle/us-baby-names for full lists
-
- Args:
- names_path: path to the csv file similar to the example type
- Returns:
- Dataset: a namedtuple of two elements: deduped names and their associated
- counts. The names contain only 26 chars and are all lower case
- """
- names_data = pd.read_csv(names_path)
- names_data.Name = names_data.Name.str.lower()
-
- name_data = names_data.groupby(by=["Name"])["Count"].sum()
- name_counts = np.array(name_data.tolist())
- names_deduped = np.array(name_data.index.tolist())
-
- Dataset = collections.namedtuple('Dataset', ['Name', 'Count'])
- return Dataset(names_deduped, name_counts)
-
-
-def _letter_to_number(letter):
- """converts letters to numbers between 1 and 27"""
- # ord of lower case 'a' is 97
- return ord(letter) - 96
-
-
-def namignizer_iterator(names, counts, batch_size, num_steps, epoch_size):
- """Takes a list of names and counts like those output from read_names, and
- makes an iterator yielding a batch_size by num_steps array of random names
- separated by an end of name token. The names are chosen randomly according
- to their counts. The batch may end mid-name
-
- Args:
- names: a set of lowercase names composed of 26 characters
- counts: a list of the frequency of those names
- batch_size: int
- num_steps: int
- epoch_size: number of batches to yield
- Yields:
- (x, y): a batch_size by num_steps array of ints representing letters, where
- x will be the input and y will be the target
- """
- name_distribution = counts / counts.sum()
-
- for i in range(epoch_size):
- data = np.zeros(batch_size * num_steps + 1)
- samples = np.random.choice(names, size=batch_size * num_steps // 2,
- replace=True, p=name_distribution)
-
- data_index = 0
- for sample in samples:
- if data_index >= batch_size * num_steps:
- break
- for letter in map(_letter_to_number, sample) + [_EON]:
- if data_index >= batch_size * num_steps:
- break
- data[data_index] = letter
- data_index += 1
-
- x = data[:batch_size * num_steps].reshape((batch_size, num_steps))
- y = data[1:batch_size * num_steps + 1].reshape((batch_size, num_steps))
-
- yield (x, y)
-
-
-def name_to_batch(name, batch_size, num_steps):
- """ Takes a single name and fills a batch with it
-
- Args:
- name: lowercase composed of 26 characters
- batch_size: int
- num_steps: int
- Returns:
- x, y: a batch_size by num_steps array of ints representing letters, where
- x will be the input and y will be the target. The array is filled up
- to the length of the string, the rest is filled with zeros
- """
- data = np.zeros(batch_size * num_steps + 1)
-
- data_index = 0
- for letter in map(_letter_to_number, name) + [_EON]:
- data[data_index] = letter
- data_index += 1
-
- x = data[:batch_size * num_steps].reshape((batch_size, num_steps))
- y = data[1:batch_size * num_steps + 1].reshape((batch_size, num_steps))
-
- return x, y
diff --git a/research/namignizer/model.py b/research/namignizer/model.py
deleted file mode 100644
index 72c5c5ecb61e8a92ec2e74b8cc7ca13bb6ace817..0000000000000000000000000000000000000000
--- a/research/namignizer/model.py
+++ /dev/null
@@ -1,136 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""RNN model with embeddings"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-
-class NamignizerModel(object):
- """The Namignizer model ~ strongly based on PTB"""
-
- def __init__(self, is_training, config):
- self.batch_size = batch_size = config.batch_size
- self.num_steps = num_steps = config.num_steps
- size = config.hidden_size
- # will always be 27
- vocab_size = config.vocab_size
-
- # placeholders for inputs
- self._input_data = tf.placeholder(tf.int32, [batch_size, num_steps])
- self._targets = tf.placeholder(tf.int32, [batch_size, num_steps])
- # weights for the loss function
- self._weights = tf.placeholder(tf.float32, [batch_size * num_steps])
-
- # lstm for our RNN cell (GRU supported too)
- lstm_cells = []
- for layer in range(config.num_layers):
- lstm_cell = tf.contrib.rnn.BasicLSTMCell(size, forget_bias=0.0)
- if is_training and config.keep_prob < 1:
- lstm_cell = tf.contrib.rnn.DropoutWrapper(
- lstm_cell, output_keep_prob=config.keep_prob)
- lstm_cells.append(lstm_cell)
- cell = tf.contrib.rnn.MultiRNNCell(lstm_cells)
-
- self._initial_state = cell.zero_state(batch_size, tf.float32)
-
- with tf.device("/cpu:0"):
- embedding = tf.get_variable("embedding", [vocab_size, size])
- inputs = tf.nn.embedding_lookup(embedding, self._input_data)
-
- if is_training and config.keep_prob < 1:
- inputs = tf.nn.dropout(inputs, config.keep_prob)
-
- outputs = []
- state = self._initial_state
- with tf.variable_scope("RNN"):
- for time_step in range(num_steps):
- if time_step > 0:
- tf.get_variable_scope().reuse_variables()
- (cell_output, state) = cell(inputs[:, time_step, :], state)
- outputs.append(cell_output)
-
- output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, size])
- softmax_w = tf.get_variable("softmax_w", [size, vocab_size])
- softmax_b = tf.get_variable("softmax_b", [vocab_size])
- logits = tf.matmul(output, softmax_w) + softmax_b
- loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
- [logits],
- [tf.reshape(self._targets, [-1])],
- [self._weights])
- self._loss = loss
- self._cost = cost = tf.reduce_sum(loss) / batch_size
- self._final_state = state
-
- # probabilities of each letter
- self._activations = tf.nn.softmax(logits)
-
- # ability to save the model
- self.saver = tf.train.Saver(tf.global_variables())
-
- if not is_training:
- return
-
- self._lr = tf.Variable(0.0, trainable=False)
- tvars = tf.trainable_variables()
- grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
- config.max_grad_norm)
- optimizer = tf.train.GradientDescentOptimizer(self.lr)
- self._train_op = optimizer.apply_gradients(zip(grads, tvars))
-
- def assign_lr(self, session, lr_value):
- session.run(tf.assign(self.lr, lr_value))
-
- @property
- def input_data(self):
- return self._input_data
-
- @property
- def targets(self):
- return self._targets
-
- @property
- def activations(self):
- return self._activations
-
- @property
- def weights(self):
- return self._weights
-
- @property
- def initial_state(self):
- return self._initial_state
-
- @property
- def cost(self):
- return self._cost
-
- @property
- def loss(self):
- return self._loss
-
- @property
- def final_state(self):
- return self._final_state
-
- @property
- def lr(self):
- return self._lr
-
- @property
- def train_op(self):
- return self._train_op
diff --git a/research/namignizer/names.py b/research/namignizer/names.py
deleted file mode 100644
index 253742716391f2f4b7a0c0cf4987e40a2aaa808f..0000000000000000000000000000000000000000
--- a/research/namignizer/names.py
+++ /dev/null
@@ -1,259 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""A library showing off sequence recognition and generation with the simple
-example of names.
-
-We use recurrent neural nets to learn complex functions able to recognize and
-generate sequences of a given form. This can be used for natural language
-syntax recognition, dynamically generating maps or puzzles and of course
-baby name generation.
-
-Before using this module, it is recommended to read the Tensorflow tutorial on
-recurrent neural nets, as it explains the basic concepts of this model, and
-will show off another module, the PTB module on which this model bases itself.
-
-Here is an overview of the functions available in this module:
-
-* RNN Module for sequence functions based on PTB
-
-* Name recognition specifically for recognizing names, but can be adapted to
- recognizing sequence patterns
-
-* Name generations specifically for generating names, but can be adapted to
- generating arbitrary sequence patterns
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import time
-
-import tensorflow as tf
-import numpy as np
-
-from model import NamignizerModel
-import data_utils
-
-
-class SmallConfig(object):
- """Small config."""
- init_scale = 0.1
- learning_rate = 1.0
- max_grad_norm = 5
- num_layers = 2
- num_steps = 20
- hidden_size = 200
- max_epoch = 4
- max_max_epoch = 13
- keep_prob = 1.0
- lr_decay = 0.5
- batch_size = 20
- vocab_size = 27
- epoch_size = 100
-
-
-class LargeConfig(object):
- """Medium config."""
- init_scale = 0.05
- learning_rate = 1.0
- max_grad_norm = 5
- num_layers = 2
- num_steps = 35
- hidden_size = 650
- max_epoch = 6
- max_max_epoch = 39
- keep_prob = 0.5
- lr_decay = 0.8
- batch_size = 20
- vocab_size = 27
- epoch_size = 100
-
-
-class TestConfig(object):
- """Tiny config, for testing."""
- init_scale = 0.1
- learning_rate = 1.0
- max_grad_norm = 1
- num_layers = 1
- num_steps = 2
- hidden_size = 2
- max_epoch = 1
- max_max_epoch = 1
- keep_prob = 1.0
- lr_decay = 0.5
- batch_size = 20
- vocab_size = 27
- epoch_size = 100
-
-
-def run_epoch(session, m, names, counts, epoch_size, eval_op, verbose=False):
- """Runs the model on the given data for one epoch
-
- Args:
- session: the tf session holding the model graph
- m: an instance of the NamignizerModel
- names: a set of lowercase names of 26 characters
- counts: a list of the frequency of the above names
- epoch_size: the number of batches to run
- eval_op: whether to change the params or not, and how to do it
- Kwargs:
- verbose: whether to print out state of training during the epoch
- Returns:
- cost: the average cost during the last stage of the epoch
- """
- start_time = time.time()
- costs = 0.0
- iters = 0
- for step, (x, y) in enumerate(data_utils.namignizer_iterator(names, counts,
- m.batch_size, m.num_steps, epoch_size)):
-
- cost, _ = session.run([m.cost, eval_op],
- {m.input_data: x,
- m.targets: y,
- m.weights: np.ones(m.batch_size * m.num_steps)})
- costs += cost
- iters += m.num_steps
-
- if verbose and step % (epoch_size // 10) == 9:
- print("%.3f perplexity: %.3f speed: %.0f lps" %
- (step * 1.0 / epoch_size, np.exp(costs / iters),
- iters * m.batch_size / (time.time() - start_time)))
-
- if step >= epoch_size:
- break
-
- return np.exp(costs / iters)
-
-
-def train(data_dir, checkpoint_path, config):
- """Trains the model with the given data
-
- Args:
- data_dir: path to the data for the model (see data_utils for data
- format)
- checkpoint_path: the path to save the trained model checkpoints
- config: one of the above configs that specify the model and how it
- should be run and trained
- Returns:
- None
- """
- # Prepare Name data.
- print("Reading Name data in %s" % data_dir)
- names, counts = data_utils.read_names(data_dir)
-
- with tf.Graph().as_default(), tf.Session() as session:
- initializer = tf.random_uniform_initializer(-config.init_scale,
- config.init_scale)
- with tf.variable_scope("model", reuse=None, initializer=initializer):
- m = NamignizerModel(is_training=True, config=config)
-
- tf.global_variables_initializer().run()
-
- for i in range(config.max_max_epoch):
- lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
- m.assign_lr(session, config.learning_rate * lr_decay)
-
- print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
- train_perplexity = run_epoch(session, m, names, counts, config.epoch_size, m.train_op,
- verbose=True)
- print("Epoch: %d Train Perplexity: %.3f" %
- (i + 1, train_perplexity))
-
- m.saver.save(session, checkpoint_path, global_step=i)
-
-
-def namignize(names, checkpoint_path, config):
- """Recognizes names and prints the Perplexity of the model for each names
- in the list
-
- Args:
- names: a list of names in the model format
- checkpoint_path: the path to restore the trained model from, should not
- include the model name, just the path to
- config: one of the above configs that specify the model and how it
- should be run and trained
- Returns:
- None
- """
- with tf.Graph().as_default(), tf.Session() as session:
-
- with tf.variable_scope("model"):
- m = NamignizerModel(is_training=False, config=config)
-
- m.saver.restore(session, checkpoint_path)
-
- for name in names:
- x, y = data_utils.name_to_batch(name, m.batch_size, m.num_steps)
-
- cost, loss, _ = session.run([m.cost, m.loss, tf.no_op()],
- {m.input_data: x,
- m.targets: y,
- m.weights: np.concatenate((
- np.ones(len(name)), np.zeros(m.batch_size * m.num_steps - len(name))))})
-
- print("Name {} gives us a perplexity of {}".format(
- name, np.exp(cost)))
-
-
-def namignator(checkpoint_path, config):
- """Generates names randomly according to a given model
-
- Args:
- checkpoint_path: the path to restore the trained model from, should not
- include the model name, just the path to
- config: one of the above configs that specify the model and how it
- should be run and trained
- Returns:
- None
- """
- # mutate the config to become a name generator config
- config.num_steps = 1
- config.batch_size = 1
-
- with tf.Graph().as_default(), tf.Session() as session:
-
- with tf.variable_scope("model"):
- m = NamignizerModel(is_training=False, config=config)
-
- m.saver.restore(session, checkpoint_path)
-
- activations, final_state, _ = session.run([m.activations, m.final_state, tf.no_op()],
- {m.input_data: np.zeros((1, 1)),
- m.targets: np.zeros((1, 1)),
- m.weights: np.ones(1)})
-
- # sample from our softmax activations
- next_letter = np.random.choice(27, p=activations[0])
- name = [next_letter]
- while next_letter != 0:
- activations, final_state, _ = session.run([m.activations, m.final_state, tf.no_op()],
- {m.input_data: [[next_letter]],
- m.targets: np.zeros((1, 1)),
- m.initial_state: final_state,
- m.weights: np.ones(1)})
-
- next_letter = np.random.choice(27, p=activations[0])
- name += [next_letter]
-
- print(map(lambda x: chr(x + 96), name))
-
-
-if __name__ == "__main__":
- train("data/SmallNames.txt", "model/namignizer", SmallConfig)
-
- namignize(["mary", "ida", "gazorbazorb", "mmmhmm", "bob"],
- tf.train.latest_checkpoint("model"), SmallConfig)
-
- namignator(tf.train.latest_checkpoint("model"), SmallConfig)
diff --git a/research/neural_gpu/README.md b/research/neural_gpu/README.md
deleted file mode 100644
index 097ef318c4e071f59e4212b0cd901907758d73e7..0000000000000000000000000000000000000000
--- a/research/neural_gpu/README.md
+++ /dev/null
@@ -1,87 +0,0 @@
-
-
-
-
-# NeuralGPU
-Code for the Neural GPU model described in http://arxiv.org/abs/1511.08228.
-The extended version was described in https://arxiv.org/abs/1610.08613.
-
-Requirements:
-* TensorFlow (see tensorflow.org for how to install)
-
-The model can be trained on the following algorithmic tasks:
-
-* `sort` - Sort a symbol list
-* `kvsort` - Sort symbol keys in dictionary
-* `id` - Return the same symbol list
-* `rev` - Reverse a symbol list
-* `rev2` - Reverse a symbol dictionary by key
-* `incr` - Add one to a symbol value
-* `add` - Long decimal addition
-* `left` - First symbol in list
-* `right` - Last symbol in list
-* `left-shift` - Left shift a symbol list
-* `right-shift` - Right shift a symbol list
-* `bmul` - Long binary multiplication
-* `mul` - Long decimal multiplication
-* `dup` - Duplicate a symbol list with padding
-* `badd` - Long binary addition
-* `qadd` - Long quaternary addition
-* `search` - Search for symbol key in dictionary
-
-It can also be trained on the WMT English-French translation task:
-
-* `wmt` - WMT English-French translation (data will be downloaded)
-
-The value range for symbols are defined by the `vocab_size` flag.
-In particular, the values are in the range `vocab_size - 1`.
-So if you set `--vocab_size=16` (the default) then `--problem=rev`
-will be reversing lists of 15 symbols, and `--problem=id` will be identity
-on a list of up to 15 symbols.
-
-
-To train the model on the binary multiplication task run:
-
-```
-python neural_gpu_trainer.py --problem=bmul
-```
-
-This trains the Extended Neural GPU, to train the original model run:
-
-```
-python neural_gpu_trainer.py --problem=bmul --beam_size=0
-```
-
-While training, interim / checkpoint model parameters will be
-written to `/tmp/neural_gpu/`.
-
-Once the amount of error gets down to what you're comfortable
-with, hit `Ctrl-C` to stop the training process. The latest
-model parameters will be in `/tmp/neural_gpu/neural_gpu.ckpt-`
-and used on any subsequent run.
-
-To evaluate a trained model on how well it decodes run:
-
-```
-python neural_gpu_trainer.py --problem=bmul --mode=1
-```
-
-To interact with a model (experimental, see code) run:
-
-```
-python neural_gpu_trainer.py --problem=bmul --mode=2
-```
-
-To train on WMT data, set a larger --nmaps and --vocab_size and avoid curriculum:
-
-```
-python neural_gpu_trainer.py --problem=wmt --vocab_size=32768 --nmaps=256
- --vec_size=256 --curriculum_seq=1.0 --max_length=60 --data_dir ~/wmt
-```
-
-With less memory, try lower batch size, e.g. `--batch_size=4`. With more GPUs
-in your system, there will be a batch on every GPU so you can run larger models.
-For example, `--batch_size=4 --num_gpus=4 --nmaps=512 --vec_size=512` will
-run a large model (512-size) on 4 GPUs, with effective batches of 4*4=16.
-
-Maintained by Lukasz Kaiser (lukaszkaiser)
diff --git a/research/neural_gpu/data_utils.py b/research/neural_gpu/data_utils.py
deleted file mode 100644
index 3c14ff701fce79408fde6505239530dc5b848dd7..0000000000000000000000000000000000000000
--- a/research/neural_gpu/data_utils.py
+++ /dev/null
@@ -1,458 +0,0 @@
-# Copyright 2015 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Neural GPU -- data generation and batching utilities."""
-
-import math
-import os
-import random
-import sys
-import time
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import program_utils
-
-FLAGS = tf.app.flags.FLAGS
-
-bins = [2 + bin_idx_i for bin_idx_i in xrange(256)]
-all_tasks = ["sort", "kvsort", "id", "rev", "rev2", "incr", "add", "left",
- "right", "left-shift", "right-shift", "bmul", "mul", "dup",
- "badd", "qadd", "search", "progeval", "progsynth"]
-log_filename = ""
-vocab, rev_vocab = None, None
-
-
-def pad(l):
- for b in bins:
- if b >= l: return b
- return bins[-1]
-
-
-def bin_for(l):
- for i, b in enumerate(bins):
- if b >= l: return i
- return len(bins) - 1
-
-
-train_set = {}
-test_set = {}
-for some_task in all_tasks:
- train_set[some_task] = []
- test_set[some_task] = []
- for all_max_len in xrange(10000):
- train_set[some_task].append([])
- test_set[some_task].append([])
-
-
-def read_tmp_file(name):
- """Read from a file with the given name in our log directory or above."""
- dirname = os.path.dirname(log_filename)
- fname = os.path.join(dirname, name + ".txt")
- if not tf.gfile.Exists(fname):
- print_out("== not found file: " + fname)
- fname = os.path.join(dirname, "../" + name + ".txt")
- if not tf.gfile.Exists(fname):
- print_out("== not found file: " + fname)
- fname = os.path.join(dirname, "../../" + name + ".txt")
- if not tf.gfile.Exists(fname):
- print_out("== not found file: " + fname)
- return None
- print_out("== found file: " + fname)
- res = []
- with tf.gfile.GFile(fname, mode="r") as f:
- for line in f:
- res.append(line.strip())
- return res
-
-
-def write_tmp_file(name, lines):
- dirname = os.path.dirname(log_filename)
- fname = os.path.join(dirname, name + ".txt")
- with tf.gfile.GFile(fname, mode="w") as f:
- for line in lines:
- f.write(line + "\n")
-
-
-def add(n1, n2, base=10):
- """Add two numbers represented as lower-endian digit lists."""
- k = max(len(n1), len(n2)) + 1
- d1 = n1 + [0 for _ in xrange(k - len(n1))]
- d2 = n2 + [0 for _ in xrange(k - len(n2))]
- res = []
- carry = 0
- for i in xrange(k):
- if d1[i] + d2[i] + carry < base:
- res.append(d1[i] + d2[i] + carry)
- carry = 0
- else:
- res.append(d1[i] + d2[i] + carry - base)
- carry = 1
- while res and res[-1] == 0:
- res = res[:-1]
- if res: return res
- return [0]
-
-
-def init_data(task, length, nbr_cases, nclass):
- """Data initialization."""
- def rand_pair(l, task):
- """Random data pair for a task. Total length should be <= l."""
- k = int((l-1)/2)
- base = 10
- if task[0] == "b": base = 2
- if task[0] == "q": base = 4
- d1 = [np.random.randint(base) for _ in xrange(k)]
- d2 = [np.random.randint(base) for _ in xrange(k)]
- if task in ["add", "badd", "qadd"]:
- res = add(d1, d2, base)
- elif task in ["mul", "bmul"]:
- d1n = sum([d * (base ** i) for i, d in enumerate(d1)])
- d2n = sum([d * (base ** i) for i, d in enumerate(d2)])
- if task == "bmul":
- res = [int(x) for x in list(reversed(str(bin(d1n * d2n))))[:-2]]
- else:
- res = [int(x) for x in list(reversed(str(d1n * d2n)))]
- else:
- sys.exit()
- sep = [12]
- if task in ["add", "badd", "qadd"]: sep = [11]
- inp = [d + 1 for d in d1] + sep + [d + 1 for d in d2]
- return inp, [r + 1 for r in res]
-
- def rand_dup_pair(l):
- """Random data pair for duplication task. Total length should be <= l."""
- k = int(l/2)
- x = [np.random.randint(nclass - 1) + 1 for _ in xrange(k)]
- inp = x + [0 for _ in xrange(l - k)]
- res = x + x + [0 for _ in xrange(l - 2*k)]
- return inp, res
-
- def rand_rev2_pair(l):
- """Random data pair for reverse2 task. Total length should be <= l."""
- inp = [(np.random.randint(nclass - 1) + 1,
- np.random.randint(nclass - 1) + 1) for _ in xrange(l/2)]
- res = [i for i in reversed(inp)]
- return [x for p in inp for x in p], [x for p in res for x in p]
-
- def rand_search_pair(l):
- """Random data pair for search task. Total length should be <= l."""
- inp = [(np.random.randint(nclass - 1) + 1,
- np.random.randint(nclass - 1) + 1) for _ in xrange(l-1/2)]
- q = np.random.randint(nclass - 1) + 1
- res = 0
- for (k, v) in reversed(inp):
- if k == q:
- res = v
- return [x for p in inp for x in p] + [q], [res]
-
- def rand_kvsort_pair(l):
- """Random data pair for key-value sort. Total length should be <= l."""
- keys = [(np.random.randint(nclass - 1) + 1, i) for i in xrange(l/2)]
- vals = [np.random.randint(nclass - 1) + 1 for _ in xrange(l/2)]
- kv = [(k, vals[i]) for (k, i) in keys]
- sorted_kv = [(k, vals[i]) for (k, i) in sorted(keys)]
- return [x for p in kv for x in p], [x for p in sorted_kv for x in p]
-
- def prog_io_pair(prog, max_len, counter=0):
- try:
- ilen = np.random.randint(max_len - 3) + 1
- bound = max(15 - (counter / 20), 1)
- inp = [random.choice(range(-bound, bound)) for _ in range(ilen)]
- inp_toks = [program_utils.prog_rev_vocab[t]
- for t in program_utils.tokenize(str(inp)) if t != ","]
- out = program_utils.evaluate(prog, {"a": inp})
- out_toks = [program_utils.prog_rev_vocab[t]
- for t in program_utils.tokenize(str(out)) if t != ","]
- if counter > 400:
- out_toks = []
- if (out_toks and out_toks[0] == program_utils.prog_rev_vocab["["] and
- len(out_toks) != len([o for o in out if o == ","]) + 3):
- raise ValueError("generated list with too long ints")
- if (out_toks and out_toks[0] != program_utils.prog_rev_vocab["["] and
- len(out_toks) > 1):
- raise ValueError("generated one int but tokenized it to many")
- if len(out_toks) > max_len:
- raise ValueError("output too long")
- return (inp_toks, out_toks)
- except ValueError:
- return prog_io_pair(prog, max_len, counter+1)
-
- def spec(inp):
- """Return the target given the input for some tasks."""
- if task == "sort":
- return sorted(inp)
- elif task == "id":
- return inp
- elif task == "rev":
- return [i for i in reversed(inp)]
- elif task == "incr":
- carry = 1
- res = []
- for i in xrange(len(inp)):
- if inp[i] + carry < nclass:
- res.append(inp[i] + carry)
- carry = 0
- else:
- res.append(1)
- carry = 1
- return res
- elif task == "left":
- return [inp[0]]
- elif task == "right":
- return [inp[-1]]
- elif task == "left-shift":
- return [inp[l-1] for l in xrange(len(inp))]
- elif task == "right-shift":
- return [inp[l+1] for l in xrange(len(inp))]
- else:
- print_out("Unknown spec for task " + str(task))
- sys.exit()
-
- l = length
- cur_time = time.time()
- total_time = 0.0
-
- is_prog = task in ["progeval", "progsynth"]
- if is_prog:
- inputs_per_prog = 5
- program_utils.make_vocab()
- progs = read_tmp_file("programs_len%d" % (l / 10))
- if not progs:
- progs = program_utils.gen(l / 10, 1.2 * nbr_cases / inputs_per_prog)
- write_tmp_file("programs_len%d" % (l / 10), progs)
- prog_ios = read_tmp_file("programs_len%d_io" % (l / 10))
- nbr_cases = min(nbr_cases, len(progs) * inputs_per_prog) / 1.2
- if not prog_ios:
- # Generate program io data.
- prog_ios = []
- for pidx, prog in enumerate(progs):
- if pidx % 500 == 0:
- print_out("== generating io pairs for program %d" % pidx)
- if pidx * inputs_per_prog > nbr_cases * 1.2:
- break
- ptoks = [program_utils.prog_rev_vocab[t]
- for t in program_utils.tokenize(prog)]
- ptoks.append(program_utils.prog_rev_vocab["_EOS"])
- plen = len(ptoks)
- for _ in xrange(inputs_per_prog):
- if task == "progeval":
- inp, out = prog_io_pair(prog, plen)
- prog_ios.append(str(inp) + "\t" + str(out) + "\t" + prog)
- elif task == "progsynth":
- plen = max(len(ptoks), 8)
- for _ in xrange(3):
- inp, out = prog_io_pair(prog, plen / 2)
- prog_ios.append(str(inp) + "\t" + str(out) + "\t" + prog)
- write_tmp_file("programs_len%d_io" % (l / 10), prog_ios)
- prog_ios_dict = {}
- for s in prog_ios:
- i, o, p = s.split("\t")
- i_clean = "".join([c for c in i if c.isdigit() or c == " "])
- o_clean = "".join([c for c in o if c.isdigit() or c == " "])
- inp = [int(x) for x in i_clean.split()]
- out = [int(x) for x in o_clean.split()]
- if inp and out:
- if p in prog_ios_dict:
- prog_ios_dict[p].append([inp, out])
- else:
- prog_ios_dict[p] = [[inp, out]]
- # Use prog_ios_dict to create data.
- progs = []
- for prog in prog_ios_dict:
- if len([c for c in prog if c == ";"]) <= (l / 10):
- progs.append(prog)
- nbr_cases = min(nbr_cases, len(progs) * inputs_per_prog) / 1.2
- print_out("== %d training cases on %d progs" % (nbr_cases, len(progs)))
- for pidx, prog in enumerate(progs):
- if pidx * inputs_per_prog > nbr_cases * 1.2:
- break
- ptoks = [program_utils.prog_rev_vocab[t]
- for t in program_utils.tokenize(prog)]
- ptoks.append(program_utils.prog_rev_vocab["_EOS"])
- plen = len(ptoks)
- dset = train_set if pidx < nbr_cases / inputs_per_prog else test_set
- for _ in xrange(inputs_per_prog):
- if task == "progeval":
- inp, out = prog_ios_dict[prog].pop()
- dset[task][bin_for(plen)].append([[ptoks, inp, [], []], [out]])
- elif task == "progsynth":
- plen, ilist = max(len(ptoks), 8), [[]]
- for _ in xrange(3):
- inp, out = prog_ios_dict[prog].pop()
- ilist.append(inp + out)
- dset[task][bin_for(plen)].append([ilist, [ptoks]])
-
- for case in xrange(0 if is_prog else nbr_cases):
- total_time += time.time() - cur_time
- cur_time = time.time()
- if l > 10000 and case % 100 == 1:
- print_out(" avg gen time %.4f s" % (total_time / float(case)))
- if task in ["add", "badd", "qadd", "bmul", "mul"]:
- i, t = rand_pair(l, task)
- train_set[task][bin_for(len(i))].append([[[], i, [], []], [t]])
- i, t = rand_pair(l, task)
- test_set[task][bin_for(len(i))].append([[[], i, [], []], [t]])
- elif task == "dup":
- i, t = rand_dup_pair(l)
- train_set[task][bin_for(len(i))].append([[i], [t]])
- i, t = rand_dup_pair(l)
- test_set[task][bin_for(len(i))].append([[i], [t]])
- elif task == "rev2":
- i, t = rand_rev2_pair(l)
- train_set[task][bin_for(len(i))].append([[i], [t]])
- i, t = rand_rev2_pair(l)
- test_set[task][bin_for(len(i))].append([[i], [t]])
- elif task == "search":
- i, t = rand_search_pair(l)
- train_set[task][bin_for(len(i))].append([[i], [t]])
- i, t = rand_search_pair(l)
- test_set[task][bin_for(len(i))].append([[i], [t]])
- elif task == "kvsort":
- i, t = rand_kvsort_pair(l)
- train_set[task][bin_for(len(i))].append([[i], [t]])
- i, t = rand_kvsort_pair(l)
- test_set[task][bin_for(len(i))].append([[i], [t]])
- elif task not in ["progeval", "progsynth"]:
- inp = [np.random.randint(nclass - 1) + 1 for i in xrange(l)]
- target = spec(inp)
- train_set[task][bin_for(l)].append([[inp], [target]])
- inp = [np.random.randint(nclass - 1) + 1 for i in xrange(l)]
- target = spec(inp)
- test_set[task][bin_for(l)].append([[inp], [target]])
-
-
-def to_symbol(i):
- """Covert ids to text."""
- if i == 0: return ""
- if i == 11: return "+"
- if i == 12: return "*"
- return str(i-1)
-
-
-def to_id(s):
- """Covert text to ids."""
- if s == "+": return 11
- if s == "*": return 12
- return int(s) + 1
-
-
-def get_batch(bin_id, batch_size, data_set, height, offset=None, preset=None):
- """Get a batch of data, training or testing."""
- inputs, targets = [], []
- pad_length = bins[bin_id]
- for b in xrange(batch_size):
- if preset is None:
- elem = random.choice(data_set[bin_id])
- if offset is not None and offset + b < len(data_set[bin_id]):
- elem = data_set[bin_id][offset + b]
- else:
- elem = preset
- inpt, targett, inpl, targetl = elem[0], elem[1], [], []
- for inp in inpt:
- inpl.append(inp + [0 for _ in xrange(pad_length - len(inp))])
- if len(inpl) == 1:
- for _ in xrange(height - 1):
- inpl.append([0 for _ in xrange(pad_length)])
- for target in targett:
- targetl.append(target + [0 for _ in xrange(pad_length - len(target))])
- inputs.append(inpl)
- targets.append(targetl)
- res_input = np.array(inputs, dtype=np.int32)
- res_target = np.array(targets, dtype=np.int32)
- assert list(res_input.shape) == [batch_size, height, pad_length]
- assert list(res_target.shape) == [batch_size, 1, pad_length]
- return res_input, res_target
-
-
-def print_out(s, newline=True):
- """Print a message out and log it to file."""
- if log_filename:
- try:
- with tf.gfile.GFile(log_filename, mode="a") as f:
- f.write(s + ("\n" if newline else ""))
- # pylint: disable=bare-except
- except:
- sys.stderr.write("Error appending to %s\n" % log_filename)
- sys.stdout.write(s + ("\n" if newline else ""))
- sys.stdout.flush()
-
-
-def decode(output):
- return [np.argmax(o, axis=1) for o in output]
-
-
-def accuracy(inpt_t, output, target_t, batch_size, nprint,
- beam_out=None, beam_scores=None):
- """Calculate output accuracy given target."""
- assert nprint < batch_size + 1
- inpt = []
- for h in xrange(inpt_t.shape[1]):
- inpt.extend([inpt_t[:, h, l] for l in xrange(inpt_t.shape[2])])
- target = [target_t[:, 0, l] for l in xrange(target_t.shape[2])]
- def tok(i):
- if rev_vocab and i < len(rev_vocab):
- return rev_vocab[i]
- return str(i - 1)
- def task_print(inp, output, target):
- stop_bound = 0
- print_len = 0
- while print_len < len(target) and target[print_len] > stop_bound:
- print_len += 1
- print_out(" i: " + " ".join([tok(i) for i in inp if i > 0]))
- print_out(" o: " +
- " ".join([tok(output[l]) for l in xrange(print_len)]))
- print_out(" t: " +
- " ".join([tok(target[l]) for l in xrange(print_len)]))
- decoded_target = target
- decoded_output = decode(output)
- # Use beam output if given and score is high enough.
- if beam_out is not None:
- for b in xrange(batch_size):
- if beam_scores[b] >= 10.0:
- for l in xrange(min(len(decoded_output), beam_out.shape[2])):
- decoded_output[l][b] = int(beam_out[b, 0, l])
- total = 0
- errors = 0
- seq = [0 for b in xrange(batch_size)]
- for l in xrange(len(decoded_output)):
- for b in xrange(batch_size):
- if decoded_target[l][b] > 0:
- total += 1
- if decoded_output[l][b] != decoded_target[l][b]:
- seq[b] = 1
- errors += 1
- e = 0 # Previous error index
- for _ in xrange(min(nprint, sum(seq))):
- while seq[e] == 0:
- e += 1
- task_print([inpt[l][e] for l in xrange(len(inpt))],
- [decoded_output[l][e] for l in xrange(len(decoded_target))],
- [decoded_target[l][e] for l in xrange(len(decoded_target))])
- e += 1
- for b in xrange(nprint - errors):
- task_print([inpt[l][b] for l in xrange(len(inpt))],
- [decoded_output[l][b] for l in xrange(len(decoded_target))],
- [decoded_target[l][b] for l in xrange(len(decoded_target))])
- return errors, total, sum(seq)
-
-
-def safe_exp(x):
- perp = 10000
- x = float(x)
- if x < 100: perp = math.exp(x)
- if perp > 10000: return 10000
- return perp
diff --git a/research/neural_gpu/neural_gpu.py b/research/neural_gpu/neural_gpu.py
deleted file mode 100644
index 55b2b3e99224b31c672014195e9ef23fa1e892f7..0000000000000000000000000000000000000000
--- a/research/neural_gpu/neural_gpu.py
+++ /dev/null
@@ -1,747 +0,0 @@
-# Copyright 2015 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""The Neural GPU Model."""
-
-import time
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from tensorflow.python.framework import function
-import data_utils as data
-
-do_jit = False # Gives more speed but experimental for now.
-jit_scope = tf.contrib.compiler.jit.experimental_jit_scope
-
-
-def conv_linear(args, kw, kh, nin, nout, rate, do_bias, bias_start, prefix):
- """Convolutional linear map."""
- if not isinstance(args, (list, tuple)):
- args = [args]
- with tf.variable_scope(prefix):
- with tf.device("/cpu:0"):
- k = tf.get_variable("CvK", [kw, kh, nin, nout])
- if len(args) == 1:
- arg = args[0]
- else:
- arg = tf.concat(axis=3, values=args)
- res = tf.nn.convolution(arg, k, dilation_rate=(rate, 1), padding="SAME")
- if not do_bias: return res
- with tf.device("/cpu:0"):
- bias_term = tf.get_variable(
- "CvB", [nout], initializer=tf.constant_initializer(bias_start))
- bias_term = tf.reshape(bias_term, [1, 1, 1, nout])
- return res + bias_term
-
-
-def sigmoid_cutoff(x, cutoff):
- """Sigmoid with cutoff, e.g., 1.2sigmoid(x) - 0.1."""
- y = tf.sigmoid(x)
- if cutoff < 1.01: return y
- d = (cutoff - 1.0) / 2.0
- return tf.minimum(1.0, tf.maximum(0.0, cutoff * y - d), name="cutoff_min")
-
-
-@function.Defun(tf.float32, noinline=True)
-def sigmoid_cutoff_12(x):
- """Sigmoid with cutoff 1.2, specialized for speed and memory use."""
- y = tf.sigmoid(x)
- return tf.minimum(1.0, tf.maximum(0.0, 1.2 * y - 0.1), name="cutoff_min_12")
-
-
-@function.Defun(tf.float32, noinline=True)
-def sigmoid_hard(x):
- """Hard sigmoid."""
- return tf.minimum(1.0, tf.maximum(0.0, 0.25 * x + 0.5))
-
-
-def place_at14(decided, selected, it):
- """Place selected at it-th coordinate of decided, dim=1 of 4."""
- slice1 = decided[:, :it, :, :]
- slice2 = decided[:, it + 1:, :, :]
- return tf.concat(axis=1, values=[slice1, selected, slice2])
-
-
-def place_at13(decided, selected, it):
- """Place selected at it-th coordinate of decided, dim=1 of 3."""
- slice1 = decided[:, :it, :]
- slice2 = decided[:, it + 1:, :]
- return tf.concat(axis=1, values=[slice1, selected, slice2])
-
-
-def tanh_cutoff(x, cutoff):
- """Tanh with cutoff, e.g., 1.1tanh(x) cut to [-1. 1]."""
- y = tf.tanh(x)
- if cutoff < 1.01: return y
- d = (cutoff - 1.0) / 2.0
- return tf.minimum(1.0, tf.maximum(-1.0, (1.0 + d) * y))
-
-
-@function.Defun(tf.float32, noinline=True)
-def tanh_hard(x):
- """Hard tanh."""
- return tf.minimum(1.0, tf.maximum(0.0, x))
-
-
-def layer_norm(x, nmaps, prefix, epsilon=1e-5):
- """Layer normalize the 4D tensor x, averaging over the last dimension."""
- with tf.variable_scope(prefix):
- scale = tf.get_variable("layer_norm_scale", [nmaps],
- initializer=tf.ones_initializer())
- bias = tf.get_variable("layer_norm_bias", [nmaps],
- initializer=tf.zeros_initializer())
- mean, variance = tf.nn.moments(x, [3], keep_dims=True)
- norm_x = (x - mean) / tf.sqrt(variance + epsilon)
- return norm_x * scale + bias
-
-
-def conv_gru(inpts, mem, kw, kh, nmaps, rate, cutoff, prefix, do_layer_norm,
- args_len=None):
- """Convolutional GRU."""
- def conv_lin(args, suffix, bias_start):
- total_args_len = args_len or len(args) * nmaps
- res = conv_linear(args, kw, kh, total_args_len, nmaps, rate, True,
- bias_start, prefix + "/" + suffix)
- if do_layer_norm:
- return layer_norm(res, nmaps, prefix + "/" + suffix)
- else:
- return res
- if cutoff == 1.2:
- reset = sigmoid_cutoff_12(conv_lin(inpts + [mem], "r", 1.0))
- gate = sigmoid_cutoff_12(conv_lin(inpts + [mem], "g", 1.0))
- elif cutoff > 10:
- reset = sigmoid_hard(conv_lin(inpts + [mem], "r", 1.0))
- gate = sigmoid_hard(conv_lin(inpts + [mem], "g", 1.0))
- else:
- reset = sigmoid_cutoff(conv_lin(inpts + [mem], "r", 1.0), cutoff)
- gate = sigmoid_cutoff(conv_lin(inpts + [mem], "g", 1.0), cutoff)
- if cutoff > 10:
- candidate = tanh_hard(conv_lin(inpts + [reset * mem], "c", 0.0))
- else:
- # candidate = tanh_cutoff(conv_lin(inpts + [reset * mem], "c", 0.0), cutoff)
- candidate = tf.tanh(conv_lin(inpts + [reset * mem], "c", 0.0))
- return gate * mem + (1 - gate) * candidate
-
-
-CHOOSE_K = 256
-
-
-def memory_call(q, l, nmaps, mem_size, vocab_size, num_gpus, update_mem):
- raise ValueError("Fill for experiments with additional memory structures.")
-
-
-def memory_run(step, nmaps, mem_size, batch_size, vocab_size,
- global_step, do_training, update_mem, decay_factor, num_gpus,
- target_emb_weights, output_w, gpu_targets_tn, it):
- """Run memory."""
- q = step[:, 0, it, :]
- mlabels = gpu_targets_tn[:, it, 0]
- res, mask, mem_loss = memory_call(
- q, mlabels, nmaps, mem_size, vocab_size, num_gpus, update_mem)
- res = tf.gather(target_emb_weights, res) * tf.expand_dims(mask[:, 0], 1)
-
- # Mix gold and original in the first steps, 20% later.
- gold = tf.nn.dropout(tf.gather(target_emb_weights, mlabels), 0.7)
- use_gold = 1.0 - tf.cast(global_step, tf.float32) / (1000. * decay_factor)
- use_gold = tf.maximum(use_gold, 0.2) * do_training
- mem = tf.cond(tf.less(tf.random_uniform([]), use_gold),
- lambda: use_gold * gold + (1.0 - use_gold) * res,
- lambda: res)
- mem = tf.reshape(mem, [-1, 1, 1, nmaps])
- return mem, mem_loss, update_mem
-
-
-@tf.RegisterGradient("CustomIdG")
-def _custom_id_grad(_, grads):
- return grads
-
-
-def quantize(t, quant_scale, max_value=1.0):
- """Quantize a tensor t with each element in [-max_value, max_value]."""
- t = tf.minimum(max_value, tf.maximum(t, -max_value))
- big = quant_scale * (t + max_value) + 0.5
- with tf.get_default_graph().gradient_override_map({"Floor": "CustomIdG"}):
- res = (tf.floor(big) / quant_scale) - max_value
- return res
-
-
-def quantize_weights_op(quant_scale, max_value):
- ops = [v.assign(quantize(v, quant_scale, float(max_value)))
- for v in tf.trainable_variables()]
- return tf.group(*ops)
-
-
-def autoenc_quantize(x, nbits, nmaps, do_training, layers=1):
- """Autoencoder into nbits vectors of bits, using noise and sigmoids."""
- enc_x = tf.reshape(x, [-1, nmaps])
- for i in xrange(layers - 1):
- enc_x = tf.layers.dense(enc_x, nmaps, name="autoenc_%d" % i)
- enc_x = tf.layers.dense(enc_x, nbits, name="autoenc_%d" % (layers - 1))
- noise = tf.truncated_normal(tf.shape(enc_x), stddev=2.0)
- dec_x = sigmoid_cutoff_12(enc_x + noise * do_training)
- dec_x = tf.reshape(dec_x, [-1, nbits])
- for i in xrange(layers):
- dec_x = tf.layers.dense(dec_x, nmaps, name="autodec_%d" % i)
- return tf.reshape(dec_x, tf.shape(x))
-
-
-def make_dense(targets, noclass, low_param):
- """Move a batch of targets to a dense 1-hot representation."""
- low = low_param / float(noclass - 1)
- high = 1.0 - low * (noclass - 1)
- targets = tf.cast(targets, tf.int64)
- return tf.one_hot(targets, depth=noclass, on_value=high, off_value=low)
-
-
-def reorder_beam(beam_size, batch_size, beam_val, output, is_first,
- tensors_to_reorder):
- """Reorder to minimize beam costs."""
- # beam_val is [batch_size x beam_size]; let b = batch_size * beam_size
- # decided is len x b x a x b
- # output is b x out_size; step is b x len x a x b;
- outputs = tf.split(axis=0, num_or_size_splits=beam_size, value=tf.nn.log_softmax(output))
- all_beam_vals, all_beam_idx = [], []
- beam_range = 1 if is_first else beam_size
- for i in xrange(beam_range):
- top_out, top_out_idx = tf.nn.top_k(outputs[i], k=beam_size)
- cur_beam_val = beam_val[:, i]
- top_out = tf.Print(top_out, [top_out, top_out_idx, beam_val, i,
- cur_beam_val], "GREPO", summarize=8)
- all_beam_vals.append(top_out + tf.expand_dims(cur_beam_val, 1))
- all_beam_idx.append(top_out_idx)
- all_beam_idx = tf.reshape(tf.transpose(tf.concat(axis=1, values=all_beam_idx), [1, 0]),
- [-1])
- top_beam, top_beam_idx = tf.nn.top_k(tf.concat(axis=1, values=all_beam_vals), k=beam_size)
- top_beam_idx = tf.Print(top_beam_idx, [top_beam, top_beam_idx],
- "GREP", summarize=8)
- reordered = [[] for _ in xrange(len(tensors_to_reorder) + 1)]
- top_out_idx = []
- for i in xrange(beam_size):
- which_idx = top_beam_idx[:, i] * batch_size + tf.range(batch_size)
- top_out_idx.append(tf.gather(all_beam_idx, which_idx))
- which_beam = top_beam_idx[:, i] / beam_size # [batch]
- which_beam = which_beam * batch_size + tf.range(batch_size)
- reordered[0].append(tf.gather(output, which_beam))
- for i, t in enumerate(tensors_to_reorder):
- reordered[i + 1].append(tf.gather(t, which_beam))
- new_tensors = [tf.concat(axis=0, values=t) for t in reordered]
- top_out_idx = tf.concat(axis=0, values=top_out_idx)
- return (top_beam, new_tensors[0], top_out_idx, new_tensors[1:])
-
-
-class NeuralGPU(object):
- """Neural GPU Model."""
-
- def __init__(self, nmaps, vec_size, niclass, noclass, dropout,
- max_grad_norm, cutoff, nconvs, kw, kh, height, mem_size,
- learning_rate, min_length, num_gpus, num_replicas,
- grad_noise_scale, sampling_rate, act_noise=0.0, do_rnn=False,
- atrous=False, beam_size=1, backward=True, do_layer_norm=False,
- autoenc_decay=1.0):
- # Feeds for parameters and ops to update them.
- self.nmaps = nmaps
- if backward:
- self.global_step = tf.Variable(0, trainable=False, name="global_step")
- self.cur_length = tf.Variable(min_length, trainable=False)
- self.cur_length_incr_op = self.cur_length.assign_add(1)
- self.lr = tf.Variable(learning_rate, trainable=False)
- self.lr_decay_op = self.lr.assign(self.lr * 0.995)
- self.do_training = tf.placeholder(tf.float32, name="do_training")
- self.update_mem = tf.placeholder(tf.int32, name="update_mem")
- self.noise_param = tf.placeholder(tf.float32, name="noise_param")
-
- # Feeds for inputs, targets, outputs, losses, etc.
- self.input = tf.placeholder(tf.int32, name="inp")
- self.target = tf.placeholder(tf.int32, name="tgt")
- self.prev_step = tf.placeholder(tf.float32, name="prev_step")
- gpu_input = tf.split(axis=0, num_or_size_splits=num_gpus, value=self.input)
- gpu_target = tf.split(axis=0, num_or_size_splits=num_gpus, value=self.target)
- gpu_prev_step = tf.split(axis=0, num_or_size_splits=num_gpus, value=self.prev_step)
- batch_size = tf.shape(gpu_input[0])[0]
-
- if backward:
- adam_lr = 0.005 * self.lr
- adam = tf.train.AdamOptimizer(adam_lr, epsilon=1e-3)
-
- def adam_update(grads):
- return adam.apply_gradients(zip(grads, tf.trainable_variables()),
- global_step=self.global_step,
- name="adam_update")
-
- # When switching from Adam to SGD we perform reverse-decay.
- if backward:
- global_step_float = tf.cast(self.global_step, tf.float32)
- sampling_decay_exponent = global_step_float / 100000.0
- sampling_decay = tf.maximum(0.05, tf.pow(0.5, sampling_decay_exponent))
- self.sampling = sampling_rate * 0.05 / sampling_decay
- else:
- self.sampling = tf.constant(0.0)
-
- # Cache variables on cpu if needed.
- if num_replicas > 1 or num_gpus > 1:
- with tf.device("/cpu:0"):
- caching_const = tf.constant(0)
- tf.get_variable_scope().set_caching_device(caching_const.op.device)
- # partitioner = tf.variable_axis_size_partitioner(1024*256*4)
- # tf.get_variable_scope().set_partitioner(partitioner)
-
- def gpu_avg(l):
- if l[0] is None:
- for elem in l:
- assert elem is None
- return 0.0
- if len(l) < 2:
- return l[0]
- return sum(l) / float(num_gpus)
-
- self.length_tensor = tf.placeholder(tf.int32, name="length")
-
- with tf.device("/cpu:0"):
- emb_weights = tf.get_variable(
- "embedding", [niclass, vec_size],
- initializer=tf.random_uniform_initializer(-1.7, 1.7))
- if beam_size > 0:
- target_emb_weights = tf.get_variable(
- "target_embedding", [noclass, nmaps],
- initializer=tf.random_uniform_initializer(-1.7, 1.7))
- e0 = tf.scatter_update(emb_weights,
- tf.constant(0, dtype=tf.int32, shape=[1]),
- tf.zeros([1, vec_size]))
- output_w = tf.get_variable("output_w", [nmaps, noclass], tf.float32)
-
- def conv_rate(layer):
- if atrous:
- return 2**layer
- return 1
-
- # pylint: disable=cell-var-from-loop
- def enc_step(step):
- """Encoder step."""
- if autoenc_decay < 1.0:
- quant_step = autoenc_quantize(step, 16, nmaps, self.do_training)
- if backward:
- exp_glob = tf.train.exponential_decay(1.0, self.global_step - 10000,
- 1000, autoenc_decay)
- dec_factor = 1.0 - exp_glob # * self.do_training
- dec_factor = tf.cond(tf.less(self.global_step, 10500),
- lambda: tf.constant(0.05), lambda: dec_factor)
- else:
- dec_factor = 1.0
- cur = tf.cond(tf.less(tf.random_uniform([]), dec_factor),
- lambda: quant_step, lambda: step)
- else:
- cur = step
- if dropout > 0.0001:
- cur = tf.nn.dropout(cur, keep_prob)
- if act_noise > 0.00001:
- cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale
- # Do nconvs-many CGRU steps.
- if do_jit and tf.get_variable_scope().reuse:
- with jit_scope():
- for layer in xrange(nconvs):
- cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
- cutoff, "ecgru_%d" % layer, do_layer_norm)
- else:
- for layer in xrange(nconvs):
- cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
- cutoff, "ecgru_%d" % layer, do_layer_norm)
- return cur
-
- zero_tgt = tf.zeros([batch_size, nmaps, 1])
- zero_tgt.set_shape([None, nmaps, 1])
-
- def dec_substep(step, decided):
- """Decoder sub-step."""
- cur = step
- if dropout > 0.0001:
- cur = tf.nn.dropout(cur, keep_prob)
- if act_noise > 0.00001:
- cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale
- # Do nconvs-many CGRU steps.
- if do_jit and tf.get_variable_scope().reuse:
- with jit_scope():
- for layer in xrange(nconvs):
- cur = conv_gru([decided], cur, kw, kh, nmaps, conv_rate(layer),
- cutoff, "dcgru_%d" % layer, do_layer_norm)
- else:
- for layer in xrange(nconvs):
- cur = conv_gru([decided], cur, kw, kh, nmaps, conv_rate(layer),
- cutoff, "dcgru_%d" % layer, do_layer_norm)
- return cur
- # pylint: enable=cell-var-from-loop
-
- def dec_step(step, it, it_int, decided, output_ta, tgts,
- mloss, nupd_in, out_idx, beam_cost):
- """Decoder step."""
- nupd, mem_loss = 0, 0.0
- if mem_size > 0:
- it_incr = tf.minimum(it+1, length - 1)
- mem, mem_loss, nupd = memory_run(
- step, nmaps, mem_size, batch_size, noclass, self.global_step,
- self.do_training, self.update_mem, 10, num_gpus,
- target_emb_weights, output_w, gpu_targets_tn, it_incr)
- step = dec_substep(step, decided)
- output_l = tf.expand_dims(tf.expand_dims(step[:, it, 0, :], 1), 1)
- # Calculate argmax output.
- output = tf.reshape(output_l, [-1, nmaps])
- # pylint: disable=cell-var-from-loop
- output = tf.matmul(output, output_w)
- if beam_size > 1:
- beam_cost, output, out, reordered = reorder_beam(
- beam_size, batch_size, beam_cost, output, it_int == 0,
- [output_l, out_idx, step, decided])
- [output_l, out_idx, step, decided] = reordered
- else:
- # Scheduled sampling.
- out = tf.multinomial(tf.stop_gradient(output), 1)
- out = tf.to_int32(tf.squeeze(out, [1]))
- out_write = output_ta.write(it, output_l[:batch_size, :, :, :])
- output = tf.gather(target_emb_weights, out)
- output = tf.reshape(output, [-1, 1, nmaps])
- output = tf.concat(axis=1, values=[output] * height)
- tgt = tgts[it, :, :, :]
- selected = tf.cond(tf.less(tf.random_uniform([]), self.sampling),
- lambda: output, lambda: tgt)
- # pylint: enable=cell-var-from-loop
- dec_write = place_at14(decided, tf.expand_dims(selected, 1), it)
- out_idx = place_at13(
- out_idx, tf.reshape(out, [beam_size * batch_size, 1, 1]), it)
- if mem_size > 0:
- mem = tf.concat(axis=2, values=[mem] * height)
- dec_write = place_at14(dec_write, mem, it_incr)
- return (step, dec_write, out_write, mloss + mem_loss, nupd_in + nupd,
- out_idx, beam_cost)
-
- # Main model construction.
- gpu_outputs = []
- gpu_losses = []
- gpu_grad_norms = []
- grads_list = []
- gpu_out_idx = []
- self.after_enc_step = []
- for gpu in xrange(num_gpus): # Multi-GPU towers, average gradients later.
- length = self.length_tensor
- length_float = tf.cast(length, tf.float32)
- if gpu > 0:
- tf.get_variable_scope().reuse_variables()
- gpu_outputs.append([])
- gpu_losses.append([])
- gpu_grad_norms.append([])
- with tf.name_scope("gpu%d" % gpu), tf.device("/gpu:%d" % gpu):
- # Main graph creation loop.
- data.print_out("Creating model.")
- start_time = time.time()
-
- # Embed inputs and calculate mask.
- with tf.device("/cpu:0"):
- tgt_shape = tf.shape(tf.squeeze(gpu_target[gpu], [1]))
- weights = tf.where(tf.squeeze(gpu_target[gpu], [1]) > 0,
- tf.ones(tgt_shape), tf.zeros(tgt_shape))
-
- # Embed inputs and targets.
- with tf.control_dependencies([e0]):
- start = tf.gather(emb_weights, gpu_input[gpu]) # b x h x l x nmaps
- gpu_targets_tn = gpu_target[gpu] # b x 1 x len
- if beam_size > 0:
- embedded_targets_tn = tf.gather(target_emb_weights,
- gpu_targets_tn)
- embedded_targets_tn = tf.transpose(
- embedded_targets_tn, [2, 0, 1, 3]) # len x b x 1 x nmaps
- embedded_targets_tn = tf.concat(axis=2, values=[embedded_targets_tn] * height)
-
- # First image comes from start by applying convolution and adding 0s.
- start = tf.transpose(start, [0, 2, 1, 3]) # Now b x len x h x vec_s
- first = conv_linear(start, 1, 1, vec_size, nmaps, 1, True, 0.0, "input")
- first = layer_norm(first, nmaps, "input")
-
- # Computation steps.
- keep_prob = dropout * 3.0 / tf.sqrt(length_float)
- keep_prob = 1.0 - self.do_training * keep_prob
- act_noise_scale = act_noise * self.do_training
-
- # Start with a convolutional gate merging previous step.
- step = conv_gru([gpu_prev_step[gpu]], first,
- kw, kh, nmaps, 1, cutoff, "first", do_layer_norm)
-
- # This is just for running a baseline RNN seq2seq model.
- if do_rnn:
- self.after_enc_step.append(step) # Not meaningful here, but needed.
- def lstm_cell():
- return tf.contrib.rnn.BasicLSTMCell(height * nmaps)
- cell = tf.contrib.rnn.MultiRNNCell(
- [lstm_cell() for _ in range(nconvs)])
- with tf.variable_scope("encoder"):
- encoder_outputs, encoder_state = tf.nn.dynamic_rnn(
- cell, tf.reshape(step, [batch_size, length, height * nmaps]),
- dtype=tf.float32, time_major=False)
-
- # Attention.
- attn = tf.layers.dense(
- encoder_outputs, height * nmaps, name="attn1")
-
- # pylint: disable=cell-var-from-loop
- @function.Defun(noinline=True)
- def attention_query(query, attn_v):
- vecs = tf.tanh(attn + tf.expand_dims(query, 1))
- mask = tf.reduce_sum(vecs * tf.reshape(attn_v, [1, 1, -1]), 2)
- mask = tf.nn.softmax(mask)
- return tf.reduce_sum(encoder_outputs * tf.expand_dims(mask, 2), 1)
-
- with tf.variable_scope("decoder"):
- def decoder_loop_fn(state__prev_cell_out__unused, cell_inp__cur_tgt):
- """Decoder loop function."""
- state, prev_cell_out, _ = state__prev_cell_out__unused
- cell_inp, cur_tgt = cell_inp__cur_tgt
- attn_q = tf.layers.dense(prev_cell_out, height * nmaps,
- name="attn_query")
- attn_res = attention_query(attn_q, tf.get_variable(
- "attn_v", [height * nmaps],
- initializer=tf.random_uniform_initializer(-0.1, 0.1)))
- concatenated = tf.reshape(tf.concat(axis=1, values=[cell_inp, attn_res]),
- [batch_size, 2 * height * nmaps])
- cell_inp = tf.layers.dense(
- concatenated, height * nmaps, name="attn_merge")
- output, new_state = cell(cell_inp, state)
-
- mem_loss = 0.0
- if mem_size > 0:
- res, mask, mem_loss = memory_call(
- output, cur_tgt, height * nmaps, mem_size, noclass,
- num_gpus, self.update_mem)
- res = tf.gather(target_emb_weights, res)
- res *= tf.expand_dims(mask[:, 0], 1)
- output = tf.layers.dense(
- tf.concat(axis=1, values=[output, res]), height * nmaps, name="rnnmem")
-
- return new_state, output, mem_loss
- # pylint: enable=cell-var-from-loop
- gpu_targets = tf.squeeze(gpu_target[gpu], [1]) # b x len
- gpu_tgt_trans = tf.transpose(gpu_targets, [1, 0])
- dec_zero = tf.zeros([batch_size, 1], dtype=tf.int32)
- dec_inp = tf.concat(axis=1, values=[dec_zero, gpu_targets])
- dec_inp = dec_inp[:, :length]
- embedded_dec_inp = tf.gather(target_emb_weights, dec_inp)
- embedded_dec_inp_proj = tf.layers.dense(
- embedded_dec_inp, height * nmaps, name="dec_proj")
- embedded_dec_inp_proj = tf.transpose(embedded_dec_inp_proj,
- [1, 0, 2])
- init_vals = (encoder_state,
- tf.zeros([batch_size, height * nmaps]), 0.0)
- _, dec_outputs, mem_losses = tf.scan(
- decoder_loop_fn, (embedded_dec_inp_proj, gpu_tgt_trans),
- initializer=init_vals)
- mem_loss = tf.reduce_mean(mem_losses)
- outputs = tf.layers.dense(dec_outputs, nmaps, name="out_proj")
- # Final convolution to get logits, list outputs.
- outputs = tf.matmul(tf.reshape(outputs, [-1, nmaps]), output_w)
- outputs = tf.reshape(outputs, [length, batch_size, noclass])
- gpu_out_idx.append(tf.argmax(outputs, 2))
- else: # Here we go with the Neural GPU.
- # Encoder.
- enc_length = length
- step = enc_step(step) # First step hard-coded.
- # pylint: disable=cell-var-from-loop
- i = tf.constant(1)
- c = lambda i, _s: tf.less(i, enc_length)
- def enc_step_lambda(i, step):
- with tf.variable_scope(tf.get_variable_scope(), reuse=True):
- new_step = enc_step(step)
- return (i + 1, new_step)
- _, step = tf.while_loop(
- c, enc_step_lambda, [i, step],
- parallel_iterations=1, swap_memory=True)
- # pylint: enable=cell-var-from-loop
-
- self.after_enc_step.append(step)
-
- # Decoder.
- if beam_size > 0:
- output_ta = tf.TensorArray(
- dtype=tf.float32, size=length, dynamic_size=False,
- infer_shape=False, name="outputs")
- out_idx = tf.zeros([beam_size * batch_size, length, 1],
- dtype=tf.int32)
- decided_t = tf.zeros([beam_size * batch_size, length,
- height, vec_size])
-
- # Prepare for beam search.
- tgts = tf.concat(axis=1, values=[embedded_targets_tn] * beam_size)
- beam_cost = tf.zeros([batch_size, beam_size])
- step = tf.concat(axis=0, values=[step] * beam_size)
- # First step hard-coded.
- step, decided_t, output_ta, mem_loss, nupd, oi, bc = dec_step(
- step, 0, 0, decided_t, output_ta, tgts, 0.0, 0, out_idx,
- beam_cost)
- tf.get_variable_scope().reuse_variables()
- # pylint: disable=cell-var-from-loop
- def step_lambda(i, step, dec_t, out_ta, ml, nu, oi, bc):
- with tf.variable_scope(tf.get_variable_scope(), reuse=True):
- s, d, t, nml, nu, oi, bc = dec_step(
- step, i, 1, dec_t, out_ta, tgts, ml, nu, oi, bc)
- return (i + 1, s, d, t, nml, nu, oi, bc)
- i = tf.constant(1)
- c = lambda i, _s, _d, _o, _ml, _nu, _oi, _bc: tf.less(i, length)
- _, step, _, output_ta, mem_loss, nupd, out_idx, _ = tf.while_loop(
- c, step_lambda,
- [i, step, decided_t, output_ta, mem_loss, nupd, oi, bc],
- parallel_iterations=1, swap_memory=True)
- # pylint: enable=cell-var-from-loop
- gpu_out_idx.append(tf.squeeze(out_idx, [2]))
- outputs = output_ta.stack()
- outputs = tf.squeeze(outputs, [2, 3]) # Now l x b x nmaps
- else:
- # If beam_size is 0 or less, we don't have a decoder.
- mem_loss = 0.0
- outputs = tf.transpose(step[:, :, 1, :], [1, 0, 2])
- gpu_out_idx.append(tf.argmax(outputs, 2))
-
- # Final convolution to get logits, list outputs.
- outputs = tf.matmul(tf.reshape(outputs, [-1, nmaps]), output_w)
- outputs = tf.reshape(outputs, [length, batch_size, noclass])
- gpu_outputs[gpu] = tf.nn.softmax(outputs)
-
- # Calculate cross-entropy loss and normalize it.
- targets_soft = make_dense(tf.squeeze(gpu_target[gpu], [1]),
- noclass, 0.1)
- targets_soft = tf.reshape(targets_soft, [-1, noclass])
- targets_hard = make_dense(tf.squeeze(gpu_target[gpu], [1]),
- noclass, 0.0)
- targets_hard = tf.reshape(targets_hard, [-1, noclass])
- output = tf.transpose(outputs, [1, 0, 2])
- xent_soft = tf.reshape(tf.nn.softmax_cross_entropy_with_logits(
- logits=tf.reshape(output, [-1, noclass]), labels=targets_soft),
- [batch_size, length])
- xent_hard = tf.reshape(tf.nn.softmax_cross_entropy_with_logits(
- logits=tf.reshape(output, [-1, noclass]), labels=targets_hard),
- [batch_size, length])
- low, high = 0.1 / float(noclass - 1), 0.9
- const = high * tf.log(high) + float(noclass - 1) * low * tf.log(low)
- weight_sum = tf.reduce_sum(weights) + 1e-20
- true_perp = tf.reduce_sum(xent_hard * weights) / weight_sum
- soft_loss = tf.reduce_sum(xent_soft * weights) / weight_sum
- perp_loss = soft_loss + const
- # Final loss: cross-entropy + shared parameter relaxation part + extra.
- mem_loss = 0.5 * tf.reduce_mean(mem_loss) / length_float
- total_loss = perp_loss + mem_loss
- gpu_losses[gpu].append(true_perp)
-
- # Gradients.
- if backward:
- data.print_out("Creating backward pass for the model.")
- grads = tf.gradients(
- total_loss, tf.trainable_variables(),
- colocate_gradients_with_ops=True)
- for g_i, g in enumerate(grads):
- if isinstance(g, tf.IndexedSlices):
- grads[g_i] = tf.convert_to_tensor(g)
- grads, norm = tf.clip_by_global_norm(grads, max_grad_norm)
- gpu_grad_norms[gpu].append(norm)
- for g in grads:
- if grad_noise_scale > 0.001:
- g += tf.truncated_normal(tf.shape(g)) * self.noise_param
- grads_list.append(grads)
- else:
- gpu_grad_norms[gpu].append(0.0)
- data.print_out("Created model for gpu %d in %.2f s."
- % (gpu, time.time() - start_time))
-
- self.updates = []
- self.after_enc_step = tf.concat(axis=0, values=self.after_enc_step) # Concat GPUs.
- if backward:
- tf.get_variable_scope()._reuse = False
- tf.get_variable_scope().set_caching_device(None)
- grads = [gpu_avg([grads_list[g][i] for g in xrange(num_gpus)])
- for i in xrange(len(grads_list[0]))]
- update = adam_update(grads)
- self.updates.append(update)
- else:
- self.updates.append(tf.no_op())
-
- self.losses = [gpu_avg([gpu_losses[g][i] for g in xrange(num_gpus)])
- for i in xrange(len(gpu_losses[0]))]
- self.out_idx = tf.concat(axis=0, values=gpu_out_idx)
- self.grad_norms = [gpu_avg([gpu_grad_norms[g][i] for g in xrange(num_gpus)])
- for i in xrange(len(gpu_grad_norms[0]))]
- self.outputs = [tf.concat(axis=1, values=[gpu_outputs[g] for g in xrange(num_gpus)])]
- self.quantize_op = quantize_weights_op(512, 8)
- if backward:
- self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=10)
-
- def step(self, sess, inp, target, do_backward_in, noise_param=None,
- beam_size=2, eos_id=2, eos_cost=0.0, update_mem=None, state=None):
- """Run a step of the network."""
- batch_size, height, length = inp.shape[0], inp.shape[1], inp.shape[2]
- do_backward = do_backward_in
- train_mode = True
- if do_backward_in is None:
- do_backward = False
- train_mode = False
- if update_mem is None:
- update_mem = do_backward
- feed_in = {}
- # print " feeding sequences of length %d" % length
- if state is None:
- state = np.zeros([batch_size, length, height, self.nmaps])
- feed_in[self.prev_step.name] = state
- feed_in[self.length_tensor.name] = length
- feed_in[self.noise_param.name] = noise_param if noise_param else 0.0
- feed_in[self.do_training.name] = 1.0 if do_backward else 0.0
- feed_in[self.update_mem.name] = 1 if update_mem else 0
- if do_backward_in is False:
- feed_in[self.sampling.name] = 0.0
- index = 0 # We're dynamic now.
- feed_out = []
- if do_backward:
- feed_out.append(self.updates[index])
- feed_out.append(self.grad_norms[index])
- if train_mode:
- feed_out.append(self.losses[index])
- feed_in[self.input.name] = inp
- feed_in[self.target.name] = target
- feed_out.append(self.outputs[index])
- if train_mode:
- # Make a full-sequence training step with one call to session.run.
- res = sess.run([self.after_enc_step] + feed_out, feed_in)
- after_enc_state, res = res[0], res[1:]
- else:
- # Make a full-sequence decoding step with one call to session.run.
- feed_in[self.sampling.name] = 1.1 # Sample every time.
- res = sess.run([self.after_enc_step, self.out_idx] + feed_out, feed_in)
- after_enc_state, out_idx = res[0], res[1]
- res = [res[2][l] for l in xrange(length)]
- outputs = [out_idx[:, i] for i in xrange(length)]
- cost = [0.0 for _ in xrange(beam_size * batch_size)]
- seen_eos = [0 for _ in xrange(beam_size * batch_size)]
- for idx, logit in enumerate(res):
- best = outputs[idx]
- for b in xrange(batch_size):
- if seen_eos[b] > 1:
- cost[b] -= eos_cost
- else:
- cost[b] += np.log(logit[b][best[b]])
- if best[b] in [eos_id]:
- seen_eos[b] += 1
- res = [[-c for c in cost]] + outputs
- # Collect and output results.
- offset = 0
- norm = None
- if do_backward:
- offset = 2
- norm = res[1]
- if train_mode:
- outputs = res[offset + 1]
- outputs = [outputs[l] for l in xrange(length)]
- return res[offset], outputs, norm, after_enc_state
diff --git a/research/neural_gpu/neural_gpu_trainer.py b/research/neural_gpu/neural_gpu_trainer.py
deleted file mode 100644
index 1f704b0da880dbde4b09bf2cc108edb034d7b1a0..0000000000000000000000000000000000000000
--- a/research/neural_gpu/neural_gpu_trainer.py
+++ /dev/null
@@ -1,1027 +0,0 @@
-# Copyright 2015 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Neural GPU."""
-
-from __future__ import print_function
-
-import math
-import os
-import random
-import sys
-import threading
-import time
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import program_utils
-import data_utils as data
-import neural_gpu as ngpu
-import wmt_utils as wmt
-
-tf.app.flags.DEFINE_float("lr", 0.1, "Learning rate.")
-tf.app.flags.DEFINE_float("init_weight", 0.8, "Initial weights deviation.")
-tf.app.flags.DEFINE_float("max_grad_norm", 4.0, "Clip gradients to this norm.")
-tf.app.flags.DEFINE_float("cutoff", 1.2, "Cutoff at the gates.")
-tf.app.flags.DEFINE_float("curriculum_ppx", 9.9, "Move curriculum if ppl < X.")
-tf.app.flags.DEFINE_float("curriculum_seq", 0.3, "Move curriculum if seq < X.")
-tf.app.flags.DEFINE_float("dropout", 0.1, "Dropout that much.")
-tf.app.flags.DEFINE_float("grad_noise_scale", 0.0, "Gradient noise scale.")
-tf.app.flags.DEFINE_float("max_sampling_rate", 0.1, "Maximal sampling rate.")
-tf.app.flags.DEFINE_float("length_norm", 0.0, "Length normalization.")
-tf.app.flags.DEFINE_float("train_beam_freq", 0.0, "Beam-based training.")
-tf.app.flags.DEFINE_float("train_beam_anneal", 20000, "How many steps anneal.")
-tf.app.flags.DEFINE_integer("eval_beam_steps", 4, "How many beam steps eval.")
-tf.app.flags.DEFINE_integer("batch_size", 32, "Batch size.")
-tf.app.flags.DEFINE_integer("steps_per_checkpoint", 100, "Steps per epoch.")
-tf.app.flags.DEFINE_integer("nmaps", 64, "Number of floats in each cell.")
-tf.app.flags.DEFINE_integer("vec_size", 64, "Size of word vectors.")
-tf.app.flags.DEFINE_integer("train_data_size", 1000, "Training examples/len.")
-tf.app.flags.DEFINE_integer("max_length", 40, "Maximum length.")
-tf.app.flags.DEFINE_integer("random_seed", 125459, "Random seed.")
-tf.app.flags.DEFINE_integer("nconvs", 2, "How many convolutions / 1 step.")
-tf.app.flags.DEFINE_integer("kw", 3, "Kernel width.")
-tf.app.flags.DEFINE_integer("kh", 3, "Kernel height.")
-tf.app.flags.DEFINE_integer("height", 4, "Height.")
-tf.app.flags.DEFINE_integer("mem_size", -1, "Memory size (sqrt)")
-tf.app.flags.DEFINE_integer("soft_mem_size", 1024, "Softmax memory this size.")
-tf.app.flags.DEFINE_integer("num_gpus", 1, "Number of GPUs to use.")
-tf.app.flags.DEFINE_integer("num_replicas", 1, "Number of replicas in use.")
-tf.app.flags.DEFINE_integer("beam_size", 1, "Beam size during decoding. "
- "If 0, no decoder, the non-extended Neural GPU.")
-tf.app.flags.DEFINE_integer("max_target_vocab", 0,
- "Maximal size of target vocabulary.")
-tf.app.flags.DEFINE_integer("decode_offset", 0, "Offset for decoding.")
-tf.app.flags.DEFINE_integer("task", -1, "Task id when running on borg.")
-tf.app.flags.DEFINE_integer("nprint", 0, "How many test examples to print out.")
-tf.app.flags.DEFINE_integer("eval_bin_print", 3, "How many bins step in eval.")
-tf.app.flags.DEFINE_integer("mode", 0, "Mode: 0-train other-decode.")
-tf.app.flags.DEFINE_bool("atrous", False, "Whether to use atrous convs.")
-tf.app.flags.DEFINE_bool("layer_norm", False, "Do layer normalization.")
-tf.app.flags.DEFINE_bool("quantize", False, "Whether to quantize variables.")
-tf.app.flags.DEFINE_bool("do_train", True, "If false, only update memory.")
-tf.app.flags.DEFINE_bool("rnn_baseline", False, "If true build an RNN instead.")
-tf.app.flags.DEFINE_bool("simple_tokenizer", False,
- "If true, tokenize on spaces only, digits are 0.")
-tf.app.flags.DEFINE_bool("normalize_digits", True,
- "Whether to normalize digits with simple tokenizer.")
-tf.app.flags.DEFINE_integer("vocab_size", 16, "Joint vocabulary size.")
-tf.app.flags.DEFINE_string("data_dir", "/tmp", "Data directory")
-tf.app.flags.DEFINE_string("train_dir", "/tmp/", "Directory to store models.")
-tf.app.flags.DEFINE_string("test_file_prefix", "", "Files to test (.en,.fr).")
-tf.app.flags.DEFINE_integer("max_train_data_size", 0,
- "Limit on the size of training data (0: no limit).")
-tf.app.flags.DEFINE_string("word_vector_file_en", "",
- "Optional file with word vectors to start training.")
-tf.app.flags.DEFINE_string("word_vector_file_fr", "",
- "Optional file with word vectors to start training.")
-tf.app.flags.DEFINE_string("problem", "wmt", "What problem are we solving?.")
-
-tf.app.flags.DEFINE_integer("ps_tasks", 0, "Number of ps tasks used.")
-tf.app.flags.DEFINE_string("master", "", "Name of the TensorFlow master.")
-
-FLAGS = tf.app.flags.FLAGS
-EXTRA_EVAL = 10
-EVAL_LEN_INCR = 8
-MAXLEN_F = 2.0
-
-
-def zero_split(tok_list, append=None):
- """Split tok_list (list of ints) on 0s, append int to all parts if given."""
- res, cur, l = [], [], 0
- for tok in tok_list:
- if tok == 0:
- if append is not None:
- cur.append(append)
- res.append(cur)
- l = max(l, len(cur))
- cur = []
- else:
- cur.append(tok)
- if append is not None:
- cur.append(append)
- res.append(cur)
- l = max(l, len(cur))
- return res, l
-
-
-def read_data(source_path, target_path, buckets, max_size=None, print_out=True):
- """Read data from source and target files and put into buckets.
-
- Args:
- source_path: path to the files with token-ids for the source language.
- target_path: path to the file with token-ids for the target language;
- it must be aligned with the source file: n-th line contains the desired
- output for n-th line from the source_path.
- buckets: the buckets to use.
- max_size: maximum number of lines to read, all other will be ignored;
- if 0 or None, data files will be read completely (no limit).
- If set to 1, no data will be returned (empty lists of the right form).
- print_out: whether to print out status or not.
-
- Returns:
- data_set: a list of length len(_buckets); data_set[n] contains a list of
- (source, target) pairs read from the provided data files that fit
- into the n-th bucket, i.e., such that len(source) < _buckets[n][0] and
- len(target) < _buckets[n][1]; source and target are lists of token-ids.
- """
- data_set = [[] for _ in buckets]
- counter = 0
- if max_size != 1:
- with tf.gfile.GFile(source_path, mode="r") as source_file:
- with tf.gfile.GFile(target_path, mode="r") as target_file:
- source, target = source_file.readline(), target_file.readline()
- while source and target and (not max_size or counter < max_size):
- counter += 1
- if counter % 100000 == 0 and print_out:
- print(" reading data line %d" % counter)
- sys.stdout.flush()
- source_ids = [int(x) for x in source.split()]
- target_ids = [int(x) for x in target.split()]
- source_ids, source_len = zero_split(source_ids)
- target_ids, target_len = zero_split(target_ids, append=wmt.EOS_ID)
- for bucket_id, size in enumerate(buckets):
- if source_len <= size and target_len <= size:
- data_set[bucket_id].append([source_ids, target_ids])
- break
- source, target = source_file.readline(), target_file.readline()
- return data_set
-
-
-global_train_set = {"wmt": []}
-train_buckets_scale = {"wmt": []}
-
-
-def calculate_buckets_scale(data_set, buckets, problem):
- """Calculate buckets scales for the given data set."""
- train_bucket_sizes = [len(data_set[b]) for b in xrange(len(buckets))]
- train_total_size = max(1, float(sum(train_bucket_sizes)))
-
- # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use
- # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to
- # the size if i-th training bucket, as used later.
- if problem not in train_buckets_scale:
- train_buckets_scale[problem] = []
- train_buckets_scale[problem].append(
- [sum(train_bucket_sizes[:i + 1]) / train_total_size
- for i in xrange(len(train_bucket_sizes))])
- return train_total_size
-
-
-def read_data_into_global(source_path, target_path, buckets,
- max_size=None, print_out=True):
- """Read data into the global variables (can be in a separate thread)."""
- # pylint: disable=global-variable-not-assigned
- global global_train_set, train_buckets_scale
- # pylint: enable=global-variable-not-assigned
- data_set = read_data(source_path, target_path, buckets, max_size, print_out)
- global_train_set["wmt"].append(data_set)
- train_total_size = calculate_buckets_scale(data_set, buckets, "wmt")
- if print_out:
- print(" Finished global data reading (%d)." % train_total_size)
-
-
-def initialize(sess=None):
- """Initialize data and model."""
- global MAXLEN_F
- # Create training directory if it does not exist.
- if not tf.gfile.IsDirectory(FLAGS.train_dir):
- data.print_out("Creating training directory %s." % FLAGS.train_dir)
- tf.gfile.MkDir(FLAGS.train_dir)
- decode_suffix = "beam%dln%d" % (FLAGS.beam_size,
- int(100 * FLAGS.length_norm))
- if FLAGS.mode == 0:
- decode_suffix = ""
- if FLAGS.task >= 0:
- data.log_filename = os.path.join(FLAGS.train_dir,
- "log%d%s" % (FLAGS.task, decode_suffix))
- else:
- data.log_filename = os.path.join(FLAGS.train_dir, "neural_gpu/log")
-
- # Set random seed.
- if FLAGS.random_seed > 0:
- seed = FLAGS.random_seed + max(0, FLAGS.task)
- tf.set_random_seed(seed)
- random.seed(seed)
- np.random.seed(seed)
-
- # Check data sizes.
- assert data.bins
- max_length = min(FLAGS.max_length, data.bins[-1])
- while len(data.bins) > 1 and data.bins[-2] >= max_length + EXTRA_EVAL:
- data.bins = data.bins[:-1]
- if sess is None and FLAGS.task == 0 and FLAGS.num_replicas > 1:
- if max_length > 60:
- max_length = max_length * 1 / 2 # Save memory on chief.
- min_length = min(14, max_length - 3) if FLAGS.problem == "wmt" else 3
- for p in FLAGS.problem.split("-"):
- if p in ["progeval", "progsynth"]:
- min_length = max(26, min_length)
- assert max_length + 1 > min_length
- while len(data.bins) > 1 and data.bins[-2] >= max_length + EXTRA_EVAL:
- data.bins = data.bins[:-1]
-
- # Create checkpoint directory if it does not exist.
- if FLAGS.mode == 0 or FLAGS.task < 0:
- checkpoint_dir = os.path.join(FLAGS.train_dir, "neural_gpu%s"
- % ("" if FLAGS.task < 0 else str(FLAGS.task)))
- else:
- checkpoint_dir = FLAGS.train_dir
- if not tf.gfile.IsDirectory(checkpoint_dir):
- data.print_out("Creating checkpoint directory %s." % checkpoint_dir)
- tf.gfile.MkDir(checkpoint_dir)
-
- # Prepare data.
- if FLAGS.problem == "wmt":
- # Prepare WMT data.
- data.print_out("Preparing WMT data in %s" % FLAGS.data_dir)
- if FLAGS.simple_tokenizer:
- MAXLEN_F = 3.5
- (en_train, fr_train, en_dev, fr_dev,
- en_path, fr_path) = wmt.prepare_wmt_data(
- FLAGS.data_dir, FLAGS.vocab_size,
- tokenizer=wmt.space_tokenizer,
- normalize_digits=FLAGS.normalize_digits)
- else:
- (en_train, fr_train, en_dev, fr_dev,
- en_path, fr_path) = wmt.prepare_wmt_data(
- FLAGS.data_dir, FLAGS.vocab_size)
-
- # Read data into buckets and compute their sizes.
- fr_vocab, rev_fr_vocab = wmt.initialize_vocabulary(fr_path)
- data.vocab = fr_vocab
- data.rev_vocab = rev_fr_vocab
- data.print_out("Reading development and training data (limit: %d)."
- % FLAGS.max_train_data_size)
- dev_set = {}
- dev_set["wmt"] = read_data(en_dev, fr_dev, data.bins)
- def data_read(size, print_out):
- read_data_into_global(en_train, fr_train, data.bins, size, print_out)
- data_read(50000, False)
- read_thread_small = threading.Thread(
- name="reading-data-small", target=lambda: data_read(900000, False))
- read_thread_small.start()
- read_thread_full = threading.Thread(
- name="reading-data-full",
- target=lambda: data_read(FLAGS.max_train_data_size, True))
- read_thread_full.start()
- data.print_out("Data reading set up.")
- else:
- # Prepare algorithmic data.
- en_path, fr_path = None, None
- tasks = FLAGS.problem.split("-")
- data_size = FLAGS.train_data_size
- for t in tasks:
- data.print_out("Generating data for %s." % t)
- if t in ["progeval", "progsynth"]:
- data.init_data(t, data.bins[-1], 20 * data_size, FLAGS.vocab_size)
- if len(program_utils.prog_vocab) > FLAGS.vocab_size - 2:
- raise ValueError("Increase vocab_size to %d for prog-tasks."
- % (len(program_utils.prog_vocab) + 2))
- data.rev_vocab = program_utils.prog_vocab
- data.vocab = program_utils.prog_rev_vocab
- else:
- for l in xrange(max_length + EXTRA_EVAL - 1):
- data.init_data(t, l, data_size, FLAGS.vocab_size)
- data.init_data(t, data.bins[-2], data_size, FLAGS.vocab_size)
- data.init_data(t, data.bins[-1], data_size, FLAGS.vocab_size)
- if t not in global_train_set:
- global_train_set[t] = []
- global_train_set[t].append(data.train_set[t])
- calculate_buckets_scale(data.train_set[t], data.bins, t)
- dev_set = data.test_set
-
- # Grid-search parameters.
- lr = FLAGS.lr
- init_weight = FLAGS.init_weight
- max_grad_norm = FLAGS.max_grad_norm
- if sess is not None and FLAGS.task > -1:
- def job_id_factor(step):
- """If jobid / step mod 3 is 0, 1, 2: say 0, 1, -1."""
- return ((((FLAGS.task / step) % 3) + 1) % 3) - 1
- lr *= math.pow(2, job_id_factor(1))
- init_weight *= math.pow(1.5, job_id_factor(3))
- max_grad_norm *= math.pow(2, job_id_factor(9))
-
- # Print out parameters.
- curriculum = FLAGS.curriculum_seq
- msg1 = ("layers %d kw %d h %d kh %d batch %d noise %.2f"
- % (FLAGS.nconvs, FLAGS.kw, FLAGS.height, FLAGS.kh,
- FLAGS.batch_size, FLAGS.grad_noise_scale))
- msg2 = ("cut %.2f lr %.3f iw %.2f cr %.2f nm %d d%.4f gn %.2f %s"
- % (FLAGS.cutoff, lr, init_weight, curriculum, FLAGS.nmaps,
- FLAGS.dropout, max_grad_norm, msg1))
- data.print_out(msg2)
-
- # Create model and initialize it.
- tf.get_variable_scope().set_initializer(
- tf.orthogonal_initializer(gain=1.8 * init_weight))
- max_sampling_rate = FLAGS.max_sampling_rate if FLAGS.mode == 0 else 0.0
- o = FLAGS.vocab_size if FLAGS.max_target_vocab < 1 else FLAGS.max_target_vocab
- ngpu.CHOOSE_K = FLAGS.soft_mem_size
- do_beam_model = FLAGS.train_beam_freq > 0.0001 and FLAGS.beam_size > 1
- beam_size = FLAGS.beam_size if FLAGS.mode > 0 and not do_beam_model else 1
- beam_size = min(beam_size, FLAGS.beam_size)
- beam_model = None
- def make_ngpu(cur_beam_size, back):
- return ngpu.NeuralGPU(
- FLAGS.nmaps, FLAGS.vec_size, FLAGS.vocab_size, o,
- FLAGS.dropout, max_grad_norm, FLAGS.cutoff, FLAGS.nconvs,
- FLAGS.kw, FLAGS.kh, FLAGS.height, FLAGS.mem_size,
- lr / math.sqrt(FLAGS.num_replicas), min_length + 3, FLAGS.num_gpus,
- FLAGS.num_replicas, FLAGS.grad_noise_scale, max_sampling_rate,
- atrous=FLAGS.atrous, do_rnn=FLAGS.rnn_baseline,
- do_layer_norm=FLAGS.layer_norm, beam_size=cur_beam_size, backward=back)
- if sess is None:
- with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
- model = make_ngpu(beam_size, True)
- if do_beam_model:
- tf.get_variable_scope().reuse_variables()
- beam_model = make_ngpu(FLAGS.beam_size, False)
- else:
- model = make_ngpu(beam_size, True)
- if do_beam_model:
- tf.get_variable_scope().reuse_variables()
- beam_model = make_ngpu(FLAGS.beam_size, False)
-
- sv = None
- if sess is None:
- # The supervisor configuration has a few overriden options.
- sv = tf.train.Supervisor(logdir=checkpoint_dir,
- is_chief=(FLAGS.task < 1),
- saver=model.saver,
- summary_op=None,
- save_summaries_secs=60,
- save_model_secs=15 * 60,
- global_step=model.global_step)
-
- config = tf.ConfigProto(allow_soft_placement=True)
- sess = sv.PrepareSession(FLAGS.master, config=config)
-
- data.print_out("Created model. Checkpoint dir %s" % checkpoint_dir)
-
- # Load model from parameters if a checkpoint exists.
- ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
- if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path + ".index"):
- data.print_out("Reading model parameters from %s"
- % ckpt.model_checkpoint_path)
- model.saver.restore(sess, ckpt.model_checkpoint_path)
- elif sv is None:
- sess.run(tf.global_variables_initializer())
- data.print_out("Initialized variables (no supervisor mode).")
- elif FLAGS.task < 1 and FLAGS.mem_size > 0:
- # sess.run(model.mem_norm_op)
- data.print_out("Created new model and normalized mem (on chief).")
-
- # Return the model and needed variables.
- return (model, beam_model, min_length, max_length, checkpoint_dir,
- (global_train_set, dev_set, en_path, fr_path), sv, sess)
-
-
-def m_step(model, beam_model, sess, batch_size, inp, target, bucket, nsteps, p):
- """Evaluation multi-step for program synthesis."""
- state, scores, hist = None, [[-11.0 for _ in xrange(batch_size)]], []
- for _ in xrange(nsteps):
- # Get the best beam (no training, just forward model).
- new_target, new_first, new_inp, new_scores = get_best_beam(
- beam_model, sess, inp, target,
- batch_size, FLAGS.beam_size, bucket, hist, p, test_mode=True)
- hist.append(new_first)
- _, _, _, state = model.step(sess, inp, new_target, False, state=state)
- inp = new_inp
- scores.append([max(scores[-1][i], new_scores[i])
- for i in xrange(batch_size)])
- # The final step with the true target.
- loss, res, _, _ = model.step(sess, inp, target, False, state=state)
- return loss, res, new_target, scores[1:]
-
-
-def single_test(bin_id, model, sess, nprint, batch_size, dev, p, print_out=True,
- offset=None, beam_model=None):
- """Test model on test data of length l using the given session."""
- if not dev[p][bin_id]:
- data.print_out(" bin %d (%d)\t%s\tppl NA errors NA seq-errors NA"
- % (bin_id, data.bins[bin_id], p))
- return 1.0, 1.0, 0.0
- inpt, target = data.get_batch(
- bin_id, batch_size, dev[p], FLAGS.height, offset)
- if FLAGS.beam_size > 1 and beam_model:
- loss, res, new_tgt, scores = m_step(
- model, beam_model, sess, batch_size, inpt, target, bin_id,
- FLAGS.eval_beam_steps, p)
- score_avgs = [sum(s) / float(len(s)) for s in scores]
- score_maxs = [max(s) for s in scores]
- score_str = ["(%.2f, %.2f)" % (score_avgs[i], score_maxs[i])
- for i in xrange(FLAGS.eval_beam_steps)]
- data.print_out(" == scores (avg, max): %s" % "; ".join(score_str))
- errors, total, seq_err = data.accuracy(inpt, res, target, batch_size,
- nprint, new_tgt, scores[-1])
- else:
- loss, res, _, _ = model.step(sess, inpt, target, False)
- errors, total, seq_err = data.accuracy(inpt, res, target, batch_size,
- nprint)
- seq_err = float(seq_err) / batch_size
- if total > 0:
- errors = float(errors) / total
- if print_out:
- data.print_out(" bin %d (%d)\t%s\tppl %.2f errors %.2f seq-errors %.2f"
- % (bin_id, data.bins[bin_id], p, data.safe_exp(loss),
- 100 * errors, 100 * seq_err))
- return (errors, seq_err, loss)
-
-
-def assign_vectors(word_vector_file, embedding_key, vocab_path, sess):
- """Assign the embedding_key variable from the given word vectors file."""
- # For words in the word vector file, set their embedding at start.
- if not tf.gfile.Exists(word_vector_file):
- data.print_out("Word vector file does not exist: %s" % word_vector_file)
- sys.exit(1)
- vocab, _ = wmt.initialize_vocabulary(vocab_path)
- vectors_variable = [v for v in tf.trainable_variables()
- if embedding_key == v.name]
- if len(vectors_variable) != 1:
- data.print_out("Word vector variable not found or too many.")
- sys.exit(1)
- vectors_variable = vectors_variable[0]
- vectors = vectors_variable.eval()
- data.print_out("Pre-setting word vectors from %s" % word_vector_file)
- with tf.gfile.GFile(word_vector_file, mode="r") as f:
- # Lines have format: dog 0.045123 -0.61323 0.413667 ...
- for line in f:
- line_parts = line.split()
- # The first part is the word.
- word = line_parts[0]
- if word in vocab:
- # Remaining parts are components of the vector.
- word_vector = np.array(map(float, line_parts[1:]))
- if len(word_vector) != FLAGS.vec_size:
- data.print_out("Warn: Word '%s', Expecting vector size %d, "
- "found %d" % (word, FLAGS.vec_size,
- len(word_vector)))
- else:
- vectors[vocab[word]] = word_vector
- # Assign the modified vectors to the vectors_variable in the graph.
- sess.run([vectors_variable.initializer],
- {vectors_variable.initializer.inputs[1]: vectors})
-
-
-def print_vectors(embedding_key, vocab_path, word_vector_file):
- """Print vectors from the given variable."""
- _, rev_vocab = wmt.initialize_vocabulary(vocab_path)
- vectors_variable = [v for v in tf.trainable_variables()
- if embedding_key == v.name]
- if len(vectors_variable) != 1:
- data.print_out("Word vector variable not found or too many.")
- sys.exit(1)
- vectors_variable = vectors_variable[0]
- vectors = vectors_variable.eval()
- l, s = vectors.shape[0], vectors.shape[1]
- data.print_out("Printing %d word vectors from %s to %s."
- % (l, embedding_key, word_vector_file))
- with tf.gfile.GFile(word_vector_file, mode="w") as f:
- # Lines have format: dog 0.045123 -0.61323 0.413667 ...
- for i in xrange(l):
- f.write(rev_vocab[i])
- for j in xrange(s):
- f.write(" %.8f" % vectors[i][j])
- f.write("\n")
-
-
-def get_bucket_id(train_buckets_scale_c, max_cur_length, data_set):
- """Get a random bucket id."""
- # Choose a bucket according to data distribution. Pick a random number
- # in [0, 1] and use the corresponding interval in train_buckets_scale.
- random_number_01 = np.random.random_sample()
- bucket_id = min([i for i in xrange(len(train_buckets_scale_c))
- if train_buckets_scale_c[i] > random_number_01])
- while bucket_id > 0 and not data_set[bucket_id]:
- bucket_id -= 1
- for _ in xrange(10 if np.random.random_sample() < 0.9 else 1):
- if data.bins[bucket_id] > max_cur_length:
- random_number_01 = min(random_number_01, np.random.random_sample())
- bucket_id = min([i for i in xrange(len(train_buckets_scale_c))
- if train_buckets_scale_c[i] > random_number_01])
- while bucket_id > 0 and not data_set[bucket_id]:
- bucket_id -= 1
- return bucket_id
-
-
-def score_beams(beams, target, inp, history, p,
- print_out=False, test_mode=False):
- """Score beams."""
- if p == "progsynth":
- return score_beams_prog(beams, target, inp, history, print_out, test_mode)
- elif test_mode:
- return beams[0], 10.0 if str(beams[0][:len(target)]) == str(target) else 0.0
- else:
- history_s = [str(h) for h in history]
- best, best_score, tgt, eos_id = None, -1000.0, target, None
- if p == "wmt":
- eos_id = wmt.EOS_ID
- if eos_id and eos_id in target:
- tgt = target[:target.index(eos_id)]
- for beam in beams:
- if eos_id and eos_id in beam:
- beam = beam[:beam.index(eos_id)]
- l = min(len(tgt), len(beam))
- score = len([i for i in xrange(l) if tgt[i] == beam[i]]) / float(len(tgt))
- hist_score = 20.0 if str([b for b in beam if b > 0]) in history_s else 0.0
- if score < 1.0:
- score -= hist_score
- if score > best_score:
- best = beam
- best_score = score
- return best, best_score
-
-
-def score_beams_prog(beams, target, inp, history, print_out=False,
- test_mode=False):
- """Score beams for program synthesis."""
- tgt_prog = linearize(target, program_utils.prog_vocab, True, 1)
- hist_progs = [linearize(h, program_utils.prog_vocab, True, 1)
- for h in history]
- tgt_set = set(target)
- if print_out:
- print("target: ", tgt_prog)
- inps, tgt_outs = [], []
- for i in xrange(3):
- ilist = [inp[i + 1, l] for l in xrange(inp.shape[1])]
- clist = [program_utils.prog_vocab[x] for x in ilist if x > 0]
- olist = clist[clist.index("]") + 1:] # outputs
- clist = clist[1:clist.index("]")] # inputs
- inps.append([int(x) for x in clist])
- if olist[0] == "[": # olist may be [int] or just int
- tgt_outs.append(str([int(x) for x in olist[1:-1]]))
- else:
- if len(olist) == 1:
- tgt_outs.append(olist[0])
- else:
- print([program_utils.prog_vocab[x] for x in ilist if x > 0])
- print(olist)
- print(tgt_prog)
- print(program_utils.evaluate(tgt_prog, {"a": inps[-1]}))
- print("AAAAA")
- tgt_outs.append(olist[0])
- if not test_mode:
- for _ in xrange(7):
- ilen = np.random.randint(len(target) - 3) + 1
- inps.append([random.choice(range(-15, 15)) for _ in range(ilen)])
- tgt_outs.extend([program_utils.evaluate(tgt_prog, {"a": inp})
- for inp in inps[3:]])
- best, best_prog, best_score = None, "", -1000.0
- for beam in beams:
- b_prog = linearize(beam, program_utils.prog_vocab, True, 1)
- b_set = set(beam)
- jsim = len(tgt_set & b_set) / float(len(tgt_set | b_set))
- b_outs = [program_utils.evaluate(b_prog, {"a": inp}) for inp in inps]
- errs = len([x for x in b_outs if x == "ERROR"])
- imatches = len([i for i in xrange(3) if b_outs[i] == tgt_outs[i]])
- perfect = 10.0 if imatches == 3 else 0.0
- hist_score = 20.0 if b_prog in hist_progs else 0.0
- if test_mode:
- score = perfect - errs
- else:
- matches = len([i for i in xrange(10) if b_outs[i] == tgt_outs[i]])
- score = perfect + matches + jsim - errs
- if score < 10.0:
- score -= hist_score
- # print b_prog
- # print "jsim: ", jsim, " errs: ", errs, " mtchs: ", matches, " s: ", score
- if score > best_score:
- best = beam
- best_prog = b_prog
- best_score = score
- if print_out:
- print("best score: ", best_score, " best prog: ", best_prog)
- return best, best_score
-
-
-def get_best_beam(beam_model, sess, inp, target, batch_size, beam_size,
- bucket, history, p, test_mode=False):
- """Run beam_model, score beams, and return the best as target and in input."""
- _, output_logits, _, _ = beam_model.step(
- sess, inp, target, None, beam_size=FLAGS.beam_size)
- new_targets, new_firsts, scores, new_inp = [], [], [], np.copy(inp)
- for b in xrange(batch_size):
- outputs = []
- history_b = [[h[b, 0, l] for l in xrange(data.bins[bucket])]
- for h in history]
- for beam_idx in xrange(beam_size):
- outputs.append([int(o[beam_idx * batch_size + b])
- for o in output_logits])
- target_t = [target[b, 0, l] for l in xrange(data.bins[bucket])]
- best, best_score = score_beams(
- outputs, [t for t in target_t if t > 0], inp[b, :, :],
- [[t for t in h if t > 0] for h in history_b], p, test_mode=test_mode)
- scores.append(best_score)
- if 1 in best: # Only until _EOS.
- best = best[:best.index(1) + 1]
- best += [0 for _ in xrange(len(target_t) - len(best))]
- new_targets.append([best])
- first, _ = score_beams(
- outputs, [t for t in target_t if t > 0], inp[b, :, :],
- [[t for t in h if t > 0] for h in history_b], p, test_mode=True)
- if 1 in first: # Only until _EOS.
- first = first[:first.index(1) + 1]
- first += [0 for _ in xrange(len(target_t) - len(first))]
- new_inp[b, 0, :] = np.array(first, dtype=np.int32)
- new_firsts.append([first])
- # Change target if we found a great answer.
- new_target = np.array(new_targets, dtype=np.int32)
- for b in xrange(batch_size):
- if scores[b] >= 10.0:
- target[b, 0, :] = new_target[b, 0, :]
- new_first = np.array(new_firsts, dtype=np.int32)
- return new_target, new_first, new_inp, scores
-
-
-def train():
- """Train the model."""
- batch_size = FLAGS.batch_size * FLAGS.num_gpus
- (model, beam_model, min_length, max_length, checkpoint_dir,
- (train_set, dev_set, en_vocab_path, fr_vocab_path), sv, sess) = initialize()
- with sess.as_default():
- quant_op = model.quantize_op
- max_cur_length = min(min_length + 3, max_length)
- prev_acc_perp = [1000000 for _ in xrange(5)]
- prev_seq_err = 1.0
- is_chief = FLAGS.task < 1
- do_report = False
-
- # Main traning loop.
- while not sv.ShouldStop():
- global_step, max_cur_length, learning_rate = sess.run(
- [model.global_step, model.cur_length, model.lr])
- acc_loss, acc_l1, acc_total, acc_errors, acc_seq_err = 0.0, 0.0, 0, 0, 0
- acc_grad_norm, step_count, step_c1, step_time = 0.0, 0, 0, 0.0
-
- # For words in the word vector file, set their embedding at start.
- bound1 = FLAGS.steps_per_checkpoint - 1
- if FLAGS.word_vector_file_en and global_step < bound1 and is_chief:
- assign_vectors(FLAGS.word_vector_file_en, "embedding:0",
- en_vocab_path, sess)
- if FLAGS.max_target_vocab < 1:
- assign_vectors(FLAGS.word_vector_file_en, "target_embedding:0",
- en_vocab_path, sess)
-
- if FLAGS.word_vector_file_fr and global_step < bound1 and is_chief:
- assign_vectors(FLAGS.word_vector_file_fr, "embedding:0",
- fr_vocab_path, sess)
- if FLAGS.max_target_vocab < 1:
- assign_vectors(FLAGS.word_vector_file_fr, "target_embedding:0",
- fr_vocab_path, sess)
-
- for _ in xrange(FLAGS.steps_per_checkpoint):
- step_count += 1
- step_c1 += 1
- global_step = int(model.global_step.eval())
- train_beam_anneal = global_step / float(FLAGS.train_beam_anneal)
- train_beam_freq = FLAGS.train_beam_freq * min(1.0, train_beam_anneal)
- p = random.choice(FLAGS.problem.split("-"))
- train_set = global_train_set[p][-1]
- bucket_id = get_bucket_id(train_buckets_scale[p][-1], max_cur_length,
- train_set)
- # Prefer longer stuff 60% of time if not wmt.
- if np.random.randint(100) < 60 and FLAGS.problem != "wmt":
- bucket1 = get_bucket_id(train_buckets_scale[p][-1], max_cur_length,
- train_set)
- bucket_id = max(bucket1, bucket_id)
-
- # Run a step and time it.
- start_time = time.time()
- inp, target = data.get_batch(bucket_id, batch_size, train_set,
- FLAGS.height)
- noise_param = math.sqrt(math.pow(global_step + 1, -0.55) *
- prev_seq_err) * FLAGS.grad_noise_scale
- # In multi-step mode, we use best from beam for middle steps.
- state, new_target, scores, history = None, None, None, []
- while (FLAGS.beam_size > 1 and
- train_beam_freq > np.random.random_sample()):
- # Get the best beam (no training, just forward model).
- new_target, new_first, new_inp, scores = get_best_beam(
- beam_model, sess, inp, target,
- batch_size, FLAGS.beam_size, bucket_id, history, p)
- history.append(new_first)
- # Training step with the previous input and the best beam as target.
- _, _, _, state = model.step(sess, inp, new_target, FLAGS.do_train,
- noise_param, update_mem=True, state=state)
- # Change input to the new one for the next step.
- inp = new_inp
- # If all results are great, stop (todo: not to wait for all?).
- if FLAGS.nprint > 1:
- print(scores)
- if sum(scores) / float(len(scores)) >= 10.0:
- break
- # The final step with the true target.
- loss, res, gnorm, _ = model.step(
- sess, inp, target, FLAGS.do_train, noise_param,
- update_mem=True, state=state)
- step_time += time.time() - start_time
- acc_grad_norm += 0.0 if gnorm is None else float(gnorm)
-
- # Accumulate statistics.
- acc_loss += loss
- acc_l1 += loss
- errors, total, seq_err = data.accuracy(
- inp, res, target, batch_size, 0, new_target, scores)
- if FLAGS.nprint > 1:
- print("seq_err: ", seq_err)
- acc_total += total
- acc_errors += errors
- acc_seq_err += seq_err
-
- # Report summary every 10 steps.
- if step_count + 3 > FLAGS.steps_per_checkpoint:
- do_report = True # Don't polute plot too early.
- if is_chief and step_count % 10 == 1 and do_report:
- cur_loss = acc_l1 / float(step_c1)
- acc_l1, step_c1 = 0.0, 0
- cur_perp = data.safe_exp(cur_loss)
- summary = tf.Summary()
- summary.value.extend(
- [tf.Summary.Value(tag="log_perplexity", simple_value=cur_loss),
- tf.Summary.Value(tag="perplexity", simple_value=cur_perp)])
- sv.SummaryComputed(sess, summary, global_step)
-
- # Normalize and print out accumulated statistics.
- acc_loss /= step_count
- step_time /= FLAGS.steps_per_checkpoint
- acc_seq_err = float(acc_seq_err) / (step_count * batch_size)
- prev_seq_err = max(0.0, acc_seq_err - 0.02) # No noise at error < 2%.
- acc_errors = float(acc_errors) / acc_total if acc_total > 0 else 1.0
- t_size = float(sum([len(x) for x in train_set])) / float(1000000)
- msg = ("step %d step-time %.2f train-size %.3f lr %.6f grad-norm %.4f"
- % (global_step + 1, step_time, t_size, learning_rate,
- acc_grad_norm / FLAGS.steps_per_checkpoint))
- data.print_out("%s len %d ppl %.6f errors %.2f sequence-errors %.2f" %
- (msg, max_cur_length, data.safe_exp(acc_loss),
- 100*acc_errors, 100*acc_seq_err))
-
- # If errors are below the curriculum threshold, move curriculum forward.
- is_good = FLAGS.curriculum_ppx > data.safe_exp(acc_loss)
- is_good = is_good and FLAGS.curriculum_seq > acc_seq_err
- if is_good and is_chief:
- if FLAGS.quantize:
- # Quantize weights.
- data.print_out(" Quantizing parameters.")
- sess.run([quant_op])
- # Increase current length (until the next with training data).
- sess.run(model.cur_length_incr_op)
- # Forget last perplexities if we're not yet at the end.
- if max_cur_length < max_length:
- prev_acc_perp.append(1000000)
-
- # Lower learning rate if we're worse than the last 5 checkpoints.
- acc_perp = data.safe_exp(acc_loss)
- if acc_perp > max(prev_acc_perp[-5:]) and is_chief:
- sess.run(model.lr_decay_op)
- prev_acc_perp.append(acc_perp)
-
- # Save checkpoint.
- if is_chief:
- checkpoint_path = os.path.join(checkpoint_dir, "neural_gpu.ckpt")
- model.saver.save(sess, checkpoint_path,
- global_step=model.global_step)
-
- # Run evaluation.
- bin_bound = 4
- for p in FLAGS.problem.split("-"):
- total_loss, total_err, tl_counter = 0.0, 0.0, 0
- for bin_id in xrange(len(data.bins)):
- if bin_id < bin_bound or bin_id % FLAGS.eval_bin_print == 1:
- err, _, loss = single_test(bin_id, model, sess, FLAGS.nprint,
- batch_size * 4, dev_set, p,
- beam_model=beam_model)
- if loss > 0.0:
- total_loss += loss
- total_err += err
- tl_counter += 1
- test_loss = total_loss / max(1, tl_counter)
- test_err = total_err / max(1, tl_counter)
- test_perp = data.safe_exp(test_loss)
- summary = tf.Summary()
- summary.value.extend(
- [tf.Summary.Value(tag="test/%s/loss" % p, simple_value=test_loss),
- tf.Summary.Value(tag="test/%s/error" % p, simple_value=test_err),
- tf.Summary.Value(tag="test/%s/perplexity" % p,
- simple_value=test_perp)])
- sv.SummaryComputed(sess, summary, global_step)
-
-
-def linearize(output, rev_fr_vocab, simple_tokenizer=None, eos_id=wmt.EOS_ID):
- # If there is an EOS symbol in outputs, cut them at that point (WMT).
- if eos_id in output:
- output = output[:output.index(eos_id)]
- # Print out French sentence corresponding to outputs.
- if simple_tokenizer or FLAGS.simple_tokenizer:
- vlen = len(rev_fr_vocab)
- def vget(o):
- if o < vlen:
- return rev_fr_vocab[o]
- return "UNK"
- return " ".join([vget(o) for o in output])
- else:
- return wmt.basic_detokenizer([rev_fr_vocab[o] for o in output])
-
-
-def evaluate():
- """Evaluate an existing model."""
- batch_size = FLAGS.batch_size * FLAGS.num_gpus
- with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
- (model, beam_model, _, _, _,
- (_, dev_set, en_vocab_path, fr_vocab_path), _, sess) = initialize(sess)
- for p in FLAGS.problem.split("-"):
- for bin_id in xrange(len(data.bins)):
- if (FLAGS.task >= 0 and bin_id > 4) or (FLAGS.nprint == 0 and
- bin_id > 8 and p == "wmt"):
- break
- single_test(bin_id, model, sess, FLAGS.nprint, batch_size, dev_set, p,
- beam_model=beam_model)
- path = FLAGS.test_file_prefix
- xid = "" if FLAGS.task < 0 else ("%.4d" % (FLAGS.task+FLAGS.decode_offset))
- en_path, fr_path = path + ".en" + xid, path + ".fr" + xid
- # Evaluate the test file if they exist.
- if path and tf.gfile.Exists(en_path) and tf.gfile.Exists(fr_path):
- data.print_out("Translating test set %s" % en_path)
- # Read lines.
- en_lines, fr_lines = [], []
- with tf.gfile.GFile(en_path, mode="r") as f:
- for line in f:
- en_lines.append(line.strip())
- with tf.gfile.GFile(fr_path, mode="r") as f:
- for line in f:
- fr_lines.append(line.strip())
- # Tokenize and convert to ids.
- en_vocab, _ = wmt.initialize_vocabulary(en_vocab_path)
- _, rev_fr_vocab = wmt.initialize_vocabulary(fr_vocab_path)
- if FLAGS.simple_tokenizer:
- en_ids = [wmt.sentence_to_token_ids(
- l, en_vocab, tokenizer=wmt.space_tokenizer,
- normalize_digits=FLAGS.normalize_digits)
- for l in en_lines]
- else:
- en_ids = [wmt.sentence_to_token_ids(l, en_vocab) for l in en_lines]
- # Translate.
- results = []
- for idx, token_ids in enumerate(en_ids):
- if idx % 5 == 0:
- data.print_out("Translating example %d of %d." % (idx, len(en_ids)))
- # Which bucket does it belong to?
- buckets = [b for b in xrange(len(data.bins))
- if data.bins[b] >= len(token_ids)]
- if buckets:
- result, result_cost = [], 100000000.0
- for bucket_id in buckets:
- if data.bins[bucket_id] > MAXLEN_F * len(token_ids) + EVAL_LEN_INCR:
- break
- # Get a 1-element batch to feed the sentence to the model.
- used_batch_size = 1 # batch_size
- inp, target = data.get_batch(
- bucket_id, used_batch_size, None, FLAGS.height,
- preset=([token_ids], [[]]))
- loss, output_logits, _, _ = model.step(
- sess, inp, target, None, beam_size=FLAGS.beam_size)
- outputs = [int(o[0]) for o in output_logits]
- loss = loss[0] - (data.bins[bucket_id] * FLAGS.length_norm)
- if FLAGS.simple_tokenizer:
- cur_out = outputs
- if wmt.EOS_ID in cur_out:
- cur_out = cur_out[:cur_out.index(wmt.EOS_ID)]
- res_tags = [rev_fr_vocab[o] for o in cur_out]
- bad_words, bad_brack = wmt.parse_constraints(token_ids, res_tags)
- loss += 1000.0 * bad_words + 100.0 * bad_brack
- # print (bucket_id, loss)
- if loss < result_cost:
- result = outputs
- result_cost = loss
- final = linearize(result, rev_fr_vocab)
- results.append("%s\t%s\n" % (final, fr_lines[idx]))
- # print result_cost
- sys.stderr.write(results[-1])
- sys.stderr.flush()
- else:
- sys.stderr.write("TOOO_LONG\t%s\n" % fr_lines[idx])
- sys.stderr.flush()
- if xid:
- decode_suffix = "beam%dln%dn" % (FLAGS.beam_size,
- int(100 * FLAGS.length_norm))
- with tf.gfile.GFile(path + ".res" + decode_suffix + xid, mode="w") as f:
- for line in results:
- f.write(line)
-
-
-def mul(l):
- res = 1.0
- for s in l:
- res *= s
- return res
-
-
-def interactive():
- """Interactively probe an existing model."""
- with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
- # Initialize model.
- (model, _, _, _, _, (_, _, en_path, fr_path), _, _) = initialize(sess)
- # Load vocabularies.
- en_vocab, rev_en_vocab = wmt.initialize_vocabulary(en_path)
- _, rev_fr_vocab = wmt.initialize_vocabulary(fr_path)
- # Print out vectors and variables.
- if FLAGS.nprint > 0 and FLAGS.word_vector_file_en:
- print_vectors("embedding:0", en_path, FLAGS.word_vector_file_en)
- if FLAGS.nprint > 0 and FLAGS.word_vector_file_fr:
- print_vectors("target_embedding:0", fr_path, FLAGS.word_vector_file_fr)
- total = 0
- for v in tf.trainable_variables():
- shape = v.get_shape().as_list()
- total += mul(shape)
- print(v.name, shape, mul(shape))
- print(total)
- # Start interactive loop.
- sys.stdout.write("Input to Neural GPU Translation Model.\n")
- sys.stdout.write("> ")
- sys.stdout.flush()
- inpt = sys.stdin.readline(), ""
- while inpt:
- cures = []
- # Get token-ids for the input sentence.
- if FLAGS.simple_tokenizer:
- token_ids = wmt.sentence_to_token_ids(
- inpt, en_vocab, tokenizer=wmt.space_tokenizer,
- normalize_digits=FLAGS.normalize_digits)
- else:
- token_ids = wmt.sentence_to_token_ids(inpt, en_vocab)
- print([rev_en_vocab[t] for t in token_ids])
- # Which bucket does it belong to?
- buckets = [b for b in xrange(len(data.bins))
- if data.bins[b] >= max(len(token_ids), len(cures))]
- if cures:
- buckets = [buckets[0]]
- if buckets:
- result, result_cost = [], 10000000.0
- for bucket_id in buckets:
- if data.bins[bucket_id] > MAXLEN_F * len(token_ids) + EVAL_LEN_INCR:
- break
- glen = 1
- for gen_idx in xrange(glen):
- # Get a 1-element batch to feed the sentence to the model.
- inp, target = data.get_batch(
- bucket_id, 1, None, FLAGS.height, preset=([token_ids], [cures]))
- loss, output_logits, _, _ = model.step(
- sess, inp, target, None, beam_size=FLAGS.beam_size,
- update_mem=False)
- # If it is a greedy decoder, outputs are argmaxes of output_logits.
- if FLAGS.beam_size > 1:
- outputs = [int(o) for o in output_logits]
- else:
- loss = loss[0] - (data.bins[bucket_id] * FLAGS.length_norm)
- outputs = [int(np.argmax(logit, axis=1))
- for logit in output_logits]
- print([rev_fr_vocab[t] for t in outputs])
- print(loss, data.bins[bucket_id])
- print(linearize(outputs, rev_fr_vocab))
- cures.append(outputs[gen_idx])
- print(cures)
- print(linearize(cures, rev_fr_vocab))
- if FLAGS.simple_tokenizer:
- cur_out = outputs
- if wmt.EOS_ID in cur_out:
- cur_out = cur_out[:cur_out.index(wmt.EOS_ID)]
- res_tags = [rev_fr_vocab[o] for o in cur_out]
- bad_words, bad_brack = wmt.parse_constraints(token_ids, res_tags)
- loss += 1000.0 * bad_words + 100.0 * bad_brack
- if loss < result_cost:
- result = outputs
- result_cost = loss
- print("FINAL", result_cost)
- print([rev_fr_vocab[t] for t in result])
- print(linearize(result, rev_fr_vocab))
- else:
- print("TOOO_LONG")
- sys.stdout.write("> ")
- sys.stdout.flush()
- inpt = sys.stdin.readline(), ""
-
-
-def main(_):
- if FLAGS.mode == 0:
- train()
- elif FLAGS.mode == 1:
- evaluate()
- else:
- interactive()
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/neural_gpu/program_utils.py b/research/neural_gpu/program_utils.py
deleted file mode 100644
index 1f49d01292012487c4a01a5832fb044a378645ff..0000000000000000000000000000000000000000
--- a/research/neural_gpu/program_utils.py
+++ /dev/null
@@ -1,444 +0,0 @@
-# Copyright 2015 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Utilities for generating program synthesis and evaluation data."""
-
-import contextlib
-import sys
-import random
-import os
-
-try:
- import StringIO
-except ImportError:
- from io import StringIO
-
-class ListType(object):
- def __init__(self, arg):
- self.arg = arg
-
- def __str__(self):
- return "[" + str(self.arg) + "]"
-
- def __eq__(self, other):
- if not isinstance(other, ListType):
- return False
- return self.arg == other.arg
-
- def __hash__(self):
- return hash(self.arg)
-
-class VarType(object):
- def __init__(self, arg):
- self.arg = arg
-
- def __str__(self):
- return str(self.arg)
-
- def __eq__(self, other):
- if not isinstance(other, VarType):
- return False
- return self.arg == other.arg
-
- def __hash__(self):
- return hash(self.arg)
-
-class FunctionType(object):
- def __init__(self, args):
- self.args = args
-
- def __str__(self):
- return str(self.args[0]) + " -> " + str(self.args[1])
-
- def __eq__(self, other):
- if not isinstance(other, FunctionType):
- return False
- return self.args == other.args
-
- def __hash__(self):
- return hash(tuple(self.args))
-
-
-class Function(object):
- def __init__(self, name, arg_types, output_type, fn_arg_types = None):
- self.name = name
- self.arg_types = arg_types
- self.fn_arg_types = fn_arg_types or []
- self.output_type = output_type
-
-Null = 100
-## Functions
-f_head = Function("c_head", [ListType("Int")], "Int")
-def c_head(xs): return xs[0] if len(xs) > 0 else Null
-
-f_last = Function("c_last", [ListType("Int")], "Int")
-def c_last(xs): return xs[-1] if len(xs) > 0 else Null
-
-f_take = Function("c_take", ["Int", ListType("Int")], ListType("Int"))
-def c_take(n, xs): return xs[:n]
-
-f_drop = Function("c_drop", ["Int", ListType("Int")], ListType("Int"))
-def c_drop(n, xs): return xs[n:]
-
-f_access = Function("c_access", ["Int", ListType("Int")], "Int")
-def c_access(n, xs): return xs[n] if n >= 0 and len(xs) > n else Null
-
-f_max = Function("c_max", [ListType("Int")], "Int")
-def c_max(xs): return max(xs) if len(xs) > 0 else Null
-
-f_min = Function("c_min", [ListType("Int")], "Int")
-def c_min(xs): return min(xs) if len(xs) > 0 else Null
-
-f_reverse = Function("c_reverse", [ListType("Int")], ListType("Int"))
-def c_reverse(xs): return list(reversed(xs))
-
-f_sort = Function("sorted", [ListType("Int")], ListType("Int"))
-# def c_sort(xs): return sorted(xs)
-
-f_sum = Function("sum", [ListType("Int")], "Int")
-# def c_sum(xs): return sum(xs)
-
-
-## Lambdas
-# Int -> Int
-def plus_one(x): return x + 1
-def minus_one(x): return x - 1
-def times_two(x): return x * 2
-def neg(x): return x * (-1)
-def div_two(x): return int(x/2)
-def sq(x): return x**2
-def times_three(x): return x * 3
-def div_three(x): return int(x/3)
-def times_four(x): return x * 4
-def div_four(x): return int(x/4)
-
-# Int -> Bool
-def pos(x): return x > 0
-def neg(x): return x < 0
-def even(x): return x%2 == 0
-def odd(x): return x%2 == 1
-
-# Int -> Int -> Int
-def add(x, y): return x + y
-def sub(x, y): return x - y
-def mul(x, y): return x * y
-
-# HOFs
-f_map = Function("map", [ListType("Int")],
- ListType("Int"),
- [FunctionType(["Int", "Int"])])
-f_filter = Function("filter", [ListType("Int")],
- ListType("Int"),
- [FunctionType(["Int", "Bool"])])
-f_count = Function("c_count", [ListType("Int")],
- "Int",
- [FunctionType(["Int", "Bool"])])
-def c_count(f, xs): return len([x for x in xs if f(x)])
-
-f_zipwith = Function("c_zipwith", [ListType("Int"), ListType("Int")],
- ListType("Int"),
- [FunctionType(["Int", "Int", "Int"])]) #FIX
-def c_zipwith(f, xs, ys): return [f(x, y) for (x, y) in zip(xs, ys)]
-
-f_scan = Function("c_scan", [ListType("Int")],
- ListType("Int"),
- [FunctionType(["Int", "Int", "Int"])])
-def c_scan(f, xs):
- out = xs
- for i in range(1, len(xs)):
- out[i] = f(xs[i], xs[i -1])
- return out
-
-@contextlib.contextmanager
-def stdoutIO(stdout=None):
- old = sys.stdout
- if stdout is None:
- stdout = StringIO.StringIO()
- sys.stdout = stdout
- yield stdout
- sys.stdout = old
-
-
-def evaluate(program_str, input_names_to_vals, default="ERROR"):
- exec_str = []
- for name, val in input_names_to_vals.iteritems():
- exec_str += name + " = " + str(val) + "; "
- exec_str += program_str
- if type(exec_str) is list:
- exec_str = "".join(exec_str)
-
- with stdoutIO() as s:
- # pylint: disable=bare-except
- try:
- exec(exec_str + " print(out)")
- return s.getvalue()[:-1]
- except:
- return default
- # pylint: enable=bare-except
-
-
-class Statement(object):
- """Statement class."""
-
- def __init__(self, fn, output_var, arg_vars, fn_args=None):
- self.fn = fn
- self.output_var = output_var
- self.arg_vars = arg_vars
- self.fn_args = fn_args or []
-
- def __str__(self):
- return "%s = %s(%s%s%s)"%(self.output_var,
- self.fn.name,
- ", ".join(self.fn_args),
- ", " if self.fn_args else "",
- ", ".join(self.arg_vars))
-
- def substitute(self, env):
- self.output_var = env.get(self.output_var, self.output_var)
- self.arg_vars = [env.get(v, v) for v in self.arg_vars]
-
-
-class ProgramGrower(object):
- """Grow programs."""
-
- def __init__(self, functions, types_to_lambdas):
- self.functions = functions
- self.types_to_lambdas = types_to_lambdas
-
- def grow_body(self, new_var_name, dependencies, types_to_vars):
- """Grow the program body."""
- choices = []
- for f in self.functions:
- if all([a in types_to_vars.keys() for a in f.arg_types]):
- choices.append(f)
-
- f = random.choice(choices)
- args = []
- for t in f.arg_types:
- possible_vars = random.choice(types_to_vars[t])
- var = random.choice(possible_vars)
- args.append(var)
- dependencies.setdefault(new_var_name, []).extend(
- [var] + (dependencies[var]))
-
- fn_args = [random.choice(self.types_to_lambdas[t]) for t in f.fn_arg_types]
- types_to_vars.setdefault(f.output_type, []).append(new_var_name)
-
- return Statement(f, new_var_name, args, fn_args)
-
- def grow(self, program_len, input_types):
- """Grow the program."""
- var_names = list(reversed(map(chr, range(97, 123))))
- dependencies = dict()
- types_to_vars = dict()
- input_names = []
- for t in input_types:
- var = var_names.pop()
- dependencies[var] = []
- types_to_vars.setdefault(t, []).append(var)
- input_names.append(var)
-
- statements = []
- for _ in range(program_len - 1):
- var = var_names.pop()
- statements.append(self.grow_body(var, dependencies, types_to_vars))
- statements.append(self.grow_body("out", dependencies, types_to_vars))
-
- new_var_names = [c for c in map(chr, range(97, 123))
- if c not in input_names]
- new_var_names.reverse()
- keep_statements = []
- env = dict()
- for s in statements:
- if s.output_var in dependencies["out"]:
- keep_statements.append(s)
- env[s.output_var] = new_var_names.pop()
- if s.output_var == "out":
- keep_statements.append(s)
-
- for k in keep_statements:
- k.substitute(env)
-
- return Program(input_names, input_types, ";".join(
- [str(k) for k in keep_statements]))
-
-
-class Program(object):
- """The program class."""
-
- def __init__(self, input_names, input_types, body):
- self.input_names = input_names
- self.input_types = input_types
- self.body = body
-
- def evaluate(self, inputs):
- """Evaluate this program."""
- if len(inputs) != len(self.input_names):
- raise AssertionError("inputs and input_names have to"
- "have the same len. inp: %s , names: %s" %
- (str(inputs), str(self.input_names)))
- inp_str = ""
- for (name, inp) in zip(self.input_names, inputs):
- inp_str += name + " = " + str(inp) + "; "
-
- with stdoutIO() as s:
- # pylint: disable=exec-used
- exec(inp_str + self.body + "; print(out)")
- # pylint: enable=exec-used
- return s.getvalue()[:-1]
-
- def flat_str(self):
- out = ""
- for s in self.body.split(";"):
- out += s + ";"
- return out
-
- def __str__(self):
- out = ""
- for (n, t) in zip(self.input_names, self.input_types):
- out += n + " = " + str(t) + "\n"
- for s in self.body.split(";"):
- out += s + "\n"
- return out
-
-
-prog_vocab = []
-prog_rev_vocab = {}
-
-
-def tokenize(string, tokens=None):
- """Tokenize the program string."""
- if tokens is None:
- tokens = prog_vocab
- tokens = sorted(tokens, key=len, reverse=True)
- out = []
- string = string.strip()
- while string:
- found = False
- for t in tokens:
- if string.startswith(t):
- out.append(t)
- string = string[len(t):]
- found = True
- break
- if not found:
- raise ValueError("Couldn't tokenize this: " + string)
- string = string.strip()
- return out
-
-
-def clean_up(output, max_val=100):
- o = eval(str(output))
- if isinstance(o, bool):
- return o
- if isinstance(o, int):
- if o >= 0:
- return min(o, max_val)
- else:
- return max(o, -1 * max_val)
- if isinstance(o, list):
- return [clean_up(l) for l in o]
-
-
-def make_vocab():
- gen(2, 0)
-
-
-def gen(max_len, how_many):
- """Generate some programs."""
- functions = [f_head, f_last, f_take, f_drop, f_access, f_max, f_min,
- f_reverse, f_sort, f_sum, f_map, f_filter, f_count, f_zipwith,
- f_scan]
-
- types_to_lambdas = {
- FunctionType(["Int", "Int"]): ["plus_one", "minus_one", "times_two",
- "div_two", "sq", "times_three",
- "div_three", "times_four", "div_four"],
- FunctionType(["Int", "Bool"]): ["pos", "neg", "even", "odd"],
- FunctionType(["Int", "Int", "Int"]): ["add", "sub", "mul"]
- }
-
- tokens = []
- for f in functions:
- tokens.append(f.name)
- for v in types_to_lambdas.values():
- tokens.extend(v)
- tokens.extend(["=", ";", ",", "(", ")", "[", "]", "Int", "out"])
- tokens.extend(map(chr, range(97, 123)))
-
- io_tokens = map(str, range(-220, 220))
- if not prog_vocab:
- prog_vocab.extend(["_PAD", "_EOS"] + tokens + io_tokens)
- for i, t in enumerate(prog_vocab):
- prog_rev_vocab[t] = i
-
- io_tokens += [",", "[", "]", ")", "(", "None"]
- grower = ProgramGrower(functions=functions,
- types_to_lambdas=types_to_lambdas)
-
- def mk_inp(l):
- return [random.choice(range(-5, 5)) for _ in range(l)]
-
- tar = [ListType("Int")]
- inps = [[mk_inp(3)], [mk_inp(5)], [mk_inp(7)], [mk_inp(15)]]
-
- save_prefix = None
- outcomes_to_programs = dict()
- tried = set()
- counter = 0
- choices = [0] if max_len == 0 else range(max_len)
- while counter < 100 * how_many and len(outcomes_to_programs) < how_many:
- counter += 1
- length = random.choice(choices)
- t = grower.grow(length, tar)
- while t in tried:
- length = random.choice(choices)
- t = grower.grow(length, tar)
- # print(t.flat_str())
- tried.add(t)
- outcomes = [clean_up(t.evaluate(i)) for i in inps]
- outcome_str = str(zip(inps, outcomes))
- if outcome_str in outcomes_to_programs:
- outcomes_to_programs[outcome_str] = min(
- [t.flat_str(), outcomes_to_programs[outcome_str]],
- key=lambda x: len(tokenize(x, tokens)))
- else:
- outcomes_to_programs[outcome_str] = t.flat_str()
- if counter % 5000 == 0:
- print("== proggen: tried: " + str(counter))
- print("== proggen: kept: " + str(len(outcomes_to_programs)))
-
- if counter % 250000 == 0 and save_prefix is not None:
- print("saving...")
- save_counter = 0
- progfilename = os.path.join(save_prefix, "prog_" + str(counter) + ".txt")
- iofilename = os.path.join(save_prefix, "io_" + str(counter) + ".txt")
- prog_token_filename = os.path.join(save_prefix,
- "prog_tokens_" + str(counter) + ".txt")
- io_token_filename = os.path.join(save_prefix,
- "io_tokens_" + str(counter) + ".txt")
- with open(progfilename, "a+") as fp, \
- open(iofilename, "a+") as fi, \
- open(prog_token_filename, "a+") as ftp, \
- open(io_token_filename, "a+") as fti:
- for (o, p) in outcomes_to_programs.iteritems():
- save_counter += 1
- if save_counter % 500 == 0:
- print("saving %d of %d" % (save_counter, len(outcomes_to_programs)))
- fp.write(p+"\n")
- fi.write(o+"\n")
- ftp.write(str(tokenize(p, tokens))+"\n")
- fti.write(str(tokenize(o, io_tokens))+"\n")
-
- return list(outcomes_to_programs.values())
diff --git a/research/neural_gpu/wmt_utils.py b/research/neural_gpu/wmt_utils.py
deleted file mode 100644
index ef831918f9c9279eb1c6e560e5730739e5fe9521..0000000000000000000000000000000000000000
--- a/research/neural_gpu/wmt_utils.py
+++ /dev/null
@@ -1,437 +0,0 @@
-# Copyright 2015 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Utilities for downloading data from WMT, tokenizing, vocabularies."""
-
-from __future__ import print_function
-
-import gzip
-import os
-import re
-import tarfile
-
-from six.moves import urllib
-import tensorflow as tf
-
-# Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_CHAR_UNK"
-_SPACE = b"_SPACE"
-_START_VOCAB = [_PAD, _GO, _EOS, _UNK, _SPACE]
-
-PAD_ID = 0
-GO_ID = 1
-EOS_ID = 2
-UNK_ID = 3
-SPACE_ID = 4
-
-# Regular expressions used to tokenize.
-_CHAR_MARKER = "_CHAR_"
-_CHAR_MARKER_LEN = len(_CHAR_MARKER)
-_SPEC_CHARS = "" + chr(226) + chr(153) + chr(128)
-_PUNCTUATION = "][.,!?\"':;%$#@&*+}{|><=/^~)(_`,0123456789" + _SPEC_CHARS + "-"
-_WORD_SPLIT = re.compile("([" + _PUNCTUATION + "])")
-_OLD_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
-
-# URLs for WMT data.
-_WMT_ENFR_TRAIN_URL = "http://www.statmt.org/wmt10/training-giga-fren.tar"
-_WMT_ENFR_DEV_URL = "http://www.statmt.org/wmt15/dev-v2.tgz"
-
-
-def maybe_download(directory, filename, url):
- """Download filename from url unless it's already in directory."""
- if not tf.gfile.Exists(directory):
- print("Creating directory %s" % directory)
- os.mkdir(directory)
- filepath = os.path.join(directory, filename)
- if not tf.gfile.Exists(filepath):
- print("Downloading %s to %s" % (url, filepath))
- filepath, _ = urllib.request.urlretrieve(url, filepath)
- statinfo = os.stat(filepath)
- print("Successfully downloaded", filename, statinfo.st_size, "bytes")
- return filepath
-
-
-def gunzip_file(gz_path, new_path):
- """Unzips from gz_path into new_path."""
- print("Unpacking %s to %s" % (gz_path, new_path))
- with gzip.open(gz_path, "rb") as gz_file:
- with open(new_path, "wb") as new_file:
- for line in gz_file:
- new_file.write(line)
-
-
-def get_wmt_enfr_train_set(directory):
- """Download the WMT en-fr training corpus to directory unless it's there."""
- train_path = os.path.join(directory, "giga-fren.release2.fixed")
- if not (tf.gfile.Exists(train_path +".fr") and
- tf.gfile.Exists(train_path +".en")):
- corpus_file = maybe_download(directory, "training-giga-fren.tar",
- _WMT_ENFR_TRAIN_URL)
- print("Extracting tar file %s" % corpus_file)
- with tarfile.open(corpus_file, "r") as corpus_tar:
- corpus_tar.extractall(directory)
- gunzip_file(train_path + ".fr.gz", train_path + ".fr")
- gunzip_file(train_path + ".en.gz", train_path + ".en")
- return train_path
-
-
-def get_wmt_enfr_dev_set(directory):
- """Download the WMT en-fr training corpus to directory unless it's there."""
- dev_name = "newstest2013"
- dev_path = os.path.join(directory, dev_name)
- if not (tf.gfile.Exists(dev_path + ".fr") and
- tf.gfile.Exists(dev_path + ".en")):
- dev_file = maybe_download(directory, "dev-v2.tgz", _WMT_ENFR_DEV_URL)
- print("Extracting tgz file %s" % dev_file)
- with tarfile.open(dev_file, "r:gz") as dev_tar:
- fr_dev_file = dev_tar.getmember("dev/" + dev_name + ".fr")
- en_dev_file = dev_tar.getmember("dev/" + dev_name + ".en")
- fr_dev_file.name = dev_name + ".fr" # Extract without "dev/" prefix.
- en_dev_file.name = dev_name + ".en"
- dev_tar.extract(fr_dev_file, directory)
- dev_tar.extract(en_dev_file, directory)
- return dev_path
-
-
-def is_char(token):
- if len(token) > _CHAR_MARKER_LEN:
- if token[:_CHAR_MARKER_LEN] == _CHAR_MARKER:
- return True
- return False
-
-
-def basic_detokenizer(tokens):
- """Reverse the process of the basic tokenizer below."""
- result = []
- previous_nospace = True
- for t in tokens:
- if is_char(t):
- result.append(t[_CHAR_MARKER_LEN:])
- previous_nospace = True
- elif t == _SPACE:
- result.append(" ")
- previous_nospace = True
- elif previous_nospace:
- result.append(t)
- previous_nospace = False
- else:
- result.extend([" ", t])
- previous_nospace = False
- return "".join(result)
-
-
-old_style = False
-
-
-def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- if old_style:
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_OLD_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
- for space_separated_fragment in sentence.strip().split():
- tokens = [t for t in re.split(_WORD_SPLIT, space_separated_fragment) if t]
- first_is_char = False
- for i, t in enumerate(tokens):
- if len(t) == 1 and t in _PUNCTUATION:
- tokens[i] = _CHAR_MARKER + t
- if i == 0:
- first_is_char = True
- if words and words[-1] != _SPACE and (first_is_char or is_char(words[-1])):
- tokens = [_SPACE] + tokens
- spaced_tokens = []
- for i, tok in enumerate(tokens):
- spaced_tokens.append(tokens[i])
- if i < len(tokens) - 1:
- if tok != _SPACE and not (is_char(tok) or is_char(tokens[i+1])):
- spaced_tokens.append(_SPACE)
- words.extend(spaced_tokens)
- return words
-
-
-def space_tokenizer(sentence):
- return sentence.strip().split()
-
-
-def is_pos_tag(token):
- """Check if token is a part-of-speech tag."""
- return(token in ["CC", "CD", "DT", "EX", "FW", "IN", "JJ", "JJR",
- "JJS", "LS", "MD", "NN", "NNS", "NNP", "NNPS", "PDT",
- "POS", "PRP", "PRP$", "RB", "RBR", "RBS", "RP", "SYM", "TO",
- "UH", "VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "WDT", "WP",
- "WP$", "WRB", ".", ",", ":", ")", "-LRB-", "(", "-RRB-",
- "HYPH", "$", "``", "''", "ADD", "AFX", "QTR", "BES", "-DFL-",
- "GW", "HVS", "NFP"])
-
-
-def parse_constraints(inpt, res):
- ntags = len(res)
- nwords = len(inpt)
- npostags = len([x for x in res if is_pos_tag(x)])
- nclose = len([x for x in res if x[0] == "/"])
- nopen = ntags - nclose - npostags
- return (abs(npostags - nwords), abs(nclose - nopen))
-
-
-def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
- tokenizer=None, normalize_digits=False):
- """Create vocabulary file (if it does not exist yet) from data file.
-
- Data file is assumed to contain one sentence per line. Each sentence is
- tokenized and digits are normalized (if normalize_digits is set).
- Vocabulary contains the most-frequent tokens up to max_vocabulary_size.
- We write it to vocabulary_path in a one-token-per-line format, so that later
- token in the first line gets id=0, second line gets id=1, and so on.
-
- Args:
- vocabulary_path: path where the vocabulary will be created.
- data_path: data file that will be used to create vocabulary.
- max_vocabulary_size: limit on the size of the created vocabulary.
- tokenizer: a function to use to tokenize each data sentence;
- if None, basic_tokenizer will be used.
- normalize_digits: Boolean; if true, all digits are replaced by 0s.
- """
- if not tf.gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from data %s" % (vocabulary_path, data_path))
- vocab, chars = {}, {}
- for c in _PUNCTUATION:
- chars[c] = 1
-
- # Read French file.
- with tf.gfile.GFile(data_path + ".fr", mode="rb") as f:
- counter = 0
- for line_in in f:
- line = " ".join(line_in.split())
- counter += 1
- if counter % 100000 == 0:
- print(" processing fr line %d" % counter)
- for c in line:
- if c in chars:
- chars[c] += 1
- else:
- chars[c] = 1
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- tokens = [t for t in tokens if not is_char(t) and t != _SPACE]
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1000000000 # We want target words first.
- else:
- vocab[word] = 1000000000
-
- # Read English file.
- with tf.gfile.GFile(data_path + ".en", mode="rb") as f:
- counter = 0
- for line_in in f:
- line = " ".join(line_in.split())
- counter += 1
- if counter % 100000 == 0:
- print(" processing en line %d" % counter)
- for c in line:
- if c in chars:
- chars[c] += 1
- else:
- chars[c] = 1
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- tokens = [t for t in tokens if not is_char(t) and t != _SPACE]
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
-
- sorted_vocab = sorted(vocab, key=vocab.get, reverse=True)
- sorted_chars = sorted(chars, key=vocab.get, reverse=True)
- sorted_chars = [_CHAR_MARKER + c for c in sorted_chars]
- vocab_list = _START_VOCAB + sorted_chars + sorted_vocab
- if tokenizer:
- vocab_list = _START_VOCAB + sorted_vocab
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with tf.gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
- """Initialize vocabulary from file.
-
- We assume the vocabulary is stored one-item-per-line, so a file:
- dog
- cat
- will result in a vocabulary {"dog": 0, "cat": 1}, and this function will
- also return the reversed-vocabulary ["dog", "cat"].
-
- Args:
- vocabulary_path: path to the file containing the vocabulary.
-
- Returns:
- a pair: the vocabulary (a dictionary mapping string to integers), and
- the reversed vocabulary (a list, which reverses the vocabulary mapping).
-
- Raises:
- ValueError: if the provided vocabulary_path does not exist.
- """
- if tf.gfile.Exists(vocabulary_path):
- rev_vocab = []
- with tf.gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
- return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
-
-
-def sentence_to_token_ids_raw(sentence, vocabulary,
- tokenizer=None, normalize_digits=old_style):
- """Convert a string to list of integers representing token-ids.
-
- For example, a sentence "I have a dog" may become tokenized into
- ["I", "have", "a", "dog"] and with vocabulary {"I": 1, "have": 2,
- "a": 4, "dog": 7"} this function will return [1, 2, 4, 7].
-
- Args:
- sentence: the sentence in bytes format to convert to token-ids.
- vocabulary: a dictionary mapping tokens to integers.
- tokenizer: a function to use to tokenize each sentence;
- if None, basic_tokenizer will be used.
- normalize_digits: Boolean; if true, all digits are replaced by 0s.
-
- Returns:
- a list of integers, the token-ids for the sentence.
- """
- if tokenizer:
- words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- result = []
- for w in words:
- if normalize_digits:
- w = re.sub(_DIGIT_RE, b"0", w)
- if w in vocabulary:
- result.append(vocabulary[w])
- else:
- if tokenizer:
- result.append(UNK_ID)
- else:
- result.append(SPACE_ID)
- for c in w:
- result.append(vocabulary.get(_CHAR_MARKER + c, UNK_ID))
- result.append(SPACE_ID)
- while result and result[0] == SPACE_ID:
- result = result[1:]
- while result and result[-1] == SPACE_ID:
- result = result[:-1]
- return result
-
-
-def sentence_to_token_ids(sentence, vocabulary,
- tokenizer=None, normalize_digits=old_style):
- """Convert a string to list of integers representing token-ids, tab=0."""
- tab_parts = sentence.strip().split("\t")
- toks = [sentence_to_token_ids_raw(t, vocabulary, tokenizer, normalize_digits)
- for t in tab_parts]
- res = []
- for t in toks:
- res.extend(t)
- res.append(0)
- return res[:-1]
-
-
-def data_to_token_ids(data_path, target_path, vocabulary_path,
- tokenizer=None, normalize_digits=False):
- """Tokenize data file and turn into token-ids using given vocabulary file.
-
- This function loads data line-by-line from data_path, calls the above
- sentence_to_token_ids, and saves the result to target_path. See comment
- for sentence_to_token_ids on the details of token-ids format.
-
- Args:
- data_path: path to the data file in one-sentence-per-line format.
- target_path: path where the file with token-ids will be created.
- vocabulary_path: path to the vocabulary file.
- tokenizer: a function to use to tokenize each sentence;
- if None, basic_tokenizer will be used.
- normalize_digits: Boolean; if true, all digits are replaced by 0s.
- """
- if not tf.gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with tf.gfile.GFile(data_path, mode="rb") as data_file:
- with tf.gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-def prepare_wmt_data(data_dir, vocabulary_size,
- tokenizer=None, normalize_digits=False):
- """Get WMT data into data_dir, create vocabularies and tokenize data.
-
- Args:
- data_dir: directory in which the data sets will be stored.
- vocabulary_size: size of the joint vocabulary to create and use.
- tokenizer: a function to use to tokenize each data sentence;
- if None, basic_tokenizer will be used.
- normalize_digits: Boolean; if true, all digits are replaced by 0s.
-
- Returns:
- A tuple of 6 elements:
- (1) path to the token-ids for English training data-set,
- (2) path to the token-ids for French training data-set,
- (3) path to the token-ids for English development data-set,
- (4) path to the token-ids for French development data-set,
- (5) path to the vocabulary file,
- (6) path to the vocabulary file (for compatibility with non-joint vocab).
- """
- # Get wmt data to the specified directory.
- train_path = get_wmt_enfr_train_set(data_dir)
- dev_path = get_wmt_enfr_dev_set(data_dir)
-
- # Create vocabularies of the appropriate sizes.
- vocab_path = os.path.join(data_dir, "vocab%d.txt" % vocabulary_size)
- create_vocabulary(vocab_path, train_path, vocabulary_size,
- tokenizer=tokenizer, normalize_digits=normalize_digits)
-
- # Create token ids for the training data.
- fr_train_ids_path = train_path + (".ids%d.fr" % vocabulary_size)
- en_train_ids_path = train_path + (".ids%d.en" % vocabulary_size)
- data_to_token_ids(train_path + ".fr", fr_train_ids_path, vocab_path,
- tokenizer=tokenizer, normalize_digits=normalize_digits)
- data_to_token_ids(train_path + ".en", en_train_ids_path, vocab_path,
- tokenizer=tokenizer, normalize_digits=normalize_digits)
-
- # Create token ids for the development data.
- fr_dev_ids_path = dev_path + (".ids%d.fr" % vocabulary_size)
- en_dev_ids_path = dev_path + (".ids%d.en" % vocabulary_size)
- data_to_token_ids(dev_path + ".fr", fr_dev_ids_path, vocab_path,
- tokenizer=tokenizer, normalize_digits=normalize_digits)
- data_to_token_ids(dev_path + ".en", en_dev_ids_path, vocab_path,
- tokenizer=tokenizer, normalize_digits=normalize_digits)
-
- return (en_train_ids_path, fr_train_ids_path,
- en_dev_ids_path, fr_dev_ids_path,
- vocab_path, vocab_path)
diff --git a/research/neural_programmer/README.md b/research/neural_programmer/README.md
deleted file mode 100644
index dcc27f6fb015ec625935a0ea37d814a2ba10d2e3..0000000000000000000000000000000000000000
--- a/research/neural_programmer/README.md
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
-# Neural Programmer
-
-Implementation of the Neural Programmer model as described in this [paper](https://openreview.net/pdf?id=ry2YOrcge).
-
-Download and extract the data from the [WikiTableQuestions](https://ppasupat.github.io/WikiTableQuestions/) site. The dataset contains
-11321, 2831, and 4344 examples for training, development, and testing respectively. We use their tokenization, number and date pre-processing. Please note that the above paper used the [initial release](https://github.com/ppasupat/WikiTableQuestions/releases/tag/v0.2) for training, development and testing.
-
-Change the `data_dir FLAG` to the location of the data.
-
-### Training
-Run `python neural_programmer.py`
-
-The models are written to `FLAGS.output_dir`.
-
-### Testing
-Run `python neural_programmer.py --evaluator_job=True`
-
-The models are loaded from `FLAGS.output_dir`. The evaluation is done on development data.
-
-In case of errors because of encoding, add `"# -*- coding: utf-8 -*-"` as the first line in `wiki_data.py`
-
-Maintained by Arvind Neelakantan (arvind2505)
diff --git a/research/neural_programmer/data_utils.py b/research/neural_programmer/data_utils.py
deleted file mode 100644
index 4df80c66ad21d2e046fabf78446dd199ae117b44..0000000000000000000000000000000000000000
--- a/research/neural_programmer/data_utils.py
+++ /dev/null
@@ -1,666 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Functions for constructing vocabulary, converting the examples to integer format and building the required masks for batch computation Author: aneelakantan (Arvind Neelakantan)
-"""
-
-from __future__ import print_function
-
-import copy
-import numbers
-import numpy as np
-import wiki_data
-
-
-def return_index(a):
- for i in range(len(a)):
- if (a[i] == 1.0):
- return i
-
-
-def construct_vocab(data, utility, add_word=False):
- ans = []
- for example in data:
- sent = ""
- for word in example.question:
- if (not (isinstance(word, numbers.Number))):
- sent += word + " "
- example.original_nc = copy.deepcopy(example.number_columns)
- example.original_wc = copy.deepcopy(example.word_columns)
- example.original_nc_names = copy.deepcopy(example.number_column_names)
- example.original_wc_names = copy.deepcopy(example.word_column_names)
- if (add_word):
- continue
- number_found = 0
- if (not (example.is_bad_example)):
- for word in example.question:
- if (isinstance(word, numbers.Number)):
- number_found += 1
- else:
- if (not (utility.word_ids.has_key(word))):
- utility.words.append(word)
- utility.word_count[word] = 1
- utility.word_ids[word] = len(utility.word_ids)
- utility.reverse_word_ids[utility.word_ids[word]] = word
- else:
- utility.word_count[word] += 1
- for col_name in example.word_column_names:
- for word in col_name:
- if (isinstance(word, numbers.Number)):
- number_found += 1
- else:
- if (not (utility.word_ids.has_key(word))):
- utility.words.append(word)
- utility.word_count[word] = 1
- utility.word_ids[word] = len(utility.word_ids)
- utility.reverse_word_ids[utility.word_ids[word]] = word
- else:
- utility.word_count[word] += 1
- for col_name in example.number_column_names:
- for word in col_name:
- if (isinstance(word, numbers.Number)):
- number_found += 1
- else:
- if (not (utility.word_ids.has_key(word))):
- utility.words.append(word)
- utility.word_count[word] = 1
- utility.word_ids[word] = len(utility.word_ids)
- utility.reverse_word_ids[utility.word_ids[word]] = word
- else:
- utility.word_count[word] += 1
-
-
-def word_lookup(word, utility):
- if (utility.word_ids.has_key(word)):
- return word
- else:
- return utility.unk_token
-
-
-def convert_to_int_2d_and_pad(a, utility):
- ans = []
- #print a
- for b in a:
- temp = []
- if (len(b) > utility.FLAGS.max_entry_length):
- b = b[0:utility.FLAGS.max_entry_length]
- for remaining in range(len(b), utility.FLAGS.max_entry_length):
- b.append(utility.dummy_token)
- assert len(b) == utility.FLAGS.max_entry_length
- for word in b:
- temp.append(utility.word_ids[word_lookup(word, utility)])
- ans.append(temp)
- #print ans
- return ans
-
-
-def convert_to_bool_and_pad(a, utility):
- a = a.tolist()
- for i in range(len(a)):
- for j in range(len(a[i])):
- if (a[i][j] < 1):
- a[i][j] = False
- else:
- a[i][j] = True
- a[i] = a[i] + [False] * (utility.FLAGS.max_elements - len(a[i]))
- return a
-
-
-seen_tables = {}
-
-
-def partial_match(question, table, number):
- answer = []
- match = {}
- for i in range(len(table)):
- temp = []
- for j in range(len(table[i])):
- temp.append(0)
- answer.append(temp)
- for i in range(len(table)):
- for j in range(len(table[i])):
- for word in question:
- if (number):
- if (word == table[i][j]):
- answer[i][j] = 1.0
- match[i] = 1.0
- else:
- if (word in table[i][j]):
- answer[i][j] = 1.0
- match[i] = 1.0
- return answer, match
-
-
-def exact_match(question, table, number):
- #performs exact match operation
- answer = []
- match = {}
- matched_indices = []
- for i in range(len(table)):
- temp = []
- for j in range(len(table[i])):
- temp.append(0)
- answer.append(temp)
- for i in range(len(table)):
- for j in range(len(table[i])):
- if (number):
- for word in question:
- if (word == table[i][j]):
- match[i] = 1.0
- answer[i][j] = 1.0
- else:
- table_entry = table[i][j]
- for k in range(len(question)):
- if (k + len(table_entry) <= len(question)):
- if (table_entry == question[k:(k + len(table_entry))]):
- #if(len(table_entry) == 1):
- #print "match: ", table_entry, question
- match[i] = 1.0
- answer[i][j] = 1.0
- matched_indices.append((k, len(table_entry)))
- return answer, match, matched_indices
-
-
-def partial_column_match(question, table, number):
- answer = []
- for i in range(len(table)):
- answer.append(0)
- for i in range(len(table)):
- for word in question:
- if (word in table[i]):
- answer[i] = 1.0
- return answer
-
-
-def exact_column_match(question, table, number):
- #performs exact match on column names
- answer = []
- matched_indices = []
- for i in range(len(table)):
- answer.append(0)
- for i in range(len(table)):
- table_entry = table[i]
- for k in range(len(question)):
- if (k + len(table_entry) <= len(question)):
- if (table_entry == question[k:(k + len(table_entry))]):
- answer[i] = 1.0
- matched_indices.append((k, len(table_entry)))
- return answer, matched_indices
-
-
-def get_max_entry(a):
- e = {}
- for w in a:
- if (w != "UNK, "):
- if (e.has_key(w)):
- e[w] += 1
- else:
- e[w] = 1
- if (len(e) > 0):
- (key, val) = sorted(e.items(), key=lambda x: -1 * x[1])[0]
- if (val > 1):
- return key
- else:
- return -1.0
- else:
- return -1.0
-
-
-def list_join(a):
- ans = ""
- for w in a:
- ans += str(w) + ", "
- return ans
-
-
-def group_by_max(table, number):
- #computes the most frequently occurring entry in a column
- answer = []
- for i in range(len(table)):
- temp = []
- for j in range(len(table[i])):
- temp.append(0)
- answer.append(temp)
- for i in range(len(table)):
- if (number):
- curr = table[i]
- else:
- curr = [list_join(w) for w in table[i]]
- max_entry = get_max_entry(curr)
- #print i, max_entry
- for j in range(len(curr)):
- if (max_entry == curr[j]):
- answer[i][j] = 1.0
- else:
- answer[i][j] = 0.0
- return answer
-
-
-def pick_one(a):
- for i in range(len(a)):
- if (1.0 in a[i]):
- return True
- return False
-
-
-def check_processed_cols(col, utility):
- return True in [
- True for y in col
- if (y != utility.FLAGS.pad_int and y !=
- utility.FLAGS.bad_number_pre_process)
- ]
-
-
-def complete_wiki_processing(data, utility, train=True):
- #convert to integers and padding
- processed_data = []
- num_bad_examples = 0
- for example in data:
- number_found = 0
- if (example.is_bad_example):
- num_bad_examples += 1
- if (not (example.is_bad_example)):
- example.string_question = example.question[:]
- #entry match
- example.processed_number_columns = example.processed_number_columns[:]
- example.processed_word_columns = example.processed_word_columns[:]
- example.word_exact_match, word_match, matched_indices = exact_match(
- example.string_question, example.original_wc, number=False)
- example.number_exact_match, number_match, _ = exact_match(
- example.string_question, example.original_nc, number=True)
- if (not (pick_one(example.word_exact_match)) and not (
- pick_one(example.number_exact_match))):
- assert len(word_match) == 0
- assert len(number_match) == 0
- example.word_exact_match, word_match = partial_match(
- example.string_question, example.original_wc, number=False)
- #group by max
- example.word_group_by_max = group_by_max(example.original_wc, False)
- example.number_group_by_max = group_by_max(example.original_nc, True)
- #column name match
- example.word_column_exact_match, wcol_matched_indices = exact_column_match(
- example.string_question, example.original_wc_names, number=False)
- example.number_column_exact_match, ncol_matched_indices = exact_column_match(
- example.string_question, example.original_nc_names, number=False)
- if (not (1.0 in example.word_column_exact_match) and not (
- 1.0 in example.number_column_exact_match)):
- example.word_column_exact_match = partial_column_match(
- example.string_question, example.original_wc_names, number=False)
- example.number_column_exact_match = partial_column_match(
- example.string_question, example.original_nc_names, number=False)
- if (len(word_match) > 0 or len(number_match) > 0):
- example.question.append(utility.entry_match_token)
- if (1.0 in example.word_column_exact_match or
- 1.0 in example.number_column_exact_match):
- example.question.append(utility.column_match_token)
- example.string_question = example.question[:]
- example.number_lookup_matrix = np.transpose(
- example.number_lookup_matrix)[:]
- example.word_lookup_matrix = np.transpose(example.word_lookup_matrix)[:]
- example.columns = example.number_columns[:]
- example.word_columns = example.word_columns[:]
- example.len_total_cols = len(example.word_column_names) + len(
- example.number_column_names)
- example.column_names = example.number_column_names[:]
- example.word_column_names = example.word_column_names[:]
- example.string_column_names = example.number_column_names[:]
- example.string_word_column_names = example.word_column_names[:]
- example.sorted_number_index = []
- example.sorted_word_index = []
- example.column_mask = []
- example.word_column_mask = []
- example.processed_column_mask = []
- example.processed_word_column_mask = []
- example.word_column_entry_mask = []
- example.question_attention_mask = []
- example.question_number = example.question_number_1 = -1
- example.question_attention_mask = []
- example.ordinal_question = []
- example.ordinal_question_one = []
- new_question = []
- if (len(example.number_columns) > 0):
- example.len_col = len(example.number_columns[0])
- else:
- example.len_col = len(example.word_columns[0])
- for (start, length) in matched_indices:
- for j in range(length):
- example.question[start + j] = utility.unk_token
- #print example.question
- for word in example.question:
- if (isinstance(word, numbers.Number) or wiki_data.is_date(word)):
- if (not (isinstance(word, numbers.Number)) and
- wiki_data.is_date(word)):
- word = word.replace("X", "").replace("-", "")
- number_found += 1
- if (number_found == 1):
- example.question_number = word
- if (len(example.ordinal_question) > 0):
- example.ordinal_question[len(example.ordinal_question) - 1] = 1.0
- else:
- example.ordinal_question.append(1.0)
- elif (number_found == 2):
- example.question_number_1 = word
- if (len(example.ordinal_question_one) > 0):
- example.ordinal_question_one[len(example.ordinal_question_one) -
- 1] = 1.0
- else:
- example.ordinal_question_one.append(1.0)
- else:
- new_question.append(word)
- example.ordinal_question.append(0.0)
- example.ordinal_question_one.append(0.0)
- example.question = [
- utility.word_ids[word_lookup(w, utility)] for w in new_question
- ]
- example.question_attention_mask = [0.0] * len(example.question)
- #when the first question number occurs before a word
- example.ordinal_question = example.ordinal_question[0:len(
- example.question)]
- example.ordinal_question_one = example.ordinal_question_one[0:len(
- example.question)]
- #question-padding
- example.question = [utility.word_ids[utility.dummy_token]] * (
- utility.FLAGS.question_length - len(example.question)
- ) + example.question
- example.question_attention_mask = [-10000.0] * (
- utility.FLAGS.question_length - len(example.question_attention_mask)
- ) + example.question_attention_mask
- example.ordinal_question = [0.0] * (utility.FLAGS.question_length -
- len(example.ordinal_question)
- ) + example.ordinal_question
- example.ordinal_question_one = [0.0] * (utility.FLAGS.question_length -
- len(example.ordinal_question_one)
- ) + example.ordinal_question_one
- if (True):
- #number columns and related-padding
- num_cols = len(example.columns)
- start = 0
- for column in example.number_columns:
- if (check_processed_cols(example.processed_number_columns[start],
- utility)):
- example.processed_column_mask.append(0.0)
- sorted_index = sorted(
- range(len(example.processed_number_columns[start])),
- key=lambda k: example.processed_number_columns[start][k],
- reverse=True)
- sorted_index = sorted_index + [utility.FLAGS.pad_int] * (
- utility.FLAGS.max_elements - len(sorted_index))
- example.sorted_number_index.append(sorted_index)
- example.columns[start] = column + [utility.FLAGS.pad_int] * (
- utility.FLAGS.max_elements - len(column))
- example.processed_number_columns[start] += [utility.FLAGS.pad_int] * (
- utility.FLAGS.max_elements -
- len(example.processed_number_columns[start]))
- start += 1
- example.column_mask.append(0.0)
- for remaining in range(num_cols, utility.FLAGS.max_number_cols):
- example.sorted_number_index.append([utility.FLAGS.pad_int] *
- (utility.FLAGS.max_elements))
- example.columns.append([utility.FLAGS.pad_int] *
- (utility.FLAGS.max_elements))
- example.processed_number_columns.append([utility.FLAGS.pad_int] *
- (utility.FLAGS.max_elements))
- example.number_exact_match.append([0.0] *
- (utility.FLAGS.max_elements))
- example.number_group_by_max.append([0.0] *
- (utility.FLAGS.max_elements))
- example.column_mask.append(-100000000.0)
- example.processed_column_mask.append(-100000000.0)
- example.number_column_exact_match.append(0.0)
- example.column_names.append([utility.dummy_token])
- #word column and related-padding
- start = 0
- word_num_cols = len(example.word_columns)
- for column in example.word_columns:
- if (check_processed_cols(example.processed_word_columns[start],
- utility)):
- example.processed_word_column_mask.append(0.0)
- sorted_index = sorted(
- range(len(example.processed_word_columns[start])),
- key=lambda k: example.processed_word_columns[start][k],
- reverse=True)
- sorted_index = sorted_index + [utility.FLAGS.pad_int] * (
- utility.FLAGS.max_elements - len(sorted_index))
- example.sorted_word_index.append(sorted_index)
- column = convert_to_int_2d_and_pad(column, utility)
- example.word_columns[start] = column + [[
- utility.word_ids[utility.dummy_token]
- ] * utility.FLAGS.max_entry_length] * (utility.FLAGS.max_elements -
- len(column))
- example.processed_word_columns[start] += [utility.FLAGS.pad_int] * (
- utility.FLAGS.max_elements -
- len(example.processed_word_columns[start]))
- example.word_column_entry_mask.append([0] * len(column) + [
- utility.word_ids[utility.dummy_token]
- ] * (utility.FLAGS.max_elements - len(column)))
- start += 1
- example.word_column_mask.append(0.0)
- for remaining in range(word_num_cols, utility.FLAGS.max_word_cols):
- example.sorted_word_index.append([utility.FLAGS.pad_int] *
- (utility.FLAGS.max_elements))
- example.word_columns.append([[utility.word_ids[utility.dummy_token]] *
- utility.FLAGS.max_entry_length] *
- (utility.FLAGS.max_elements))
- example.word_column_entry_mask.append(
- [utility.word_ids[utility.dummy_token]] *
- (utility.FLAGS.max_elements))
- example.word_exact_match.append([0.0] * (utility.FLAGS.max_elements))
- example.word_group_by_max.append([0.0] * (utility.FLAGS.max_elements))
- example.processed_word_columns.append([utility.FLAGS.pad_int] *
- (utility.FLAGS.max_elements))
- example.word_column_mask.append(-100000000.0)
- example.processed_word_column_mask.append(-100000000.0)
- example.word_column_exact_match.append(0.0)
- example.word_column_names.append([utility.dummy_token] *
- utility.FLAGS.max_entry_length)
- seen_tables[example.table_key] = 1
- #convert column and word column names to integers
- example.column_ids = convert_to_int_2d_and_pad(example.column_names,
- utility)
- example.word_column_ids = convert_to_int_2d_and_pad(
- example.word_column_names, utility)
- for i_em in range(len(example.number_exact_match)):
- example.number_exact_match[i_em] = example.number_exact_match[
- i_em] + [0.0] * (utility.FLAGS.max_elements -
- len(example.number_exact_match[i_em]))
- example.number_group_by_max[i_em] = example.number_group_by_max[
- i_em] + [0.0] * (utility.FLAGS.max_elements -
- len(example.number_group_by_max[i_em]))
- for i_em in range(len(example.word_exact_match)):
- example.word_exact_match[i_em] = example.word_exact_match[
- i_em] + [0.0] * (utility.FLAGS.max_elements -
- len(example.word_exact_match[i_em]))
- example.word_group_by_max[i_em] = example.word_group_by_max[
- i_em] + [0.0] * (utility.FLAGS.max_elements -
- len(example.word_group_by_max[i_em]))
- example.exact_match = example.number_exact_match + example.word_exact_match
- example.group_by_max = example.number_group_by_max + example.word_group_by_max
- example.exact_column_match = example.number_column_exact_match + example.word_column_exact_match
- #answer and related mask, padding
- if (example.is_lookup):
- example.answer = example.calc_answer
- example.number_print_answer = example.number_lookup_matrix.tolist()
- example.word_print_answer = example.word_lookup_matrix.tolist()
- for i_answer in range(len(example.number_print_answer)):
- example.number_print_answer[i_answer] = example.number_print_answer[
- i_answer] + [0.0] * (utility.FLAGS.max_elements -
- len(example.number_print_answer[i_answer]))
- for i_answer in range(len(example.word_print_answer)):
- example.word_print_answer[i_answer] = example.word_print_answer[
- i_answer] + [0.0] * (utility.FLAGS.max_elements -
- len(example.word_print_answer[i_answer]))
- example.number_lookup_matrix = convert_to_bool_and_pad(
- example.number_lookup_matrix, utility)
- example.word_lookup_matrix = convert_to_bool_and_pad(
- example.word_lookup_matrix, utility)
- for remaining in range(num_cols, utility.FLAGS.max_number_cols):
- example.number_lookup_matrix.append([False] *
- utility.FLAGS.max_elements)
- example.number_print_answer.append([0.0] * utility.FLAGS.max_elements)
- for remaining in range(word_num_cols, utility.FLAGS.max_word_cols):
- example.word_lookup_matrix.append([False] *
- utility.FLAGS.max_elements)
- example.word_print_answer.append([0.0] * utility.FLAGS.max_elements)
- example.print_answer = example.number_print_answer + example.word_print_answer
- else:
- example.answer = example.calc_answer
- example.print_answer = [[0.0] * (utility.FLAGS.max_elements)] * (
- utility.FLAGS.max_number_cols + utility.FLAGS.max_word_cols)
- #question_number masks
- if (example.question_number == -1):
- example.question_number_mask = np.zeros([utility.FLAGS.max_elements])
- else:
- example.question_number_mask = np.ones([utility.FLAGS.max_elements])
- if (example.question_number_1 == -1):
- example.question_number_one_mask = -10000.0
- else:
- example.question_number_one_mask = np.float64(0.0)
- if (example.len_col > utility.FLAGS.max_elements):
- continue
- processed_data.append(example)
- return processed_data
-
-
-def add_special_words(utility):
- utility.words.append(utility.entry_match_token)
- utility.word_ids[utility.entry_match_token] = len(utility.word_ids)
- utility.reverse_word_ids[utility.word_ids[
- utility.entry_match_token]] = utility.entry_match_token
- utility.entry_match_token_id = utility.word_ids[utility.entry_match_token]
- print("entry match token: ", utility.word_ids[
- utility.entry_match_token], utility.entry_match_token_id)
- utility.words.append(utility.column_match_token)
- utility.word_ids[utility.column_match_token] = len(utility.word_ids)
- utility.reverse_word_ids[utility.word_ids[
- utility.column_match_token]] = utility.column_match_token
- utility.column_match_token_id = utility.word_ids[utility.column_match_token]
- print("entry match token: ", utility.word_ids[
- utility.column_match_token], utility.column_match_token_id)
- utility.words.append(utility.dummy_token)
- utility.word_ids[utility.dummy_token] = len(utility.word_ids)
- utility.reverse_word_ids[utility.word_ids[
- utility.dummy_token]] = utility.dummy_token
- utility.dummy_token_id = utility.word_ids[utility.dummy_token]
- utility.words.append(utility.unk_token)
- utility.word_ids[utility.unk_token] = len(utility.word_ids)
- utility.reverse_word_ids[utility.word_ids[
- utility.unk_token]] = utility.unk_token
-
-
-def perform_word_cutoff(utility):
- if (utility.FLAGS.word_cutoff > 0):
- for word in utility.word_ids.keys():
- if (utility.word_count.has_key(word) and utility.word_count[word] <
- utility.FLAGS.word_cutoff and word != utility.unk_token and
- word != utility.dummy_token and word != utility.entry_match_token and
- word != utility.column_match_token):
- utility.word_ids.pop(word)
- utility.words.remove(word)
-
-
-def word_dropout(question, utility):
- if (utility.FLAGS.word_dropout_prob > 0.0):
- new_question = []
- for i in range(len(question)):
- if (question[i] != utility.dummy_token_id and
- utility.random.random() > utility.FLAGS.word_dropout_prob):
- new_question.append(utility.word_ids[utility.unk_token])
- else:
- new_question.append(question[i])
- return new_question
- else:
- return question
-
-
-def generate_feed_dict(data, curr, batch_size, gr, train=False, utility=None):
- #prepare feed dict dictionary
- feed_dict = {}
- feed_examples = []
- for j in range(batch_size):
- feed_examples.append(data[curr + j])
- if (train):
- feed_dict[gr.batch_question] = [
- word_dropout(feed_examples[j].question, utility)
- for j in range(batch_size)
- ]
- else:
- feed_dict[gr.batch_question] = [
- feed_examples[j].question for j in range(batch_size)
- ]
- feed_dict[gr.batch_question_attention_mask] = [
- feed_examples[j].question_attention_mask for j in range(batch_size)
- ]
- feed_dict[
- gr.batch_answer] = [feed_examples[j].answer for j in range(batch_size)]
- feed_dict[gr.batch_number_column] = [
- feed_examples[j].columns for j in range(batch_size)
- ]
- feed_dict[gr.batch_processed_number_column] = [
- feed_examples[j].processed_number_columns for j in range(batch_size)
- ]
- feed_dict[gr.batch_processed_sorted_index_number_column] = [
- feed_examples[j].sorted_number_index for j in range(batch_size)
- ]
- feed_dict[gr.batch_processed_sorted_index_word_column] = [
- feed_examples[j].sorted_word_index for j in range(batch_size)
- ]
- feed_dict[gr.batch_question_number] = np.array(
- [feed_examples[j].question_number for j in range(batch_size)]).reshape(
- (batch_size, 1))
- feed_dict[gr.batch_question_number_one] = np.array(
- [feed_examples[j].question_number_1 for j in range(batch_size)]).reshape(
- (batch_size, 1))
- feed_dict[gr.batch_question_number_mask] = [
- feed_examples[j].question_number_mask for j in range(batch_size)
- ]
- feed_dict[gr.batch_question_number_one_mask] = np.array(
- [feed_examples[j].question_number_one_mask for j in range(batch_size)
- ]).reshape((batch_size, 1))
- feed_dict[gr.batch_print_answer] = [
- feed_examples[j].print_answer for j in range(batch_size)
- ]
- feed_dict[gr.batch_exact_match] = [
- feed_examples[j].exact_match for j in range(batch_size)
- ]
- feed_dict[gr.batch_group_by_max] = [
- feed_examples[j].group_by_max for j in range(batch_size)
- ]
- feed_dict[gr.batch_column_exact_match] = [
- feed_examples[j].exact_column_match for j in range(batch_size)
- ]
- feed_dict[gr.batch_ordinal_question] = [
- feed_examples[j].ordinal_question for j in range(batch_size)
- ]
- feed_dict[gr.batch_ordinal_question_one] = [
- feed_examples[j].ordinal_question_one for j in range(batch_size)
- ]
- feed_dict[gr.batch_number_column_mask] = [
- feed_examples[j].column_mask for j in range(batch_size)
- ]
- feed_dict[gr.batch_number_column_names] = [
- feed_examples[j].column_ids for j in range(batch_size)
- ]
- feed_dict[gr.batch_processed_word_column] = [
- feed_examples[j].processed_word_columns for j in range(batch_size)
- ]
- feed_dict[gr.batch_word_column_mask] = [
- feed_examples[j].word_column_mask for j in range(batch_size)
- ]
- feed_dict[gr.batch_word_column_names] = [
- feed_examples[j].word_column_ids for j in range(batch_size)
- ]
- feed_dict[gr.batch_word_column_entry_mask] = [
- feed_examples[j].word_column_entry_mask for j in range(batch_size)
- ]
- return feed_dict
diff --git a/research/neural_programmer/model.py b/research/neural_programmer/model.py
deleted file mode 100644
index 610d66699e6e41188be58cc1f623c030d243c689..0000000000000000000000000000000000000000
--- a/research/neural_programmer/model.py
+++ /dev/null
@@ -1,679 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Author: aneelakantan (Arvind Neelakantan)
-"""
-
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-import nn_utils
-
-
-class Graph():
-
- def __init__(self, utility, batch_size, max_passes, mode="train"):
- self.utility = utility
- self.data_type = self.utility.tf_data_type[self.utility.FLAGS.data_type]
- self.max_elements = self.utility.FLAGS.max_elements
- max_elements = self.utility.FLAGS.max_elements
- self.num_cols = self.utility.FLAGS.max_number_cols
- self.num_word_cols = self.utility.FLAGS.max_word_cols
- self.question_length = self.utility.FLAGS.question_length
- self.batch_size = batch_size
- self.max_passes = max_passes
- self.mode = mode
- self.embedding_dims = self.utility.FLAGS.embedding_dims
- #input question and a mask
- self.batch_question = tf.placeholder(tf.int32,
- [batch_size, self.question_length])
- self.batch_question_attention_mask = tf.placeholder(
- self.data_type, [batch_size, self.question_length])
- #ground truth scalar answer and lookup answer
- self.batch_answer = tf.placeholder(self.data_type, [batch_size])
- self.batch_print_answer = tf.placeholder(
- self.data_type,
- [batch_size, self.num_cols + self.num_word_cols, max_elements])
- #number columns and its processed version
- self.batch_number_column = tf.placeholder(
- self.data_type, [batch_size, self.num_cols, max_elements
- ]) #columns with numeric entries
- self.batch_processed_number_column = tf.placeholder(
- self.data_type, [batch_size, self.num_cols, max_elements])
- self.batch_processed_sorted_index_number_column = tf.placeholder(
- tf.int32, [batch_size, self.num_cols, max_elements])
- #word columns and its processed version
- self.batch_processed_word_column = tf.placeholder(
- self.data_type, [batch_size, self.num_word_cols, max_elements])
- self.batch_processed_sorted_index_word_column = tf.placeholder(
- tf.int32, [batch_size, self.num_word_cols, max_elements])
- self.batch_word_column_entry_mask = tf.placeholder(
- tf.int32, [batch_size, self.num_word_cols, max_elements])
- #names of word and number columns along with their mask
- self.batch_word_column_names = tf.placeholder(
- tf.int32,
- [batch_size, self.num_word_cols, self.utility.FLAGS.max_entry_length])
- self.batch_word_column_mask = tf.placeholder(
- self.data_type, [batch_size, self.num_word_cols])
- self.batch_number_column_names = tf.placeholder(
- tf.int32,
- [batch_size, self.num_cols, self.utility.FLAGS.max_entry_length])
- self.batch_number_column_mask = tf.placeholder(self.data_type,
- [batch_size, self.num_cols])
- #exact match and group by max operation
- self.batch_exact_match = tf.placeholder(
- self.data_type,
- [batch_size, self.num_cols + self.num_word_cols, max_elements])
- self.batch_column_exact_match = tf.placeholder(
- self.data_type, [batch_size, self.num_cols + self.num_word_cols])
- self.batch_group_by_max = tf.placeholder(
- self.data_type,
- [batch_size, self.num_cols + self.num_word_cols, max_elements])
- #numbers in the question along with their position. This is used to compute arguments to the comparison operations
- self.batch_question_number = tf.placeholder(self.data_type, [batch_size, 1])
- self.batch_question_number_one = tf.placeholder(self.data_type,
- [batch_size, 1])
- self.batch_question_number_mask = tf.placeholder(
- self.data_type, [batch_size, max_elements])
- self.batch_question_number_one_mask = tf.placeholder(self.data_type,
- [batch_size, 1])
- self.batch_ordinal_question = tf.placeholder(
- self.data_type, [batch_size, self.question_length])
- self.batch_ordinal_question_one = tf.placeholder(
- self.data_type, [batch_size, self.question_length])
-
- def LSTM_question_embedding(self, sentence, sentence_length):
- #LSTM processes the input question
- lstm_params = "question_lstm"
- hidden_vectors = []
- sentence = self.batch_question
- question_hidden = tf.zeros(
- [self.batch_size, self.utility.FLAGS.embedding_dims], self.data_type)
- question_c_hidden = tf.zeros(
- [self.batch_size, self.utility.FLAGS.embedding_dims], self.data_type)
- if (self.utility.FLAGS.rnn_dropout > 0.0):
- if (self.mode == "train"):
- rnn_dropout_mask = tf.cast(
- tf.random_uniform(
- tf.shape(question_hidden), minval=0.0, maxval=1.0) <
- self.utility.FLAGS.rnn_dropout,
- self.data_type) / self.utility.FLAGS.rnn_dropout
- else:
- rnn_dropout_mask = tf.ones_like(question_hidden)
- for question_iterator in range(self.question_length):
- curr_word = sentence[:, question_iterator]
- question_vector = nn_utils.apply_dropout(
- nn_utils.get_embedding(curr_word, self.utility, self.params),
- self.utility.FLAGS.dropout, self.mode)
- question_hidden, question_c_hidden = nn_utils.LSTMCell(
- question_vector, question_hidden, question_c_hidden, lstm_params,
- self.params)
- if (self.utility.FLAGS.rnn_dropout > 0.0):
- question_hidden = question_hidden * rnn_dropout_mask
- hidden_vectors.append(tf.expand_dims(question_hidden, 0))
- hidden_vectors = tf.concat(axis=0, values=hidden_vectors)
- return question_hidden, hidden_vectors
-
- def history_recurrent_step(self, curr_hprev, hprev):
- #A single RNN step for controller or history RNN
- return tf.tanh(
- tf.matmul(
- tf.concat(axis=1, values=[hprev, curr_hprev]), self.params[
- "history_recurrent"])) + self.params["history_recurrent_bias"]
-
- def question_number_softmax(self, hidden_vectors):
- #Attention on quetsion to decide the question number to passed to comparison ops
- def compute_ans(op_embedding, comparison):
- op_embedding = tf.expand_dims(op_embedding, 0)
- #dot product of operation embedding with hidden state to the left of the number occurrence
- first = tf.transpose(
- tf.matmul(op_embedding,
- tf.transpose(
- tf.reduce_sum(hidden_vectors * tf.tile(
- tf.expand_dims(
- tf.transpose(self.batch_ordinal_question), 2),
- [1, 1, self.utility.FLAGS.embedding_dims]), 0))))
- second = self.batch_question_number_one_mask + tf.transpose(
- tf.matmul(op_embedding,
- tf.transpose(
- tf.reduce_sum(hidden_vectors * tf.tile(
- tf.expand_dims(
- tf.transpose(self.batch_ordinal_question_one), 2
- ), [1, 1, self.utility.FLAGS.embedding_dims]), 0))))
- question_number_softmax = tf.nn.softmax(tf.concat(axis=1, values=[first, second]))
- if (self.mode == "test"):
- cond = tf.equal(question_number_softmax,
- tf.reshape(
- tf.reduce_max(question_number_softmax, 1),
- [self.batch_size, 1]))
- question_number_softmax = tf.where(
- cond,
- tf.fill(tf.shape(question_number_softmax), 1.0),
- tf.fill(tf.shape(question_number_softmax), 0.0))
- question_number_softmax = tf.cast(question_number_softmax,
- self.data_type)
- ans = tf.reshape(
- tf.reduce_sum(question_number_softmax * tf.concat(
- axis=1, values=[self.batch_question_number, self.batch_question_number_one]),
- 1), [self.batch_size, 1])
- return ans
-
- def compute_op_position(op_name):
- for i in range(len(self.utility.operations_set)):
- if (op_name == self.utility.operations_set[i]):
- return i
-
- def compute_question_number(op_name):
- op_embedding = tf.nn.embedding_lookup(self.params_unit,
- compute_op_position(op_name))
- return compute_ans(op_embedding, op_name)
-
- curr_greater_question_number = compute_question_number("greater")
- curr_lesser_question_number = compute_question_number("lesser")
- curr_geq_question_number = compute_question_number("geq")
- curr_leq_question_number = compute_question_number("leq")
- return curr_greater_question_number, curr_lesser_question_number, curr_geq_question_number, curr_leq_question_number
-
- def perform_attention(self, context_vector, hidden_vectors, length, mask):
- #Performs attention on hiddent_vectors using context vector
- context_vector = tf.tile(
- tf.expand_dims(context_vector, 0), [length, 1, 1]) #time * bs * d
- attention_softmax = tf.nn.softmax(
- tf.transpose(tf.reduce_sum(context_vector * hidden_vectors, 2)) +
- mask) #batch_size * time
- attention_softmax = tf.tile(
- tf.expand_dims(tf.transpose(attention_softmax), 2),
- [1, 1, self.embedding_dims])
- ans_vector = tf.reduce_sum(attention_softmax * hidden_vectors, 0)
- return ans_vector
-
- #computes embeddings for column names using parameters of question module
- def get_column_hidden_vectors(self):
- #vector representations for the column names
- self.column_hidden_vectors = tf.reduce_sum(
- nn_utils.get_embedding(self.batch_number_column_names, self.utility,
- self.params), 2)
- self.word_column_hidden_vectors = tf.reduce_sum(
- nn_utils.get_embedding(self.batch_word_column_names, self.utility,
- self.params), 2)
-
- def create_summary_embeddings(self):
- #embeddings for each text entry in the table using parameters of the question module
- self.summary_text_entry_embeddings = tf.reduce_sum(
- tf.expand_dims(self.batch_exact_match, 3) * tf.expand_dims(
- tf.expand_dims(
- tf.expand_dims(
- nn_utils.get_embedding(self.utility.entry_match_token_id,
- self.utility, self.params), 0), 1),
- 2), 2)
-
- def compute_column_softmax(self, column_controller_vector, time_step):
- #compute softmax over all the columns using column controller vector
- column_controller_vector = tf.tile(
- tf.expand_dims(column_controller_vector, 1),
- [1, self.num_cols + self.num_word_cols, 1]) #max_cols * bs * d
- column_controller_vector = nn_utils.apply_dropout(
- column_controller_vector, self.utility.FLAGS.dropout, self.mode)
- self.full_column_hidden_vectors = tf.concat(
- axis=1, values=[self.column_hidden_vectors, self.word_column_hidden_vectors])
- self.full_column_hidden_vectors += self.summary_text_entry_embeddings
- self.full_column_hidden_vectors = nn_utils.apply_dropout(
- self.full_column_hidden_vectors, self.utility.FLAGS.dropout, self.mode)
- column_logits = tf.reduce_sum(
- column_controller_vector * self.full_column_hidden_vectors, 2) + (
- self.params["word_match_feature_column_name"] *
- self.batch_column_exact_match) + self.full_column_mask
- column_softmax = tf.nn.softmax(column_logits) #batch_size * max_cols
- return column_softmax
-
- def compute_first_or_last(self, select, first=True):
- #perform first ot last operation on row select with probabilistic row selection
- answer = tf.zeros_like(select)
- running_sum = tf.zeros([self.batch_size, 1], self.data_type)
- for i in range(self.max_elements):
- if (first):
- current = tf.slice(select, [0, i], [self.batch_size, 1])
- else:
- current = tf.slice(select, [0, self.max_elements - 1 - i],
- [self.batch_size, 1])
- curr_prob = current * (1 - running_sum)
- curr_prob = curr_prob * tf.cast(curr_prob >= 0.0, self.data_type)
- running_sum += curr_prob
- temp_ans = []
- curr_prob = tf.expand_dims(tf.reshape(curr_prob, [self.batch_size]), 0)
- for i_ans in range(self.max_elements):
- if (not (first) and i_ans == self.max_elements - 1 - i):
- temp_ans.append(curr_prob)
- elif (first and i_ans == i):
- temp_ans.append(curr_prob)
- else:
- temp_ans.append(tf.zeros_like(curr_prob))
- temp_ans = tf.transpose(tf.concat(axis=0, values=temp_ans))
- answer += temp_ans
- return answer
-
- def make_hard_softmax(self, softmax):
- #converts soft selection to hard selection. used at test time
- cond = tf.equal(
- softmax, tf.reshape(tf.reduce_max(softmax, 1), [self.batch_size, 1]))
- softmax = tf.where(
- cond, tf.fill(tf.shape(softmax), 1.0), tf.fill(tf.shape(softmax), 0.0))
- softmax = tf.cast(softmax, self.data_type)
- return softmax
-
- def compute_max_or_min(self, select, maxi=True):
- #computes the argmax and argmin of a column with probabilistic row selection
- answer = tf.zeros([
- self.batch_size, self.num_cols + self.num_word_cols, self.max_elements
- ], self.data_type)
- sum_prob = tf.zeros([self.batch_size, self.num_cols + self.num_word_cols],
- self.data_type)
- for j in range(self.max_elements):
- if (maxi):
- curr_pos = j
- else:
- curr_pos = self.max_elements - 1 - j
- select_index = tf.slice(self.full_processed_sorted_index_column,
- [0, 0, curr_pos], [self.batch_size, -1, 1])
- select_mask = tf.equal(
- tf.tile(
- tf.expand_dims(
- tf.tile(
- tf.expand_dims(tf.range(self.max_elements), 0),
- [self.batch_size, 1]), 1),
- [1, self.num_cols + self.num_word_cols, 1]), select_index)
- curr_prob = tf.expand_dims(select, 1) * tf.cast(
- select_mask, self.data_type) * self.select_bad_number_mask
- curr_prob = curr_prob * tf.expand_dims((1 - sum_prob), 2)
- curr_prob = curr_prob * tf.expand_dims(
- tf.cast((1 - sum_prob) > 0.0, self.data_type), 2)
- answer = tf.where(select_mask, curr_prob, answer)
- sum_prob += tf.reduce_sum(curr_prob, 2)
- return answer
-
- def perform_operations(self, softmax, full_column_softmax, select,
- prev_select_1, curr_pass):
- #performs all the 15 operations. computes scalar output, lookup answer and row selector
- column_softmax = tf.slice(full_column_softmax, [0, 0],
- [self.batch_size, self.num_cols])
- word_column_softmax = tf.slice(full_column_softmax, [0, self.num_cols],
- [self.batch_size, self.num_word_cols])
- init_max = self.compute_max_or_min(select, maxi=True)
- init_min = self.compute_max_or_min(select, maxi=False)
- #operations that are column independent
- count = tf.reshape(tf.reduce_sum(select, 1), [self.batch_size, 1])
- select_full_column_softmax = tf.tile(
- tf.expand_dims(full_column_softmax, 2),
- [1, 1, self.max_elements
- ]) #BS * (max_cols + max_word_cols) * max_elements
- select_word_column_softmax = tf.tile(
- tf.expand_dims(word_column_softmax, 2),
- [1, 1, self.max_elements]) #BS * max_word_cols * max_elements
- select_greater = tf.reduce_sum(
- self.init_select_greater * select_full_column_softmax,
- 1) * self.batch_question_number_mask #BS * max_elements
- select_lesser = tf.reduce_sum(
- self.init_select_lesser * select_full_column_softmax,
- 1) * self.batch_question_number_mask #BS * max_elements
- select_geq = tf.reduce_sum(
- self.init_select_geq * select_full_column_softmax,
- 1) * self.batch_question_number_mask #BS * max_elements
- select_leq = tf.reduce_sum(
- self.init_select_leq * select_full_column_softmax,
- 1) * self.batch_question_number_mask #BS * max_elements
- select_max = tf.reduce_sum(init_max * select_full_column_softmax,
- 1) #BS * max_elements
- select_min = tf.reduce_sum(init_min * select_full_column_softmax,
- 1) #BS * max_elements
- select_prev = tf.concat(axis=1, values=[
- tf.slice(select, [0, 1], [self.batch_size, self.max_elements - 1]),
- tf.cast(tf.zeros([self.batch_size, 1]), self.data_type)
- ])
- select_next = tf.concat(axis=1, values=[
- tf.cast(tf.zeros([self.batch_size, 1]), self.data_type), tf.slice(
- select, [0, 0], [self.batch_size, self.max_elements - 1])
- ])
- select_last_rs = self.compute_first_or_last(select, False)
- select_first_rs = self.compute_first_or_last(select, True)
- select_word_match = tf.reduce_sum(self.batch_exact_match *
- select_full_column_softmax, 1)
- select_group_by_max = tf.reduce_sum(self.batch_group_by_max *
- select_full_column_softmax, 1)
- length_content = 1
- length_select = 13
- length_print = 1
- values = tf.concat(axis=1, values=[count])
- softmax_content = tf.slice(softmax, [0, 0],
- [self.batch_size, length_content])
- #compute scalar output
- output = tf.reduce_sum(tf.multiply(softmax_content, values), 1)
- #compute lookup answer
- softmax_print = tf.slice(softmax, [0, length_content + length_select],
- [self.batch_size, length_print])
- curr_print = select_full_column_softmax * tf.tile(
- tf.expand_dims(select, 1),
- [1, self.num_cols + self.num_word_cols, 1
- ]) #BS * max_cols * max_elements (conisders only column)
- self.batch_lookup_answer = curr_print * tf.tile(
- tf.expand_dims(softmax_print, 2),
- [1, self.num_cols + self.num_word_cols, self.max_elements
- ]) #BS * max_cols * max_elements
- self.batch_lookup_answer = self.batch_lookup_answer * self.select_full_mask
- #compute row select
- softmax_select = tf.slice(softmax, [0, length_content],
- [self.batch_size, length_select])
- select_lists = [
- tf.expand_dims(select_prev, 1), tf.expand_dims(select_next, 1),
- tf.expand_dims(select_first_rs, 1), tf.expand_dims(select_last_rs, 1),
- tf.expand_dims(select_group_by_max, 1),
- tf.expand_dims(select_greater, 1), tf.expand_dims(select_lesser, 1),
- tf.expand_dims(select_geq, 1), tf.expand_dims(select_leq, 1),
- tf.expand_dims(select_max, 1), tf.expand_dims(select_min, 1),
- tf.expand_dims(select_word_match, 1),
- tf.expand_dims(self.reset_select, 1)
- ]
- select = tf.reduce_sum(
- tf.tile(tf.expand_dims(softmax_select, 2), [1, 1, self.max_elements]) *
- tf.concat(axis=1, values=select_lists), 1)
- select = select * self.select_whole_mask
- return output, select
-
- def one_pass(self, select, question_embedding, hidden_vectors, hprev,
- prev_select_1, curr_pass):
- #Performs one timestep which involves selecting an operation and a column
- attention_vector = self.perform_attention(
- hprev, hidden_vectors, self.question_length,
- self.batch_question_attention_mask) #batch_size * embedding_dims
- controller_vector = tf.nn.relu(
- tf.matmul(hprev, self.params["controller_prev"]) + tf.matmul(
- tf.concat(axis=1, values=[question_embedding, attention_vector]), self.params[
- "controller"]))
- column_controller_vector = tf.nn.relu(
- tf.matmul(hprev, self.params["column_controller_prev"]) + tf.matmul(
- tf.concat(axis=1, values=[question_embedding, attention_vector]), self.params[
- "column_controller"]))
- controller_vector = nn_utils.apply_dropout(
- controller_vector, self.utility.FLAGS.dropout, self.mode)
- self.operation_logits = tf.matmul(controller_vector,
- tf.transpose(self.params_unit))
- softmax = tf.nn.softmax(self.operation_logits)
- soft_softmax = softmax
- #compute column softmax: bs * max_columns
- weighted_op_representation = tf.transpose(
- tf.matmul(tf.transpose(self.params_unit), tf.transpose(softmax)))
- column_controller_vector = tf.nn.relu(
- tf.matmul(
- tf.concat(axis=1, values=[
- column_controller_vector, weighted_op_representation
- ]), self.params["break_conditional"]))
- full_column_softmax = self.compute_column_softmax(column_controller_vector,
- curr_pass)
- soft_column_softmax = full_column_softmax
- if (self.mode == "test"):
- full_column_softmax = self.make_hard_softmax(full_column_softmax)
- softmax = self.make_hard_softmax(softmax)
- output, select = self.perform_operations(softmax, full_column_softmax,
- select, prev_select_1, curr_pass)
- return output, select, softmax, soft_softmax, full_column_softmax, soft_column_softmax
-
- def compute_lookup_error(self, val):
- #computes lookup error.
- cond = tf.equal(self.batch_print_answer, val)
- inter = tf.where(
- cond, self.init_print_error,
- tf.tile(
- tf.reshape(tf.constant(1e10, self.data_type), [1, 1, 1]), [
- self.batch_size, self.utility.FLAGS.max_word_cols +
- self.utility.FLAGS.max_number_cols,
- self.utility.FLAGS.max_elements
- ]))
- return tf.reduce_min(tf.reduce_min(inter, 1), 1) * tf.cast(
- tf.greater(
- tf.reduce_sum(tf.reduce_sum(tf.cast(cond, self.data_type), 1), 1),
- 0.0), self.data_type)
-
- def soft_min(self, x, y):
- return tf.maximum(-1.0 * (1 / (
- self.utility.FLAGS.soft_min_value + 0.0)) * tf.log(
- tf.exp(-self.utility.FLAGS.soft_min_value * x) + tf.exp(
- -self.utility.FLAGS.soft_min_value * y)), tf.zeros_like(x))
-
- def error_computation(self):
- #computes the error of each example in a batch
- math_error = 0.5 * tf.square(tf.subtract(self.scalar_output, self.batch_answer))
- #scale math error
- math_error = math_error / self.rows
- math_error = tf.minimum(math_error, self.utility.FLAGS.max_math_error *
- tf.ones(tf.shape(math_error), self.data_type))
- self.init_print_error = tf.where(
- self.batch_gold_select, -1 * tf.log(self.batch_lookup_answer + 1e-300 +
- self.invert_select_full_mask), -1 *
- tf.log(1 - self.batch_lookup_answer)) * self.select_full_mask
- print_error_1 = self.init_print_error * tf.cast(
- tf.equal(self.batch_print_answer, 0.0), self.data_type)
- print_error = tf.reduce_sum(tf.reduce_sum((print_error_1), 1), 1)
- for val in range(1, 58):
- print_error += self.compute_lookup_error(val + 0.0)
- print_error = print_error * self.utility.FLAGS.print_cost / self.num_entries
- if (self.mode == "train"):
- error = tf.where(
- tf.logical_and(
- tf.not_equal(self.batch_answer, 0.0),
- tf.not_equal(
- tf.reduce_sum(tf.reduce_sum(self.batch_print_answer, 1), 1),
- 0.0)),
- self.soft_min(math_error, print_error),
- tf.where(
- tf.not_equal(self.batch_answer, 0.0), math_error, print_error))
- else:
- error = tf.where(
- tf.logical_and(
- tf.equal(self.scalar_output, 0.0),
- tf.equal(
- tf.reduce_sum(tf.reduce_sum(self.batch_lookup_answer, 1), 1),
- 0.0)),
- tf.ones_like(math_error),
- tf.where(
- tf.equal(self.scalar_output, 0.0), print_error, math_error))
- return error
-
- def batch_process(self):
- #Computes loss and fraction of correct examples in a batch.
- self.params_unit = nn_utils.apply_dropout(
- self.params["unit"], self.utility.FLAGS.dropout, self.mode)
- batch_size = self.batch_size
- max_passes = self.max_passes
- num_timesteps = 1
- max_elements = self.max_elements
- select = tf.cast(
- tf.fill([self.batch_size, max_elements], 1.0), self.data_type)
- hprev = tf.cast(
- tf.fill([self.batch_size, self.embedding_dims], 0.0),
- self.data_type) #running sum of the hidden states of the model
- output = tf.cast(tf.fill([self.batch_size, 1], 0.0),
- self.data_type) #output of the model
- correct = tf.cast(
- tf.fill([1], 0.0), self.data_type
- ) #to compute accuracy, returns number of correct examples for this batch
- total_error = 0.0
- prev_select_1 = tf.zeros_like(select)
- self.create_summary_embeddings()
- self.get_column_hidden_vectors()
- #get question embedding
- question_embedding, hidden_vectors = self.LSTM_question_embedding(
- self.batch_question, self.question_length)
- #compute arguments for comparison operation
- greater_question_number, lesser_question_number, geq_question_number, leq_question_number = self.question_number_softmax(
- hidden_vectors)
- self.init_select_greater = tf.cast(
- tf.greater(self.full_processed_column,
- tf.expand_dims(greater_question_number, 2)), self.
- data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
- self.init_select_lesser = tf.cast(
- tf.less(self.full_processed_column,
- tf.expand_dims(lesser_question_number, 2)), self.
- data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
- self.init_select_geq = tf.cast(
- tf.greater_equal(self.full_processed_column,
- tf.expand_dims(geq_question_number, 2)), self.
- data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
- self.init_select_leq = tf.cast(
- tf.less_equal(self.full_processed_column,
- tf.expand_dims(leq_question_number, 2)), self.
- data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
- self.init_select_word_match = 0
- if (self.utility.FLAGS.rnn_dropout > 0.0):
- if (self.mode == "train"):
- history_rnn_dropout_mask = tf.cast(
- tf.random_uniform(
- tf.shape(hprev), minval=0.0, maxval=1.0) <
- self.utility.FLAGS.rnn_dropout,
- self.data_type) / self.utility.FLAGS.rnn_dropout
- else:
- history_rnn_dropout_mask = tf.ones_like(hprev)
- select = select * self.select_whole_mask
- self.batch_log_prob = tf.zeros([self.batch_size], dtype=self.data_type)
- #Perform max_passes and at each pass select operation and column
- for curr_pass in range(max_passes):
- print("step: ", curr_pass)
- output, select, softmax, soft_softmax, column_softmax, soft_column_softmax = self.one_pass(
- select, question_embedding, hidden_vectors, hprev, prev_select_1,
- curr_pass)
- prev_select_1 = select
- #compute input to history RNN
- input_op = tf.transpose(
- tf.matmul(
- tf.transpose(self.params_unit), tf.transpose(
- soft_softmax))) #weighted average of emebdding of operations
- input_col = tf.reduce_sum(
- tf.expand_dims(soft_column_softmax, 2) *
- self.full_column_hidden_vectors, 1)
- history_input = tf.concat(axis=1, values=[input_op, input_col])
- history_input = nn_utils.apply_dropout(
- history_input, self.utility.FLAGS.dropout, self.mode)
- hprev = self.history_recurrent_step(history_input, hprev)
- if (self.utility.FLAGS.rnn_dropout > 0.0):
- hprev = hprev * history_rnn_dropout_mask
- self.scalar_output = output
- error = self.error_computation()
- cond = tf.less(error, 0.0001, name="cond")
- correct_add = tf.where(
- cond, tf.fill(tf.shape(cond), 1.0), tf.fill(tf.shape(cond), 0.0))
- correct = tf.reduce_sum(correct_add)
- error = error / batch_size
- total_error = tf.reduce_sum(error)
- total_correct = correct / batch_size
- return total_error, total_correct
-
- def compute_error(self):
- #Sets mask variables and performs batch processing
- self.batch_gold_select = self.batch_print_answer > 0.0
- self.full_column_mask = tf.concat(
- axis=1, values=[self.batch_number_column_mask, self.batch_word_column_mask])
- self.full_processed_column = tf.concat(
- axis=1,
- values=[self.batch_processed_number_column, self.batch_processed_word_column])
- self.full_processed_sorted_index_column = tf.concat(axis=1, values=[
- self.batch_processed_sorted_index_number_column,
- self.batch_processed_sorted_index_word_column
- ])
- self.select_bad_number_mask = tf.cast(
- tf.logical_and(
- tf.not_equal(self.full_processed_column,
- self.utility.FLAGS.pad_int),
- tf.not_equal(self.full_processed_column,
- self.utility.FLAGS.bad_number_pre_process)),
- self.data_type)
- self.select_mask = tf.cast(
- tf.logical_not(
- tf.equal(self.batch_number_column, self.utility.FLAGS.pad_int)),
- self.data_type)
- self.select_word_mask = tf.cast(
- tf.logical_not(
- tf.equal(self.batch_word_column_entry_mask,
- self.utility.dummy_token_id)), self.data_type)
- self.select_full_mask = tf.concat(
- axis=1, values=[self.select_mask, self.select_word_mask])
- self.select_whole_mask = tf.maximum(
- tf.reshape(
- tf.slice(self.select_mask, [0, 0, 0],
- [self.batch_size, 1, self.max_elements]),
- [self.batch_size, self.max_elements]),
- tf.reshape(
- tf.slice(self.select_word_mask, [0, 0, 0],
- [self.batch_size, 1, self.max_elements]),
- [self.batch_size, self.max_elements]))
- self.invert_select_full_mask = tf.cast(
- tf.concat(axis=1, values=[
- tf.equal(self.batch_number_column, self.utility.FLAGS.pad_int),
- tf.equal(self.batch_word_column_entry_mask,
- self.utility.dummy_token_id)
- ]), self.data_type)
- self.batch_lookup_answer = tf.zeros(tf.shape(self.batch_gold_select))
- self.reset_select = self.select_whole_mask
- self.rows = tf.reduce_sum(self.select_whole_mask, 1)
- self.num_entries = tf.reshape(
- tf.reduce_sum(tf.reduce_sum(self.select_full_mask, 1), 1),
- [self.batch_size])
- self.final_error, self.final_correct = self.batch_process()
- return self.final_error
-
- def create_graph(self, params, global_step):
- #Creates the graph to compute error, gradient computation and updates parameters
- self.params = params
- batch_size = self.batch_size
- learning_rate = tf.cast(self.utility.FLAGS.learning_rate, self.data_type)
- self.total_cost = self.compute_error()
- optimize_params = self.params.values()
- optimize_names = self.params.keys()
- print("optimize params ", optimize_names)
- if (self.utility.FLAGS.l2_regularizer > 0.0):
- reg_cost = 0.0
- for ind_param in self.params.keys():
- reg_cost += tf.nn.l2_loss(self.params[ind_param])
- self.total_cost += self.utility.FLAGS.l2_regularizer * reg_cost
- grads = tf.gradients(self.total_cost, optimize_params, name="gradients")
- grad_norm = 0.0
- for p, name in zip(grads, optimize_names):
- print("grads: ", p, name)
- if isinstance(p, tf.IndexedSlices):
- grad_norm += tf.reduce_sum(p.values * p.values)
- elif not (p == None):
- grad_norm += tf.reduce_sum(p * p)
- grad_norm = tf.sqrt(grad_norm)
- max_grad_norm = np.float32(self.utility.FLAGS.clip_gradients).astype(
- self.utility.np_data_type[self.utility.FLAGS.data_type])
- grad_scale = tf.minimum(
- tf.cast(1.0, self.data_type), max_grad_norm / grad_norm)
- clipped_grads = list()
- for p in grads:
- if isinstance(p, tf.IndexedSlices):
- tmp = p.values * grad_scale
- clipped_grads.append(tf.IndexedSlices(tmp, p.indices))
- elif not (p == None):
- clipped_grads.append(p * grad_scale)
- else:
- clipped_grads.append(p)
- grads = clipped_grads
- self.global_step = global_step
- params_list = self.params.values()
- params_list.append(self.global_step)
- adam = tf.train.AdamOptimizer(
- learning_rate,
- epsilon=tf.cast(self.utility.FLAGS.eps, self.data_type),
- use_locking=True)
- self.step = adam.apply_gradients(zip(grads, optimize_params),
- global_step=self.global_step)
- self.init_op = tf.global_variables_initializer()
diff --git a/research/neural_programmer/neural_programmer.py b/research/neural_programmer/neural_programmer.py
deleted file mode 100644
index 145ca13d6ac8ce80d651f902440bfb3240f1c7a2..0000000000000000000000000000000000000000
--- a/research/neural_programmer/neural_programmer.py
+++ /dev/null
@@ -1,239 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Implementation of the Neural Programmer model described in https://openreview.net/pdf?id=ry2YOrcge
-
-This file calls functions to load & pre-process data, construct the TF graph
-and performs training or evaluation as specified by the flag evaluator_job
-Author: aneelakantan (Arvind Neelakantan)
-"""
-from __future__ import print_function
-
-import time
-from random import Random
-import numpy as np
-import tensorflow as tf
-import model
-import wiki_data
-import parameters
-import data_utils
-
-tf.flags.DEFINE_integer("train_steps", 100001, "Number of steps to train")
-tf.flags.DEFINE_integer("eval_cycle", 500,
- "Evaluate model at every eval_cycle steps")
-tf.flags.DEFINE_integer("max_elements", 100,
- "maximum rows that are considered for processing")
-tf.flags.DEFINE_integer(
- "max_number_cols", 15,
- "maximum number columns that are considered for processing")
-tf.flags.DEFINE_integer(
- "max_word_cols", 25,
- "maximum number columns that are considered for processing")
-tf.flags.DEFINE_integer("question_length", 62, "maximum question length")
-tf.flags.DEFINE_integer("max_entry_length", 1, "")
-tf.flags.DEFINE_integer("max_passes", 4, "number of operation passes")
-tf.flags.DEFINE_integer("embedding_dims", 256, "")
-tf.flags.DEFINE_integer("batch_size", 20, "")
-tf.flags.DEFINE_float("clip_gradients", 1.0, "")
-tf.flags.DEFINE_float("eps", 1e-6, "")
-tf.flags.DEFINE_float("param_init", 0.1, "")
-tf.flags.DEFINE_float("learning_rate", 0.001, "")
-tf.flags.DEFINE_float("l2_regularizer", 0.0001, "")
-tf.flags.DEFINE_float("print_cost", 50.0,
- "weighting factor in the objective function")
-tf.flags.DEFINE_string("job_id", "temp", """job id""")
-tf.flags.DEFINE_string("output_dir", "../model/",
- """output_dir""")
-tf.flags.DEFINE_string("data_dir", "../data/",
- """data_dir""")
-tf.flags.DEFINE_integer("write_every", 500, "wrtie every N")
-tf.flags.DEFINE_integer("param_seed", 150, "")
-tf.flags.DEFINE_integer("python_seed", 200, "")
-tf.flags.DEFINE_float("dropout", 0.8, "dropout keep probability")
-tf.flags.DEFINE_float("rnn_dropout", 0.9,
- "dropout keep probability for rnn connections")
-tf.flags.DEFINE_float("pad_int", -20000.0,
- "number columns are padded with pad_int")
-tf.flags.DEFINE_string("data_type", "double", "float or double")
-tf.flags.DEFINE_float("word_dropout_prob", 0.9, "word dropout keep prob")
-tf.flags.DEFINE_integer("word_cutoff", 10, "")
-tf.flags.DEFINE_integer("vocab_size", 10800, "")
-tf.flags.DEFINE_boolean("evaluator_job", False,
- "wehther to run as trainer/evaluator")
-tf.flags.DEFINE_float(
- "bad_number_pre_process", -200000.0,
- "number that is added to a corrupted table entry in a number column")
-tf.flags.DEFINE_float("max_math_error", 3.0,
- "max square loss error that is considered")
-tf.flags.DEFINE_float("soft_min_value", 5.0, "")
-FLAGS = tf.flags.FLAGS
-
-
-class Utility:
- #holds FLAGS and other variables that are used in different files
- def __init__(self):
- global FLAGS
- self.FLAGS = FLAGS
- self.unk_token = "UNK"
- self.entry_match_token = "entry_match"
- self.column_match_token = "column_match"
- self.dummy_token = "dummy_token"
- self.tf_data_type = {}
- self.tf_data_type["double"] = tf.float64
- self.tf_data_type["float"] = tf.float32
- self.np_data_type = {}
- self.np_data_type["double"] = np.float64
- self.np_data_type["float"] = np.float32
- self.operations_set = ["count"] + [
- "prev", "next", "first_rs", "last_rs", "group_by_max", "greater",
- "lesser", "geq", "leq", "max", "min", "word-match"
- ] + ["reset_select"] + ["print"]
- self.word_ids = {}
- self.reverse_word_ids = {}
- self.word_count = {}
- self.random = Random(FLAGS.python_seed)
-
-
-def evaluate(sess, data, batch_size, graph, i):
- #computes accuracy
- num_examples = 0.0
- gc = 0.0
- for j in range(0, len(data) - batch_size + 1, batch_size):
- [ct] = sess.run([graph.final_correct],
- feed_dict=data_utils.generate_feed_dict(data, j, batch_size,
- graph))
- gc += ct * batch_size
- num_examples += batch_size
- print("dev set accuracy after ", i, " : ", gc / num_examples)
- print(num_examples, len(data))
- print("--------")
-
-
-def Train(graph, utility, batch_size, train_data, sess, model_dir,
- saver):
- #performs training
- curr = 0
- train_set_loss = 0.0
- utility.random.shuffle(train_data)
- start = time.time()
- for i in range(utility.FLAGS.train_steps):
- curr_step = i
- if (i > 0 and i % FLAGS.write_every == 0):
- model_file = model_dir + "/model_" + str(i)
- saver.save(sess, model_file)
- if curr + batch_size >= len(train_data):
- curr = 0
- utility.random.shuffle(train_data)
- step, cost_value = sess.run(
- [graph.step, graph.total_cost],
- feed_dict=data_utils.generate_feed_dict(
- train_data, curr, batch_size, graph, train=True, utility=utility))
- curr = curr + batch_size
- train_set_loss += cost_value
- if (i > 0 and i % FLAGS.eval_cycle == 0):
- end = time.time()
- time_taken = end - start
- print("step ", i, " ", time_taken, " seconds ")
- start = end
- print(" printing train set loss: ", train_set_loss / utility.FLAGS.eval_cycle)
- train_set_loss = 0.0
-
-
-def master(train_data, dev_data, utility):
- #creates TF graph and calls trainer or evaluator
- batch_size = utility.FLAGS.batch_size
- model_dir = utility.FLAGS.output_dir + "/model" + utility.FLAGS.job_id + "/"
- #create all paramters of the model
- param_class = parameters.Parameters(utility)
- params, global_step, init = param_class.parameters(utility)
- key = "test" if (FLAGS.evaluator_job) else "train"
- graph = model.Graph(utility, batch_size, utility.FLAGS.max_passes, mode=key)
- graph.create_graph(params, global_step)
- prev_dev_error = 0.0
- final_loss = 0.0
- final_accuracy = 0.0
- #start session
- with tf.Session() as sess:
- sess.run(init.name)
- sess.run(graph.init_op.name)
- to_save = params.copy()
- saver = tf.train.Saver(to_save, max_to_keep=500)
- if (FLAGS.evaluator_job):
- while True:
- selected_models = {}
- file_list = tf.gfile.ListDirectory(model_dir)
- for model_file in file_list:
- if ("checkpoint" in model_file or "index" in model_file or
- "meta" in model_file):
- continue
- if ("data" in model_file):
- model_file = model_file.split(".")[0]
- model_step = int(
- model_file.split("_")[len(model_file.split("_")) - 1])
- selected_models[model_step] = model_file
- file_list = sorted(selected_models.items(), key=lambda x: x[0])
- if (len(file_list) > 0):
- file_list = file_list[0:len(file_list) - 1]
- print("list of models: ", file_list)
- for model_file in file_list:
- model_file = model_file[1]
- print("restoring: ", model_file)
- saver.restore(sess, model_dir + "/" + model_file)
- model_step = int(
- model_file.split("_")[len(model_file.split("_")) - 1])
- print("evaluating on dev ", model_file, model_step)
- evaluate(sess, dev_data, batch_size, graph, model_step)
- else:
- ckpt = tf.train.get_checkpoint_state(model_dir)
- print("model dir: ", model_dir)
- if (not (tf.gfile.IsDirectory(utility.FLAGS.output_dir))):
- print("create dir: ", utility.FLAGS.output_dir)
- tf.gfile.MkDir(utility.FLAGS.output_dir)
- if (not (tf.gfile.IsDirectory(model_dir))):
- print("create dir: ", model_dir)
- tf.gfile.MkDir(model_dir)
- Train(graph, utility, batch_size, train_data, sess, model_dir,
- saver)
-
-def main(args):
- utility = Utility()
- train_name = "random-split-1-train.examples"
- dev_name = "random-split-1-dev.examples"
- test_name = "pristine-unseen-tables.examples"
- #load data
- dat = wiki_data.WikiQuestionGenerator(train_name, dev_name, test_name, FLAGS.data_dir)
- train_data, dev_data, test_data = dat.load()
- utility.words = []
- utility.word_ids = {}
- utility.reverse_word_ids = {}
- #construct vocabulary
- data_utils.construct_vocab(train_data, utility)
- data_utils.construct_vocab(dev_data, utility, True)
- data_utils.construct_vocab(test_data, utility, True)
- data_utils.add_special_words(utility)
- data_utils.perform_word_cutoff(utility)
- #convert data to int format and pad the inputs
- train_data = data_utils.complete_wiki_processing(train_data, utility, True)
- dev_data = data_utils.complete_wiki_processing(dev_data, utility, False)
- test_data = data_utils.complete_wiki_processing(test_data, utility, False)
- print("# train examples ", len(train_data))
- print("# dev examples ", len(dev_data))
- print("# test examples ", len(test_data))
- print("running open source")
- #construct TF graph and train or evaluate
- master(train_data, dev_data, utility)
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/neural_programmer/nn_utils.py b/research/neural_programmer/nn_utils.py
deleted file mode 100644
index 2f3a1a98bf7f71631410fc88982b336d33a02f52..0000000000000000000000000000000000000000
--- a/research/neural_programmer/nn_utils.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Author: aneelakantan (Arvind Neelakantan)
-"""
-
-import tensorflow as tf
-
-def get_embedding(word, utility, params):
- return tf.nn.embedding_lookup(params["word"], word)
-
-
-def apply_dropout(x, dropout_rate, mode):
- if (dropout_rate > 0.0):
- if (mode == "train"):
- x = tf.nn.dropout(x, dropout_rate)
- else:
- x = x
- return x
-
-
-def LSTMCell(x, mprev, cprev, key, params):
- """Create an LSTM cell.
-
- Implements the equations in pg.2 from
- "Long Short-Term Memory Based Recurrent Neural Network Architectures
- For Large Vocabulary Speech Recognition",
- Hasim Sak, Andrew Senior, Francoise Beaufays.
-
- Args:
- w: A dictionary of the weights and optional biases as returned
- by LSTMParametersSplit().
- x: Inputs to this cell.
- mprev: m_{t-1}, the recurrent activations (same as the output)
- from the previous cell.
- cprev: c_{t-1}, the cell activations from the previous cell.
- keep_prob: Keep probability on the input and the outputs of a cell.
-
- Returns:
- m: Outputs of this cell.
- c: Cell Activations.
- """
-
- i = tf.matmul(x, params[key + "_ix"]) + tf.matmul(mprev, params[key + "_im"])
- i = tf.nn.bias_add(i, params[key + "_i"])
- f = tf.matmul(x, params[key + "_fx"]) + tf.matmul(mprev, params[key + "_fm"])
- f = tf.nn.bias_add(f, params[key + "_f"])
- c = tf.matmul(x, params[key + "_cx"]) + tf.matmul(mprev, params[key + "_cm"])
- c = tf.nn.bias_add(c, params[key + "_c"])
- o = tf.matmul(x, params[key + "_ox"]) + tf.matmul(mprev, params[key + "_om"])
- o = tf.nn.bias_add(o, params[key + "_o"])
- i = tf.sigmoid(i, name="i_gate")
- f = tf.sigmoid(f, name="f_gate")
- o = tf.sigmoid(o, name="o_gate")
- c = f * cprev + i * tf.tanh(c)
- m = o * c
- return m, c
diff --git a/research/neural_programmer/parameters.py b/research/neural_programmer/parameters.py
deleted file mode 100644
index c576ae822b2d93c561381e27fe65afd2902b564e..0000000000000000000000000000000000000000
--- a/research/neural_programmer/parameters.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Author: aneelakantan (Arvind Neelakantan)
-"""
-
-import numpy as np
-import tensorflow as tf
-
-
-class Parameters:
-
- def __init__(self, u):
- self.utility = u
- self.init_seed_counter = 0
- self.word_init = {}
-
- def parameters(self, utility):
- params = {}
- inits = []
- embedding_dims = self.utility.FLAGS.embedding_dims
- params["unit"] = tf.Variable(
- self.RandomUniformInit([len(utility.operations_set), embedding_dims]))
- params["word"] = tf.Variable(
- self.RandomUniformInit([utility.FLAGS.vocab_size, embedding_dims]))
- params["word_match_feature_column_name"] = tf.Variable(
- self.RandomUniformInit([1]))
- params["controller"] = tf.Variable(
- self.RandomUniformInit([2 * embedding_dims, embedding_dims]))
- params["column_controller"] = tf.Variable(
- self.RandomUniformInit([2 * embedding_dims, embedding_dims]))
- params["column_controller_prev"] = tf.Variable(
- self.RandomUniformInit([embedding_dims, embedding_dims]))
- params["controller_prev"] = tf.Variable(
- self.RandomUniformInit([embedding_dims, embedding_dims]))
- global_step = tf.Variable(1, name="global_step")
- #weigths of question and history RNN (or LSTM)
- key_list = ["question_lstm"]
- for key in key_list:
- # Weights going from inputs to nodes.
- for wgts in ["ix", "fx", "cx", "ox"]:
- params[key + "_" + wgts] = tf.Variable(
- self.RandomUniformInit([embedding_dims, embedding_dims]))
- # Weights going from nodes to nodes.
- for wgts in ["im", "fm", "cm", "om"]:
- params[key + "_" + wgts] = tf.Variable(
- self.RandomUniformInit([embedding_dims, embedding_dims]))
- #Biases for the gates and cell
- for bias in ["i", "f", "c", "o"]:
- if (bias == "f"):
- print("forget gate bias")
- params[key + "_" + bias] = tf.Variable(
- tf.random_uniform([embedding_dims], 1.0, 1.1, self.utility.
- tf_data_type[self.utility.FLAGS.data_type]))
- else:
- params[key + "_" + bias] = tf.Variable(
- self.RandomUniformInit([embedding_dims]))
- params["history_recurrent"] = tf.Variable(
- self.RandomUniformInit([3 * embedding_dims, embedding_dims]))
- params["history_recurrent_bias"] = tf.Variable(
- self.RandomUniformInit([1, embedding_dims]))
- params["break_conditional"] = tf.Variable(
- self.RandomUniformInit([2 * embedding_dims, embedding_dims]))
- init = tf.global_variables_initializer()
- return params, global_step, init
-
- def RandomUniformInit(self, shape):
- """Returns a RandomUniform Tensor between -param_init and param_init."""
- param_seed = self.utility.FLAGS.param_seed
- self.init_seed_counter += 1
- return tf.random_uniform(
- shape, -1.0 *
- (np.float32(self.utility.FLAGS.param_init)
- ).astype(self.utility.np_data_type[self.utility.FLAGS.data_type]),
- (np.float32(self.utility.FLAGS.param_init)
- ).astype(self.utility.np_data_type[self.utility.FLAGS.data_type]),
- self.utility.tf_data_type[self.utility.FLAGS.data_type],
- param_seed + self.init_seed_counter)
diff --git a/research/neural_programmer/wiki_data.py b/research/neural_programmer/wiki_data.py
deleted file mode 100644
index c91637ca1ae537526ebddf4408b0fccd22d0f5e1..0000000000000000000000000000000000000000
--- a/research/neural_programmer/wiki_data.py
+++ /dev/null
@@ -1,532 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Loads the WikiQuestions dataset.
-
-An example consists of question, table. Additionally, we store the processed
-columns which store the entries after performing number, date and other
-preprocessing as done in the baseline.
-columns, column names and processed columns are split into word and number
-columns.
-lookup answer (or matrix) is also split into number and word lookup matrix
-Author: aneelakantan (Arvind Neelakantan)
-"""
-from __future__ import print_function
-
-import math
-import os
-import re
-import numpy as np
-import unicodedata as ud
-import tensorflow as tf
-
-bad_number = -200000.0 #number that is added to a corrupted table entry in a number column
-
-def is_nan_or_inf(number):
- return math.isnan(number) or math.isinf(number)
-
-def strip_accents(s):
- u = unicode(s, "utf-8")
- u_new = ''.join(c for c in ud.normalize('NFKD', u) if ud.category(c) != 'Mn')
- return u_new.encode("utf-8")
-
-
-def correct_unicode(string):
- string = strip_accents(string)
- string = re.sub("\xc2\xa0", " ", string).strip()
- string = re.sub("\xe2\x80\x93", "-", string).strip()
- #string = re.sub(ur'[\u0300-\u036F]', "", string)
- string = re.sub("‚", ",", string)
- string = re.sub("…", "...", string)
- #string = re.sub("[·・]", ".", string)
- string = re.sub("ˆ", "^", string)
- string = re.sub("˜", "~", string)
- string = re.sub("‹", "<", string)
- string = re.sub("›", ">", string)
- #string = re.sub("[‘’´`]", "'", string)
- #string = re.sub("[“â€Â«Â»]", "\"", string)
- #string = re.sub("[•†‡]", "", string)
- #string = re.sub("[â€â€‘–—]", "-", string)
- string = re.sub(r'[\u2E00-\uFFFF]', "", string)
- string = re.sub("\\s+", " ", string).strip()
- return string
-
-
-def simple_normalize(string):
- string = correct_unicode(string)
- # Citations
- string = re.sub("\[(nb ?)?\d+\]", "", string)
- string = re.sub("\*+$", "", string)
- # Year in parenthesis
- string = re.sub("\(\d* ?-? ?\d*\)", "", string)
- string = re.sub("^\"(.*)\"$", "", string)
- return string
-
-
-def full_normalize(string):
- #print "an: ", string
- string = simple_normalize(string)
- # Remove trailing info in brackets
- string = re.sub("\[[^\]]*\]", "", string)
- # Remove most unicode characters in other languages
- string = re.sub(r'[\u007F-\uFFFF]', "", string.strip())
- # Remove trailing info in parenthesis
- string = re.sub("\([^)]*\)$", "", string.strip())
- string = final_normalize(string)
- # Get rid of question marks
- string = re.sub("\?", "", string).strip()
- # Get rid of trailing colons (usually occur in column titles)
- string = re.sub("\:$", " ", string).strip()
- # Get rid of slashes
- string = re.sub(r"/", " ", string).strip()
- string = re.sub(r"\\", " ", string).strip()
- # Replace colon, slash, and dash with space
- # Note: need better replacement for this when parsing time
- string = re.sub(r"\:", " ", string).strip()
- string = re.sub("/", " ", string).strip()
- string = re.sub("-", " ", string).strip()
- # Convert empty strings to UNK
- # Important to do this last or near last
- if not string:
- string = "UNK"
- return string
-
-def final_normalize(string):
- # Remove leading and trailing whitespace
- string = re.sub("\\s+", " ", string).strip()
- # Convert entirely to lowercase
- string = string.lower()
- # Get rid of strangely escaped newline characters
- string = re.sub("\\\\n", " ", string).strip()
- # Get rid of quotation marks
- string = re.sub(r"\"", "", string).strip()
- string = re.sub(r"\'", "", string).strip()
- string = re.sub(r"`", "", string).strip()
- # Get rid of *
- string = re.sub("\*", "", string).strip()
- return string
-
-def is_number(x):
- try:
- f = float(x)
- return not is_nan_or_inf(f)
- except ValueError:
- return False
- except TypeError:
- return False
-
-
-class WikiExample(object):
-
- def __init__(self, id, question, answer, table_key):
- self.question_id = id
- self.question = question
- self.answer = answer
- self.table_key = table_key
- self.lookup_matrix = []
- self.is_bad_example = False
- self.is_word_lookup = False
- self.is_ambiguous_word_lookup = False
- self.is_number_lookup = False
- self.is_number_calc = False
- self.is_unknown_answer = False
-
-
-class TableInfo(object):
-
- def __init__(self, word_columns, word_column_names, word_column_indices,
- number_columns, number_column_names, number_column_indices,
- processed_word_columns, processed_number_columns, orig_columns):
- self.word_columns = word_columns
- self.word_column_names = word_column_names
- self.word_column_indices = word_column_indices
- self.number_columns = number_columns
- self.number_column_names = number_column_names
- self.number_column_indices = number_column_indices
- self.processed_word_columns = processed_word_columns
- self.processed_number_columns = processed_number_columns
- self.orig_columns = orig_columns
-
-
-class WikiQuestionLoader(object):
-
- def __init__(self, data_name, root_folder):
- self.root_folder = root_folder
- self.data_folder = os.path.join(self.root_folder, "data")
- self.examples = []
- self.data_name = data_name
-
- def num_questions(self):
- return len(self.examples)
-
- def load_qa(self):
- data_source = os.path.join(self.data_folder, self.data_name)
- f = tf.gfile.GFile(data_source, "r")
- id_regex = re.compile("\(id ([^\)]*)\)")
- for line in f:
- id_match = id_regex.search(line)
- id = id_match.group(1)
- self.examples.append(id)
-
- def load(self):
- self.load_qa()
-
-
-def is_date(word):
- if (not (bool(re.search("[a-z0-9]", word, re.IGNORECASE)))):
- return False
- if (len(word) != 10):
- return False
- if (word[4] != "-"):
- return False
- if (word[7] != "-"):
- return False
- for i in range(len(word)):
- if (not (word[i] == "X" or word[i] == "x" or word[i] == "-" or re.search(
- "[0-9]", word[i]))):
- return False
- return True
-
-
-class WikiQuestionGenerator(object):
-
- def __init__(self, train_name, dev_name, test_name, root_folder):
- self.train_name = train_name
- self.dev_name = dev_name
- self.test_name = test_name
- self.train_loader = WikiQuestionLoader(train_name, root_folder)
- self.dev_loader = WikiQuestionLoader(dev_name, root_folder)
- self.test_loader = WikiQuestionLoader(test_name, root_folder)
- self.bad_examples = 0
- self.root_folder = root_folder
- self.data_folder = os.path.join(self.root_folder, "annotated/data")
- self.annotated_examples = {}
- self.annotated_tables = {}
- self.annotated_word_reject = {}
- self.annotated_word_reject["-lrb-"] = 1
- self.annotated_word_reject["-rrb-"] = 1
- self.annotated_word_reject["UNK"] = 1
-
- def is_money(self, word):
- if (not (bool(re.search("[a-z0-9]", word, re.IGNORECASE)))):
- return False
- for i in range(len(word)):
- if (not (word[i] == "E" or word[i] == "." or re.search("[0-9]",
- word[i]))):
- return False
- return True
-
- def remove_consecutive(self, ner_tags, ner_values):
- for i in range(len(ner_tags)):
- if ((ner_tags[i] == "NUMBER" or ner_tags[i] == "MONEY" or
- ner_tags[i] == "PERCENT" or ner_tags[i] == "DATE") and
- i + 1 < len(ner_tags) and ner_tags[i] == ner_tags[i + 1] and
- ner_values[i] == ner_values[i + 1] and ner_values[i] != ""):
- word = ner_values[i]
- word = word.replace(">", "").replace("<", "").replace("=", "").replace(
- "%", "").replace("~", "").replace("$", "").replace("£", "").replace(
- "€", "")
- if (re.search("[A-Z]", word) and not (is_date(word)) and not (
- self.is_money(word))):
- ner_values[i] = "A"
- else:
- ner_values[i] = ","
- return ner_tags, ner_values
-
- def pre_process_sentence(self, tokens, ner_tags, ner_values):
- sentence = []
- tokens = tokens.split("|")
- ner_tags = ner_tags.split("|")
- ner_values = ner_values.split("|")
- ner_tags, ner_values = self.remove_consecutive(ner_tags, ner_values)
- #print "old: ", tokens
- for i in range(len(tokens)):
- word = tokens[i]
- if (ner_values[i] != "" and
- (ner_tags[i] == "NUMBER" or ner_tags[i] == "MONEY" or
- ner_tags[i] == "PERCENT" or ner_tags[i] == "DATE")):
- word = ner_values[i]
- word = word.replace(">", "").replace("<", "").replace("=", "").replace(
- "%", "").replace("~", "").replace("$", "").replace("£", "").replace(
- "€", "")
- if (re.search("[A-Z]", word) and not (is_date(word)) and not (
- self.is_money(word))):
- word = tokens[i]
- if (is_number(ner_values[i])):
- word = float(ner_values[i])
- elif (is_number(word)):
- word = float(word)
- if (tokens[i] == "score"):
- word = "score"
- if (is_number(word)):
- word = float(word)
- if (not (self.annotated_word_reject.has_key(word))):
- if (is_number(word) or is_date(word) or self.is_money(word)):
- sentence.append(word)
- else:
- word = full_normalize(word)
- if (not (self.annotated_word_reject.has_key(word)) and
- bool(re.search("[a-z0-9]", word, re.IGNORECASE))):
- m = re.search(",", word)
- sentence.append(word.replace(",", ""))
- if (len(sentence) == 0):
- sentence.append("UNK")
- return sentence
-
- def load_annotated_data(self, in_file):
- self.annotated_examples = {}
- self.annotated_tables = {}
- f = tf.gfile.GFile(in_file, "r")
- counter = 0
- for line in f:
- if (counter > 0):
- line = line.strip()
- (question_id, utterance, context, target_value, tokens, lemma_tokens,
- pos_tags, ner_tags, ner_values, target_canon) = line.split("\t")
- question = self.pre_process_sentence(tokens, ner_tags, ner_values)
- target_canon = target_canon.split("|")
- self.annotated_examples[question_id] = WikiExample(
- question_id, question, target_canon, context)
- self.annotated_tables[context] = []
- counter += 1
- print("Annotated examples loaded ", len(self.annotated_examples))
- f.close()
-
- def is_number_column(self, a):
- for w in a:
- if (len(w) != 1):
- return False
- if (not (is_number(w[0]))):
- return False
- return True
-
- def convert_table(self, table):
- answer = []
- for i in range(len(table)):
- temp = []
- for j in range(len(table[i])):
- temp.append(" ".join([str(w) for w in table[i][j]]))
- answer.append(temp)
- return answer
-
- def load_annotated_tables(self):
- for table in self.annotated_tables.keys():
- annotated_table = table.replace("csv", "annotated")
- orig_columns = []
- processed_columns = []
- f = tf.gfile.GFile(os.path.join(self.root_folder, annotated_table), "r")
- counter = 0
- for line in f:
- if (counter > 0):
- line = line.strip()
- line = line + "\t" * (13 - len(line.split("\t")))
- (row, col, read_id, content, tokens, lemma_tokens, pos_tags, ner_tags,
- ner_values, number, date, num2, read_list) = line.split("\t")
- counter += 1
- f.close()
- max_row = int(row)
- max_col = int(col)
- for i in range(max_col + 1):
- orig_columns.append([])
- processed_columns.append([])
- for j in range(max_row + 1):
- orig_columns[i].append(bad_number)
- processed_columns[i].append(bad_number)
- #print orig_columns
- f = tf.gfile.GFile(os.path.join(self.root_folder, annotated_table), "r")
- counter = 0
- column_names = []
- for line in f:
- if (counter > 0):
- line = line.strip()
- line = line + "\t" * (13 - len(line.split("\t")))
- (row, col, read_id, content, tokens, lemma_tokens, pos_tags, ner_tags,
- ner_values, number, date, num2, read_list) = line.split("\t")
- entry = self.pre_process_sentence(tokens, ner_tags, ner_values)
- if (row == "-1"):
- column_names.append(entry)
- else:
- orig_columns[int(col)][int(row)] = entry
- if (len(entry) == 1 and is_number(entry[0])):
- processed_columns[int(col)][int(row)] = float(entry[0])
- else:
- for single_entry in entry:
- if (is_number(single_entry)):
- processed_columns[int(col)][int(row)] = float(single_entry)
- break
- nt = ner_tags.split("|")
- nv = ner_values.split("|")
- for i_entry in range(len(tokens.split("|"))):
- if (nt[i_entry] == "DATE" and
- is_number(nv[i_entry].replace("-", "").replace("X", ""))):
- processed_columns[int(col)][int(row)] = float(nv[
- i_entry].replace("-", "").replace("X", ""))
- #processed_columns[int(col)][int(row)] = float(nv[i_entry])
- if (len(entry) == 1 and (is_number(entry[0]) or is_date(entry[0]) or
- self.is_money(entry[0]))):
- if (len(entry) == 1 and not (is_number(entry[0])) and
- is_date(entry[0])):
- entry[0] = entry[0].replace("X", "x")
- counter += 1
- word_columns = []
- processed_word_columns = []
- word_column_names = []
- word_column_indices = []
- number_columns = []
- processed_number_columns = []
- number_column_names = []
- number_column_indices = []
- for i in range(max_col + 1):
- if (self.is_number_column(orig_columns[i])):
- number_column_indices.append(i)
- number_column_names.append(column_names[i])
- temp = []
- for w in orig_columns[i]:
- if (is_number(w[0])):
- temp.append(w[0])
- number_columns.append(temp)
- processed_number_columns.append(processed_columns[i])
- else:
- word_column_indices.append(i)
- word_column_names.append(column_names[i])
- word_columns.append(orig_columns[i])
- processed_word_columns.append(processed_columns[i])
- table_info = TableInfo(
- word_columns, word_column_names, word_column_indices, number_columns,
- number_column_names, number_column_indices, processed_word_columns,
- processed_number_columns, orig_columns)
- self.annotated_tables[table] = table_info
- f.close()
-
- def answer_classification(self):
- lookup_questions = 0
- number_lookup_questions = 0
- word_lookup_questions = 0
- ambiguous_lookup_questions = 0
- number_questions = 0
- bad_questions = 0
- ice_bad_questions = 0
- tot = 0
- got = 0
- ice = {}
- with tf.gfile.GFile(
- self.root_folder + "/arvind-with-norms-2.tsv", mode="r") as f:
- lines = f.readlines()
- for line in lines:
- line = line.strip()
- if (not (self.annotated_examples.has_key(line.split("\t")[0]))):
- continue
- if (len(line.split("\t")) == 4):
- line = line + "\t" * (5 - len(line.split("\t")))
- if (not (is_number(line.split("\t")[2]))):
- ice_bad_questions += 1
- (example_id, ans_index, ans_raw, process_answer,
- matched_cells) = line.split("\t")
- if (ice.has_key(example_id)):
- ice[example_id].append(line.split("\t"))
- else:
- ice[example_id] = [line.split("\t")]
- for q_id in self.annotated_examples.keys():
- tot += 1
- example = self.annotated_examples[q_id]
- table_info = self.annotated_tables[example.table_key]
- # Figure out if the answer is numerical or lookup
- n_cols = len(table_info.orig_columns)
- n_rows = len(table_info.orig_columns[0])
- example.lookup_matrix = np.zeros((n_rows, n_cols))
- exact_matches = {}
- for (example_id, ans_index, ans_raw, process_answer,
- matched_cells) in ice[q_id]:
- for match_cell in matched_cells.split("|"):
- if (len(match_cell.split(",")) == 2):
- (row, col) = match_cell.split(",")
- row = int(row)
- col = int(col)
- if (row >= 0):
- exact_matches[ans_index] = 1
- answer_is_in_table = len(exact_matches) == len(example.answer)
- if (answer_is_in_table):
- for (example_id, ans_index, ans_raw, process_answer,
- matched_cells) in ice[q_id]:
- for match_cell in matched_cells.split("|"):
- if (len(match_cell.split(",")) == 2):
- (row, col) = match_cell.split(",")
- row = int(row)
- col = int(col)
- example.lookup_matrix[row, col] = float(ans_index) + 1.0
- example.lookup_number_answer = 0.0
- if (answer_is_in_table):
- lookup_questions += 1
- if len(example.answer) == 1 and is_number(example.answer[0]):
- example.number_answer = float(example.answer[0])
- number_lookup_questions += 1
- example.is_number_lookup = True
- else:
- #print "word lookup"
- example.calc_answer = example.number_answer = 0.0
- word_lookup_questions += 1
- example.is_word_lookup = True
- else:
- if (len(example.answer) == 1 and is_number(example.answer[0])):
- example.number_answer = example.answer[0]
- example.is_number_calc = True
- else:
- bad_questions += 1
- example.is_bad_example = True
- example.is_unknown_answer = True
- example.is_lookup = example.is_word_lookup or example.is_number_lookup
- if not example.is_word_lookup and not example.is_bad_example:
- number_questions += 1
- example.calc_answer = example.answer[0]
- example.lookup_number_answer = example.calc_answer
- # Split up the lookup matrix into word part and number part
- number_column_indices = table_info.number_column_indices
- word_column_indices = table_info.word_column_indices
- example.word_columns = table_info.word_columns
- example.number_columns = table_info.number_columns
- example.word_column_names = table_info.word_column_names
- example.processed_number_columns = table_info.processed_number_columns
- example.processed_word_columns = table_info.processed_word_columns
- example.number_column_names = table_info.number_column_names
- example.number_lookup_matrix = example.lookup_matrix[:,
- number_column_indices]
- example.word_lookup_matrix = example.lookup_matrix[:, word_column_indices]
-
- def load(self):
- train_data = []
- dev_data = []
- test_data = []
- self.load_annotated_data(
- os.path.join(self.data_folder, "training.annotated"))
- self.load_annotated_tables()
- self.answer_classification()
- self.train_loader.load()
- self.dev_loader.load()
- for i in range(self.train_loader.num_questions()):
- example = self.train_loader.examples[i]
- example = self.annotated_examples[example]
- train_data.append(example)
- for i in range(self.dev_loader.num_questions()):
- example = self.dev_loader.examples[i]
- dev_data.append(self.annotated_examples[example])
-
- self.load_annotated_data(
- os.path.join(self.data_folder, "pristine-unseen-tables.annotated"))
- self.load_annotated_tables()
- self.answer_classification()
- self.test_loader.load()
- for i in range(self.test_loader.num_questions()):
- example = self.test_loader.examples[i]
- test_data.append(self.annotated_examples[example])
- return train_data, dev_data, test_data
diff --git a/research/next_frame_prediction/README.md b/research/next_frame_prediction/README.md
deleted file mode 100644
index 9aa9b6fc5a3146a5e24ce53422d985570891d42b..0000000000000000000000000000000000000000
--- a/research/next_frame_prediction/README.md
+++ /dev/null
@@ -1,89 +0,0 @@
-
-
-
-
-Visual Dynamics: Probabilistic Future Frame Synthesis via Cross Convolutional Networks.
-
-Introduction
-
-https://arxiv.org/pdf/1607.02586v1.pdf
-
-This is an implementation based on my understanding, with small
-variations. It doesn't necessarily represents the paper published
-by the original authors.
-
-Authors: Xin Pan, Anelia Angelova
-
-Results:
-
-
-
-
-
-
-
-Prerequisite:
-
-1. Install TensorFlow (r0.12), Bazel.
-
-2. Download the Sprites dataset or generate moving object dataset.
-
-Sprites data is located here:
-
-http://www.scottreed.info/files/nips2015-analogy-data.tar.gz
-
-Convert .mat files into images and use sprites_gen.py to convert them
-to tf.SequenceExample.
-
-How to run:
-
-```shell
-$ ls -R
-.:
-data next_frame_prediction WORKSPACE
-
-./data:
-tfrecords tfrecords_test
-
-./next_frame_prediction:
-cross_conv g3doc README.md
-
-./next_frame_prediction/cross_conv:
-BUILD eval.py objects_gen.py model.py reader.py sprites_gen.py train.py
-
-./next_frame_prediction/g3doc:
-cross_conv2.png cross_conv3.png cross_conv.png
-
-
-# Build everything.
-$ bazel build -c opt next_frame_prediction/...
-
-# The following example runs the generated 2d objects.
-# For Sprites dataset, image_size should be 60, norm_scale should be 255.0.
-# Batch size is normally 16~64, depending on your memory size.
-
-# Run training.
-$ bazel-bin/next_frame_prediction/cross_conv/train \
- --batch_size=1 \
- --data_filepattern=data/tfrecords \
- --image_size=64 \
- --log_root=/tmp/predict
-
-step: 1, loss: 24.428671
-step: 2, loss: 19.211605
-step: 3, loss: 5.543143
-step: 4, loss: 3.035339
-step: 5, loss: 1.771392
-step: 6, loss: 2.099824
-step: 7, loss: 1.747665
-step: 8, loss: 1.572436
-step: 9, loss: 1.586816
-step: 10, loss: 1.434191
-
-# Run eval.
-$ bazel-bin/next_frame_prediction/cross_conv/eval \
- --batch_size=1 \
- --data_filepattern=data/tfrecords_test \
- --image_size=64 \
- --log_root=/tmp/predict
-```
diff --git a/research/next_frame_prediction/cross_conv/BUILD b/research/next_frame_prediction/cross_conv/BUILD
deleted file mode 100644
index b435087f34f6ffbeba016119c60724d8ac3eb180..0000000000000000000000000000000000000000
--- a/research/next_frame_prediction/cross_conv/BUILD
+++ /dev/null
@@ -1,48 +0,0 @@
-licenses(["notice"]) # Apache 2.0
-
-package_group(
- name = "internal",
- packages = [
- "//next_frame_prediction/...",
- ],
-)
-
-package(default_visibility = [":internal"])
-
-py_library(
- name = "model",
- srcs = ["model.py"],
-)
-
-py_library(
- name = "reader",
- srcs = ["reader.py"],
-)
-
-py_binary(
- name = "train",
- srcs = ["train.py"],
- deps = [
- ":model",
- ":reader",
- ],
-)
-
-py_binary(
- name = "eval",
- srcs = ["eval.py"],
- deps = [
- ":model",
- ":reader",
- ],
-)
-
-py_binary(
- name = "example_gen",
- srcs = ["example_gen.py"],
-)
-
-py_binary(
- name = "sprites_gen",
- srcs = ["sprites_gen.py"],
-)
diff --git a/research/next_frame_prediction/cross_conv/eval.py b/research/next_frame_prediction/cross_conv/eval.py
deleted file mode 100644
index 17ebc0e0edd2911f828cbb145ee40a06db8795b5..0000000000000000000000000000000000000000
--- a/research/next_frame_prediction/cross_conv/eval.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Eval Cross Convolutional Model."""
-import io
-import os
-import sys
-import time
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import model as cross_conv_model
-import reader
-
-FLAGS = tf.flags.FLAGS
-tf.flags.DEFINE_string('log_root', '/tmp/moving_obj', 'The root dir of output.')
-tf.flags.DEFINE_string('data_filepattern',
- 'est',
- 'training data file pattern.')
-tf.flags.DEFINE_integer('batch_size', 1, 'Batch size.')
-tf.flags.DEFINE_integer('image_size', 64, 'Image height and width.')
-tf.flags.DEFINE_float('norm_scale', 1.0, 'Normalize the original image')
-tf.flags.DEFINE_float('scale', 10.0,
- 'Scale the image after norm_scale and move the diff '
- 'to the positive realm.')
-tf.flags.DEFINE_integer('sequence_length', 2, 'tf.SequenceExample length.')
-tf.flags.DEFINE_integer('eval_batch_count', 100,
- 'Average the result this number of examples.')
-tf.flags.DEFINE_bool('l2_loss', True, 'If true, include l2_loss.')
-tf.flags.DEFINE_bool('reconstr_loss', False, 'If true, include reconstr_loss.')
-tf.flags.DEFINE_bool('kl_loss', True, 'If true, include KL loss.')
-
-slim = tf.contrib.slim
-
-
-def _Eval():
- params = dict()
- params['batch_size'] = FLAGS.batch_size
- params['seq_len'] = FLAGS.sequence_length
- params['image_size'] = FLAGS.image_size
- params['is_training'] = False
- params['norm_scale'] = FLAGS.norm_scale
- params['scale'] = FLAGS.scale
- params['l2_loss'] = FLAGS.l2_loss
- params['reconstr_loss'] = FLAGS.reconstr_loss
- params['kl_loss'] = FLAGS.kl_loss
-
- eval_dir = os.path.join(FLAGS.log_root, 'eval')
-
- images = reader.ReadInput(
- FLAGS.data_filepattern, shuffle=False, params=params)
- images *= params['scale']
- # Increase the value makes training much faster.
- image_diff_list = reader.SequenceToImageAndDiff(images)
- model = cross_conv_model.CrossConvModel(image_diff_list, params)
- model.Build()
-
- summary_writer = tf.summary.FileWriter(eval_dir)
- saver = tf.train.Saver()
- sess = tf.Session('', config=tf.ConfigProto(allow_soft_placement=True))
- tf.train.start_queue_runners(sess)
-
- while True:
- time.sleep(60)
- try:
- ckpt_state = tf.train.get_checkpoint_state(FLAGS.log_root)
- except tf.errors.OutOfRangeError as e:
- sys.stderr.write('Cannot restore checkpoint: %s\n' % e)
- continue
- if not (ckpt_state and ckpt_state.model_checkpoint_path):
- sys.stderr.write('No model to eval yet at %s\n' % FLAGS.log_root)
- continue
- sys.stderr.write('Loading checkpoint %s\n' %
- ckpt_state.model_checkpoint_path)
- saver.restore(sess, ckpt_state.model_checkpoint_path)
- # Use the empirical distribution of z from training set.
- if not tf.gfile.Exists(os.path.join(FLAGS.log_root, 'z_mean.npy')):
- sys.stderr.write('No z at %s\n' % FLAGS.log_root)
- continue
-
- with tf.gfile.Open(os.path.join(FLAGS.log_root, 'z_mean.npy')) as f:
- sample_z_mean = np.load(io.BytesIO(f.read()))
- with tf.gfile.Open(
- os.path.join(FLAGS.log_root, 'z_stddev_log.npy')) as f:
- sample_z_stddev_log = np.load(io.BytesIO(f.read()))
-
- total_loss = 0.0
- for _ in xrange(FLAGS.eval_batch_count):
- loss_val, total_steps, summaries = sess.run(
- [model.loss, model.global_step, model.summary_op],
- feed_dict={model.z_mean: sample_z_mean,
- model.z_stddev_log: sample_z_stddev_log})
- total_loss += loss_val
-
- summary_writer.add_summary(summaries, total_steps)
- sys.stderr.write('steps: %d, loss: %f\n' %
- (total_steps, total_loss / FLAGS.eval_batch_count))
-
-
-def main(_):
- _Eval()
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/next_frame_prediction/cross_conv/example_gen.py b/research/next_frame_prediction/cross_conv/example_gen.py
deleted file mode 100644
index bcda0bc405a60c3116e8c488cae92f502720fec4..0000000000000000000000000000000000000000
--- a/research/next_frame_prediction/cross_conv/example_gen.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Generate examples of two objects moving in different directions."""
-import random
-import sys
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-
-tf.flags.DEFINE_string('out_file', '',
- 'Output file for the tfrecords.')
-
-
-def _add_object(obj_type, image, image2, xpos, ypos):
- """Add a moving obj to two consecutive images."""
- obj_size = random.randint(8, 10)
- channel = random.randint(0, 2)
- move = random.randint(6, 10)
-
- obj = np.zeros([obj_size, obj_size, 3])
- if obj_type == 'rectangle':
- xpos2 = xpos + move
- ypos2 = ypos
- for i in xrange(obj_size):
- obj[i, 0:i+1, channel] = [1.0 for _ in xrange(i+1)]
- elif obj_type == 'square':
- xpos2 = xpos
- ypos2 = ypos + move
- obj[:, :, channel] = 1.0
-
- for x in xrange(obj_size):
- for y in xrange(obj_size):
- if obj[x, y, channel] == 1.0:
- image[xpos+x, ypos+y, channel] = 1.0
- image2[xpos2+x, ypos2+y, channel] = 1.0
-
-
-def _images_to_example(image, image2):
- """Convert two consecutive images to SequenceExample."""
- example = tf.SequenceExample()
- feature_list = example.feature_lists.feature_list['moving_objs']
- feature = feature_list.feature.add()
- feature.float_list.value.extend(np.reshape(image, [-1]).tolist())
- feature = feature_list.feature.add()
- feature.float_list.value.extend(np.reshape(image2, [-1]).tolist())
- return example
-
-
-def generate_input():
- """Generate tfrecords."""
- writer = tf.python_io.TFRecordWriter(tf.flags.FLAGS.out_file)
- writer2 = tf.python_io.TFRecordWriter(tf.flags.FLAGS.out_file + '_test')
-
- examples = []
- for xpos in xrange(0, 40, 3):
- for ypos in xrange(0, 40, 3):
- for xpos2 in xrange(0, 40, 3):
- for ypos2 in xrange(0, 40, 3):
- image = np.zeros([64, 64, 3])
- image2 = np.zeros([64, 64, 3])
- _add_object('rectangle', image, image2, xpos, ypos)
- _add_object('square', image, image2, xpos2, ypos2)
- examples.append(_images_to_example(image, image2))
-
- sys.stderr.write('Finish generating examples.\n')
- random.shuffle(examples)
- for count, ex in enumerate(examples):
- if count % 10 == 0:
- writer2.write(ex.SerializeToString())
- else:
- writer.write(ex.SerializeToString())
-
-def main(_):
- generate_input()
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/next_frame_prediction/cross_conv/model.py b/research/next_frame_prediction/cross_conv/model.py
deleted file mode 100644
index 7b48e446e18b70fec87142f6834f33332287d02e..0000000000000000000000000000000000000000
--- a/research/next_frame_prediction/cross_conv/model.py
+++ /dev/null
@@ -1,233 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Cross Convolutional Model.
-
-https://arxiv.org/pdf/1607.02586v1.pdf
-"""
-import math
-import sys
-
-from six.moves import xrange
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-
-class CrossConvModel(object):
-
- def __init__(self, image_diff_list, params):
- """Constructor.
-
- Args:
- image_diff_list: A list of (image, diff) tuples, with shape
- [batch_size, image_size, image_size, 3] and image_sizes as
- [32, 64, 128, 256].
- params: Dict of parameters.
- """
- self.images = [i for (i, _) in image_diff_list]
- # Move the diff to the positive realm.
- self.diffs = [(d + params['scale']) / 2 for (i, d) in image_diff_list]
- self.params = params
-
- def Build(self):
- with tf.device('/gpu:0'):
- with slim.arg_scope([slim.conv2d],
- activation_fn=tf.nn.relu,
- normalizer_fn=slim.batch_norm,
- normalizer_params={'is_training':
- self.params['is_training']}):
- self._BuildMotionKernel()
- encoded_images = self._BuildImageEncoder()
- cross_conved_images = self._CrossConv(encoded_images)
- self._BuildImageDecoder(cross_conved_images)
- self._BuildLoss()
-
- image = self.images[1]
- diff = self.diffs[1]
-
- self.global_step = tf.Variable(0, name='global_step', trainable=False)
-
- if self.params['is_training']:
- self._BuildTrainOp()
-
- diff = diff * 2.0 - self.params['scale']
- diff_output = self.diff_output * 2.0 - self.params['scale']
- concat_image = tf.concat(
- axis=1, values=[image, image + diff_output, image + diff, diff_output])
- tf.summary.image('origin_predict_expect_predictdiff', concat_image)
- self.summary_op = tf.summary.merge_all()
- return self.loss
-
- def _BuildTrainOp(self):
- lrn_rate = tf.maximum(
- 0.01, # min_lr_rate.
- tf.train.exponential_decay(
- self.params['learning_rate'], self.global_step, 10000, 0.5))
- tf.summary.scalar('learning rate', lrn_rate)
- optimizer = tf.train.GradientDescentOptimizer(lrn_rate)
- self.train_op = slim.learning.create_train_op(
- self.loss, optimizer, global_step=self.global_step)
-
- def _BuildLoss(self):
- # 1. reconstr_loss seems doesn't do better than l2 loss.
- # 2. Only works when using reduce_mean. reduce_sum doesn't work.
- # 3. It seems kl loss doesn't play an important role.
- self.loss = 0
- with tf.variable_scope('loss'):
- if self.params['l2_loss']:
- l2_loss = tf.reduce_mean(tf.square(self.diff_output - self.diffs[1]))
- tf.summary.scalar('l2_loss', l2_loss)
- self.loss += l2_loss
- if self.params['reconstr_loss']:
- reconstr_loss = (-tf.reduce_mean(
- self.diffs[1] * (1e-10 + self.diff_output) +
- (1-self.diffs[1]) * tf.log(1e-10 + 1 - self.diff_output)))
- reconstr_loss = tf.check_numerics(reconstr_loss, 'reconstr_loss')
- tf.summary.scalar('reconstr_loss', reconstr_loss)
- self.loss += reconstr_loss
- if self.params['kl_loss']:
- kl_loss = (0.5 * tf.reduce_mean(
- tf.square(self.z_mean) + tf.square(self.z_stddev) -
- 2 * self.z_stddev_log - 1))
- tf.summary.scalar('kl_loss', kl_loss)
- self.loss += kl_loss
-
- tf.summary.scalar('loss', self.loss)
-
- def _BuildMotionKernel(self):
- image = self.images[-2]
- diff = self.diffs[-2]
- shape = image.get_shape().as_list()
- assert shape[1] == shape[2] and shape[1] == 128
- batch_size = shape[0]
-
- net = tf.concat(axis=3, values=[image, diff])
- with tf.variable_scope('motion_encoder'):
- with slim.arg_scope([slim.conv2d], padding='VALID'):
- net = slim.conv2d(net, 96, [5, 5], stride=1)
- net = slim.max_pool2d(net, [2, 2])
- net = slim.conv2d(net, 96, [5, 5], stride=1)
- net = slim.max_pool2d(net, [2, 2])
- net = slim.conv2d(net, 128, [5, 5], stride=1)
- net = slim.conv2d(net, 128, [5, 5], stride=1)
- net = slim.max_pool2d(net, [2, 2])
- net = slim.conv2d(net, 256, [4, 4], stride=1)
- net = slim.conv2d(net, 256, [3, 3], stride=1)
-
- z = tf.reshape(net, shape=[batch_size, -1])
- self.z_mean, self.z_stddev_log = tf.split(
- axis=1, num_or_size_splits=2, value=z)
- self.z_stddev = tf.exp(self.z_stddev_log)
-
- epsilon = tf.random_normal(
- self.z_mean.get_shape().as_list(), 0, 1, dtype=tf.float32)
- kernel = self.z_mean + tf.multiply(self.z_stddev, epsilon)
-
- width = int(math.sqrt(kernel.get_shape().as_list()[1] // 128))
- kernel = tf.reshape(kernel, [batch_size, width, width, 128])
- with tf.variable_scope('kernel_decoder'):
- with slim.arg_scope([slim.conv2d], padding='SAME'):
- kernel = slim.conv2d(kernel, 128, [5, 5], stride=1)
- self.kernel = slim.conv2d(kernel, 128, [5, 5], stride=1)
-
- sys.stderr.write('kernel shape: %s\n' % kernel.get_shape())
-
- def _BuildImageEncoder(self):
- feature_maps = []
- for (i, image) in enumerate(self.images):
- with tf.variable_scope('image_encoder_%d' % i):
- with slim.arg_scope([slim.conv2d, slim.max_pool2d], padding='SAME'):
- net = slim.conv2d(image, 64, [5, 5], stride=1)
- net = slim.conv2d(net, 64, [5, 5], stride=1)
- net = slim.max_pool2d(net, [5, 5])
- net = slim.conv2d(net, 64, [5, 5], stride=1)
- net = slim.conv2d(net, 32, [5, 5], stride=1)
- net = slim.max_pool2d(net, [2, 2])
- sys.stderr.write('image_conv shape: %s\n' % net.get_shape())
- feature_maps.append(net)
- return feature_maps
-
- def _CrossConvHelper(self, encoded_image, kernel):
- """Cross Convolution.
-
- The encoded image and kernel are of the same shape. Namely
- [batch_size, image_size, image_size, channels]. They are split
- into [image_size, image_size] image squares [kernel_size, kernel_size]
- kernel squares. kernel squares are used to convolute image squares.
- """
- images = tf.expand_dims(encoded_image, 0)
- kernels = tf.expand_dims(kernel, 3)
- return tf.nn.depthwise_conv2d(images, kernels, [1, 1, 1, 1], 'SAME')
-
- def _CrossConv(self, encoded_images):
- """Apply the motion kernel on the encoded_images."""
- cross_conved_images = []
- kernels = tf.split(axis=3, num_or_size_splits=4, value=self.kernel)
- for (i, encoded_image) in enumerate(encoded_images):
- with tf.variable_scope('cross_conv_%d' % i):
- kernel = kernels[i]
-
- encoded_image = tf.unstack(encoded_image, axis=0)
- kernel = tf.unstack(kernel, axis=0)
- assert len(encoded_image) == len(kernel)
- assert len(encoded_image) == self.params['batch_size']
- conved_image = []
- for j in xrange(len(encoded_image)):
- conved_image.append(self._CrossConvHelper(
- encoded_image[j], kernel[j]))
- cross_conved_images.append(tf.concat(axis=0, values=conved_image))
- sys.stderr.write('cross_conved shape: %s\n' %
- cross_conved_images[-1].get_shape())
- return cross_conved_images
-
- def _Deconv(self, net, out_filters, kernel_size, stride):
- shape = net.get_shape().as_list()
- in_filters = shape[3]
- kernel_shape = [kernel_size, kernel_size, out_filters, in_filters]
-
- weights = tf.get_variable(
- name='weights',
- shape=kernel_shape,
- dtype=tf.float32,
- initializer=tf.truncated_normal_initializer(stddev=0.01))
-
-
- out_height = shape[1] * stride
- out_width = shape[2] * stride
- batch_size = shape[0]
-
- output_shape = [batch_size, out_height, out_width, out_filters]
- net = tf.nn.conv2d_transpose(net, weights, output_shape,
- [1, stride, stride, 1], padding='SAME')
- slim.batch_norm(net)
- return net
-
- def _BuildImageDecoder(self, cross_conved_images):
- """Decode the cross_conved feature maps into the predicted images."""
- nets = []
- for i, cross_conved_image in enumerate(cross_conved_images):
- with tf.variable_scope('image_decoder_%d' % i):
- stride = 64 / cross_conved_image.get_shape().as_list()[1]
- # TODO(xpan): Alternative solution for upsampling?
- nets.append(self._Deconv(
- cross_conved_image, 64, kernel_size=3, stride=stride))
-
- net = tf.concat(axis=3, values=nets)
- net = slim.conv2d(net, 128, [9, 9], padding='SAME', stride=1)
- net = slim.conv2d(net, 128, [1, 1], padding='SAME', stride=1)
- net = slim.conv2d(net, 3, [1, 1], padding='SAME', stride=1)
- self.diff_output = net
- sys.stderr.write('diff_output shape: %s\n' % self.diff_output.get_shape())
diff --git a/research/next_frame_prediction/cross_conv/reader.py b/research/next_frame_prediction/cross_conv/reader.py
deleted file mode 100644
index ab4ab698dda938f182be0019168aa132c1e3c5af..0000000000000000000000000000000000000000
--- a/research/next_frame_prediction/cross_conv/reader.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Read image sequence."""
-
-from six.moves import xrange
-import tensorflow as tf
-
-
-def SequenceToImageAndDiff(images):
- """Convert image sequence batch into image and diff batch.
-
- Each image pair is converted to the first image and their diff.
- Batch size will increase if sequence length is larger than 2.
-
- Args:
- images: Image sequence with shape
- [batch_size, seq_len, image_size, image_size, channel]
-
- Returns:
- the list of (image, diff) tuples with shape
- [batch_size2, image_size, image_size, channel]. image_sizes are
- [32, 64, 128, 256].
- """
- image_diff_list = []
- image_seq = tf.unstack(images, axis=1)
- for size in [32, 64, 128, 256]:
- resized_images = [
- tf.image.resize_images(i, [size, size]) for i in image_seq]
- diffs = []
- for i in xrange(0, len(resized_images)-1):
- diffs.append(resized_images[i+1] - resized_images[i])
- image_diff_list.append(
- (tf.concat(axis=0, values=resized_images[:-1]), tf.concat(axis=0, values=diffs)))
- return image_diff_list
-
-
-def ReadInput(data_filepattern, shuffle, params):
- """Read the tf.SequenceExample tfrecord files.
-
- Args:
- data_filepattern: tf.SequenceExample tfrecord filepattern.
- shuffle: Whether to shuffle the examples.
- params: parameter dict.
-
- Returns:
- image sequence batch [batch_size, seq_len, image_size, image_size, channel].
- """
- image_size = params['image_size']
- filenames = tf.gfile.Glob(data_filepattern)
- filename_queue = tf.train.string_input_producer(filenames, shuffle=shuffle)
- reader = tf.TFRecordReader()
- _, example = reader.read(filename_queue)
- feature_sepc = {
- 'moving_objs': tf.FixedLenSequenceFeature(
- shape=[image_size * image_size * 3], dtype=tf.float32)}
- _, features = tf.parse_single_sequence_example(
- example, sequence_features=feature_sepc)
- moving_objs = tf.reshape(
- features['moving_objs'], [params['seq_len'], image_size, image_size, 3])
- if shuffle:
- examples = tf.train.shuffle_batch(
- [moving_objs],
- batch_size=params['batch_size'],
- num_threads=64,
- capacity=params['batch_size'] * 100,
- min_after_dequeue=params['batch_size'] * 4)
- else:
- examples = tf.train.batch([moving_objs],
- batch_size=params['batch_size'],
- num_threads=16,
- capacity=params['batch_size'])
- examples /= params['norm_scale']
- return examples
diff --git a/research/next_frame_prediction/cross_conv/sprites_gen.py b/research/next_frame_prediction/cross_conv/sprites_gen.py
deleted file mode 100644
index 0d36c255cd93a90797272d7a80389f16fc6f3702..0000000000000000000000000000000000000000
--- a/research/next_frame_prediction/cross_conv/sprites_gen.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Generate the sprites tfrecords from raw_images."""
-import os
-import random
-import re
-import sys
-
-import numpy as np
-import scipy.misc
-from six.moves import xrange
-import tensorflow as tf
-
-
-tf.flags.DEFINE_string('data_filepattern', '', 'The raw images.')
-tf.flags.DEFINE_string('out_file', '',
- 'File name for the tfrecord output.')
-
-
-def _read_images():
- """Read images from image files into data structure."""
- sprites = dict()
- files = tf.gfile.Glob(tf.flags.FLAGS.data_filepattern)
- for f in files:
- image = scipy.misc.imread(f)
- m = re.search('image_([0-9]+)_([0-9]+)_([0-9]+).jpg', os.path.basename(f))
- if m.group(1) not in sprites:
- sprites[m.group(1)] = dict()
- character = sprites[m.group(1)]
- if m.group(2) not in character:
- character[m.group(2)] = dict()
- pose = character[m.group(2)]
- pose[int(m.group(3))] = image
- return sprites
-
-
-def _images_to_example(image, image2):
- """Convert 2 consecutive image to a SequenceExample."""
- example = tf.SequenceExample()
- feature_list = example.feature_lists.feature_list['moving_objs']
- feature = feature_list.feature.add()
- feature.float_list.value.extend(np.reshape(image, [-1]).tolist())
- feature = feature_list.feature.add()
- feature.float_list.value.extend(np.reshape(image2, [-1]).tolist())
- return example
-
-
-def generate_input():
- """Generate tfrecords."""
- sprites = _read_images()
- sys.stderr.write('Finish reading images.\n')
- train_writer = tf.python_io.TFRecordWriter(
- tf.flags.FLAGS.out_file.replace('sprites', 'sprites_train'))
- test_writer = tf.python_io.TFRecordWriter(
- tf.flags.FLAGS.out_file.replace('sprites', 'sprites_test'))
-
- train_examples = []
- test_examples = []
- for i in sprites:
- if int(i) < 24:
- examples = test_examples
- else:
- examples = train_examples
-
- character = sprites[i]
- for j in character.keys():
- pose = character[j]
- for k in xrange(1, len(pose), 1):
- image = pose[k]
- image2 = pose[k+1]
- examples.append(_images_to_example(image, image2))
-
- sys.stderr.write('Finish generating examples: %d, %d.\n' %
- (len(train_examples), len(test_examples)))
- random.shuffle(train_examples)
- _ = [train_writer.write(ex.SerializeToString()) for ex in train_examples]
- _ = [test_writer.write(ex.SerializeToString()) for ex in test_examples]
-
-
-def main(_):
- generate_input()
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/next_frame_prediction/cross_conv/train.py b/research/next_frame_prediction/cross_conv/train.py
deleted file mode 100644
index 5b9973f52cc3946b3396c1e0b87fda19901735f6..0000000000000000000000000000000000000000
--- a/research/next_frame_prediction/cross_conv/train.py
+++ /dev/null
@@ -1,122 +0,0 @@
-# Copyright 2016 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Train the cross convolutional model."""
-import os
-import sys
-
-import numpy as np
-import tensorflow as tf
-
-import model as cross_conv_model
-import reader
-
-FLAGS = tf.flags.FLAGS
-tf.flags.DEFINE_string('master', '', 'Session address.')
-tf.flags.DEFINE_string('log_root', '/tmp/moving_obj', 'The root dir of output.')
-tf.flags.DEFINE_string('data_filepattern', '',
- 'training data file pattern.')
-tf.flags.DEFINE_integer('image_size', 64, 'Image height and width.')
-tf.flags.DEFINE_integer('batch_size', 1, 'Batch size.')
-tf.flags.DEFINE_float('norm_scale', 1.0, 'Normalize the original image')
-tf.flags.DEFINE_float('scale', 10.0,
- 'Scale the image after norm_scale and move the diff '
- 'to the positive realm.')
-tf.flags.DEFINE_integer('sequence_length', 2, 'tf.SequenceExample length.')
-tf.flags.DEFINE_float('learning_rate', 0.8, 'Learning rate.')
-tf.flags.DEFINE_bool('l2_loss', True, 'If true, include l2_loss.')
-tf.flags.DEFINE_bool('reconstr_loss', False, 'If true, include reconstr_loss.')
-tf.flags.DEFINE_bool('kl_loss', True, 'If true, include KL loss.')
-
-slim = tf.contrib.slim
-
-
-def _Train():
- params = dict()
- params['batch_size'] = FLAGS.batch_size
- params['seq_len'] = FLAGS.sequence_length
- params['image_size'] = FLAGS.image_size
- params['is_training'] = True
- params['norm_scale'] = FLAGS.norm_scale
- params['scale'] = FLAGS.scale
- params['learning_rate'] = FLAGS.learning_rate
- params['l2_loss'] = FLAGS.l2_loss
- params['reconstr_loss'] = FLAGS.reconstr_loss
- params['kl_loss'] = FLAGS.kl_loss
-
- train_dir = os.path.join(FLAGS.log_root, 'train')
-
- images = reader.ReadInput(FLAGS.data_filepattern, shuffle=True, params=params)
- images *= params['scale']
- # Increase the value makes training much faster.
- image_diff_list = reader.SequenceToImageAndDiff(images)
- model = cross_conv_model.CrossConvModel(image_diff_list, params)
- model.Build()
- tf.contrib.tfprof.model_analyzer.print_model_analysis(tf.get_default_graph())
-
- summary_writer = tf.summary.FileWriter(train_dir)
- sv = tf.train.Supervisor(logdir=FLAGS.log_root,
- summary_op=None,
- is_chief=True,
- save_model_secs=60,
- global_step=model.global_step)
- sess = sv.prepare_or_wait_for_session(
- FLAGS.master, config=tf.ConfigProto(allow_soft_placement=True))
-
- total_loss = 0.0
- step = 0
- sample_z_mean = np.zeros(model.z_mean.get_shape().as_list())
- sample_z_stddev_log = np.zeros(model.z_stddev_log.get_shape().as_list())
- sample_step = 0
-
- while True:
- _, loss_val, total_steps, summaries, z_mean, z_stddev_log = sess.run(
- [model.train_op, model.loss, model.global_step,
- model.summary_op,
- model.z_mean, model.z_stddev_log])
-
- sample_z_mean += z_mean
- sample_z_stddev_log += z_stddev_log
- total_loss += loss_val
- step += 1
- sample_step += 1
-
- if step % 100 == 0:
- summary_writer.add_summary(summaries, total_steps)
- sys.stderr.write('step: %d, loss: %f\n' %
- (total_steps, total_loss / step))
- total_loss = 0.0
- step = 0
-
- # Sampled z is used for eval.
- # It seems 10k is better than 1k. Maybe try 100k next?
- if sample_step % 10000 == 0:
- with tf.gfile.Open(os.path.join(FLAGS.log_root, 'z_mean.npy'), 'w') as f:
- np.save(f, sample_z_mean / sample_step)
- with tf.gfile.Open(
- os.path.join(FLAGS.log_root, 'z_stddev_log.npy'), 'w') as f:
- np.save(f, sample_z_stddev_log / sample_step)
- sample_z_mean = np.zeros(model.z_mean.get_shape().as_list())
- sample_z_stddev_log = np.zeros(
- model.z_stddev_log.get_shape().as_list())
- sample_step = 0
-
-
-def main(_):
- _Train()
-
-
-if __name__ == '__main__':
- tf.app.run()
diff --git a/research/next_frame_prediction/g3doc/cross_conv.png b/research/next_frame_prediction/g3doc/cross_conv.png
deleted file mode 100644
index 13915f944188adf0b0a3dc85219fce7bcb5e7de9..0000000000000000000000000000000000000000
Binary files a/research/next_frame_prediction/g3doc/cross_conv.png and /dev/null differ
diff --git a/research/next_frame_prediction/g3doc/cross_conv2.png b/research/next_frame_prediction/g3doc/cross_conv2.png
deleted file mode 100644
index c4b5e8e9d6169a1a908858a91fdc6467ae03ea2a..0000000000000000000000000000000000000000
Binary files a/research/next_frame_prediction/g3doc/cross_conv2.png and /dev/null differ
diff --git a/research/next_frame_prediction/g3doc/cross_conv3.png b/research/next_frame_prediction/g3doc/cross_conv3.png
deleted file mode 100644
index 054d7d1edf2043c50a3ea8d332cc83a8dcb32c9b..0000000000000000000000000000000000000000
Binary files a/research/next_frame_prediction/g3doc/cross_conv3.png and /dev/null differ
diff --git a/research/object_detection/README.md b/research/object_detection/README.md
index 9e9bf549861c2ae5aff7c386ce2efe6ca2bb1ff9..5a79bcc456884fe63a55d30eb6926a39f3d94d73 100644
--- a/research/object_detection/README.md
+++ b/research/object_detection/README.md
@@ -54,19 +54,55 @@ Note: The models we provide in [TF2 Zoo](g3doc/tf2_detection_zoo.md) and
[TF1 Zoo](g3doc/tf1_detection_zoo.md) are specific to the TensorFlow major
version and are not interoperable.
-Please select one of the two links below for TensorFlow version specific
+Please select one of the links below for TensorFlow version-specific
documentation of the Object Detection API:
+### Tensorflow 2.x
+ *
+ Object Detection API TensorFlow 2
+ *
+ TensorFlow 2 Model Zoo
+
+### Tensorflow 1.x
+ *
+ Object Detection API TensorFlow 1
+ *
+ TensorFlow 1 Model Zoo
+
+## Whats New
-| [](g3doc/tf2.md) | [](g3doc/tf2_detection_zoo.md) |
-|---|---|
-| [](g3doc/tf1.md) | [](g3doc/tf1_detection_zoo.md) |
+### DeepMAC architecture
-
+We have released our new architecture, **DeepMAC**, desgined for partially
+supervised instance segmentation. DeepMAC stands for Deep Mask-heads
+Above CenterNet, and is based on our CenterNet implementation. In our
+[paper](https://arxiv.org/abs/2104.00613) we show that DeepMAC achieves
+state-of-the-art results for the partially supervised instance segmentation
+task without using any specialty modules or losses; just better mask-head
+architectures. The findings from our paper are not specific to CenterNet and
+can also be applied to Mask R-CNN or without any detector at all.
+Please see links below for more details
-## Whats New
+* [DeepMAC documentation](g3doc/deepmac.md).
+* [Mask RCNN code](https://github.com/tensorflow/models/tree/master/official/vision/beta/projects/deepmac_maskrcnn)
+ in TF Model garden code base.
+* [DeepMAC Colab](./colab_tutorials/deepmac_colab.ipynb) that lets you run a
+ pre-trained DeepMAC model on user-specified boxes. Note that you are not
+ restricted to COCO classes!
+* Project website - [git.io/deepmac](https://git.io/deepmac)
+
+Thanks to contributors: Vighnesh Birodkar, Zhichao Lu, Siyang Li,
+ Vivek Rathod, Jonathan Huang
+
+
+### Mobile Inference for TF2 models
+
+TF2 OD API models can now be converted to TensorFlow Lite! Only SSD models
+currently supported. See documentation.
+
+**Thanks to contributors**: Sachin Joglekar
### TensorFlow 2 Support
@@ -104,6 +140,18 @@ Sergi Caelles Prat, Shan Yang, Sudheendra Vijayanarasimhan, Tina Tian, Tomer
Kaftan, Vighnesh Birodkar, Vishnu Banna, Vivek Rathod, Yanhui Liang, Yiming Shi,
Yixin Shi, Yu-hui Chen, Zhichao Lu.
+### MobileDet GPU
+
+We have released SSDLite with MobileDet GPU backbone, which achieves 17% mAP
+higher than the MobileNetV2 SSDLite (27.5 mAP vs 23.5 mAP) on a NVIDIA Jetson
+Xavier at comparable latency (3.2ms vs 3.3ms).
+
+Along with the model definition, we are also releasing model checkpoints trained
+on the COCO dataset.
+
+Thanks to contributors: Yongzhe Wang, Bo Chen, Hanxiao Liu, Le An
+(NVIDIA), Yu-Te Cheng (NVIDIA), Oliver Knieps (NVIDIA), and Josh Park (NVIDIA).
+
### Context R-CNN
We have released [Context R-CNN](https://arxiv.org/abs/1912.03538), a model that
diff --git a/research/object_detection/builders/box_predictor_builder.py b/research/object_detection/builders/box_predictor_builder.py
index 029649d8d9dd68877adac6bb971d5fd024f62246..d0f994c8eca1313e34c3c5133c56abe8be14f784 100644
--- a/research/object_detection/builders/box_predictor_builder.py
+++ b/research/object_detection/builders/box_predictor_builder.py
@@ -329,6 +329,8 @@ def build_weight_shared_convolutional_keras_box_predictor(
share_prediction_tower=False,
apply_batch_norm=True,
use_depthwise=False,
+ apply_conv_hyperparams_to_heads=False,
+ apply_conv_hyperparams_pointwise=False,
score_converter_fn=tf.identity,
box_encodings_clip_range=None,
name='WeightSharedConvolutionalBoxPredictor',
@@ -369,6 +371,14 @@ def build_weight_shared_convolutional_keras_box_predictor(
apply_batch_norm: Whether to apply batch normalization to conv layers in
this predictor.
use_depthwise: Whether to use depthwise separable conv2d instead of conv2d.
+ apply_conv_hyperparams_to_heads: Whether to apply conv_hyperparams to
+ depthwise seperable convolution layers in the box and class heads. By
+ default, the conv_hyperparams are only applied to layers in the predictor
+ tower when using depthwise separable convolutions.
+ apply_conv_hyperparams_pointwise: Whether to apply the conv_hyperparams to
+ the pointwise_initializer and pointwise_regularizer when using depthwise
+ separable convolutions. By default, conv_hyperparams are only applied to
+ the depthwise initializer and regularizer when use_depthwise is true.
score_converter_fn: Callable score converter to perform elementwise op on
class scores.
box_encodings_clip_range: Min and max values for clipping the box_encodings.
@@ -391,6 +401,7 @@ def build_weight_shared_convolutional_keras_box_predictor(
conv_hyperparams=conv_hyperparams,
num_predictions_per_location=num_predictions_per_location,
use_depthwise=use_depthwise,
+ apply_conv_hyperparams_to_heads=apply_conv_hyperparams_to_heads,
box_encodings_clip_range=box_encodings_clip_range,
name='WeightSharedConvolutionalBoxHead')
class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead(
@@ -403,6 +414,7 @@ def build_weight_shared_convolutional_keras_box_predictor(
num_predictions_per_location=num_predictions_per_location,
class_prediction_bias_init=class_prediction_bias_init,
use_depthwise=use_depthwise,
+ apply_conv_hyperparams_to_heads=apply_conv_hyperparams_to_heads,
score_converter_fn=score_converter_fn,
name='WeightSharedConvolutionalClassHead')
other_heads = {}
@@ -423,6 +435,7 @@ def build_weight_shared_convolutional_keras_box_predictor(
apply_batch_norm=apply_batch_norm,
share_prediction_tower=share_prediction_tower,
use_depthwise=use_depthwise,
+ apply_conv_hyperparams_pointwise=apply_conv_hyperparams_pointwise,
name=name))
@@ -920,6 +933,10 @@ def build_keras(hyperparams_fn, freeze_batchnorm, inplace_batchnorm_update,
share_prediction_tower=config_box_predictor.share_prediction_tower,
apply_batch_norm=apply_batch_norm,
use_depthwise=config_box_predictor.use_depthwise,
+ apply_conv_hyperparams_to_heads=(
+ config_box_predictor.apply_conv_hyperparams_to_heads),
+ apply_conv_hyperparams_pointwise=(
+ config_box_predictor.apply_conv_hyperparams_pointwise),
score_converter_fn=score_converter_fn,
box_encodings_clip_range=box_encodings_clip_range,
keyword_args=keyword_args)
diff --git a/research/object_detection/builders/dataset_builder.py b/research/object_detection/builders/dataset_builder.py
index c1c1ce3ecd17c2625585cd83f080b49c0150151a..2a47c628645145be0b11425d5d7ec97f59b80544 100644
--- a/research/object_detection/builders/dataset_builder.py
+++ b/research/object_detection/builders/dataset_builder.py
@@ -27,6 +27,7 @@ from __future__ import division
from __future__ import print_function
import functools
+import math
import tensorflow.compat.v1 as tf
from object_detection.builders import decoder_builder
@@ -50,20 +51,24 @@ def make_initializable_iterator(dataset):
return iterator
-def read_dataset(file_read_func, input_files, config,
- filename_shard_fn=None):
+def _read_dataset_internal(file_read_func,
+ input_files,
+ num_readers,
+ config,
+ filename_shard_fn=None):
"""Reads a dataset, and handles repetition and shuffling.
Args:
- file_read_func: Function to use in tf_data.parallel_interleave, to
- read every individual file into a tf.data.Dataset.
+ file_read_func: Function to use in tf_data.parallel_interleave, to read
+ every individual file into a tf.data.Dataset.
input_files: A list of file paths to read.
+ num_readers: Number of readers to use.
config: A input_reader_builder.InputReader object.
- filename_shard_fn: optional, A funciton used to shard filenames across
- replicas. This function takes as input a TF dataset of filenames and
- is expected to return its sharded version. It is useful when the
- dataset is being loaded on one of possibly many replicas and we want
- to evenly shard the files between the replicas.
+ filename_shard_fn: optional, A function used to shard filenames across
+ replicas. This function takes as input a TF dataset of filenames and is
+ expected to return its sharded version. It is useful when the dataset is
+ being loaded on one of possibly many replicas and we want to evenly shard
+ the files between the replicas.
Returns:
A tf.data.Dataset of (undecoded) tf-records based on config.
@@ -71,12 +76,12 @@ def read_dataset(file_read_func, input_files, config,
Raises:
RuntimeError: If no files are found at the supplied path(s).
"""
- # Shard, shuffle, and read files.
filenames = tf.gfile.Glob(input_files)
+ tf.logging.info('Reading record datasets for input file: %s' % input_files)
+ tf.logging.info('Number of filenames to read: %s' % len(filenames))
if not filenames:
raise RuntimeError('Did not find any input files matching the glob pattern '
'{}'.format(input_files))
- num_readers = config.num_readers
if num_readers > len(filenames):
num_readers = len(filenames)
tf.logging.warning('num_readers has been reduced to %d to match input file '
@@ -103,6 +108,63 @@ def read_dataset(file_read_func, input_files, config,
return records_dataset
+def read_dataset(file_read_func, input_files, config, filename_shard_fn=None):
+ """Reads multiple datasets with sampling.
+
+ Args:
+ file_read_func: Function to use in tf_data.parallel_interleave, to read
+ every individual file into a tf.data.Dataset.
+ input_files: A list of file paths to read.
+ config: A input_reader_builder.InputReader object.
+ filename_shard_fn: optional, A function used to shard filenames across
+ replicas. This function takes as input a TF dataset of filenames and is
+ expected to return its sharded version. It is useful when the dataset is
+ being loaded on one of possibly many replicas and we want to evenly shard
+ the files between the replicas.
+
+ Returns:
+ A tf.data.Dataset of (undecoded) tf-records based on config.
+
+ Raises:
+ RuntimeError: If no files are found at the supplied path(s).
+ """
+ if config.sample_from_datasets_weights:
+ tf.logging.info('Reading weighted datasets: %s' % input_files)
+ if len(input_files) != len(config.sample_from_datasets_weights):
+ raise ValueError('Expected the number of input files to be the same as '
+ 'the number of dataset sample weights. But got '
+ '[input_files, sample_from_datasets_weights]: [' +
+ input_files + ', ' +
+ str(config.sample_from_datasets_weights) + ']')
+ tf.logging.info('Sampling from datasets %s with weights %s' %
+ (input_files, config.sample_from_datasets_weights))
+ records_datasets = []
+ dataset_weights = []
+ for i, input_file in enumerate(input_files):
+ weight = config.sample_from_datasets_weights[i]
+ num_readers = math.ceil(config.num_readers *
+ weight /
+ sum(config.sample_from_datasets_weights))
+ tf.logging.info(
+ 'Num readers for dataset [%s]: %d', input_file, num_readers)
+ if num_readers == 0:
+ tf.logging.info('Skipping dataset due to zero weights: %s', input_file)
+ continue
+ tf.logging.info(
+ 'Num readers for dataset [%s]: %d', input_file, num_readers)
+ records_dataset = _read_dataset_internal(file_read_func, [input_file],
+ num_readers, config,
+ filename_shard_fn)
+ dataset_weights.append(weight)
+ records_datasets.append(records_dataset)
+ return tf.data.experimental.sample_from_datasets(records_datasets,
+ dataset_weights)
+ else:
+ tf.logging.info('Reading unweighted datasets: %s' % input_files)
+ return _read_dataset_internal(file_read_func, input_files,
+ config.num_readers, config, filename_shard_fn)
+
+
def shard_function_for_context(input_context):
"""Returns a function that shards filenames based on the input context."""
@@ -195,7 +257,8 @@ def build(input_reader_config, batch_size=None, transform_input_data_fn=None,
dataset = dataset_map_fn(dataset, transform_input_data_fn,
batch_size, input_reader_config)
if batch_size:
- dataset = dataset.batch(batch_size, drop_remainder=True)
+ dataset = dataset.batch(batch_size,
+ drop_remainder=input_reader_config.drop_remainder)
dataset = dataset.prefetch(input_reader_config.num_prefetch_batches)
return dataset
diff --git a/research/object_detection/builders/dataset_builder_test.py b/research/object_detection/builders/dataset_builder_test.py
index eb2cdb3ccbd891e5f089281d9b506d636d26d6a9..c47218a008d2ae192a78614be8063184f35e1629 100644
--- a/research/object_detection/builders/dataset_builder_test.py
+++ b/research/object_detection/builders/dataset_builder_test.py
@@ -532,6 +532,9 @@ class ReadDatasetTest(test_case.TestCase):
return get_iterator_next_for_testing(dataset, self.is_tf2())
+ def _assert_item_count(self, data, item, percentage):
+ self.assertAlmostEqual(data.count(item)/len(data), percentage, places=1)
+
def test_make_initializable_iterator_with_hashTable(self):
def graph_fn():
@@ -554,6 +557,88 @@ class ReadDatasetTest(test_case.TestCase):
result = self.execute(graph_fn, [])
self.assertAllEqual(result, [-1, 100, 1, 100])
+ def test_read_dataset_sample_from_datasets_weights_equal_weight(self):
+ """Ensure that the files' values are equally-weighted."""
+ config = input_reader_pb2.InputReader()
+ config.num_readers = 2
+ config.shuffle = False
+ config.sample_from_datasets_weights.extend([0.5, 0.5])
+
+ def graph_fn():
+ return self._get_dataset_next(
+ [self._path_template % '0', self._path_template % '1'],
+ config,
+ batch_size=1000)
+
+ data = list(self.execute(graph_fn, []))
+ self.assertEqual(len(data), 1000)
+ self._assert_item_count(data, 1, 0.25)
+ self._assert_item_count(data, 10, 0.25)
+ self._assert_item_count(data, 2, 0.25)
+ self._assert_item_count(data, 20, 0.25)
+
+ def test_read_dataset_sample_from_datasets_weights_non_normalized(self):
+ """Ensure that the values are equally-weighted when not normalized."""
+ config = input_reader_pb2.InputReader()
+ config.num_readers = 2
+ config.shuffle = False
+ # Values are not normalized to sum to 1. In this case, it's a 50/50 split
+ # with each dataset having weight of 1.
+ config.sample_from_datasets_weights.extend([1, 1])
+
+ def graph_fn():
+ return self._get_dataset_next(
+ [self._path_template % '0', self._path_template % '1'],
+ config,
+ batch_size=1000)
+
+ data = list(self.execute(graph_fn, []))
+ self.assertEqual(len(data), 1000)
+ self._assert_item_count(data, 1, 0.25)
+ self._assert_item_count(data, 10, 0.25)
+ self._assert_item_count(data, 2, 0.25)
+ self._assert_item_count(data, 20, 0.25)
+
+ def test_read_dataset_sample_from_datasets_weights_zero_weight(self):
+ """Ensure that the files' values are equally-weighted."""
+ config = input_reader_pb2.InputReader()
+ config.num_readers = 2
+ config.shuffle = False
+ config.sample_from_datasets_weights.extend([1.0, 0.0])
+
+ def graph_fn():
+ return self._get_dataset_next(
+ [self._path_template % '0', self._path_template % '1'],
+ config,
+ batch_size=1000)
+
+ data = list(self.execute(graph_fn, []))
+ self.assertEqual(len(data), 1000)
+ self._assert_item_count(data, 1, 0.5)
+ self._assert_item_count(data, 10, 0.5)
+ self._assert_item_count(data, 2, 0.0)
+ self._assert_item_count(data, 20, 0.0)
+
+ def test_read_dataset_sample_from_datasets_weights_unbalanced(self):
+ """Ensure that the files' values are equally-weighted."""
+ config = input_reader_pb2.InputReader()
+ config.num_readers = 2
+ config.shuffle = False
+ config.sample_from_datasets_weights.extend([0.1, 0.9])
+
+ def graph_fn():
+ return self._get_dataset_next(
+ [self._path_template % '0', self._path_template % '1'],
+ config,
+ batch_size=1000)
+
+ data = list(self.execute(graph_fn, []))
+ self.assertEqual(len(data), 1000)
+ self._assert_item_count(data, 1, 0.05)
+ self._assert_item_count(data, 10, 0.05)
+ self._assert_item_count(data, 2, 0.45)
+ self._assert_item_count(data, 20, 0.45)
+
def test_read_dataset(self):
config = input_reader_pb2.InputReader()
config.num_readers = 1
diff --git a/research/object_detection/builders/decoder_builder.py b/research/object_detection/builders/decoder_builder.py
index c0895051ac1a5dcdd8bd1528f13a7d5f909b306f..43986c35e248160b9c5741d4f8460b2f41655276 100644
--- a/research/object_detection/builders/decoder_builder.py
+++ b/research/object_detection/builders/decoder_builder.py
@@ -59,12 +59,16 @@ def build(input_reader_config):
num_additional_channels=input_reader_config.num_additional_channels,
num_keypoints=input_reader_config.num_keypoints,
expand_hierarchy_labels=input_reader_config.expand_labels_hierarchy,
- load_dense_pose=input_reader_config.load_dense_pose)
+ load_dense_pose=input_reader_config.load_dense_pose,
+ load_track_id=input_reader_config.load_track_id,
+ load_keypoint_depth_features=input_reader_config
+ .load_keypoint_depth_features)
return decoder
elif input_type == input_reader_pb2.InputType.Value('TF_SEQUENCE_EXAMPLE'):
decoder = tf_sequence_example_decoder.TfSequenceExampleDecoder(
label_map_proto_file=label_map_proto_file,
- load_context_features=input_reader_config.load_context_features)
+ load_context_features=input_reader_config.load_context_features,
+ load_context_image_ids=input_reader_config.load_context_image_ids)
return decoder
raise ValueError('Unsupported input_type in config.')
diff --git a/research/object_detection/builders/decoder_builder_test.py b/research/object_detection/builders/decoder_builder_test.py
index d45285fd19f7648ab4d9365b155ba35a2ce0d3ed..6b13a3b3e70a5ff539a02b3d7c0aa182a61e090c 100644
--- a/research/object_detection/builders/decoder_builder_test.py
+++ b/research/object_detection/builders/decoder_builder_test.py
@@ -65,6 +65,8 @@ class DecoderBuilderTest(test_case.TestCase):
'image/object/bbox/ymax': dataset_util.float_list_feature([1.0]),
'image/object/class/label': dataset_util.int64_list_feature([2]),
'image/object/mask': dataset_util.float_list_feature(flat_mask),
+ 'image/object/keypoint/x': dataset_util.float_list_feature([1.0, 1.0]),
+ 'image/object/keypoint/y': dataset_util.float_list_feature([1.0, 1.0])
}
if has_additional_channels:
additional_channels_key = 'image/additional_channels/encoded'
@@ -188,6 +190,28 @@ class DecoderBuilderTest(test_case.TestCase):
masks = self.execute_cpu(graph_fn, [])
self.assertAllEqual((1, 4, 5), masks.shape)
+ def test_build_tf_record_input_reader_and_load_keypoint_depth(self):
+ input_reader_text_proto = """
+ load_keypoint_depth_features: true
+ num_keypoints: 2
+ tf_record_input_reader {}
+ """
+ input_reader_proto = input_reader_pb2.InputReader()
+ text_format.Parse(input_reader_text_proto, input_reader_proto)
+
+ decoder = decoder_builder.build(input_reader_proto)
+ serialized_example = self._make_serialized_tf_example()
+
+ def graph_fn():
+ tensor_dict = decoder.decode(serialized_example)
+ return (tensor_dict[fields.InputDataFields.groundtruth_keypoint_depths],
+ tensor_dict[
+ fields.InputDataFields.groundtruth_keypoint_depth_weights])
+
+ (kpts_depths, kpts_depth_weights) = self.execute_cpu(graph_fn, [])
+ self.assertAllEqual((1, 2), kpts_depths.shape)
+ self.assertAllEqual((1, 2), kpts_depth_weights.shape)
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/builders/hyperparams_builder.py b/research/object_detection/builders/hyperparams_builder.py
index 90aef43ac1bd92fb86dbd730cdb0420858572c18..9fdf4450abd919c9d09aefbf22ff1a24a4726078 100644
--- a/research/object_detection/builders/hyperparams_builder.py
+++ b/research/object_detection/builders/hyperparams_builder.py
@@ -20,7 +20,11 @@ import tf_slim as slim
from object_detection.core import freezable_batch_norm
from object_detection.protos import hyperparams_pb2
from object_detection.utils import context_manager
+from object_detection.utils import tf_version
+# pylint: disable=g-import-not-at-top
+if tf_version.is_tf2():
+ from object_detection.core import freezable_sync_batch_norm
# pylint: enable=g-import-not-at-top
@@ -60,9 +64,14 @@ class KerasLayerHyperparams(object):
'hyperparams_pb.Hyperparams.')
self._batch_norm_params = None
+ self._use_sync_batch_norm = False
if hyperparams_config.HasField('batch_norm'):
self._batch_norm_params = _build_keras_batch_norm_params(
hyperparams_config.batch_norm)
+ elif hyperparams_config.HasField('sync_batch_norm'):
+ self._use_sync_batch_norm = True
+ self._batch_norm_params = _build_keras_batch_norm_params(
+ hyperparams_config.sync_batch_norm)
self._force_use_bias = hyperparams_config.force_use_bias
self._activation_fn = _build_activation_fn(hyperparams_config.activation)
@@ -81,6 +90,9 @@ class KerasLayerHyperparams(object):
def use_batch_norm(self):
return self._batch_norm_params is not None
+ def use_sync_batch_norm(self):
+ return self._use_sync_batch_norm
+
def force_use_bias(self):
return self._force_use_bias
@@ -133,10 +145,12 @@ class KerasLayerHyperparams(object):
is False)
"""
if self.use_batch_norm():
- return freezable_batch_norm.FreezableBatchNorm(
- training=training,
- **self.batch_norm_params(**overrides)
- )
+ if self._use_sync_batch_norm:
+ return freezable_sync_batch_norm.FreezableSyncBatchNorm(
+ training=training, **self.batch_norm_params(**overrides))
+ else:
+ return freezable_batch_norm.FreezableBatchNorm(
+ training=training, **self.batch_norm_params(**overrides))
else:
return tf.keras.layers.Lambda(tf.identity)
@@ -154,6 +168,20 @@ class KerasLayerHyperparams(object):
else:
return tf.keras.layers.Lambda(tf.identity, name=name)
+ def get_regularizer_weight(self):
+ """Returns the l1 or l2 regularizer weight.
+
+ Returns: A float value corresponding to the l1 or l2 regularization weight,
+ or None if neither l1 or l2 regularization is defined.
+ """
+ regularizer = self._op_params['kernel_regularizer']
+ if hasattr(regularizer, 'l1'):
+ return float(regularizer.l1)
+ elif hasattr(regularizer, 'l2'):
+ return float(regularizer.l2)
+ else:
+ return None
+
def params(self, include_activation=False, **overrides):
"""Returns a dict containing the layer construction hyperparameters to use.
@@ -219,6 +247,10 @@ def build(hyperparams_config, is_training):
raise ValueError('Hyperparams force_use_bias only supported by '
'KerasLayerHyperparams.')
+ if hyperparams_config.HasField('sync_batch_norm'):
+ raise ValueError('Hyperparams sync_batch_norm only supported by '
+ 'KerasLayerHyperparams.')
+
normalizer_fn = None
batch_norm_params = None
if hyperparams_config.HasField('batch_norm'):
@@ -327,7 +359,7 @@ def _build_initializer(initializer, build_for_keras=False):
operators. If false builds for Slim.
Returns:
- tf initializer.
+ tf initializer or string corresponding to the tf keras initializer name.
Raises:
ValueError: On unknown initializer.
@@ -383,6 +415,13 @@ def _build_initializer(initializer, build_for_keras=False):
factor=initializer.variance_scaling_initializer.factor,
mode=mode,
uniform=initializer.variance_scaling_initializer.uniform)
+ if initializer_oneof == 'keras_initializer_by_name':
+ if build_for_keras:
+ return initializer.keras_initializer_by_name
+ else:
+ raise ValueError(
+ 'Unsupported non-Keras usage of keras_initializer_by_name: {}'.format(
+ initializer.keras_initializer_by_name))
if initializer_oneof is None:
return None
raise ValueError('Unknown initializer function: {}'.format(
diff --git a/research/object_detection/builders/hyperparams_builder_test.py b/research/object_detection/builders/hyperparams_builder_test.py
index e48ac23bcb547c9729038b901a9612d3712d69cb..d21f52dc5fab256cd86a0f0cd379f3fb6702a38c 100644
--- a/research/object_detection/builders/hyperparams_builder_test.py
+++ b/research/object_detection/builders/hyperparams_builder_test.py
@@ -558,7 +558,7 @@ class KerasHyperparamsBuilderTest(tf.test.TestCase):
result = regularizer(tf.constant(weights)).numpy()
self.assertAllClose(np.abs(weights).sum() * 0.5, result)
- def test_return_l2_regularizer_weights_keras(self):
+ def test_return_l2_regularized_weights_keras(self):
conv_hyperparams_text_proto = """
regularizer {
l2_regularizer {
@@ -580,6 +580,63 @@ class KerasHyperparamsBuilderTest(tf.test.TestCase):
result = regularizer(tf.constant(weights)).numpy()
self.assertAllClose(np.power(weights, 2).sum() / 2.0 * 0.42, result)
+ def test_return_l1_regularizer_weight_keras(self):
+ conv_hyperparams_text_proto = """
+ regularizer {
+ l1_regularizer {
+ weight: 0.5
+ }
+ }
+ initializer {
+ truncated_normal_initializer {
+ }
+ }
+ """
+ conv_hyperparams_proto = hyperparams_pb2.Hyperparams()
+ text_format.Parse(conv_hyperparams_text_proto, conv_hyperparams_proto)
+ keras_config = hyperparams_builder.KerasLayerHyperparams(
+ conv_hyperparams_proto)
+
+ regularizer_weight = keras_config.get_regularizer_weight()
+ self.assertIsInstance(regularizer_weight, float)
+ self.assertAlmostEqual(regularizer_weight, 0.5)
+
+ def test_return_l2_regularizer_weight_keras(self):
+ conv_hyperparams_text_proto = """
+ regularizer {
+ l2_regularizer {
+ weight: 0.5
+ }
+ }
+ initializer {
+ truncated_normal_initializer {
+ }
+ }
+ """
+ conv_hyperparams_proto = hyperparams_pb2.Hyperparams()
+ text_format.Parse(conv_hyperparams_text_proto, conv_hyperparams_proto)
+ keras_config = hyperparams_builder.KerasLayerHyperparams(
+ conv_hyperparams_proto)
+
+ regularizer_weight = keras_config.get_regularizer_weight()
+ self.assertIsInstance(regularizer_weight, float)
+ self.assertAlmostEqual(regularizer_weight, 0.25)
+
+ def test_return_undefined_regularizer_weight_keras(self):
+ conv_hyperparams_text_proto = """
+ initializer {
+ truncated_normal_initializer {
+ }
+ }
+ """
+ conv_hyperparams_proto = hyperparams_pb2.Hyperparams()
+ text_format.Parse(conv_hyperparams_text_proto, conv_hyperparams_proto)
+ keras_config = hyperparams_builder.KerasLayerHyperparams(
+ conv_hyperparams_proto)
+
+ regularizer_weight = keras_config.get_regularizer_weight()
+ self.assertIsNone(regularizer_weight)
+
def test_return_non_default_batch_norm_params_keras(
self):
conv_hyperparams_text_proto = """
@@ -973,5 +1030,26 @@ class KerasHyperparamsBuilderTest(tf.test.TestCase):
self._assert_variance_in_range(initializer, shape=[100, 40],
variance=0.64, tol=1e-1)
+ def test_keras_initializer_by_name(self):
+ conv_hyperparams_text_proto = """
+ regularizer {
+ l2_regularizer {
+ }
+ }
+ initializer {
+ keras_initializer_by_name: "glorot_uniform"
+ }
+ """
+ conv_hyperparams_proto = hyperparams_pb2.Hyperparams()
+ text_format.Parse(conv_hyperparams_text_proto, conv_hyperparams_proto)
+ keras_config = hyperparams_builder.KerasLayerHyperparams(
+ conv_hyperparams_proto)
+ initializer_arg = keras_config.params()['kernel_initializer']
+ conv_layer = tf.keras.layers.Conv2D(
+ filters=16, kernel_size=3, **keras_config.params())
+ self.assertEqual(initializer_arg, 'glorot_uniform')
+ self.assertIsInstance(conv_layer.kernel_initializer,
+ type(tf.keras.initializers.get('glorot_uniform')))
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/builders/input_reader_builder.py b/research/object_detection/builders/input_reader_builder.py
index c7755177e70d528984ea425f21fb9afaf11d9eaa..50a8becbe0542cfe916946a7c31aa4c6af4b9ac0 100644
--- a/research/object_detection/builders/input_reader_builder.py
+++ b/research/object_detection/builders/input_reader_builder.py
@@ -85,7 +85,8 @@ def build(input_reader_config):
elif input_type == input_reader_pb2.InputType.Value('TF_SEQUENCE_EXAMPLE'):
decoder = tf_sequence_example_decoder.TfSequenceExampleDecoder(
label_map_proto_file=label_map_proto_file,
- load_context_features=input_reader_config.load_context_features)
+ load_context_features=input_reader_config.load_context_features,
+ load_context_image_ids=input_reader_config.load_context_image_ids)
return decoder.decode(string_tensor)
raise ValueError('Unsupported input_type.')
raise ValueError('Unsupported input_reader_config.')
diff --git a/research/object_detection/builders/losses_builder.py b/research/object_detection/builders/losses_builder.py
index 5a69c9b602c95ab6c8368638b2e38448ae113b9c..5aec7e192104cda8672f34bc2230efb725938a7c 100644
--- a/research/object_detection/builders/losses_builder.py
+++ b/research/object_detection/builders/losses_builder.py
@@ -204,6 +204,9 @@ def _build_localization_loss(loss_config):
if loss_type == 'l1_localization_loss':
return losses.L1LocalizationLoss()
+ if loss_type == 'weighted_giou':
+ return losses.WeightedGIOULocalizationLoss()
+
raise ValueError('Empty loss config.')
@@ -227,7 +230,7 @@ def _build_classification_loss(loss_config):
if loss_type == 'weighted_sigmoid':
return losses.WeightedSigmoidClassificationLoss()
- if loss_type == 'weighted_sigmoid_focal':
+ elif loss_type == 'weighted_sigmoid_focal':
config = loss_config.weighted_sigmoid_focal
alpha = None
if config.HasField('alpha'):
@@ -236,25 +239,31 @@ def _build_classification_loss(loss_config):
gamma=config.gamma,
alpha=alpha)
- if loss_type == 'weighted_softmax':
+ elif loss_type == 'weighted_softmax':
config = loss_config.weighted_softmax
return losses.WeightedSoftmaxClassificationLoss(
logit_scale=config.logit_scale)
- if loss_type == 'weighted_logits_softmax':
+ elif loss_type == 'weighted_logits_softmax':
config = loss_config.weighted_logits_softmax
return losses.WeightedSoftmaxClassificationAgainstLogitsLoss(
logit_scale=config.logit_scale)
- if loss_type == 'bootstrapped_sigmoid':
+ elif loss_type == 'bootstrapped_sigmoid':
config = loss_config.bootstrapped_sigmoid
return losses.BootstrappedSigmoidClassificationLoss(
alpha=config.alpha,
bootstrap_type=('hard' if config.hard_bootstrap else 'soft'))
- if loss_type == 'penalty_reduced_logistic_focal_loss':
+ elif loss_type == 'penalty_reduced_logistic_focal_loss':
config = loss_config.penalty_reduced_logistic_focal_loss
return losses.PenaltyReducedLogisticFocalLoss(
alpha=config.alpha, beta=config.beta)
- raise ValueError('Empty loss config.')
+ elif loss_type == 'weighted_dice_classification_loss':
+ config = loss_config.weighted_dice_classification_loss
+ return losses.WeightedDiceClassificationLoss(
+ squared_normalization=config.squared_normalization)
+
+ else:
+ raise ValueError('Empty loss config.')
diff --git a/research/object_detection/builders/losses_builder_test.py b/research/object_detection/builders/losses_builder_test.py
index b37b7f3195427b951e2c508f0df191f176b9d835..07c01653b20532782e808030e452e8df84e533a1 100644
--- a/research/object_detection/builders/losses_builder_test.py
+++ b/research/object_detection/builders/losses_builder_test.py
@@ -97,6 +97,23 @@ class LocalizationLossBuilderTest(tf.test.TestCase):
self.assertIsInstance(localization_loss,
losses.WeightedIOULocalizationLoss)
+ def test_build_weighted_giou_localization_loss(self):
+ losses_text_proto = """
+ localization_loss {
+ weighted_giou {
+ }
+ }
+ classification_loss {
+ weighted_softmax {
+ }
+ }
+ """
+ losses_proto = losses_pb2.Loss()
+ text_format.Merge(losses_text_proto, losses_proto)
+ _, localization_loss, _, _, _, _, _ = losses_builder.build(losses_proto)
+ self.assertIsInstance(localization_loss,
+ losses.WeightedGIOULocalizationLoss)
+
def test_anchorwise_output(self):
losses_text_proto = """
localization_loss {
@@ -298,6 +315,45 @@ class ClassificationLossBuilderTest(tf.test.TestCase):
with self.assertRaises(ValueError):
losses_builder.build(losses_proto)
+ def test_build_penalty_reduced_logistic_focal_loss(self):
+ losses_text_proto = """
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ """
+ losses_proto = losses_pb2.Loss()
+ text_format.Merge(losses_text_proto, losses_proto)
+ classification_loss, _, _, _, _, _, _ = losses_builder.build(losses_proto)
+ self.assertIsInstance(classification_loss,
+ losses.PenaltyReducedLogisticFocalLoss)
+ self.assertAlmostEqual(classification_loss._alpha, 2.0)
+ self.assertAlmostEqual(classification_loss._beta, 4.0)
+
+ def test_build_dice_loss(self):
+ losses_text_proto = """
+ classification_loss {
+ weighted_dice_classification_loss {
+ squared_normalization: true
+ }
+ }
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ """
+ losses_proto = losses_pb2.Loss()
+ text_format.Merge(losses_text_proto, losses_proto)
+ classification_loss, _, _, _, _, _, _ = losses_builder.build(losses_proto)
+ self.assertIsInstance(classification_loss,
+ losses.WeightedDiceClassificationLoss)
+ assert classification_loss._squared_normalization
class HardExampleMinerBuilderTest(tf.test.TestCase):
diff --git a/research/object_detection/builders/model_builder.py b/research/object_detection/builders/model_builder.py
index d69b7bc7c2546a6461a2d75d3576d725d748d409..08c65e0763ab2450a368f280859c6b329b31d7a6 100644
--- a/research/object_detection/builders/model_builder.py
+++ b/research/object_detection/builders/model_builder.py
@@ -17,6 +17,9 @@
import functools
import sys
+
+from absl import logging
+
from object_detection.builders import anchor_generator_builder
from object_detection.builders import box_coder_builder
from object_detection.builders import box_predictor_builder
@@ -39,6 +42,7 @@ from object_detection.protos import losses_pb2
from object_detection.protos import model_pb2
from object_detection.utils import label_map_util
from object_detection.utils import ops
+from object_detection.utils import spatial_transform_ops as spatial_ops
from object_detection.utils import tf_version
## Feature Extractors for TF
@@ -48,6 +52,8 @@ from object_detection.utils import tf_version
# pylint: disable=g-import-not-at-top
if tf_version.is_tf2():
from object_detection.models import center_net_hourglass_feature_extractor
+ from object_detection.models import center_net_mobilenet_v2_feature_extractor
+ from object_detection.models import center_net_mobilenet_v2_fpn_feature_extractor
from object_detection.models import center_net_resnet_feature_extractor
from object_detection.models import center_net_resnet_v1_fpn_feature_extractor
from object_detection.models import faster_rcnn_inception_resnet_v2_keras_feature_extractor as frcnn_inc_res_keras
@@ -138,13 +144,34 @@ if tf_version.is_tf2():
}
CENTER_NET_EXTRACTOR_FUNCTION_MAP = {
- 'resnet_v2_50': center_net_resnet_feature_extractor.resnet_v2_50,
- 'resnet_v2_101': center_net_resnet_feature_extractor.resnet_v2_101,
+ 'resnet_v2_50':
+ center_net_resnet_feature_extractor.resnet_v2_50,
+ 'resnet_v2_101':
+ center_net_resnet_feature_extractor.resnet_v2_101,
+ 'resnet_v1_18_fpn':
+ center_net_resnet_v1_fpn_feature_extractor.resnet_v1_18_fpn,
+ 'resnet_v1_34_fpn':
+ center_net_resnet_v1_fpn_feature_extractor.resnet_v1_34_fpn,
'resnet_v1_50_fpn':
center_net_resnet_v1_fpn_feature_extractor.resnet_v1_50_fpn,
'resnet_v1_101_fpn':
center_net_resnet_v1_fpn_feature_extractor.resnet_v1_101_fpn,
- 'hourglass_104': center_net_hourglass_feature_extractor.hourglass_104,
+ 'hourglass_10':
+ center_net_hourglass_feature_extractor.hourglass_10,
+ 'hourglass_20':
+ center_net_hourglass_feature_extractor.hourglass_20,
+ 'hourglass_32':
+ center_net_hourglass_feature_extractor.hourglass_32,
+ 'hourglass_52':
+ center_net_hourglass_feature_extractor.hourglass_52,
+ 'hourglass_104':
+ center_net_hourglass_feature_extractor.hourglass_104,
+ 'mobilenet_v2':
+ center_net_mobilenet_v2_feature_extractor.mobilenet_v2,
+ 'mobilenet_v2_fpn':
+ center_net_mobilenet_v2_fpn_feature_extractor.mobilenet_v2_fpn,
+ 'mobilenet_v2_fpn_sep_conv':
+ center_net_mobilenet_v2_fpn_feature_extractor.mobilenet_v2_fpn,
}
FEATURE_EXTRACTOR_MAPS = [
@@ -220,9 +247,12 @@ if tf_version.is_tf1():
frcnn_resnet_v1.FasterRCNNResnet152FeatureExtractor,
}
+ CENTER_NET_EXTRACTOR_FUNCTION_MAP = {}
+
FEATURE_EXTRACTOR_MAPS = [
SSD_FEATURE_EXTRACTOR_CLASS_MAP,
- FASTER_RCNN_FEATURE_EXTRACTOR_CLASS_MAP
+ FASTER_RCNN_FEATURE_EXTRACTOR_CLASS_MAP,
+ CENTER_NET_EXTRACTOR_FUNCTION_MAP
]
@@ -515,9 +545,31 @@ def _build_faster_rcnn_keras_feature_extractor(
feature_type))
feature_extractor_class = FASTER_RCNN_KERAS_FEATURE_EXTRACTOR_CLASS_MAP[
feature_type]
+
+ kwargs = {}
+
+ if feature_extractor_config.HasField('conv_hyperparams'):
+ kwargs.update({
+ 'conv_hyperparams':
+ hyperparams_builder.KerasLayerHyperparams(
+ feature_extractor_config.conv_hyperparams),
+ 'override_base_feature_extractor_hyperparams':
+ feature_extractor_config.override_base_feature_extractor_hyperparams
+ })
+
+ if feature_extractor_config.HasField('fpn'):
+ kwargs.update({
+ 'fpn_min_level':
+ feature_extractor_config.fpn.min_level,
+ 'fpn_max_level':
+ feature_extractor_config.fpn.max_level,
+ 'additional_layer_depth':
+ feature_extractor_config.fpn.additional_layer_depth,
+ })
+
return feature_extractor_class(
is_training, first_stage_features_stride,
- batch_norm_trainable)
+ batch_norm_trainable, **kwargs)
def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
@@ -648,8 +700,9 @@ def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
second_stage_localization_loss_weight)
crop_and_resize_fn = (
- ops.matmul_crop_and_resize if frcnn_config.use_matmul_crop_and_resize
- else ops.native_crop_and_resize)
+ spatial_ops.multilevel_matmul_crop_and_resize
+ if frcnn_config.use_matmul_crop_and_resize
+ else spatial_ops.multilevel_native_crop_and_resize)
clip_anchors_to_image = (
frcnn_config.clip_anchors_to_image)
@@ -719,7 +772,9 @@ def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
'return_raw_detections_during_predict':
frcnn_config.return_raw_detections_during_predict,
'output_final_box_features':
- frcnn_config.output_final_box_features
+ frcnn_config.output_final_box_features,
+ 'output_final_box_rpn_features':
+ frcnn_config.output_final_box_rpn_features,
}
if ((not is_keras and isinstance(second_stage_box_predictor,
@@ -736,7 +791,19 @@ def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
'attention_bottleneck_dimension':
context_config.attention_bottleneck_dimension,
'attention_temperature':
- context_config.attention_temperature
+ context_config.attention_temperature,
+ 'use_self_attention':
+ context_config.use_self_attention,
+ 'use_long_term_attention':
+ context_config.use_long_term_attention,
+ 'self_attention_in_sequence':
+ context_config.self_attention_in_sequence,
+ 'num_attention_heads':
+ context_config.num_attention_heads,
+ 'num_attention_layers':
+ context_config.num_attention_layers,
+ 'attention_position':
+ context_config.attention_position
})
return context_rcnn_meta_arch.ContextRCNNMetaArch(
initial_crop_size=initial_crop_size,
@@ -792,6 +859,25 @@ def keypoint_proto_to_params(kp_config, keypoint_map_dict):
for label, value in kp_config.keypoint_label_to_std.items():
keypoint_std_dev_dict[label] = value
keypoint_std_dev = [keypoint_std_dev_dict[label] for label in keypoint_labels]
+ if kp_config.HasField('heatmap_head_params'):
+ heatmap_head_num_filters = list(kp_config.heatmap_head_params.num_filters)
+ heatmap_head_kernel_sizes = list(kp_config.heatmap_head_params.kernel_sizes)
+ else:
+ heatmap_head_num_filters = [256]
+ heatmap_head_kernel_sizes = [3]
+ if kp_config.HasField('offset_head_params'):
+ offset_head_num_filters = list(kp_config.offset_head_params.num_filters)
+ offset_head_kernel_sizes = list(kp_config.offset_head_params.kernel_sizes)
+ else:
+ offset_head_num_filters = [256]
+ offset_head_kernel_sizes = [3]
+ if kp_config.HasField('regress_head_params'):
+ regress_head_num_filters = list(kp_config.regress_head_params.num_filters)
+ regress_head_kernel_sizes = list(
+ kp_config.regress_head_params.kernel_sizes)
+ else:
+ regress_head_num_filters = [256]
+ regress_head_kernel_sizes = [3]
return center_net_meta_arch.KeypointEstimationParams(
task_name=kp_config.task_name,
class_id=label_map_item.id - CLASS_ID_OFFSET,
@@ -814,7 +900,19 @@ def keypoint_proto_to_params(kp_config, keypoint_map_dict):
candidate_search_scale=kp_config.candidate_search_scale,
candidate_ranking_mode=kp_config.candidate_ranking_mode,
offset_peak_radius=kp_config.offset_peak_radius,
- per_keypoint_offset=kp_config.per_keypoint_offset)
+ per_keypoint_offset=kp_config.per_keypoint_offset,
+ predict_depth=kp_config.predict_depth,
+ per_keypoint_depth=kp_config.per_keypoint_depth,
+ keypoint_depth_loss_weight=kp_config.keypoint_depth_loss_weight,
+ score_distance_offset=kp_config.score_distance_offset,
+ clip_out_of_frame_keypoints=kp_config.clip_out_of_frame_keypoints,
+ rescore_instances=kp_config.rescore_instances,
+ heatmap_head_num_filters=heatmap_head_num_filters,
+ heatmap_head_kernel_sizes=heatmap_head_kernel_sizes,
+ offset_head_num_filters=offset_head_num_filters,
+ offset_head_kernel_sizes=offset_head_kernel_sizes,
+ regress_head_num_filters=regress_head_num_filters,
+ regress_head_kernel_sizes=regress_head_kernel_sizes)
def object_detection_proto_to_params(od_config):
@@ -844,13 +942,26 @@ def object_center_proto_to_params(oc_config):
losses_pb2.WeightedL2LocalizationLoss())
loss.classification_loss.CopyFrom(oc_config.classification_loss)
classification_loss, _, _, _, _, _, _ = (losses_builder.build(loss))
+ keypoint_weights_for_center = []
+ if oc_config.keypoint_weights_for_center:
+ keypoint_weights_for_center = list(oc_config.keypoint_weights_for_center)
+
+ if oc_config.HasField('center_head_params'):
+ center_head_num_filters = list(oc_config.center_head_params.num_filters)
+ center_head_kernel_sizes = list(oc_config.center_head_params.kernel_sizes)
+ else:
+ center_head_num_filters = [256]
+ center_head_kernel_sizes = [3]
return center_net_meta_arch.ObjectCenterParams(
classification_loss=classification_loss,
object_center_loss_weight=oc_config.object_center_loss_weight,
heatmap_bias_init=oc_config.heatmap_bias_init,
min_box_overlap_iou=oc_config.min_box_overlap_iou,
max_box_predictions=oc_config.max_box_predictions,
- use_labeled_classes=oc_config.use_labeled_classes)
+ use_labeled_classes=oc_config.use_labeled_classes,
+ keypoint_weights_for_center=keypoint_weights_for_center,
+ center_head_num_filters=center_head_num_filters,
+ center_head_kernel_sizes=center_head_kernel_sizes)
def mask_proto_to_params(mask_config):
@@ -886,6 +997,39 @@ def densepose_proto_to_params(densepose_config):
heatmap_bias_init=densepose_config.heatmap_bias_init)
+def tracking_proto_to_params(tracking_config):
+ """Converts CenterNet.TrackEstimation proto to parameter namedtuple."""
+ loss = losses_pb2.Loss()
+ # Add dummy localization loss to avoid the loss_builder throwing error.
+ # TODO(yuhuic): update the loss builder to take the localization loss
+ # directly.
+ loss.localization_loss.weighted_l2.CopyFrom(
+ losses_pb2.WeightedL2LocalizationLoss())
+ loss.classification_loss.CopyFrom(tracking_config.classification_loss)
+ classification_loss, _, _, _, _, _, _ = losses_builder.build(loss)
+ return center_net_meta_arch.TrackParams(
+ num_track_ids=tracking_config.num_track_ids,
+ reid_embed_size=tracking_config.reid_embed_size,
+ classification_loss=classification_loss,
+ num_fc_layers=tracking_config.num_fc_layers,
+ task_loss_weight=tracking_config.task_loss_weight)
+
+
+def temporal_offset_proto_to_params(temporal_offset_config):
+ """Converts CenterNet.TemporalOffsetEstimation proto to param-tuple."""
+ loss = losses_pb2.Loss()
+ # Add dummy classification loss to avoid the loss_builder throwing error.
+ # TODO(yuhuic): update the loss builder to take the classification loss
+ # directly.
+ loss.classification_loss.weighted_sigmoid.CopyFrom(
+ losses_pb2.WeightedSigmoidClassificationLoss())
+ loss.localization_loss.CopyFrom(temporal_offset_config.localization_loss)
+ _, localization_loss, _, _, _, _, _ = losses_builder.build(loss)
+ return center_net_meta_arch.TemporalOffsetParams(
+ localization_loss=localization_loss,
+ task_loss_weight=temporal_offset_config.task_loss_weight)
+
+
def _build_center_net_model(center_net_config, is_training, add_summaries):
"""Build a CenterNet detection model.
@@ -903,7 +1047,7 @@ def _build_center_net_model(center_net_config, is_training, add_summaries):
center_net_config.image_resizer)
_check_feature_extractor_exists(center_net_config.feature_extractor.type)
feature_extractor = _build_center_net_feature_extractor(
- center_net_config.feature_extractor)
+ center_net_config.feature_extractor, is_training)
object_center_params = object_center_proto_to_params(
center_net_config.object_center_params)
@@ -943,6 +1087,20 @@ def _build_center_net_model(center_net_config, is_training, add_summaries):
densepose_params = densepose_proto_to_params(
center_net_config.densepose_estimation_task)
+ track_params = None
+ if center_net_config.HasField('track_estimation_task'):
+ track_params = tracking_proto_to_params(
+ center_net_config.track_estimation_task)
+
+ temporal_offset_params = None
+ if center_net_config.HasField('temporal_offset_task'):
+ temporal_offset_params = temporal_offset_proto_to_params(
+ center_net_config.temporal_offset_task)
+ non_max_suppression_fn = None
+ if center_net_config.HasField('post_processing'):
+ non_max_suppression_fn, _ = post_processing_builder.build(
+ center_net_config.post_processing)
+
return center_net_meta_arch.CenterNetMetaArch(
is_training=is_training,
add_summaries=add_summaries,
@@ -953,22 +1111,35 @@ def _build_center_net_model(center_net_config, is_training, add_summaries):
object_detection_params=object_detection_params,
keypoint_params_dict=keypoint_params_dict,
mask_params=mask_params,
- densepose_params=densepose_params)
+ densepose_params=densepose_params,
+ track_params=track_params,
+ temporal_offset_params=temporal_offset_params,
+ use_depthwise=center_net_config.use_depthwise,
+ compute_heatmap_sparse=center_net_config.compute_heatmap_sparse,
+ non_max_suppression_fn=non_max_suppression_fn)
-def _build_center_net_feature_extractor(
- feature_extractor_config):
+def _build_center_net_feature_extractor(feature_extractor_config, is_training):
"""Build a CenterNet feature extractor from the given config."""
if feature_extractor_config.type not in CENTER_NET_EXTRACTOR_FUNCTION_MAP:
raise ValueError('\'{}\' is not a known CenterNet feature extractor type'
.format(feature_extractor_config.type))
+ # For backwards compatibility:
+ use_separable_conv = (
+ feature_extractor_config.use_separable_conv or
+ feature_extractor_config.type == 'mobilenet_v2_fpn_sep_conv')
+ kwargs = {
+ 'channel_means': list(feature_extractor_config.channel_means),
+ 'channel_stds': list(feature_extractor_config.channel_stds),
+ 'bgr_ordering': feature_extractor_config.bgr_ordering,
+ 'depth_multiplier': feature_extractor_config.depth_multiplier,
+ 'use_separable_conv': use_separable_conv,
+ }
+
return CENTER_NET_EXTRACTOR_FUNCTION_MAP[feature_extractor_config.type](
- channel_means=list(feature_extractor_config.channel_means),
- channel_stds=list(feature_extractor_config.channel_stds),
- bgr_ordering=feature_extractor_config.bgr_ordering
- )
+ **kwargs)
META_ARCH_BUILDER_MAP = {
diff --git a/research/object_detection/builders/model_builder_tf2_test.py b/research/object_detection/builders/model_builder_tf2_test.py
index 9cbefdc0f1f598b380570d0b0ab140c29855d8d0..9d386fdd353f51a404f2b1bc5592d9f06446b792 100644
--- a/research/object_detection/builders/model_builder_tf2_test.py
+++ b/research/object_detection/builders/model_builder_tf2_test.py
@@ -18,20 +18,23 @@
import os
import unittest
+from absl.testing import parameterized
import tensorflow.compat.v1 as tf
from google.protobuf import text_format
from object_detection.builders import model_builder
from object_detection.builders import model_builder_test
from object_detection.core import losses
-from object_detection.models import center_net_resnet_feature_extractor
+from object_detection.models import center_net_hourglass_feature_extractor
+from object_detection.models.keras_models import hourglass_network
from object_detection.protos import center_net_pb2
from object_detection.protos import model_pb2
from object_detection.utils import tf_version
@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
-class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
+class ModelBuilderTF2Test(
+ model_builder_test.ModelBuilderTest, parameterized.TestCase):
def default_ssd_feature_extractor(self):
return 'ssd_resnet50_v1_fpn_keras'
@@ -78,7 +81,7 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
f.write(keypoint_spec_text)
return keypoint_label_map_path
- def get_fake_keypoint_proto(self):
+ def get_fake_keypoint_proto(self, customize_head_params=False):
task_proto_txt = """
task_name: "human_pose"
task_loss_weight: 0.9
@@ -116,12 +119,30 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
candidate_ranking_mode: "score_distance_ratio"
offset_peak_radius: 3
per_keypoint_offset: true
+ predict_depth: true
+ per_keypoint_depth: true
+ keypoint_depth_loss_weight: 0.3
"""
+ if customize_head_params:
+ task_proto_txt += """
+ heatmap_head_params {
+ num_filters: 64
+ num_filters: 32
+ kernel_sizes: 5
+ kernel_sizes: 3
+ }
+ offset_head_params {
+ num_filters: 128
+ num_filters: 64
+ kernel_sizes: 5
+ kernel_sizes: 3
+ }
+ """
config = text_format.Merge(task_proto_txt,
center_net_pb2.CenterNet.KeypointEstimation())
return config
- def get_fake_object_center_proto(self):
+ def get_fake_object_center_proto(self, customize_head_params=False):
proto_txt = """
object_center_loss_weight: 0.5
heatmap_bias_init: 3.14
@@ -134,6 +155,35 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
}
}
"""
+ if customize_head_params:
+ proto_txt += """
+ center_head_params {
+ num_filters: 64
+ num_filters: 32
+ kernel_sizes: 5
+ kernel_sizes: 3
+ }
+ """
+ return text_format.Merge(proto_txt,
+ center_net_pb2.CenterNet.ObjectCenterParams())
+
+ def get_fake_object_center_from_keypoints_proto(self):
+ proto_txt = """
+ object_center_loss_weight: 0.5
+ heatmap_bias_init: 3.14
+ min_box_overlap_iou: 0.2
+ max_box_predictions: 15
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 3.0
+ beta: 4.0
+ }
+ }
+ keypoint_weights_for_center: 1.0
+ keypoint_weights_for_center: 0.0
+ keypoint_weights_for_center: 1.0
+ keypoint_weights_for_center: 0.0
+ """
return text_format.Merge(proto_txt,
center_net_pb2.CenterNet.ObjectCenterParams())
@@ -186,13 +236,17 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
return text_format.Merge(proto_txt,
center_net_pb2.CenterNet.DensePoseEstimation())
- def test_create_center_net_model(self):
+ @parameterized.parameters(
+ {'customize_head_params': True},
+ {'customize_head_params': False}
+ )
+ def test_create_center_net_model(self, customize_head_params):
"""Test building a CenterNet model from proto txt."""
proto_txt = """
center_net {
num_classes: 10
feature_extractor {
- type: "resnet_v2_101"
+ type: "hourglass_52"
channel_stds: [4, 5, 6]
bgr_ordering: true
}
@@ -208,11 +262,13 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
# Set up the configuration proto.
config = text_format.Merge(proto_txt, model_pb2.DetectionModel())
config.center_net.object_center_params.CopyFrom(
- self.get_fake_object_center_proto())
+ self.get_fake_object_center_proto(
+ customize_head_params=customize_head_params))
config.center_net.object_detection_task.CopyFrom(
self.get_fake_object_detection_proto())
config.center_net.keypoint_estimation_task.append(
- self.get_fake_keypoint_proto())
+ self.get_fake_keypoint_proto(
+ customize_head_params=customize_head_params))
config.center_net.keypoint_label_map_path = (
self.get_fake_label_map_file_path())
config.center_net.mask_estimation_task.CopyFrom(
@@ -233,6 +289,12 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
self.assertAlmostEqual(
model._center_params.heatmap_bias_init, 3.14, places=4)
self.assertEqual(model._center_params.max_box_predictions, 15)
+ if customize_head_params:
+ self.assertEqual(model._center_params.center_head_num_filters, [64, 32])
+ self.assertEqual(model._center_params.center_head_kernel_sizes, [5, 3])
+ else:
+ self.assertEqual(model._center_params.center_head_num_filters, [256])
+ self.assertEqual(model._center_params.center_head_kernel_sizes, [3])
# Check object detection related parameters.
self.assertAlmostEqual(model._od_params.offset_loss_weight, 0.1)
@@ -264,6 +326,21 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
self.assertEqual(kp_params.candidate_ranking_mode, 'score_distance_ratio')
self.assertEqual(kp_params.offset_peak_radius, 3)
self.assertEqual(kp_params.per_keypoint_offset, True)
+ self.assertEqual(kp_params.predict_depth, True)
+ self.assertEqual(kp_params.per_keypoint_depth, True)
+ self.assertAlmostEqual(kp_params.keypoint_depth_loss_weight, 0.3)
+ if customize_head_params:
+ # Set by the config.
+ self.assertEqual(kp_params.heatmap_head_num_filters, [64, 32])
+ self.assertEqual(kp_params.heatmap_head_kernel_sizes, [5, 3])
+ self.assertEqual(kp_params.offset_head_num_filters, [128, 64])
+ self.assertEqual(kp_params.offset_head_kernel_sizes, [5, 3])
+ else:
+ # Default values:
+ self.assertEqual(kp_params.heatmap_head_num_filters, [256])
+ self.assertEqual(kp_params.heatmap_head_kernel_sizes, [3])
+ self.assertEqual(kp_params.offset_head_num_filters, [256])
+ self.assertEqual(kp_params.offset_head_kernel_sizes, [3])
# Check mask related parameters.
self.assertAlmostEqual(model._mask_params.task_loss_weight, 0.7)
@@ -292,11 +369,58 @@ class ModelBuilderTF2Test(model_builder_test.ModelBuilderTest):
# Check feature extractor parameters.
self.assertIsInstance(
- model._feature_extractor,
- center_net_resnet_feature_extractor.CenterNetResnetFeatureExtractor)
+ model._feature_extractor, center_net_hourglass_feature_extractor
+ .CenterNetHourglassFeatureExtractor)
self.assertAllClose(model._feature_extractor._channel_means, [0, 0, 0])
self.assertAllClose(model._feature_extractor._channel_stds, [4, 5, 6])
self.assertTrue(model._feature_extractor._bgr_ordering)
+ backbone = model._feature_extractor._network
+ self.assertIsInstance(backbone, hourglass_network.HourglassNetwork)
+ self.assertTrue(backbone.num_hourglasses, 1)
+
+ def test_create_center_net_model_from_keypoints(self):
+ """Test building a CenterNet model from proto txt."""
+ proto_txt = """
+ center_net {
+ num_classes: 10
+ feature_extractor {
+ type: "hourglass_52"
+ channel_stds: [4, 5, 6]
+ bgr_ordering: true
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 512
+ max_dimension: 512
+ pad_to_max_dimension: true
+ }
+ }
+ }
+ """
+ # Set up the configuration proto.
+ config = text_format.Merge(proto_txt, model_pb2.DetectionModel())
+ # Only add object center and keypoint estimation configs here.
+ config.center_net.object_center_params.CopyFrom(
+ self.get_fake_object_center_from_keypoints_proto())
+ config.center_net.keypoint_estimation_task.append(
+ self.get_fake_keypoint_proto())
+ config.center_net.keypoint_label_map_path = (
+ self.get_fake_label_map_file_path())
+
+ # Build the model from the configuration.
+ model = model_builder.build(config, is_training=True)
+
+ # Check object center related parameters.
+ self.assertEqual(model._num_classes, 10)
+ self.assertEqual(model._center_params.keypoint_weights_for_center,
+ [1.0, 0.0, 1.0, 0.0])
+
+ # Check keypoint estimation related parameters.
+ kp_params = model._kp_params_dict['human_pose']
+ self.assertAlmostEqual(kp_params.task_loss_weight, 0.9)
+ self.assertEqual(kp_params.keypoint_indices, [0, 1, 2, 3])
+ self.assertEqual(kp_params.keypoint_labels,
+ ['nose', 'left_shoulder', 'right_shoulder', 'hip'])
if __name__ == '__main__':
diff --git a/research/object_detection/builders/optimizer_builder.py b/research/object_detection/builders/optimizer_builder.py
index d602bad1292e222b5cbc532a873299dd918ef011..f24747aa9bac11574612db938d87d9bc50d2e9ea 100644
--- a/research/object_detection/builders/optimizer_builder.py
+++ b/research/object_detection/builders/optimizer_builder.py
@@ -18,6 +18,12 @@
import tensorflow.compat.v1 as tf
from object_detection.utils import learning_schedules
+from object_detection.utils import tf_version
+
+# pylint: disable=g-import-not-at-top
+if tf_version.is_tf2():
+ from official.modeling.optimization import ema_optimizer
+# pylint: enable=g-import-not-at-top
try:
from tensorflow.contrib import opt as tf_opt # pylint: disable=g-import-not-at-top
@@ -130,7 +136,9 @@ def build_optimizers_tf_v2(optimizer_config, global_step=None):
raise ValueError('Optimizer %s not supported.' % optimizer_type)
if optimizer_config.use_moving_average:
- raise ValueError('Moving average not supported in eager mode.')
+ optimizer = ema_optimizer.ExponentialMovingAverage(
+ optimizer=optimizer,
+ average_decay=optimizer_config.moving_average_decay)
return optimizer, summary_vars
diff --git a/research/object_detection/builders/optimizer_builder_tf2_test.py b/research/object_detection/builders/optimizer_builder_tf2_test.py
index 2c555f9a0f4c22b7c27955c92eaa3655c8fae5c6..5ae125fa0489b80fa76b76d2af4521aed15ae6e2 100644
--- a/research/object_detection/builders/optimizer_builder_tf2_test.py
+++ b/research/object_detection/builders/optimizer_builder_tf2_test.py
@@ -82,7 +82,7 @@ class OptimizerBuilderV2Test(tf.test.TestCase):
optimizer, _ = optimizer_builder.build(optimizer_proto)
self.assertIsInstance(optimizer, tf.keras.optimizers.Adam)
- def testMovingAverageOptimizerUnsupported(self):
+ def testBuildMovingAverageOptimizer(self):
optimizer_text_proto = """
adam_optimizer: {
learning_rate: {
@@ -95,8 +95,8 @@ class OptimizerBuilderV2Test(tf.test.TestCase):
"""
optimizer_proto = optimizer_pb2.Optimizer()
text_format.Merge(optimizer_text_proto, optimizer_proto)
- with self.assertRaises(ValueError):
- optimizer_builder.build(optimizer_proto)
+ optimizer, _ = optimizer_builder.build(optimizer_proto)
+ self.assertIsInstance(optimizer, tf.keras.optimizers.Optimizer)
if __name__ == '__main__':
diff --git a/research/object_detection/builders/post_processing_builder.py b/research/object_detection/builders/post_processing_builder.py
index 18795f58ccb4a382bdc457c2d15e069d1bb52662..c61f6891e29eeced8ba5fbe3f78fe1c95eb60501 100644
--- a/research/object_detection/builders/post_processing_builder.py
+++ b/research/object_detection/builders/post_processing_builder.py
@@ -103,7 +103,8 @@ def _build_non_max_suppressor(nms_config):
use_partitioned_nms=nms_config.use_partitioned_nms,
use_combined_nms=nms_config.use_combined_nms,
change_coordinate_frame=nms_config.change_coordinate_frame,
- use_hard_nms=nms_config.use_hard_nms)
+ use_hard_nms=nms_config.use_hard_nms,
+ use_cpu_nms=nms_config.use_cpu_nms)
return non_max_suppressor_fn
diff --git a/research/object_detection/builders/preprocessor_builder.py b/research/object_detection/builders/preprocessor_builder.py
index b61239d2e1ec87c232fbfdb53d0cb3c39da26e3e..9e59d94aa72a884806ac2e9f41244bccd8632aaf 100644
--- a/research/object_detection/builders/preprocessor_builder.py
+++ b/research/object_detection/builders/preprocessor_builder.py
@@ -89,8 +89,6 @@ PREPROCESSING_FUNCTION_MAP = {
preprocessor.random_adjust_saturation,
'random_distort_color':
preprocessor.random_distort_color,
- 'random_jitter_boxes':
- preprocessor.random_jitter_boxes,
'random_crop_to_aspect_ratio':
preprocessor.random_crop_to_aspect_ratio,
'random_black_patches':
@@ -109,6 +107,8 @@ PREPROCESSING_FUNCTION_MAP = {
preprocessor.subtract_channel_mean,
'convert_class_logits_to_softmax':
preprocessor.convert_class_logits_to_softmax,
+ 'adjust_gamma':
+ preprocessor.adjust_gamma,
}
@@ -123,6 +123,16 @@ RESIZE_METHOD_MAP = {
}
+def get_random_jitter_kwargs(proto):
+ return {
+ 'ratio':
+ proto.ratio,
+ 'jitter_mode':
+ preprocessor_pb2.RandomJitterBoxes.JitterMode.Name(proto.jitter_mode
+ ).lower()
+ }
+
+
def build(preprocessor_step_config):
"""Builds preprocessing step based on the configuration.
@@ -425,4 +435,8 @@ def build(preprocessor_step_config):
'output_size': config.output_size,
}
+ if step_type == 'random_jitter_boxes':
+ config = preprocessor_step_config.random_jitter_boxes
+ kwargs = get_random_jitter_kwargs(config)
+ return preprocessor.random_jitter_boxes, kwargs
raise ValueError('Unknown preprocessing step.')
diff --git a/research/object_detection/builders/preprocessor_builder_test.py b/research/object_detection/builders/preprocessor_builder_test.py
index 9e90344d0478229fa95355b53ecfa5f876325936..396adec5673f817478578500d8c7728c06c25855 100644
--- a/research/object_detection/builders/preprocessor_builder_test.py
+++ b/research/object_detection/builders/preprocessor_builder_test.py
@@ -216,13 +216,14 @@ class PreprocessorBuilderTest(tf.test.TestCase):
preprocessor_text_proto = """
random_jitter_boxes {
ratio: 0.1
+ jitter_mode: SHRINK
}
"""
preprocessor_proto = preprocessor_pb2.PreprocessingStep()
text_format.Merge(preprocessor_text_proto, preprocessor_proto)
function, args = preprocessor_builder.build(preprocessor_proto)
self.assertEqual(function, preprocessor.random_jitter_boxes)
- self.assert_dictionary_close(args, {'ratio': 0.1})
+ self.assert_dictionary_close(args, {'ratio': 0.1, 'jitter_mode': 'shrink'})
def test_build_random_crop_image(self):
preprocessor_text_proto = """
@@ -753,6 +754,19 @@ class PreprocessorBuilderTest(tf.test.TestCase):
'max_border': 128
})
+ def test_adjust_gamma(self):
+ preprocessor_text_proto = """
+ adjust_gamma {
+ gamma: 2.2
+ gain: 2.0
+ }
+ """
+ preprocessor_proto = preprocessor_pb2.PreprocessingStep()
+ text_format.Parse(preprocessor_text_proto, preprocessor_proto)
+ function, args = preprocessor_builder.build(preprocessor_proto)
+ self.assertEqual(function, preprocessor.adjust_gamma)
+ self.assert_dictionary_close(args, {'gamma': 2.2, 'gain': 2.0})
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/colab_tutorials/centernet_on_device.ipynb b/research/object_detection/colab_tutorials/centernet_on_device.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..bb3af3034490b12fc68cb09e8903c01d07ac31d6
--- /dev/null
+++ b/research/object_detection/colab_tutorials/centernet_on_device.ipynb
@@ -0,0 +1,762 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "name": "centernet_on_mobile.ipynb",
+ "provenance": [],
+ "collapsed_sections": [],
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pDIqEDDxWAh2"
+ },
+ "source": [
+ "#Introduction\r\n",
+ "\r\n",
+ "Welcome to the **CenterNet on-device with TensorFlow Lite** Colab. Here, we demonstrate how you can run a mobile-optimized version of the [CenterNet](https://arxiv.org/abs/1904.08189) architecture with [TensorFlow Lite](https://www.tensorflow.org/lite) (a.k.a. TFLite). \r\n",
+ "\r\n",
+ "Users can use this notebook as a reference for obtaining TFLite version of CenterNet for *Object Detection* or [*Keypoint detection*](https://cocodataset.org/#keypoints-2020). The code also shows how to perform pre-/post-processing & inference with TFLite's Python API.\r\n",
+ "\r\n",
+ "**NOTE:** CenterNet support in TFLite is still experimental, and currently works with floating-point inference only."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "3LQWTJ-BWzmW"
+ },
+ "source": [
+ "# Set Up"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gx84EpH7INPj"
+ },
+ "source": [
+ "## Libraries & Imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "EU_hXi7IW9QC"
+ },
+ "source": [
+ "!pip install tf-nightly"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "ZTU9_JcOZz-J"
+ },
+ "source": [
+ "import os\r\n",
+ "import pathlib\r\n",
+ "\r\n",
+ "# Clone the tensorflow models repository if it doesn't already exist\r\n",
+ "if \"models\" in pathlib.Path.cwd().parts:\r\n",
+ " while \"models\" in pathlib.Path.cwd().parts:\r\n",
+ " os.chdir('..')\r\n",
+ "elif not pathlib.Path('models').exists():\r\n",
+ " !git clone --depth 1 https://github.com/tensorflow/models"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "laJxis1WZ2xj"
+ },
+ "source": [
+ "# Install the Object Detection API\r\n",
+ "%%bash\r\n",
+ "cd models/research/\r\n",
+ "protoc object_detection/protos/*.proto --python_out=.\r\n",
+ "cp object_detection/packages/tf2/setup.py .\r\n",
+ "python -m pip install ."
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "je0LrJNjDsk9"
+ },
+ "source": [
+ "import matplotlib\r\n",
+ "import matplotlib.pyplot as plt\r\n",
+ "\r\n",
+ "import os\r\n",
+ "import random\r\n",
+ "import io\r\n",
+ "import imageio\r\n",
+ "import glob\r\n",
+ "import scipy.misc\r\n",
+ "import numpy as np\r\n",
+ "from six import BytesIO\r\n",
+ "from PIL import Image, ImageDraw, ImageFont\r\n",
+ "from IPython.display import display, Javascript\r\n",
+ "from IPython.display import Image as IPyImage\r\n",
+ "\r\n",
+ "import tensorflow as tf\r\n",
+ "\r\n",
+ "from object_detection.utils import label_map_util\r\n",
+ "from object_detection.utils import config_util\r\n",
+ "from object_detection.utils import visualization_utils as viz_utils\r\n",
+ "from object_detection.utils import colab_utils\r\n",
+ "from object_detection.utils import config_util\r\n",
+ "from object_detection.builders import model_builder\r\n",
+ "\r\n",
+ "%matplotlib inline"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "O5IXwbhhH0bs"
+ },
+ "source": [
+ "## Test Image from COCO\r\n",
+ "\r\n",
+ "We use a sample image from the COCO'17 validation dataset that contains people, to showcase inference with CenterNet."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "h-JuG84HDvm-"
+ },
+ "source": [
+ "# Download COCO'17 validation set for test image\r\n",
+ "%%bash\r\n",
+ "mkdir -p coco && cd coco\r\n",
+ "wget -q -N http://images.cocodataset.org/zips/val2017.zip\r\n",
+ "unzip -q -o val2017.zip && rm *.zip\r\n",
+ "cd .."
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "peX5mPGmEj8s"
+ },
+ "source": [
+ "# Print the image we are going to test on as a sanity check.\r\n",
+ "\r\n",
+ "def load_image_into_numpy_array(path):\r\n",
+ " \"\"\"Load an image from file into a numpy array.\r\n",
+ "\r\n",
+ " Puts image into numpy array to feed into tensorflow graph.\r\n",
+ " Note that by convention we put it into a numpy array with shape\r\n",
+ " (height, width, channels), where channels=3 for RGB.\r\n",
+ "\r\n",
+ " Args:\r\n",
+ " path: a file path.\r\n",
+ "\r\n",
+ " Returns:\r\n",
+ " uint8 numpy array with shape (img_height, img_width, 3)\r\n",
+ " \"\"\"\r\n",
+ " img_data = tf.io.gfile.GFile(path, 'rb').read()\r\n",
+ " image = Image.open(BytesIO(img_data))\r\n",
+ " (im_width, im_height) = image.size\r\n",
+ " return np.array(image.getdata()).reshape(\r\n",
+ " (im_height, im_width, 3)).astype(np.uint8)\r\n",
+ "\r\n",
+ "image_path = 'coco/val2017/000000013729.jpg'\r\n",
+ "plt.figure(figsize = (30, 20))\r\n",
+ "plt.imshow(load_image_into_numpy_array(image_path))"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6cqOdvfrR1vW"
+ },
+ "source": [
+ "## Utilities for Inference\r\n",
+ "\r\n",
+ "The `detect` function shown below describes how input and output tensors from CenterNet (obtained in subsequent sections) can be processed. This logic can be ported to other languages depending on your application (for e.g. to Java for Android apps)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "13Ouc2C3RyqR"
+ },
+ "source": [
+ "def detect(interpreter, input_tensor, include_keypoint=False):\r\n",
+ " \"\"\"Run detection on an input image.\r\n",
+ "\r\n",
+ " Args:\r\n",
+ " interpreter: tf.lite.Interpreter\r\n",
+ " input_tensor: A [1, height, width, 3] Tensor of type tf.float32.\r\n",
+ " Note that height and width can be anything since the image will be\r\n",
+ " immediately resized according to the needs of the model within this\r\n",
+ " function.\r\n",
+ " include_keypoint: True if model supports keypoints output. See\r\n",
+ " https://cocodataset.org/#keypoints-2020\r\n",
+ "\r\n",
+ " Returns:\r\n",
+ " A sequence containing the following output tensors:\r\n",
+ " boxes: a numpy array of shape [N, 4]\r\n",
+ " classes: a numpy array of shape [N]. Note that class indices are \r\n",
+ " 1-based, and match the keys in the label map.\r\n",
+ " scores: a numpy array of shape [N] or None. If scores=None, then\r\n",
+ " this function assumes that the boxes to be plotted are groundtruth\r\n",
+ " boxes and plot all boxes as black with no classes or scores.\r\n",
+ " category_index: a dict containing category dictionaries (each holding\r\n",
+ " category index `id` and category name `name`) keyed by category \r\n",
+ " indices.\r\n",
+ " If include_keypoints is True, the following are also returned:\r\n",
+ " keypoints: (optional) a numpy array of shape [N, 17, 2] representing\r\n",
+ " the yx-coordinates of the detection 17 COCO human keypoints\r\n",
+ " (https://cocodataset.org/#keypoints-2020) in normalized image frame\r\n",
+ " (i.e. [0.0, 1.0]). \r\n",
+ " keypoint_scores: (optional) a numpy array of shape [N, 17] representing the\r\n",
+ " keypoint prediction confidence scores.\r\n",
+ " \"\"\"\r\n",
+ " input_details = interpreter.get_input_details()\r\n",
+ " output_details = interpreter.get_output_details()\r\n",
+ "\r\n",
+ " interpreter.set_tensor(input_details[0]['index'], input_tensor.numpy())\r\n",
+ "\r\n",
+ " interpreter.invoke()\r\n",
+ "\r\n",
+ " boxes = interpreter.get_tensor(output_details[0]['index'])\r\n",
+ " classes = interpreter.get_tensor(output_details[1]['index'])\r\n",
+ " scores = interpreter.get_tensor(output_details[2]['index'])\r\n",
+ " num_detections = interpreter.get_tensor(output_details[3]['index'])\r\n",
+ "\r\n",
+ " if include_keypoint:\r\n",
+ " kpts = interpreter.get_tensor(output_details[4]['index'])\r\n",
+ " kpts_scores = interpreter.get_tensor(output_details[5]['index'])\r\n",
+ " return boxes, classes, scores, num_detections, kpts, kpts_scores\r\n",
+ " else:\r\n",
+ " return boxes, classes, scores, num_detections\r\n",
+ "\r\n",
+ "# Utility for visualizing results\r\n",
+ "def plot_detections(image_np,\r\n",
+ " boxes,\r\n",
+ " classes,\r\n",
+ " scores,\r\n",
+ " category_index,\r\n",
+ " keypoints=None,\r\n",
+ " keypoint_scores=None,\r\n",
+ " figsize=(12, 16),\r\n",
+ " image_name=None):\r\n",
+ " \"\"\"Wrapper function to visualize detections.\r\n",
+ "\r\n",
+ " Args:\r\n",
+ " image_np: uint8 numpy array with shape (img_height, img_width, 3)\r\n",
+ " boxes: a numpy array of shape [N, 4]\r\n",
+ " classes: a numpy array of shape [N]. Note that class indices are 1-based,\r\n",
+ " and match the keys in the label map.\r\n",
+ " scores: a numpy array of shape [N] or None. If scores=None, then\r\n",
+ " this function assumes that the boxes to be plotted are groundtruth\r\n",
+ " boxes and plot all boxes as black with no classes or scores.\r\n",
+ " category_index: a dict containing category dictionaries (each holding\r\n",
+ " category index `id` and category name `name`) keyed by category indices.\r\n",
+ " keypoints: (optional) a numpy array of shape [N, 17, 2] representing the \r\n",
+ " yx-coordinates of the detection 17 COCO human keypoints\r\n",
+ " (https://cocodataset.org/#keypoints-2020) in normalized image frame\r\n",
+ " (i.e. [0.0, 1.0]). \r\n",
+ " keypoint_scores: (optional) anumpy array of shape [N, 17] representing the\r\n",
+ " keypoint prediction confidence scores.\r\n",
+ " figsize: size for the figure.\r\n",
+ " image_name: a name for the image file.\r\n",
+ " \"\"\"\r\n",
+ "\r\n",
+ " keypoint_edges = [(0, 1),\r\n",
+ " (0, 2),\r\n",
+ " (1, 3),\r\n",
+ " (2, 4),\r\n",
+ " (0, 5),\r\n",
+ " (0, 6),\r\n",
+ " (5, 7),\r\n",
+ " (7, 9),\r\n",
+ " (6, 8),\r\n",
+ " (8, 10),\r\n",
+ " (5, 6),\r\n",
+ " (5, 11),\r\n",
+ " (6, 12),\r\n",
+ " (11, 12),\r\n",
+ " (11, 13),\r\n",
+ " (13, 15),\r\n",
+ " (12, 14),\r\n",
+ " (14, 16)]\r\n",
+ " image_np_with_annotations = image_np.copy()\r\n",
+ " # Only visualize objects that get a score > 0.3.\r\n",
+ " viz_utils.visualize_boxes_and_labels_on_image_array(\r\n",
+ " image_np_with_annotations,\r\n",
+ " boxes,\r\n",
+ " classes,\r\n",
+ " scores,\r\n",
+ " category_index,\r\n",
+ " keypoints=keypoints,\r\n",
+ " keypoint_scores=keypoint_scores,\r\n",
+ " keypoint_edges=keypoint_edges,\r\n",
+ " use_normalized_coordinates=True,\r\n",
+ " min_score_thresh=0.3)\r\n",
+ " if image_name:\r\n",
+ " plt.imsave(image_name, image_np_with_annotations)\r\n",
+ " else:\r\n",
+ " return image_np_with_annotations"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "3cNYi8HuIWzO"
+ },
+ "source": [
+ "# Object Detection"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "azdDCdWQMSoH"
+ },
+ "source": [
+ "## Download Model from Detection Zoo\r\n",
+ "\r\n",
+ "**NOTE:** Not all CenterNet models from the [TF2 Detection Zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) work with TFLite, only the [MobileNet-based version](http://download.tensorflow.org/models/object_detection/tf2/20210210/centernet_mobilenetv2fpn_512x512_coco17_od.tar.gz) does.\r\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "Sywt8MKzIeOi"
+ },
+ "source": [
+ "# Get mobile-friendly CenterNet for Object Detection\r\n",
+ "# See TensorFlow 2 Detection Model Zoo for more details:\r\n",
+ "# https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md\r\n",
+ "\r\n",
+ "%%bash\r\n",
+ "wget http://download.tensorflow.org/models/object_detection/tf2/20210210/centernet_mobilenetv2fpn_512x512_coco17_od.tar.gz\r\n",
+ "tar -xf centernet_mobilenetv2fpn_512x512_coco17_od.tar.gz\r\n",
+ "rm centernet_mobilenetv2fpn_512x512_coco17_od.tar.gz*"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MiRrVpTnLvsk"
+ },
+ "source": [
+ "Now that we have downloaded the CenterNet model that uses MobileNet as a backbone, we can obtain a TensorFlow Lite model from it. \r\n",
+ "\r\n",
+ "The downloaded archive already contains `model.tflite` that works with TensorFlow Lite, but we re-generate the model in the next sub-section to account for cases where you might re-train the model on your own dataset (with corresponding changes to `pipeline.config` & `checkpoint` directory)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "jT0bruuxM496"
+ },
+ "source": [
+ "## Generate TensorFlow Lite Model\r\n",
+ "\r\n",
+ "First, we invoke `export_tflite_graph_tf2.py` to generate a TFLite-friendly intermediate SavedModel. This will then be passed to the TensorFlow Lite Converter for generating the final model.\r\n",
+ "\r\n",
+ "This is similar to what we do for [SSD architectures](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/running_on_mobile_tf2.md)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "jpcCjiQ_JrU5",
+ "collapsed": true
+ },
+ "source": [
+ "%%bash\r\n",
+ "# Export the intermediate SavedModel that outputs 10 detections & takes in an \r\n",
+ "# image of dim 320x320.\r\n",
+ "# Modify these parameters according to your needs.\r\n",
+ "\r\n",
+ "python models/research/object_detection/export_tflite_graph_tf2.py \\\r\n",
+ " --pipeline_config_path=centernet_mobilenetv2_fpn_od/pipeline.config \\\r\n",
+ " --trained_checkpoint_dir=centernet_mobilenetv2_fpn_od/checkpoint \\\r\n",
+ " --output_directory=centernet_mobilenetv2_fpn_od/tflite \\\r\n",
+ " --centernet_include_keypoints=false \\\r\n",
+ " --max_detections=10 \\\r\n",
+ " --config_override=\" \\\r\n",
+ " model{ \\\r\n",
+ " center_net { \\\r\n",
+ " image_resizer { \\\r\n",
+ " fixed_shape_resizer { \\\r\n",
+ " height: 320 \\\r\n",
+ " width: 320 \\\r\n",
+ " } \\\r\n",
+ " } \\\r\n",
+ " } \\\r\n",
+ " }\""
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "zhhP6HL8PUGq"
+ },
+ "source": [
+ "# Generate TensorFlow Lite model using the converter.\r\n",
+ "%%bash\r\n",
+ "tflite_convert --output_file=centernet_mobilenetv2_fpn_od/model.tflite \\\r\n",
+ " --saved_model_dir=centernet_mobilenetv2_fpn_od/tflite/saved_model"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gj1Q_e_2Rn5i"
+ },
+ "source": [
+ "## TensorFlow Lite Inference\r\n",
+ "\r\n",
+ "Use a TensorFlow Lite Interpreter to detect objects in the test image."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "uV9t9icURsei"
+ },
+ "source": [
+ "%matplotlib inline\r\n",
+ "\r\n",
+ "# Load the TFLite model and allocate tensors.\r\n",
+ "model_path = 'centernet_mobilenetv2_fpn_od/model.tflite'\r\n",
+ "label_map_path = 'centernet_mobilenetv2_fpn_od/label_map.txt'\r\n",
+ "image_path = 'coco/val2017/000000013729.jpg'\r\n",
+ "\r\n",
+ "# Initialize TensorFlow Lite Interpreter.\r\n",
+ "interpreter = tf.lite.Interpreter(model_path=model_path)\r\n",
+ "interpreter.allocate_tensors()\r\n",
+ "\r\n",
+ "# Label map can be used to figure out what class ID maps to what\r\n",
+ "# label. `label_map.txt` is human-readable.\r\n",
+ "category_index = label_map_util.create_category_index_from_labelmap(\r\n",
+ " label_map_path)\r\n",
+ "\r\n",
+ "label_id_offset = 1\r\n",
+ "\r\n",
+ "image = tf.io.read_file(image_path)\r\n",
+ "image = tf.compat.v1.image.decode_jpeg(image)\r\n",
+ "image = tf.expand_dims(image, axis=0)\r\n",
+ "image_numpy = image.numpy()\r\n",
+ "\r\n",
+ "input_tensor = tf.convert_to_tensor(image_numpy, dtype=tf.float32)\r\n",
+ "# Note that CenterNet doesn't require any pre-processing except resizing to the\r\n",
+ "# input size that the TensorFlow Lite Interpreter was generated with.\r\n",
+ "input_tensor = tf.image.resize(input_tensor, (320, 320))\r\n",
+ "boxes, classes, scores, num_detections = detect(interpreter, input_tensor)\r\n",
+ "\r\n",
+ "vis_image = plot_detections(\r\n",
+ " image_numpy[0],\r\n",
+ " boxes[0],\r\n",
+ " classes[0].astype(np.uint32) + label_id_offset,\r\n",
+ " scores[0],\r\n",
+ " category_index)\r\n",
+ "plt.figure(figsize = (30, 20))\r\n",
+ "plt.imshow(vis_image)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "DefXu4JXVxPD"
+ },
+ "source": [
+ "# Keypoints\r\n",
+ "\r\n",
+ "Unlike SSDs, CenterNet also supports COCO [Keypoint detection](https://cocodataset.org/#keypoints-2020). To be more specific, the 'keypoints' version of CenterNet shown here provides keypoints as a `[N, 17, 2]`-shaped tensor representing the (normalized) yx-coordinates of 17 COCO human keypoints.\r\n",
+ "\r\n",
+ "See the `detect()` function in the **Utilities for Inference** section to better understand the keypoints output."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "xu47DkrDV18O"
+ },
+ "source": [
+ "## Download Model from Detection Zoo\r\n",
+ "\r\n",
+ "**NOTE:** Not all CenterNet models from the [TF2 Detection Zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md) work with TFLite, only the [MobileNet-based version](http://download.tensorflow.org/models/object_detection/tf2/20210210/centernet_mobilenetv2fpn_512x512_coco17_od.tar.gz) does."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "sd7f64WjWD7z"
+ },
+ "source": [
+ "# Get mobile-friendly CenterNet for Keypoint detection task.\r\n",
+ "# See TensorFlow 2 Detection Model Zoo for more details:\r\n",
+ "# https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md\r\n",
+ "\r\n",
+ "%%bash\r\n",
+ "wget http://download.tensorflow.org/models/object_detection/tf2/20210210/centernet_mobilenetv2fpn_512x512_coco17_kpts.tar.gz\r\n",
+ "tar -xf centernet_mobilenetv2fpn_512x512_coco17_kpts.tar.gz\r\n",
+ "rm centernet_mobilenetv2fpn_512x512_coco17_kpts.tar.gz*"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "NSFc-xSLX1ZC"
+ },
+ "source": [
+ "## Generate TensorFlow Lite Model\r\n",
+ "\r\n",
+ "As before, we leverage `export_tflite_graph_tf2.py` to generate a TFLite-friendly intermediate SavedModel. This will then be passed to the TFLite converter to generating the final model.\r\n",
+ "\r\n",
+ "Note that we need to include an additional `keypoint_label_map_path` parameter for exporting the keypoints outputs."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "8kEhwYynX-cD"
+ },
+ "source": [
+ "%%bash\r\n",
+ "# Export the intermediate SavedModel that outputs 10 detections & takes in an \r\n",
+ "# image of dim 320x320.\r\n",
+ "# Modify these parameters according to your needs.\r\n",
+ "\r\n",
+ "python models/research/object_detection/export_tflite_graph_tf2.py \\\r\n",
+ " --pipeline_config_path=centernet_mobilenetv2_fpn_kpts/pipeline.config \\\r\n",
+ " --trained_checkpoint_dir=centernet_mobilenetv2_fpn_kpts/checkpoint \\\r\n",
+ " --output_directory=centernet_mobilenetv2_fpn_kpts/tflite \\\r\n",
+ " --centernet_include_keypoints=true \\\r\n",
+ " --keypoint_label_map_path=centernet_mobilenetv2_fpn_kpts/label_map.txt \\\r\n",
+ " --max_detections=10 \\\r\n",
+ " --config_override=\" \\\r\n",
+ " model{ \\\r\n",
+ " center_net { \\\r\n",
+ " image_resizer { \\\r\n",
+ " fixed_shape_resizer { \\\r\n",
+ " height: 320 \\\r\n",
+ " width: 320 \\\r\n",
+ " } \\\r\n",
+ " } \\\r\n",
+ " } \\\r\n",
+ " }\""
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "TJtsyMlLY1DU"
+ },
+ "source": [
+ "# Generate TensorFlow Lite model using the converter.\r\n",
+ "\r\n",
+ "%%bash\r\n",
+ "tflite_convert --output_file=centernet_mobilenetv2_fpn_kpts/model.tflite \\\r\n",
+ " --saved_model_dir=centernet_mobilenetv2_fpn_kpts/tflite/saved_model"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nJCxPBjYZSk6"
+ },
+ "source": [
+ "## TensorFlow Lite Inference\r\n",
+ "\r\n",
+ "Use a TensorFlow Lite Interpreter to detect people & their keypoints in the test image."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "F2GpD7r8ZUzx"
+ },
+ "source": [
+ "%matplotlib inline\r\n",
+ "\r\n",
+ "# Load the TFLite model and allocate tensors.\r\n",
+ "model_path = 'centernet_mobilenetv2_fpn_kpts/model.tflite'\r\n",
+ "image_path = 'coco/val2017/000000013729.jpg'\r\n",
+ "\r\n",
+ "# Initialize TensorFlow Lite Interpreter.\r\n",
+ "interpreter = tf.lite.Interpreter(model_path=model_path)\r\n",
+ "interpreter.allocate_tensors()\r\n",
+ "\r\n",
+ "# Keypoints are only relevant for people, so we only care about that\r\n",
+ "# category Id here.\r\n",
+ "category_index = {1: {'id': 1, 'name': 'person'}}\r\n",
+ "\r\n",
+ "label_id_offset = 1\r\n",
+ "\r\n",
+ "image = tf.io.read_file(image_path)\r\n",
+ "image = tf.compat.v1.image.decode_jpeg(image)\r\n",
+ "image = tf.expand_dims(image, axis=0)\r\n",
+ "image_numpy = image.numpy()\r\n",
+ "\r\n",
+ "input_tensor = tf.convert_to_tensor(image_numpy, dtype=tf.float32)\r\n",
+ "# Note that CenterNet doesn't require any pre-processing except resizing to\r\n",
+ "# input size that the TensorFlow Lite Interpreter was generated with.\r\n",
+ "input_tensor = tf.image.resize(input_tensor, (320, 320))\r\n",
+ "(boxes, classes, scores, num_detections, kpts, kpts_scores) = detect(\r\n",
+ " interpreter, input_tensor, include_keypoint=True)\r\n",
+ "\r\n",
+ "vis_image = plot_detections(\r\n",
+ " image_numpy[0],\r\n",
+ " boxes[0],\r\n",
+ " classes[0].astype(np.uint32) + label_id_offset,\r\n",
+ " scores[0],\r\n",
+ " category_index,\r\n",
+ " keypoints=kpts[0],\r\n",
+ " keypoint_scores=kpts_scores[0])\r\n",
+ "plt.figure(figsize = (30, 20))\r\n",
+ "plt.imshow(vis_image)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "59Y3q6AC9C7s"
+ },
+ "source": [
+ "# Running On Mobile\r\n",
+ "\r\n",
+ "As mentioned earlier, both the above models can be run on mobile phones with TensorFlow Lite. See our [**inference documentation**](https://www.tensorflow.org/lite/guide/inference) for general guidelines on platform-specific APIs & leveraging hardware acceleration. Both the object-detection & keypoint-detection versions of CenterNet are compatible with our [GPU delegate](https://www.tensorflow.org/lite/performance/gpu). *We are working on developing quantized versions of this model.*\r\n",
+ "\r\n",
+ "To leverage *object-detection* in your Android app, the simplest way is to use TFLite's [**ObjectDetector Task API**](https://www.tensorflow.org/lite/inference_with_metadata/task_library/object_detector). It is a high-level API that encapsulates complex but common image processing and post processing logic. Inference can be done in 5 lines of code. It is supported in Java for Android and C++ for native code. *We are working on building the Swift API for iOS, as well as the support for the keypoint-detection model.*\r\n",
+ "\r\n",
+ "To use the Task API, the model needs to be packed with [TFLite Metadata](https://www.tensorflow.org/lite/convert/metadata). This metadata helps the inference code perform the correct pre & post processing as required by the model. Use the following code to create the metadata."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "8T_qzv6lDN_a"
+ },
+ "source": [
+ "!pip install tflite_support_nightly"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "CTZhBmfWDQ3z"
+ },
+ "source": [
+ "from tflite_support.metadata_writers import object_detector\n",
+ "from tflite_support.metadata_writers import writer_utils\n",
+ "\n",
+ "ObjectDetectorWriter = object_detector.MetadataWriter\n",
+ "\n",
+ "_MODEL_PATH = \"centernet_mobilenetv2_fpn_od/model.tflite\"\n",
+ "_SAVE_TO_PATH = \"centernet_mobilenetv2_fpn_od/model_with_metadata.tflite\"\n",
+ "_LABEL_PATH = \"centernet_mobilenetv2_fpn_od/tflite_label_map.txt\"\n",
+ "\n",
+ "# We need to convert Detection API's labelmap into what the Task API needs:\n",
+ "# a txt file with one class name on each line from index 0 to N.\n",
+ "# The first '0' class indicates the background.\n",
+ "# This code assumes COCO detection which has 90 classes, you can write a label\n",
+ "# map file for your model if re-trained.\n",
+ "od_label_map_path = 'centernet_mobilenetv2_fpn_od/label_map.txt'\n",
+ "category_index = label_map_util.create_category_index_from_labelmap(\n",
+ " label_map_path)\n",
+ "f = open(_LABEL_PATH, 'w')\n",
+ "for class_id in range(1, 91):\n",
+ " if class_id not in category_index:\n",
+ " f.write('???\\n')\n",
+ " continue\n",
+ " name = category_index[class_id]['name']\n",
+ " f.write(name+'\\n')\n",
+ "f.close()\n",
+ "\n",
+ "writer = ObjectDetectorWriter.create_for_inference(\n",
+ " writer_utils.load_file(_MODEL_PATH), input_norm_mean=[0], \n",
+ " input_norm_std=[1], label_file_paths=[_LABEL_PATH])\n",
+ "writer_utils.save_file(writer.populate(), _SAVE_TO_PATH)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "b2tc7awzDUHr"
+ },
+ "source": [
+ "Visualize the metadata just created by the following code:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "_SRqVdZNDYF1"
+ },
+ "source": [
+ "from tflite_support import metadata\n",
+ "\n",
+ "displayer = metadata.MetadataDisplayer.with_model_file(_SAVE_TO_PATH)\n",
+ "print(\"Metadata populated:\")\n",
+ "print(displayer.get_metadata_json())\n",
+ "print(\"=============================\")\n",
+ "print(\"Associated file(s) populated:\")\n",
+ "print(displayer.get_packed_associated_file_list())"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "SPUNsg9eDjWT"
+ },
+ "source": [
+ "See more information about *object-detection* models from our [public documentation](https://www.tensorflow.org/lite/examples/object_detection/overview). The [Object Detection example app](https://github.com/tensorflow/examples/tree/master/lite/examples/object_detection) is a good starting point for integrating that model into your Android and iOS app. You can find [examples](https://github.com/tensorflow/examples/tree/master/lite/examples/object_detection/android#switch-between-inference-solutions-task-library-vs-tflite-interpreter) of using both the TFLite Task Library and TFLite Interpreter API."
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/research/object_detection/colab_tutorials/deepmac_colab.ipynb b/research/object_detection/colab_tutorials/deepmac_colab.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..cc2bd1dff248a5ff68758cf346269da6ff3ad3fc
--- /dev/null
+++ b/research/object_detection/colab_tutorials/deepmac_colab.ipynb
@@ -0,0 +1,341 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "name": "deepmac_demo.ipynb",
+ "provenance": [],
+ "collapsed_sections": []
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "language_info": {
+ "name": "python"
+ },
+ "accelerator": "GPU"
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "P-esW81yhfCN"
+ },
+ "source": [
+ "# Novel class segmentation demo with Deep-MAC\n",
+ "\n",
+ "Welcome to the Novel class segmentation (with Deep-MAC) demo --- this colab loads a Deep-MAC model and tests it interactively with user-specified boxes. Deep-MAC was only trained to detect and segment COCO classes, but generalizes well when segmenting within user-specified boxes of unseen classes.\n",
+ "\n",
+ "Estimated time to run through this colab (with GPU): 10-15 minutes.\n",
+ "Note that the bulk of this time is in installing Tensorflow and downloading\n",
+ "the checkpoint then running inference for the first time. Once you've done\n",
+ "all that, running on new images is very fast."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Kq1eGNssiW31"
+ },
+ "source": [
+ "# Prerequisites\n",
+ "\n",
+ "Please change runtime to GPU."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "UT7N0HJhiRKr"
+ },
+ "source": [
+ "# Installation and Imports\n",
+ "\n",
+ "This takes 3-4 minutes."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "nNdls0Pe0UPK"
+ },
+ "source": [
+ "!pip install -U --pre tensorflow==\"2.2.0\"\n",
+ "\n",
+ "import os\n",
+ "import pathlib\n",
+ "\n",
+ "# Clone the tensorflow models repository if it doesn't already exist\n",
+ "if \"models\" in pathlib.Path.cwd().parts:\n",
+ " while \"models\" in pathlib.Path.cwd().parts:\n",
+ " os.chdir('..')\n",
+ "elif not pathlib.Path('models').exists():\n",
+ " !git clone --depth 1 https://github.com/tensorflow/models\n"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "WwjV9clX0n7S"
+ },
+ "source": [
+ "# Install the Object Detection API\n",
+ "%%bash\n",
+ "cd models/research/\n",
+ "protoc object_detection/protos/*.proto --python_out=.\n",
+ "cp object_detection/packages/tf2/setup.py .\n",
+ "python -m pip install ."
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "sfrrno2L0sRR"
+ },
+ "source": [
+ "import glob\n",
+ "import io\n",
+ "import logging\n",
+ "import os\n",
+ "import random\n",
+ "import warnings\n",
+ "\n",
+ "import imageio\n",
+ "from IPython.display import display, Javascript\n",
+ "from IPython.display import Image as IPyImage\n",
+ "import matplotlib\n",
+ "from matplotlib import patches\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from object_detection.utils import colab_utils\n",
+ "from object_detection.utils import ops\n",
+ "from object_detection.utils import visualization_utils as viz_utils\n",
+ "from PIL import Image, ImageDraw, ImageFont\n",
+ "import scipy.misc\n",
+ "from six import BytesIO\n",
+ "from skimage import color\n",
+ "from skimage import transform\n",
+ "from skimage import util\n",
+ "from skimage.color import rgb_colors\n",
+ "import tensorflow as tf\n",
+ "\n",
+ "%matplotlib inline\n",
+ "\n",
+ "COLORS = ([rgb_colors.cyan, rgb_colors.orange, rgb_colors.pink,\n",
+ " rgb_colors.purple, rgb_colors.limegreen , rgb_colors.crimson] +\n",
+ " [(color) for (name, color) in color.color_dict.items()])\n",
+ "random.shuffle(COLORS)\n",
+ "\n",
+ "logging.disable(logging.WARNING)\n",
+ "\n",
+ "\n",
+ "def read_image(path):\n",
+ " \"\"\"Read an image and optionally resize it for better plotting.\"\"\"\n",
+ " with tf.io.gfile.GFile(path, 'rb') as f:\n",
+ " img = Image.open(f)\n",
+ " return np.array(img, dtype=np.uint8)\n",
+ "\n",
+ "\n",
+ "def resize_for_display(image, max_height=600):\n",
+ " height, width, _ = image.shape\n",
+ " width = int(width * max_height / height)\n",
+ " with warnings.catch_warnings():\n",
+ " warnings.simplefilter(\"ignore\", UserWarning)\n",
+ " return util.img_as_ubyte(transform.resize(image, (height, width)))\n",
+ "\n",
+ "\n",
+ "def get_mask_prediction_function(model):\n",
+ " \"\"\"Get single image mask prediction function using a model.\"\"\"\n",
+ "\n",
+ " @tf.function\n",
+ " def predict_masks(image, boxes):\n",
+ " height, width, _ = image.shape.as_list()\n",
+ " batch = image[tf.newaxis]\n",
+ " boxes = boxes[tf.newaxis]\n",
+ "\n",
+ " detections = model(batch, boxes)\n",
+ " masks = detections['detection_masks']\n",
+ "\n",
+ " return ops.reframe_box_masks_to_image_masks(masks[0], boxes[0],\n",
+ " height, width)\n",
+ "\n",
+ " return predict_masks\n",
+ "\n",
+ "\n",
+ "def plot_image_annotations(image, boxes, masks, darken_image=0.5):\n",
+ " fig, ax = plt.subplots(figsize=(16, 12))\n",
+ " ax.set_axis_off()\n",
+ " image = (image * darken_image).astype(np.uint8)\n",
+ " ax.imshow(image)\n",
+ "\n",
+ " height, width, _ = image.shape\n",
+ "\n",
+ " num_colors = len(COLORS)\n",
+ " color_index = 0\n",
+ "\n",
+ " for box, mask in zip(boxes, masks):\n",
+ " ymin, xmin, ymax, xmax = box\n",
+ " ymin *= height\n",
+ " ymax *= height\n",
+ " xmin *= width\n",
+ " xmax *= width\n",
+ "\n",
+ " color = COLORS[color_index]\n",
+ " color = np.array(color)\n",
+ " rect = patches.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,\n",
+ " linewidth=2.5, edgecolor=color, facecolor='none')\n",
+ " ax.add_patch(rect)\n",
+ " mask = (mask > 0.5).astype(np.float32)\n",
+ " color_image = np.ones_like(image) * color[np.newaxis, np.newaxis, :]\n",
+ " color_and_mask = np.concatenate(\n",
+ " [color_image, mask[:, :, np.newaxis]], axis=2)\n",
+ "\n",
+ " ax.imshow(color_and_mask, alpha=0.5)\n",
+ "\n",
+ " color_index = (color_index + 1) % num_colors\n",
+ "\n",
+ " return ax"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ry9yq8zsi0Gg"
+ },
+ "source": [
+ "# Load Deep-MAC Model\n",
+ "\n",
+ "This can take up to 5 minutes."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "PZ-wnbYu05K8"
+ },
+ "source": [
+ "print('Downloading and untarring model')\n",
+ "!wget http://download.tensorflow.org/models/object_detection/tf2/20210329/deepmac_1024x1024_coco17.tar.gz\n",
+ "!cp deepmac_1024x1024_coco17.tar.gz models/research/object_detection/test_data/\n",
+ "!tar -xzf models/research/object_detection/test_data/deepmac_1024x1024_coco17.tar.gz\n",
+ "!mv deepmac_1024x1024_coco17 models/research/object_detection/test_data/\n",
+ "model_path = 'models/research/object_detection/test_data/deepmac_1024x1024_coco17/saved_model'\n",
+ "\n",
+ "print('Loading SavedModel')\n",
+ "model = tf.keras.models.load_model(model_path)\n",
+ "prediction_function = get_mask_prediction_function(model)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ilXkYOB_NUSc"
+ },
+ "source": [
+ "# Load image"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "txj4UkoDNaOq"
+ },
+ "source": [
+ "image_path = 'models/research/object_detection/test_images/image3.jpg'\n",
+ "image = read_image(image_path)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zyhudgYUjcvE"
+ },
+ "source": [
+ "# Annotate an image with one or more boxes\n",
+ "\n",
+ "This model is trained on COCO categories, but we encourage you to try segmenting\n",
+ "anything you want!\n",
+ "\n",
+ "Don't forget to hit **submit** when done."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "aZvY4At0074j"
+ },
+ "source": [
+ "display_image = resize_for_display(image)\n",
+ "\n",
+ "boxes_list = []\n",
+ "colab_utils.annotate([display_image], boxes_list)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "gUUG7NPBJMoa"
+ },
+ "source": [
+ "# In case you didn't want to label...\n",
+ "\n",
+ "Run this cell only if you didn't annotate anything above and would prefer to just use our preannotated boxes. Don't forget to uncomment.\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "lupqTv1HJK5K"
+ },
+ "source": [
+ "# boxes_list = [np.array([[0.000, 0.160, 0.362, 0.812],\n",
+ "# [0.340, 0.286, 0.472, 0.619],\n",
+ "# [0.437, 0.008, 0.650, 0.263],\n",
+ "# [0.382, 0.003, 0.538, 0.594],\n",
+ "# [0.518, 0.444, 0.625,0.554]], dtype=np.float32)]"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Ak1WO93NjvN-"
+ },
+ "source": [
+ "# Visualize mask predictions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "vdzuKnpj1A3L"
+ },
+ "source": [
+ "%matplotlib inline\n",
+ "\n",
+ "boxes = boxes_list[0]\n",
+ "masks = prediction_function(tf.convert_to_tensor(image),\n",
+ " tf.convert_to_tensor(boxes, dtype=tf.float32))\n",
+ "plot_image_annotations(image, boxes, masks.numpy())\n",
+ "plt.show()"
+ ],
+ "execution_count": null,
+ "outputs": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/research/object_detection/colab_tutorials/eager_few_shot_od_training_tflite.ipynb b/research/object_detection/colab_tutorials/eager_few_shot_od_training_tflite.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..b47d4bdb4f1bc81c8e8727721fbb9db732de8e22
--- /dev/null
+++ b/research/object_detection/colab_tutorials/eager_few_shot_od_training_tflite.ipynb
@@ -0,0 +1,730 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "rOvvWAVTkMR7"
+ },
+ "source": [
+ "# Introduction\n",
+ "\n",
+ "Welcome to the **Few Shot Object Detection for TensorFlow Lite** Colab. Here, we demonstrate fine tuning of a SSD architecture (pre-trained on COCO) on very few examples of a *novel* class. We will then generate a (downloadable) TensorFlow Lite model for on-device inference.\n",
+ "\n",
+ "**NOTE:** This Colab is meant for the few-shot detection use-case. To train a model on a large dataset, please follow the [TF2 training](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_training_and_evaluation.md#training) documentation and then [convert](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/running_on_mobile_tf2.md) the model to TensorFlow Lite."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "3U2sv0upw04O"
+ },
+ "source": [
+ "# Set Up"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vPs64QA1Zdov"
+ },
+ "source": [
+ "## Imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "H0rKBV4uZacD"
+ },
+ "outputs": [],
+ "source": [
+ "# Support for TF2 models was added after TF 2.3.\n",
+ "!pip install tf-nightly"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "oi28cqGGFWnY"
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import pathlib\n",
+ "\n",
+ "# Clone the tensorflow models repository if it doesn't already exist\n",
+ "if \"models\" in pathlib.Path.cwd().parts:\n",
+ " while \"models\" in pathlib.Path.cwd().parts:\n",
+ " os.chdir('..')\n",
+ "elif not pathlib.Path('models').exists():\n",
+ " !git clone --depth 1 https://github.com/tensorflow/models"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "NwdsBdGhFanc"
+ },
+ "outputs": [],
+ "source": [
+ "# Install the Object Detection API\n",
+ "%%bash\n",
+ "cd models/research/\n",
+ "protoc object_detection/protos/*.proto --python_out=.\n",
+ "cp object_detection/packages/tf2/setup.py .\n",
+ "python -m pip install ."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "uZcqD4NLdnf4"
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import os\n",
+ "import random\n",
+ "import io\n",
+ "import imageio\n",
+ "import glob\n",
+ "import scipy.misc\n",
+ "import numpy as np\n",
+ "from six import BytesIO\n",
+ "from PIL import Image, ImageDraw, ImageFont\n",
+ "from IPython.display import display, Javascript\n",
+ "from IPython.display import Image as IPyImage\n",
+ "\n",
+ "import tensorflow as tf\n",
+ "\n",
+ "from object_detection.utils import label_map_util\n",
+ "from object_detection.utils import config_util\n",
+ "from object_detection.utils import visualization_utils as viz_utils\n",
+ "from object_detection.utils import colab_utils\n",
+ "from object_detection.utils import config_util\n",
+ "from object_detection.builders import model_builder\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "IogyryF2lFBL"
+ },
+ "source": [
+ "##Utilities"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "-y9R0Xllefec"
+ },
+ "outputs": [],
+ "source": [
+ "def load_image_into_numpy_array(path):\n",
+ " \"\"\"Load an image from file into a numpy array.\n",
+ "\n",
+ " Puts image into numpy array to feed into tensorflow graph.\n",
+ " Note that by convention we put it into a numpy array with shape\n",
+ " (height, width, channels), where channels=3 for RGB.\n",
+ "\n",
+ " Args:\n",
+ " path: a file path.\n",
+ "\n",
+ " Returns:\n",
+ " uint8 numpy array with shape (img_height, img_width, 3)\n",
+ " \"\"\"\n",
+ " img_data = tf.io.gfile.GFile(path, 'rb').read()\n",
+ " image = Image.open(BytesIO(img_data))\n",
+ " (im_width, im_height) = image.size\n",
+ " return np.array(image.getdata()).reshape(\n",
+ " (im_height, im_width, 3)).astype(np.uint8)\n",
+ "\n",
+ "def plot_detections(image_np,\n",
+ " boxes,\n",
+ " classes,\n",
+ " scores,\n",
+ " category_index,\n",
+ " figsize=(12, 16),\n",
+ " image_name=None):\n",
+ " \"\"\"Wrapper function to visualize detections.\n",
+ "\n",
+ " Args:\n",
+ " image_np: uint8 numpy array with shape (img_height, img_width, 3)\n",
+ " boxes: a numpy array of shape [N, 4]\n",
+ " classes: a numpy array of shape [N]. Note that class indices are 1-based,\n",
+ " and match the keys in the label map.\n",
+ " scores: a numpy array of shape [N] or None. If scores=None, then\n",
+ " this function assumes that the boxes to be plotted are groundtruth\n",
+ " boxes and plot all boxes as black with no classes or scores.\n",
+ " category_index: a dict containing category dictionaries (each holding\n",
+ " category index `id` and category name `name`) keyed by category indices.\n",
+ " figsize: size for the figure.\n",
+ " image_name: a name for the image file.\n",
+ " \"\"\"\n",
+ " image_np_with_annotations = image_np.copy()\n",
+ " viz_utils.visualize_boxes_and_labels_on_image_array(\n",
+ " image_np_with_annotations,\n",
+ " boxes,\n",
+ " classes,\n",
+ " scores,\n",
+ " category_index,\n",
+ " use_normalized_coordinates=True,\n",
+ " min_score_thresh=0.8)\n",
+ " if image_name:\n",
+ " plt.imsave(image_name, image_np_with_annotations)\n",
+ " else:\n",
+ " plt.imshow(image_np_with_annotations)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "sSaXL28TZfk1"
+ },
+ "source": [
+ "## Rubber Ducky data\n",
+ "\n",
+ "We will start with some toy data consisting of 5 images of a rubber\n",
+ "ducky. Note that the [COCO](https://cocodataset.org/#explore) dataset contains a number of animals, but notably, it does *not* contain rubber duckies (or even ducks for that matter), so this is a novel class."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "SQy3ND7EpFQM"
+ },
+ "outputs": [],
+ "source": [
+ "# Load images and visualize\n",
+ "train_image_dir = 'models/research/object_detection/test_images/ducky/train/'\n",
+ "train_images_np = []\n",
+ "for i in range(1, 6):\n",
+ " image_path = os.path.join(train_image_dir, 'robertducky' + str(i) + '.jpg')\n",
+ " train_images_np.append(load_image_into_numpy_array(image_path))\n",
+ "\n",
+ "plt.rcParams['axes.grid'] = False\n",
+ "plt.rcParams['xtick.labelsize'] = False\n",
+ "plt.rcParams['ytick.labelsize'] = False\n",
+ "plt.rcParams['xtick.top'] = False\n",
+ "plt.rcParams['xtick.bottom'] = False\n",
+ "plt.rcParams['ytick.left'] = False\n",
+ "plt.rcParams['ytick.right'] = False\n",
+ "plt.rcParams['figure.figsize'] = [14, 7]\n",
+ "\n",
+ "for idx, train_image_np in enumerate(train_images_np):\n",
+ " plt.subplot(2, 3, idx+1)\n",
+ " plt.imshow(train_image_np)\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "LbOe9Ym7xMGV"
+ },
+ "source": [
+ "# Transfer Learning\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Dqb_yjAo3cO_"
+ },
+ "source": [
+ "## Data Preparation\n",
+ "\n",
+ "First, we populate the groundtruth with pre-annotated bounding boxes.\n",
+ "\n",
+ "We then add the class annotations (for simplicity, we assume a single 'Duck' class in this colab; though it should be straightforward to extend this to handle multiple classes). We also convert everything to the format that the training\n",
+ "loop below expects (e.g., everything converted to tensors, classes converted to one-hot representations, etc.)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "wIAT6ZUmdHOC"
+ },
+ "outputs": [],
+ "source": [
+ "gt_boxes = [\n",
+ " np.array([[0.436, 0.591, 0.629, 0.712]], dtype=np.float32),\n",
+ " np.array([[0.539, 0.583, 0.73, 0.71]], dtype=np.float32),\n",
+ " np.array([[0.464, 0.414, 0.626, 0.548]], dtype=np.float32),\n",
+ " np.array([[0.313, 0.308, 0.648, 0.526]], dtype=np.float32),\n",
+ " np.array([[0.256, 0.444, 0.484, 0.629]], dtype=np.float32)\n",
+ "]\n",
+ "\n",
+ "# By convention, our non-background classes start counting at 1. Given\n",
+ "# that we will be predicting just one class, we will therefore assign it a\n",
+ "# `class id` of 1.\n",
+ "duck_class_id = 1\n",
+ "num_classes = 1\n",
+ "\n",
+ "category_index = {duck_class_id: {'id': duck_class_id, 'name': 'rubber_ducky'}}\n",
+ "\n",
+ "# Convert class labels to one-hot; convert everything to tensors.\n",
+ "# The `label_id_offset` here shifts all classes by a certain number of indices;\n",
+ "# we do this here so that the model receives one-hot labels where non-background\n",
+ "# classes start counting at the zeroth index. This is ordinarily just handled\n",
+ "# automatically in our training binaries, but we need to reproduce it here.\n",
+ "label_id_offset = 1\n",
+ "train_image_tensors = []\n",
+ "gt_classes_one_hot_tensors = []\n",
+ "gt_box_tensors = []\n",
+ "for (train_image_np, gt_box_np) in zip(\n",
+ " train_images_np, gt_boxes):\n",
+ " train_image_tensors.append(tf.expand_dims(tf.convert_to_tensor(\n",
+ " train_image_np, dtype=tf.float32), axis=0))\n",
+ " gt_box_tensors.append(tf.convert_to_tensor(gt_box_np, dtype=tf.float32))\n",
+ " zero_indexed_groundtruth_classes = tf.convert_to_tensor(\n",
+ " np.ones(shape=[gt_box_np.shape[0]], dtype=np.int32) - label_id_offset)\n",
+ " gt_classes_one_hot_tensors.append(tf.one_hot(\n",
+ " zero_indexed_groundtruth_classes, num_classes))\n",
+ "print('Done prepping data.')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "b3_Z3mJWN9KJ"
+ },
+ "source": [
+ "Let's just visualize the rubber duckies as a sanity check\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "YBD6l-E4N71y"
+ },
+ "outputs": [],
+ "source": [
+ "dummy_scores = np.array([1.0], dtype=np.float32) # give boxes a score of 100%\n",
+ "\n",
+ "plt.figure(figsize=(30, 15))\n",
+ "for idx in range(5):\n",
+ " plt.subplot(2, 3, idx+1)\n",
+ " plot_detections(\n",
+ " train_images_np[idx],\n",
+ " gt_boxes[idx],\n",
+ " np.ones(shape=[gt_boxes[idx].shape[0]], dtype=np.int32),\n",
+ " dummy_scores, category_index)\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ghDAsqfoZvPh"
+ },
+ "source": [
+ "## Load mobile-friendly model\n",
+ "\n",
+ "In this cell we build a mobile-friendly single-stage detection architecture (SSD MobileNet V2 FPN-Lite) and restore all but the classification layer at the top (which will be randomly initialized).\n",
+ "\n",
+ "**NOTE**: TensorFlow Lite only supports SSD models for now.\n",
+ "\n",
+ "For simplicity, we have hardcoded a number of things in this colab for the specific SSD architecture at hand (including assuming that the image size will always be 320x320), however it is not difficult to generalize to other model configurations (`pipeline.config` in the zip downloaded from the [Model Zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.)).\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "9J16r3NChD-7"
+ },
+ "outputs": [],
+ "source": [
+ "# Download the checkpoint and put it into models/research/object_detection/test_data/\n",
+ "\n",
+ "!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8.tar.gz\n",
+ "!tar -xf ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8.tar.gz\n",
+ "!if [ -d \"models/research/object_detection/test_data/checkpoint\" ]; then rm -Rf models/research/object_detection/test_data/checkpoint; fi\n",
+ "!mkdir models/research/object_detection/test_data/checkpoint\n",
+ "!mv ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8/checkpoint models/research/object_detection/test_data/"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "RyT4BUbaMeG-"
+ },
+ "outputs": [],
+ "source": [
+ "tf.keras.backend.clear_session()\n",
+ "\n",
+ "print('Building model and restoring weights for fine-tuning...', flush=True)\n",
+ "num_classes = 1\n",
+ "pipeline_config = 'models/research/object_detection/configs/tf2/ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8.config'\n",
+ "checkpoint_path = 'models/research/object_detection/test_data/checkpoint/ckpt-0'\n",
+ "\n",
+ "# This will be where we save checkpoint \u0026 config for TFLite conversion later.\n",
+ "output_directory = 'output/'\n",
+ "output_checkpoint_dir = os.path.join(output_directory, 'checkpoint')\n",
+ "\n",
+ "# Load pipeline config and build a detection model.\n",
+ "#\n",
+ "# Since we are working off of a COCO architecture which predicts 90\n",
+ "# class slots by default, we override the `num_classes` field here to be just\n",
+ "# one (for our new rubber ducky class).\n",
+ "configs = config_util.get_configs_from_pipeline_file(pipeline_config)\n",
+ "model_config = configs['model']\n",
+ "model_config.ssd.num_classes = num_classes\n",
+ "model_config.ssd.freeze_batchnorm = True\n",
+ "detection_model = model_builder.build(\n",
+ " model_config=model_config, is_training=True)\n",
+ "# Save new pipeline config\n",
+ "pipeline_proto = config_util.create_pipeline_proto_from_configs(configs)\n",
+ "config_util.save_pipeline_config(pipeline_proto, output_directory)\n",
+ "\n",
+ "# Set up object-based checkpoint restore --- SSD has two prediction\n",
+ "# `heads` --- one for classification, the other for box regression. We will\n",
+ "# restore the box regression head but initialize the classification head\n",
+ "# from scratch (we show the omission below by commenting out the line that\n",
+ "# we would add if we wanted to restore both heads)\n",
+ "fake_box_predictor = tf.compat.v2.train.Checkpoint(\n",
+ " _base_tower_layers_for_heads=detection_model._box_predictor._base_tower_layers_for_heads,\n",
+ " # _prediction_heads=detection_model._box_predictor._prediction_heads,\n",
+ " # (i.e., the classification head that we *will not* restore)\n",
+ " _box_prediction_head=detection_model._box_predictor._box_prediction_head,\n",
+ " )\n",
+ "fake_model = tf.compat.v2.train.Checkpoint(\n",
+ " _feature_extractor=detection_model._feature_extractor,\n",
+ " _box_predictor=fake_box_predictor)\n",
+ "ckpt = tf.compat.v2.train.Checkpoint(model=fake_model)\n",
+ "ckpt.restore(checkpoint_path).expect_partial()\n",
+ "\n",
+ "# To save checkpoint for TFLite conversion.\n",
+ "exported_ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)\n",
+ "ckpt_manager = tf.train.CheckpointManager(\n",
+ " exported_ckpt, output_checkpoint_dir, max_to_keep=1)\n",
+ "\n",
+ "# Run model through a dummy image so that variables are created\n",
+ "image, shapes = detection_model.preprocess(tf.zeros([1, 320, 320, 3]))\n",
+ "prediction_dict = detection_model.predict(image, shapes)\n",
+ "_ = detection_model.postprocess(prediction_dict, shapes)\n",
+ "print('Weights restored!')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "pCkWmdoZZ0zJ"
+ },
+ "source": [
+ "## Eager training loop (Fine-tuning)\n",
+ "\n",
+ "Some of the parameters in this block have been set empirically: for example, `learning_rate`, `num_batches` \u0026 `momentum` for SGD. These are just a starting point, you will have to tune these for your data \u0026 model architecture to get the best results.\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "nyHoF4mUrv5-"
+ },
+ "outputs": [],
+ "source": [
+ "tf.keras.backend.set_learning_phase(True)\n",
+ "\n",
+ "# These parameters can be tuned; since our training set has 5 images\n",
+ "# it doesn't make sense to have a much larger batch size, though we could\n",
+ "# fit more examples in memory if we wanted to.\n",
+ "batch_size = 5\n",
+ "learning_rate = 0.15\n",
+ "num_batches = 1000\n",
+ "\n",
+ "# Select variables in top layers to fine-tune.\n",
+ "trainable_variables = detection_model.trainable_variables\n",
+ "to_fine_tune = []\n",
+ "prefixes_to_train = [\n",
+ " 'WeightSharedConvolutionalBoxPredictor/WeightSharedConvolutionalBoxHead',\n",
+ " 'WeightSharedConvolutionalBoxPredictor/WeightSharedConvolutionalClassHead']\n",
+ "for var in trainable_variables:\n",
+ " if any([var.name.startswith(prefix) for prefix in prefixes_to_train]):\n",
+ " to_fine_tune.append(var)\n",
+ "\n",
+ "# Set up forward + backward pass for a single train step.\n",
+ "def get_model_train_step_function(model, optimizer, vars_to_fine_tune):\n",
+ " \"\"\"Get a tf.function for training step.\"\"\"\n",
+ "\n",
+ " # Use tf.function for a bit of speed.\n",
+ " # Comment out the tf.function decorator if you want the inside of the\n",
+ " # function to run eagerly.\n",
+ " @tf.function\n",
+ " def train_step_fn(image_tensors,\n",
+ " groundtruth_boxes_list,\n",
+ " groundtruth_classes_list):\n",
+ " \"\"\"A single training iteration.\n",
+ "\n",
+ " Args:\n",
+ " image_tensors: A list of [1, height, width, 3] Tensor of type tf.float32.\n",
+ " Note that the height and width can vary across images, as they are\n",
+ " reshaped within this function to be 320x320.\n",
+ " groundtruth_boxes_list: A list of Tensors of shape [N_i, 4] with type\n",
+ " tf.float32 representing groundtruth boxes for each image in the batch.\n",
+ " groundtruth_classes_list: A list of Tensors of shape [N_i, num_classes]\n",
+ " with type tf.float32 representing groundtruth boxes for each image in\n",
+ " the batch.\n",
+ "\n",
+ " Returns:\n",
+ " A scalar tensor representing the total loss for the input batch.\n",
+ " \"\"\"\n",
+ " shapes = tf.constant(batch_size * [[320, 320, 3]], dtype=tf.int32)\n",
+ " model.provide_groundtruth(\n",
+ " groundtruth_boxes_list=groundtruth_boxes_list,\n",
+ " groundtruth_classes_list=groundtruth_classes_list)\n",
+ " with tf.GradientTape() as tape:\n",
+ " preprocessed_images = tf.concat(\n",
+ " [detection_model.preprocess(image_tensor)[0]\n",
+ " for image_tensor in image_tensors], axis=0)\n",
+ " prediction_dict = model.predict(preprocessed_images, shapes)\n",
+ " losses_dict = model.loss(prediction_dict, shapes)\n",
+ " total_loss = losses_dict['Loss/localization_loss'] + losses_dict['Loss/classification_loss']\n",
+ " gradients = tape.gradient(total_loss, vars_to_fine_tune)\n",
+ " optimizer.apply_gradients(zip(gradients, vars_to_fine_tune))\n",
+ " return total_loss\n",
+ "\n",
+ " return train_step_fn\n",
+ "\n",
+ "optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate, momentum=0.9)\n",
+ "train_step_fn = get_model_train_step_function(\n",
+ " detection_model, optimizer, to_fine_tune)\n",
+ "\n",
+ "print('Start fine-tuning!', flush=True)\n",
+ "for idx in range(num_batches):\n",
+ " # Grab keys for a random subset of examples\n",
+ " all_keys = list(range(len(train_images_np)))\n",
+ " random.shuffle(all_keys)\n",
+ " example_keys = all_keys[:batch_size]\n",
+ "\n",
+ " # Note that we do not do data augmentation in this demo. If you want a\n",
+ " # a fun exercise, we recommend experimenting with random horizontal flipping\n",
+ " # and random cropping :)\n",
+ " gt_boxes_list = [gt_box_tensors[key] for key in example_keys]\n",
+ " gt_classes_list = [gt_classes_one_hot_tensors[key] for key in example_keys]\n",
+ " image_tensors = [train_image_tensors[key] for key in example_keys]\n",
+ "\n",
+ " # Training step (forward pass + backwards pass)\n",
+ " total_loss = train_step_fn(image_tensors, gt_boxes_list, gt_classes_list)\n",
+ "\n",
+ " if idx % 100 == 0:\n",
+ " print('batch ' + str(idx) + ' of ' + str(num_batches)\n",
+ " + ', loss=' + str(total_loss.numpy()), flush=True)\n",
+ "\n",
+ "print('Done fine-tuning!')\n",
+ "\n",
+ "ckpt_manager.save()\n",
+ "print('Checkpoint saved!')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "cYk1_9Fc2lZO"
+ },
+ "source": [
+ "# Export \u0026 run with TensorFlow Lite\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "y0nsDVEd9SuX"
+ },
+ "source": [
+ "## Model Conversion\n",
+ "\n",
+ "First, we invoke the `export_tflite_graph_tf2.py` script to generate a TFLite-friendly intermediate SavedModel. This will then be passed to the TensorFlow Lite Converter for generating the final model.\n",
+ "\n",
+ "To know more about this process, please look at [this documentation](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/running_on_mobile_tf2.md)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "dyrqHSQQ7WKE"
+ },
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "python models/research/object_detection/export_tflite_graph_tf2.py \\\n",
+ " --pipeline_config_path output/pipeline.config \\\n",
+ " --trained_checkpoint_dir output/checkpoint \\\n",
+ " --output_directory tflite"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "m5hjPyR78bgs"
+ },
+ "outputs": [],
+ "source": [
+ "!tflite_convert --saved_model_dir=tflite/saved_model --output_file=tflite/model.tflite"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "WHlXL1x_Z3tc"
+ },
+ "source": [
+ "## Test .tflite model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "WcE6OwrHQJya"
+ },
+ "outputs": [],
+ "source": [
+ "test_image_dir = 'models/research/object_detection/test_images/ducky/test/'\n",
+ "test_images_np = []\n",
+ "for i in range(1, 50):\n",
+ " image_path = os.path.join(test_image_dir, 'out' + str(i) + '.jpg')\n",
+ " test_images_np.append(np.expand_dims(\n",
+ " load_image_into_numpy_array(image_path), axis=0))\n",
+ "\n",
+ "# Again, uncomment this decorator if you want to run inference eagerly\n",
+ "def detect(interpreter, input_tensor):\n",
+ " \"\"\"Run detection on an input image.\n",
+ "\n",
+ " Args:\n",
+ " interpreter: tf.lite.Interpreter\n",
+ " input_tensor: A [1, height, width, 3] Tensor of type tf.float32.\n",
+ " Note that height and width can be anything since the image will be\n",
+ " immediately resized according to the needs of the model within this\n",
+ " function.\n",
+ "\n",
+ " Returns:\n",
+ " A dict containing 3 Tensors (`detection_boxes`, `detection_classes`,\n",
+ " and `detection_scores`).\n",
+ " \"\"\"\n",
+ " input_details = interpreter.get_input_details()\n",
+ " output_details = interpreter.get_output_details()\n",
+ "\n",
+ " # We use the original model for pre-processing, since the TFLite model doesn't\n",
+ " # include pre-processing.\n",
+ " preprocessed_image, shapes = detection_model.preprocess(input_tensor)\n",
+ " interpreter.set_tensor(input_details[0]['index'], preprocessed_image.numpy())\n",
+ "\n",
+ " interpreter.invoke()\n",
+ "\n",
+ " boxes = interpreter.get_tensor(output_details[0]['index'])\n",
+ " classes = interpreter.get_tensor(output_details[1]['index'])\n",
+ " scores = interpreter.get_tensor(output_details[2]['index'])\n",
+ " return boxes, classes, scores\n",
+ "\n",
+ "# Load the TFLite model and allocate tensors.\n",
+ "interpreter = tf.lite.Interpreter(model_path=\"tflite/model.tflite\")\n",
+ "interpreter.allocate_tensors()\n",
+ "\n",
+ "# Note that the first frame will trigger tracing of the tf.function, which will\n",
+ "# take some time, after which inference should be fast.\n",
+ "\n",
+ "label_id_offset = 1\n",
+ "for i in range(len(test_images_np)):\n",
+ " input_tensor = tf.convert_to_tensor(test_images_np[i], dtype=tf.float32)\n",
+ " boxes, classes, scores = detect(interpreter, input_tensor)\n",
+ "\n",
+ " plot_detections(\n",
+ " test_images_np[i][0],\n",
+ " boxes[0],\n",
+ " classes[0].astype(np.uint32) + label_id_offset,\n",
+ " scores[0],\n",
+ " category_index, figsize=(15, 20), image_name=\"gif_frame_\" + ('%02d' % i) + \".jpg\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ZkMPOSQE0x8C"
+ },
+ "outputs": [],
+ "source": [
+ "imageio.plugins.freeimage.download()\n",
+ "\n",
+ "anim_file = 'duckies_test.gif'\n",
+ "\n",
+ "filenames = glob.glob('gif_frame_*.jpg')\n",
+ "filenames = sorted(filenames)\n",
+ "last = -1\n",
+ "images = []\n",
+ "for filename in filenames:\n",
+ " image = imageio.imread(filename)\n",
+ " images.append(image)\n",
+ "\n",
+ "imageio.mimsave(anim_file, images, 'GIF-FI', fps=5)\n",
+ "\n",
+ "display(IPyImage(open(anim_file, 'rb').read()))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "yzaHWsS58_PQ"
+ },
+ "source": [
+ "## (Optional) Download model\n",
+ "\n",
+ "This model can be run on-device with **TensorFlow Lite**. Look at [our SSD model signature](https://www.tensorflow.org/lite/models/object_detection/overview#uses_and_limitations) to understand how to interpret the model IO tensors. Our [Object Detection example](https://github.com/tensorflow/examples/tree/master/lite/examples/object_detection) is a good starting point for integrating the model into your mobile app.\n",
+ "\n",
+ "Refer to TFLite's [inference documentation](https://www.tensorflow.org/lite/guide/inference) for more details."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "gZ6vac3RAY3j"
+ },
+ "outputs": [],
+ "source": [
+ "from google.colab import files\n",
+ "files.download('tflite/model.tflite') "
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "collapsed_sections": [],
+ "name": "eager_few_shot_od_training_tflite.ipynb",
+ "provenance": [],
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/research/object_detection/colab_tutorials/inference_from_saved_model_tf2_colab.ipynb b/research/object_detection/colab_tutorials/inference_from_saved_model_tf2_colab.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..1e88f4c5d52435297bd2ba8c8bb47a2f3c346f30
--- /dev/null
+++ b/research/object_detection/colab_tutorials/inference_from_saved_model_tf2_colab.ipynb
@@ -0,0 +1,313 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "name": "inference_from_saved_model_tf2_colab.ipynb",
+ "provenance": [],
+ "collapsed_sections": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "cT5cdSLPX0ui"
+ },
+ "source": [
+ "# Intro to Object Detection Colab\n",
+ "\n",
+ "Welcome to the object detection colab! This demo will take you through the steps of running an \"out-of-the-box\" detection model in SavedModel format on a collection of images.\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "vPs64QA1Zdov"
+ },
+ "source": [
+ "Imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "OBzb04bdNGM8",
+ "colab_type": "code",
+ "colab": {}
+ },
+ "source": [
+ "!pip install -U --pre tensorflow==\"2.2.0\""
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "NgSXyvKSNHIl",
+ "colab_type": "code",
+ "colab": {}
+ },
+ "source": [
+ "import os\n",
+ "import pathlib\n",
+ "\n",
+ "# Clone the tensorflow models repository if it doesn't already exist\n",
+ "if \"models\" in pathlib.Path.cwd().parts:\n",
+ " while \"models\" in pathlib.Path.cwd().parts:\n",
+ " os.chdir('..')\n",
+ "elif not pathlib.Path('models').exists():\n",
+ " !git clone --depth 1 https://github.com/tensorflow/models"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "rhpPgW7TNLs6",
+ "colab_type": "code",
+ "colab": {}
+ },
+ "source": [
+ "# Install the Object Detection API\n",
+ "%%bash\n",
+ "cd models/research/\n",
+ "protoc object_detection/protos/*.proto --python_out=.\n",
+ "cp object_detection/packages/tf2/setup.py .\n",
+ "python -m pip install ."
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab_type": "code",
+ "id": "yn5_uV1HLvaz",
+ "colab": {}
+ },
+ "source": [
+ "import io\n",
+ "import os\n",
+ "import scipy.misc\n",
+ "import numpy as np\n",
+ "import six\n",
+ "import time\n",
+ "\n",
+ "from six import BytesIO\n",
+ "\n",
+ "import matplotlib\n",
+ "import matplotlib.pyplot as plt\n",
+ "from PIL import Image, ImageDraw, ImageFont\n",
+ "\n",
+ "import tensorflow as tf\n",
+ "from object_detection.utils import visualization_utils as viz_utils\n",
+ "\n",
+ "%matplotlib inline"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab_type": "code",
+ "id": "-y9R0Xllefec",
+ "colab": {}
+ },
+ "source": [
+ "def load_image_into_numpy_array(path):\n",
+ " \"\"\"Load an image from file into a numpy array.\n",
+ "\n",
+ " Puts image into numpy array to feed into tensorflow graph.\n",
+ " Note that by convention we put it into a numpy array with shape\n",
+ " (height, width, channels), where channels=3 for RGB.\n",
+ "\n",
+ " Args:\n",
+ " path: a file path (this can be local or on colossus)\n",
+ "\n",
+ " Returns:\n",
+ " uint8 numpy array with shape (img_height, img_width, 3)\n",
+ " \"\"\"\n",
+ " img_data = tf.io.gfile.GFile(path, 'rb').read()\n",
+ " image = Image.open(BytesIO(img_data))\n",
+ " (im_width, im_height) = image.size\n",
+ " return np.array(image.getdata()).reshape(\n",
+ " (im_height, im_width, 3)).astype(np.uint8)\n",
+ "\n",
+ "# Load the COCO Label Map\n",
+ "category_index = {\n",
+ " 1: {'id': 1, 'name': 'person'},\n",
+ " 2: {'id': 2, 'name': 'bicycle'},\n",
+ " 3: {'id': 3, 'name': 'car'},\n",
+ " 4: {'id': 4, 'name': 'motorcycle'},\n",
+ " 5: {'id': 5, 'name': 'airplane'},\n",
+ " 6: {'id': 6, 'name': 'bus'},\n",
+ " 7: {'id': 7, 'name': 'train'},\n",
+ " 8: {'id': 8, 'name': 'truck'},\n",
+ " 9: {'id': 9, 'name': 'boat'},\n",
+ " 10: {'id': 10, 'name': 'traffic light'},\n",
+ " 11: {'id': 11, 'name': 'fire hydrant'},\n",
+ " 13: {'id': 13, 'name': 'stop sign'},\n",
+ " 14: {'id': 14, 'name': 'parking meter'},\n",
+ " 15: {'id': 15, 'name': 'bench'},\n",
+ " 16: {'id': 16, 'name': 'bird'},\n",
+ " 17: {'id': 17, 'name': 'cat'},\n",
+ " 18: {'id': 18, 'name': 'dog'},\n",
+ " 19: {'id': 19, 'name': 'horse'},\n",
+ " 20: {'id': 20, 'name': 'sheep'},\n",
+ " 21: {'id': 21, 'name': 'cow'},\n",
+ " 22: {'id': 22, 'name': 'elephant'},\n",
+ " 23: {'id': 23, 'name': 'bear'},\n",
+ " 24: {'id': 24, 'name': 'zebra'},\n",
+ " 25: {'id': 25, 'name': 'giraffe'},\n",
+ " 27: {'id': 27, 'name': 'backpack'},\n",
+ " 28: {'id': 28, 'name': 'umbrella'},\n",
+ " 31: {'id': 31, 'name': 'handbag'},\n",
+ " 32: {'id': 32, 'name': 'tie'},\n",
+ " 33: {'id': 33, 'name': 'suitcase'},\n",
+ " 34: {'id': 34, 'name': 'frisbee'},\n",
+ " 35: {'id': 35, 'name': 'skis'},\n",
+ " 36: {'id': 36, 'name': 'snowboard'},\n",
+ " 37: {'id': 37, 'name': 'sports ball'},\n",
+ " 38: {'id': 38, 'name': 'kite'},\n",
+ " 39: {'id': 39, 'name': 'baseball bat'},\n",
+ " 40: {'id': 40, 'name': 'baseball glove'},\n",
+ " 41: {'id': 41, 'name': 'skateboard'},\n",
+ " 42: {'id': 42, 'name': 'surfboard'},\n",
+ " 43: {'id': 43, 'name': 'tennis racket'},\n",
+ " 44: {'id': 44, 'name': 'bottle'},\n",
+ " 46: {'id': 46, 'name': 'wine glass'},\n",
+ " 47: {'id': 47, 'name': 'cup'},\n",
+ " 48: {'id': 48, 'name': 'fork'},\n",
+ " 49: {'id': 49, 'name': 'knife'},\n",
+ " 50: {'id': 50, 'name': 'spoon'},\n",
+ " 51: {'id': 51, 'name': 'bowl'},\n",
+ " 52: {'id': 52, 'name': 'banana'},\n",
+ " 53: {'id': 53, 'name': 'apple'},\n",
+ " 54: {'id': 54, 'name': 'sandwich'},\n",
+ " 55: {'id': 55, 'name': 'orange'},\n",
+ " 56: {'id': 56, 'name': 'broccoli'},\n",
+ " 57: {'id': 57, 'name': 'carrot'},\n",
+ " 58: {'id': 58, 'name': 'hot dog'},\n",
+ " 59: {'id': 59, 'name': 'pizza'},\n",
+ " 60: {'id': 60, 'name': 'donut'},\n",
+ " 61: {'id': 61, 'name': 'cake'},\n",
+ " 62: {'id': 62, 'name': 'chair'},\n",
+ " 63: {'id': 63, 'name': 'couch'},\n",
+ " 64: {'id': 64, 'name': 'potted plant'},\n",
+ " 65: {'id': 65, 'name': 'bed'},\n",
+ " 67: {'id': 67, 'name': 'dining table'},\n",
+ " 70: {'id': 70, 'name': 'toilet'},\n",
+ " 72: {'id': 72, 'name': 'tv'},\n",
+ " 73: {'id': 73, 'name': 'laptop'},\n",
+ " 74: {'id': 74, 'name': 'mouse'},\n",
+ " 75: {'id': 75, 'name': 'remote'},\n",
+ " 76: {'id': 76, 'name': 'keyboard'},\n",
+ " 77: {'id': 77, 'name': 'cell phone'},\n",
+ " 78: {'id': 78, 'name': 'microwave'},\n",
+ " 79: {'id': 79, 'name': 'oven'},\n",
+ " 80: {'id': 80, 'name': 'toaster'},\n",
+ " 81: {'id': 81, 'name': 'sink'},\n",
+ " 82: {'id': 82, 'name': 'refrigerator'},\n",
+ " 84: {'id': 84, 'name': 'book'},\n",
+ " 85: {'id': 85, 'name': 'clock'},\n",
+ " 86: {'id': 86, 'name': 'vase'},\n",
+ " 87: {'id': 87, 'name': 'scissors'},\n",
+ " 88: {'id': 88, 'name': 'teddy bear'},\n",
+ " 89: {'id': 89, 'name': 'hair drier'},\n",
+ " 90: {'id': 90, 'name': 'toothbrush'},\n",
+ "}"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "QwcBC2TlPSwg",
+ "colab_type": "code",
+ "colab": {}
+ },
+ "source": [
+ "# Download the saved model and put it into models/research/object_detection/test_data/\n",
+ "!wget http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d5_coco17_tpu-32.tar.gz\n",
+ "!tar -xf efficientdet_d5_coco17_tpu-32.tar.gz\n",
+ "!mv efficientdet_d5_coco17_tpu-32/ models/research/object_detection/test_data/"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab_type": "code",
+ "id": "Z2p-PmKLYCVU",
+ "colab": {}
+ },
+ "source": [
+ "start_time = time.time()\n",
+ "tf.keras.backend.clear_session()\n",
+ "detect_fn = tf.saved_model.load('models/research/object_detection/test_data/efficientdet_d5_coco17_tpu-32/saved_model/')\n",
+ "end_time = time.time()\n",
+ "elapsed_time = end_time - start_time\n",
+ "print('Elapsed time: ' + str(elapsed_time) + 's')"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab_type": "code",
+ "id": "vukkhd5-9NSL",
+ "colab": {}
+ },
+ "source": [
+ "import time\n",
+ "\n",
+ "image_dir = 'models/research/object_detection/test_images'\n",
+ "\n",
+ "elapsed = []\n",
+ "for i in range(2):\n",
+ " image_path = os.path.join(image_dir, 'image' + str(i + 1) + '.jpg')\n",
+ " image_np = load_image_into_numpy_array(image_path)\n",
+ " input_tensor = np.expand_dims(image_np, 0)\n",
+ " start_time = time.time()\n",
+ " detections = detect_fn(input_tensor)\n",
+ " end_time = time.time()\n",
+ " elapsed.append(end_time - start_time)\n",
+ "\n",
+ " plt.rcParams['figure.figsize'] = [42, 21]\n",
+ " label_id_offset = 1\n",
+ " image_np_with_detections = image_np.copy()\n",
+ " viz_utils.visualize_boxes_and_labels_on_image_array(\n",
+ " image_np_with_detections,\n",
+ " detections['detection_boxes'][0].numpy(),\n",
+ " detections['detection_classes'][0].numpy().astype(np.int32),\n",
+ " detections['detection_scores'][0].numpy(),\n",
+ " category_index,\n",
+ " use_normalized_coordinates=True,\n",
+ " max_boxes_to_draw=200,\n",
+ " min_score_thresh=.40,\n",
+ " agnostic_mode=False)\n",
+ " plt.subplot(2, 1, i+1)\n",
+ " plt.imshow(image_np_with_detections)\n",
+ "\n",
+ "mean_elapsed = sum(elapsed) / float(len(elapsed))\n",
+ "print('Elapsed time: ' + str(mean_elapsed) + ' second per image')"
+ ],
+ "execution_count": null,
+ "outputs": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb b/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb
index b4eac1ade9a65bf3dd1c8ba584d368afcfe9003b..0dd1253207f26f31aa7a23471399553574b6eae3 100644
--- a/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb
+++ b/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb
@@ -10,11 +10,11 @@
"# Object Detection API Demo\n",
"\n",
"\u003ctable align=\"left\"\u003e\u003ctd\u003e\n",
- " \u003ca target=\"_blank\" href=\"https://colab.sandbox.google.com/github/tensorflow/models/blob/master/research/object_detection/colab_tutorials/colab_tutorials/object_detection_tutorial.ipynb\"\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://colab.sandbox.google.com/github/tensorflow/models/blob/master/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb\"\u003e\n",
" \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\n",
" \u003c/a\u003e\n",
"\u003c/td\u003e\u003ctd\u003e\n",
- " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/models/blob/master/research/object_detection/colab_tutorials/colab_tutorials/object_detection_tutorial.ipynb\"\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/models/blob/master/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb\"\u003e\n",
" \u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
"\u003c/td\u003e\u003c/table\u003e"
]
diff --git a/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_coco_tpu-128.config b/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_coco_tpu-128.config
new file mode 100644
index 0000000000000000000000000000000000000000..a97db04ede0bc751c8b58fe99ec5bd0943f22ffc
--- /dev/null
+++ b/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_coco_tpu-128.config
@@ -0,0 +1,210 @@
+# DeepMAC meta architecture from the "The surprising impact of mask-head
+# architecture on novel class segmentation" [1] paper with an Hourglass-100[2]
+# mask head. This config is trained on all COCO classes and achieves a
+# mask mAP of 39.4% on the COCO testdev-2017 set.
+# [1]: https://arxiv.org/abs/2104.00613
+# [2]: https://arxiv.org/abs/1904.07850
+
+# Train on TPU-128
+
+model {
+ center_net {
+ num_classes: 90
+ feature_extractor {
+ type: "hourglass_104"
+ bgr_ordering: true
+ channel_means: [104.01362025, 114.03422265, 119.9165958 ]
+ channel_stds: [73.6027665 , 69.89082075, 70.9150767 ]
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 1024
+ max_dimension: 1024
+ pad_to_max_dimension: true
+ }
+ }
+ object_detection_task {
+ task_loss_weight: 1.0
+ offset_loss_weight: 1.0
+ scale_loss_weight: 0.1
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ object_center_loss_weight: 1.0
+ min_box_overlap_iou: 0.7
+ max_box_predictions: 100
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+
+ deepmac_mask_estimation {
+ dim: 32
+ task_loss_weight: 5.0
+ pixel_embedding_dim: 16
+ mask_size: 32
+ use_xy: true
+ use_instance_embedding: true
+ network_type: "hourglass100"
+ classification_loss {
+ weighted_sigmoid {}
+ }
+ }
+ }
+}
+
+train_config: {
+
+ batch_size: 128
+ num_steps: 50000
+
+ data_augmentation_options {
+ random_horizontal_flip {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_hue {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_contrast {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_saturation {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_brightness {
+ }
+ }
+
+ data_augmentation_options {
+ random_square_crop_by_scale {
+ scale_min: 0.6
+ scale_max: 1.3
+ }
+ }
+
+ optimizer {
+ adam_optimizer: {
+ epsilon: 1e-7 # Match tf.keras.optimizers.Adam's default.
+ learning_rate: {
+ cosine_decay_learning_rate {
+ learning_rate_base: 1e-3
+ total_steps: 50000
+ warmup_learning_rate: 2.5e-4
+ warmup_steps: 5000
+ }
+ }
+ }
+ use_moving_average: false
+ }
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+
+ fine_tune_checkpoint_version: V2
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED/ckpt-51"
+ fine_tune_checkpoint_type: "fine_tune"
+}
+
+train_input_reader: {
+ load_instance_masks: true
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ mask_type: PNG_MASKS
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+}
+
+eval_config: {
+ metrics_set: "coco_detection_metrics"
+ metrics_set: "coco_mask_metrics"
+ include_metrics_per_category: true
+ use_moving_averages: false
+ batch_size: 1;
+ super_categories {
+ key: "VOC"
+ value: "person,bicycle,car,motorcycle,airplane,bus,train,boat,bird,cat,"
+ "dog,horse,sheep,cow,bottle,chair,couch,potted plant,dining table,tv"
+ }
+ super_categories {
+ key: "NonVOC"
+ value: "truck,traffic light,fire hydrant,stop sign,parking meter,bench,"
+ "elephant,bear,zebra,giraffe,backpack,umbrella,handbag,tie,suitcase,"
+ "frisbee,skis,snowboard,sports ball,kite,baseball bat,baseball glove,"
+ "skateboard,surfboard,tennis racket,wine glass,cup,fork,knife,spoon,bowl,"
+ "banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,cake,bed,"
+ "toilet,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,"
+ "sink,refrigerator,book,clock,vase,scissors,teddy bear,hair drier,"
+ "toothbrush"
+ }
+ super_categories {
+ key: "person"
+ value: "person"
+ }
+ super_categories {
+ key: "vehicle"
+ value: "bicycle,car,motorcycle,airplane,bus,train,truck,boat"
+ }
+ super_categories {
+ key: "outdoor"
+ value: "traffic light,fire hydrant,stop sign,parking meter,bench"
+ }
+ super_categories {
+ key: "animal"
+ value: "bird,cat,dog,horse,sheep,cow,elephant,bear,zebra,giraffe"
+ }
+ super_categories {
+ key: "accessory"
+ value: "backpack,umbrella,handbag,tie,suitcase"
+ }
+ super_categories {
+ key: "sports"
+ value: "frisbee,skis,snowboard,sports ball,kite,baseball bat,"
+ "baseball glove,skateboard,surfboard,tennis racket"
+ }
+ super_categories {
+ key: "kitchen"
+ value: "bottle,wine glass,cup,fork,knife,spoon,bowl"
+ }
+ super_categories {
+ key: "food"
+ value: "banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,"
+ "cake"
+ }
+ super_categories {
+ key: "furniture"
+ value: "chair,couch,potted plant,bed,dining table,toilet"
+ }
+ super_categories {
+ key: "electronic"
+ value: "tv,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,"
+ "sink,refrigerator"
+ }
+ super_categories {
+ key: "indoor"
+ value: "book,clock,vase,scissors,teddy bear,hair drier,toothbrush"
+ }
+}
+
+eval_input_reader: {
+ load_instance_masks: true
+ mask_type: PNG_MASKS
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ shuffle: false
+ num_epochs: 1
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/val2017-?????-of-00032.tfrecord"
+ }
+}
diff --git a/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_non_voc_only_tpu-128.config b/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_non_voc_only_tpu-128.config
new file mode 100644
index 0000000000000000000000000000000000000000..8262c444e2eecd4982a2bc1ee81edf18fc81ba99
--- /dev/null
+++ b/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_non_voc_only_tpu-128.config
@@ -0,0 +1,273 @@
+# DeepMAC meta architecture from the "The surprising impact of mask-head
+# architecture on novel class segmentation" [1] paper with an Hourglass-100[2]
+# mask head. This config is trained on masks from the non-VOC classes and
+# achieves a mask mAP of 39.1% on the VOC classes.
+# [1]: https://arxiv.org/abs/2104.00613
+# [2]: https://arxiv.org/abs/1904.07850
+
+# Train on TPU-128
+
+model {
+ center_net {
+ num_classes: 90
+ feature_extractor {
+ type: "hourglass_104"
+ bgr_ordering: true
+ channel_means: [104.01362025, 114.03422265, 119.9165958 ]
+ channel_stds: [73.6027665 , 69.89082075, 70.9150767 ]
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 1024
+ max_dimension: 1024
+ pad_to_max_dimension: true
+ }
+ }
+ object_detection_task {
+ task_loss_weight: 1.0
+ offset_loss_weight: 1.0
+ scale_loss_weight: 0.1
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ object_center_loss_weight: 1.0
+ min_box_overlap_iou: 0.7
+ max_box_predictions: 100
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+
+ deepmac_mask_estimation {
+ dim: 32
+ task_loss_weight: 5.0
+ pixel_embedding_dim: 16
+ mask_size: 32
+ use_xy: true
+ use_instance_embedding: true
+ network_type: "hourglass100"
+ classification_loss {
+ weighted_sigmoid {}
+ }
+
+ allowed_masked_classes_ids: [
+ 8,
+ 10,
+ 11,
+ 13,
+ 14,
+ 15,
+ 22,
+ 23,
+ 24,
+ 25,
+ 27,
+ 28,
+ 31,
+ 32,
+ 33,
+ 34,
+ 35,
+ 36,
+ 37,
+ 38,
+ 39,
+ 40,
+ 41,
+ 42,
+ 43,
+ 46,
+ 47,
+ 48,
+ 49,
+ 50,
+ 51,
+ 52,
+ 53,
+ 54,
+ 55,
+ 56,
+ 57,
+ 58,
+ 59,
+ 60,
+ 61,
+ 65,
+ 70,
+ 73,
+ 74,
+ 75,
+ 76,
+ 77,
+ 78,
+ 79,
+ 80,
+ 81,
+ 82,
+ 84,
+ 85,
+ 86,
+ 87,
+ 88,
+ 89,
+ 90
+ ]
+ }
+ }
+}
+
+train_config: {
+
+ batch_size: 128
+ num_steps: 50000
+
+ data_augmentation_options {
+ random_horizontal_flip {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_hue {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_contrast {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_saturation {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_brightness {
+ }
+ }
+
+ data_augmentation_options {
+ random_square_crop_by_scale {
+ scale_min: 0.6
+ scale_max: 1.3
+ }
+ }
+
+ optimizer {
+ adam_optimizer: {
+ epsilon: 1e-7 # Match tf.keras.optimizers.Adam's default.
+ learning_rate: {
+ cosine_decay_learning_rate {
+ learning_rate_base: 1e-3
+ total_steps: 50000
+ warmup_learning_rate: 2.5e-4
+ warmup_steps: 5000
+ }
+ }
+ }
+ use_moving_average: false
+ }
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+
+ fine_tune_checkpoint_version: V2
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED/ckpt-51"
+ fine_tune_checkpoint_type: "fine_tune"
+}
+
+train_input_reader: {
+ load_instance_masks: true
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ mask_type: PNG_MASKS
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+}
+
+eval_config: {
+ metrics_set: "coco_detection_metrics"
+ metrics_set: "coco_mask_metrics"
+ include_metrics_per_category: true
+ use_moving_averages: false
+ batch_size: 1;
+ super_categories {
+ key: "VOC"
+ value: "person,bicycle,car,motorcycle,airplane,bus,train,boat,bird,cat,"
+ "dog,horse,sheep,cow,bottle,chair,couch,potted plant,dining table,tv"
+ }
+ super_categories {
+ key: "NonVOC"
+ value: "truck,traffic light,fire hydrant,stop sign,parking meter,bench,"
+ "elephant,bear,zebra,giraffe,backpack,umbrella,handbag,tie,suitcase,"
+ "frisbee,skis,snowboard,sports ball,kite,baseball bat,baseball glove,"
+ "skateboard,surfboard,tennis racket,wine glass,cup,fork,knife,spoon,bowl,"
+ "banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,cake,bed,"
+ "toilet,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,"
+ "sink,refrigerator,book,clock,vase,scissors,teddy bear,hair drier,"
+ "toothbrush"
+ }
+ super_categories {
+ key: "person"
+ value: "person"
+ }
+ super_categories {
+ key: "vehicle"
+ value: "bicycle,car,motorcycle,airplane,bus,train,truck,boat"
+ }
+ super_categories {
+ key: "outdoor"
+ value: "traffic light,fire hydrant,stop sign,parking meter,bench"
+ }
+ super_categories {
+ key: "animal"
+ value: "bird,cat,dog,horse,sheep,cow,elephant,bear,zebra,giraffe"
+ }
+ super_categories {
+ key: "accessory"
+ value: "backpack,umbrella,handbag,tie,suitcase"
+ }
+ super_categories {
+ key: "sports"
+ value: "frisbee,skis,snowboard,sports ball,kite,baseball bat,"
+ "baseball glove,skateboard,surfboard,tennis racket"
+ }
+ super_categories {
+ key: "kitchen"
+ value: "bottle,wine glass,cup,fork,knife,spoon,bowl"
+ }
+ super_categories {
+ key: "food"
+ value: "banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,"
+ "cake"
+ }
+ super_categories {
+ key: "furniture"
+ value: "chair,couch,potted plant,bed,dining table,toilet"
+ }
+ super_categories {
+ key: "electronic"
+ value: "tv,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,"
+ "sink,refrigerator"
+ }
+ super_categories {
+ key: "indoor"
+ value: "book,clock,vase,scissors,teddy bear,hair drier,toothbrush"
+ }
+}
+
+eval_input_reader: {
+ load_instance_masks: true
+ mask_type: PNG_MASKS
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ shuffle: false
+ num_epochs: 1
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/val2017-?????-of-00032.tfrecord"
+ }
+}
diff --git a/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_voc_only_tpu-128.config b/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_voc_only_tpu-128.config
new file mode 100644
index 0000000000000000000000000000000000000000..f2c7f7c3bcd1af9d65920c21e8df295af10b2875
--- /dev/null
+++ b/research/object_detection/configs/tf2/center_net_deepmac_1024x1024_voc_only_tpu-128.config
@@ -0,0 +1,234 @@
+# DeepMAC meta architecture from the "The surprising impact of mask-head
+# architecture on novel class segmentation" [1] paper with an Hourglass-100[2]
+# mask head. This config is only trained on masks from the VOC classes in COCO
+# and achieves a mask mAP of 35.5% on non-VOC classes.
+# [1]: https://arxiv.org/abs/2104.00613
+# [2]: https://arxiv.org/abs/1904.07850
+
+# Train on TPU-128
+
+model {
+ center_net {
+ num_classes: 90
+ feature_extractor {
+ type: "hourglass_104"
+ bgr_ordering: true
+ channel_means: [104.01362025, 114.03422265, 119.9165958 ]
+ channel_stds: [73.6027665 , 69.89082075, 70.9150767 ]
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 1024
+ max_dimension: 1024
+ pad_to_max_dimension: true
+ }
+ }
+ object_detection_task {
+ task_loss_weight: 1.0
+ offset_loss_weight: 1.0
+ scale_loss_weight: 0.1
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ object_center_loss_weight: 1.0
+ min_box_overlap_iou: 0.7
+ max_box_predictions: 100
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+
+ deepmac_mask_estimation {
+ dim: 32
+ task_loss_weight: 5.0
+ pixel_embedding_dim: 16
+ mask_size: 32
+ use_xy: true
+ use_instance_embedding: true
+ network_type: "hourglass100"
+ classification_loss {
+ weighted_sigmoid {}
+ }
+
+ allowed_masked_classes_ids: [
+ 1, # person
+ 2, # bicycle
+ 3, # car
+ 4, # motorcycle/motorbike
+ 5, # airplane/aeroplane,
+ 6, # bus
+ 7, # train
+ 9, # boat
+ 16, # bird
+ 17, # cat
+ 18, # dog
+ 19, # horse
+ 20, # sheep
+ 21, # cow
+ 44, # bottle
+ 62, # chair
+ 63, # couch/sofa
+ 64, # potted plant
+ 67, # dining table
+ 72 # tvmonitor
+ ]
+ }
+ }
+}
+
+train_config: {
+
+ batch_size: 128
+ num_steps: 50000
+
+ data_augmentation_options {
+ random_horizontal_flip {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_hue {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_contrast {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_saturation {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_brightness {
+ }
+ }
+
+ data_augmentation_options {
+ random_square_crop_by_scale {
+ scale_min: 0.6
+ scale_max: 1.3
+ }
+ }
+
+ optimizer {
+ adam_optimizer: {
+ epsilon: 1e-7 # Match tf.keras.optimizers.Adam's default.
+ learning_rate: {
+ cosine_decay_learning_rate {
+ learning_rate_base: 1e-3
+ total_steps: 50000
+ warmup_learning_rate: 2.5e-4
+ warmup_steps: 5000
+ }
+ }
+ }
+ use_moving_average: false
+ }
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+
+ fine_tune_checkpoint_version: V2
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED/ckpt-51"
+ fine_tune_checkpoint_type: "fine_tune"
+}
+
+train_input_reader: {
+ load_instance_masks: true
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ mask_type: PNG_MASKS
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+}
+
+eval_config: {
+ metrics_set: "coco_detection_metrics"
+ metrics_set: "coco_mask_metrics"
+ include_metrics_per_category: true
+ use_moving_averages: false
+ batch_size: 1;
+ super_categories {
+ key: "VOC"
+ value: "person,bicycle,car,motorcycle,airplane,bus,train,boat,bird,cat,"
+ "dog,horse,sheep,cow,bottle,chair,couch,potted plant,dining table,tv"
+ }
+ super_categories {
+ key: "NonVOC"
+ value: "truck,traffic light,fire hydrant,stop sign,parking meter,bench,"
+ "elephant,bear,zebra,giraffe,backpack,umbrella,handbag,tie,suitcase,"
+ "frisbee,skis,snowboard,sports ball,kite,baseball bat,baseball glove,"
+ "skateboard,surfboard,tennis racket,wine glass,cup,fork,knife,spoon,bowl,"
+ "banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,cake,bed,"
+ "toilet,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,"
+ "sink,refrigerator,book,clock,vase,scissors,teddy bear,hair drier,"
+ "toothbrush"
+ }
+ super_categories {
+ key: "person"
+ value: "person"
+ }
+ super_categories {
+ key: "vehicle"
+ value: "bicycle,car,motorcycle,airplane,bus,train,truck,boat"
+ }
+ super_categories {
+ key: "outdoor"
+ value: "traffic light,fire hydrant,stop sign,parking meter,bench"
+ }
+ super_categories {
+ key: "animal"
+ value: "bird,cat,dog,horse,sheep,cow,elephant,bear,zebra,giraffe"
+ }
+ super_categories {
+ key: "accessory"
+ value: "backpack,umbrella,handbag,tie,suitcase"
+ }
+ super_categories {
+ key: "sports"
+ value: "frisbee,skis,snowboard,sports ball,kite,baseball bat,"
+ "baseball glove,skateboard,surfboard,tennis racket"
+ }
+ super_categories {
+ key: "kitchen"
+ value: "bottle,wine glass,cup,fork,knife,spoon,bowl"
+ }
+ super_categories {
+ key: "food"
+ value: "banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,"
+ "cake"
+ }
+ super_categories {
+ key: "furniture"
+ value: "chair,couch,potted plant,bed,dining table,toilet"
+ }
+ super_categories {
+ key: "electronic"
+ value: "tv,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,"
+ "sink,refrigerator"
+ }
+ super_categories {
+ key: "indoor"
+ value: "book,clock,vase,scissors,teddy bear,hair drier,toothbrush"
+ }
+}
+
+eval_input_reader: {
+ load_instance_masks: true
+ mask_type: PNG_MASKS
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ shuffle: false
+ num_epochs: 1
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/val2017-?????-of-00032.tfrecord"
+ }
+}
+
diff --git a/research/object_detection/configs/tf2/center_net_deepmac_512x512_voc_only_tpu-32.config b/research/object_detection/configs/tf2/center_net_deepmac_512x512_voc_only_tpu-32.config
new file mode 100644
index 0000000000000000000000000000000000000000..1b5891e21362d0de016afc88d24e0a8956100b84
--- /dev/null
+++ b/research/object_detection/configs/tf2/center_net_deepmac_512x512_voc_only_tpu-32.config
@@ -0,0 +1,233 @@
+# DeepMAC meta architecture from the "The surprising impact of mask-head
+# architecture on novel class segmentation" [1] paper with an Hourglass-52[2]
+# mask head. This config is only trained on masks from the VOC classes in COCO
+# and achieves a mask mAP of 32.5% on non-VOC classes.
+# [1]: https://arxiv.org/abs/2104.00613
+# [2]: https://arxiv.org/abs/1904.07850
+
+# Train on TPU-32
+
+model {
+ center_net {
+ num_classes: 90
+ feature_extractor {
+ type: "hourglass_104"
+ bgr_ordering: true
+ channel_means: [104.01362025, 114.03422265, 119.9165958 ]
+ channel_stds: [73.6027665 , 69.89082075, 70.9150767 ]
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 512
+ max_dimension: 512
+ pad_to_max_dimension: true
+ }
+ }
+ object_detection_task {
+ task_loss_weight: 1.0
+ offset_loss_weight: 1.0
+ scale_loss_weight: 0.1
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ object_center_loss_weight: 1.0
+ min_box_overlap_iou: 0.7
+ max_box_predictions: 100
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+
+ deepmac_mask_estimation {
+ dim: 32
+ task_loss_weight: 5.0
+ pixel_embedding_dim: 16
+ mask_size: 32
+ use_xy: true
+ use_instance_embedding: true
+ network_type: "hourglass52"
+ classification_loss {
+ weighted_sigmoid {}
+ }
+
+ allowed_masked_classes_ids: [
+ 1, # person
+ 2, # bicycle
+ 3, # car
+ 4, # motorcycle/motorbike
+ 5, # airplane/aeroplane,
+ 6, # bus
+ 7, # train
+ 9, # boat
+ 16, # bird
+ 17, # cat
+ 18, # dog
+ 19, # horse
+ 20, # sheep
+ 21, # cow
+ 44, # bottle
+ 62, # chair
+ 63, # couch/sofa
+ 64, # potted plant
+ 67, # dining table
+ 72 # tvmonitor
+ ]
+ }
+ }
+}
+
+train_config: {
+
+ batch_size: 128
+ num_steps: 50000
+
+ data_augmentation_options {
+ random_horizontal_flip {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_hue {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_contrast {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_saturation {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_brightness {
+ }
+ }
+
+ data_augmentation_options {
+ random_square_crop_by_scale {
+ scale_min: 0.6
+ scale_max: 1.3
+ }
+ }
+
+ optimizer {
+ adam_optimizer: {
+ epsilon: 1e-7 # Match tf.keras.optimizers.Adam's default.
+ learning_rate: {
+ cosine_decay_learning_rate {
+ learning_rate_base: 1e-3
+ total_steps: 50000
+ warmup_learning_rate: 2.5e-4
+ warmup_steps: 5000
+ }
+ }
+ }
+ use_moving_average: false
+ }
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+
+ fine_tune_checkpoint_version: V2
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED/ckpt-1"
+ fine_tune_checkpoint_type: "detection"
+}
+
+train_input_reader: {
+ load_instance_masks: true
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ mask_type: PNG_MASKS
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+}
+
+eval_config: {
+ metrics_set: "coco_detection_metrics"
+ metrics_set: "coco_mask_metrics"
+ include_metrics_per_category: true
+ use_moving_averages: false
+ batch_size: 1;
+ super_categories {
+ key: "VOC"
+ value: "person,bicycle,car,motorcycle,airplane,bus,train,boat,bird,cat,"
+ "dog,horse,sheep,cow,bottle,chair,couch,potted plant,dining table,tv"
+ }
+ super_categories {
+ key: "NonVOC"
+ value: "truck,traffic light,fire hydrant,stop sign,parking meter,bench,"
+ "elephant,bear,zebra,giraffe,backpack,umbrella,handbag,tie,suitcase,"
+ "frisbee,skis,snowboard,sports ball,kite,baseball bat,baseball glove,"
+ "skateboard,surfboard,tennis racket,wine glass,cup,fork,knife,spoon,bowl,"
+ "banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,cake,bed,"
+ "toilet,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,"
+ "sink,refrigerator,book,clock,vase,scissors,teddy bear,hair drier,"
+ "toothbrush"
+ }
+ super_categories {
+ key: "person"
+ value: "person"
+ }
+ super_categories {
+ key: "vehicle"
+ value: "bicycle,car,motorcycle,airplane,bus,train,truck,boat"
+ }
+ super_categories {
+ key: "outdoor"
+ value: "traffic light,fire hydrant,stop sign,parking meter,bench"
+ }
+ super_categories {
+ key: "animal"
+ value: "bird,cat,dog,horse,sheep,cow,elephant,bear,zebra,giraffe"
+ }
+ super_categories {
+ key: "accessory"
+ value: "backpack,umbrella,handbag,tie,suitcase"
+ }
+ super_categories {
+ key: "sports"
+ value: "frisbee,skis,snowboard,sports ball,kite,baseball bat,"
+ "baseball glove,skateboard,surfboard,tennis racket"
+ }
+ super_categories {
+ key: "kitchen"
+ value: "bottle,wine glass,cup,fork,knife,spoon,bowl"
+ }
+ super_categories {
+ key: "food"
+ value: "banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,"
+ "cake"
+ }
+ super_categories {
+ key: "furniture"
+ value: "chair,couch,potted plant,bed,dining table,toilet"
+ }
+ super_categories {
+ key: "electronic"
+ value: "tv,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,"
+ "sink,refrigerator"
+ }
+ super_categories {
+ key: "indoor"
+ value: "book,clock,vase,scissors,teddy bear,hair drier,toothbrush"
+ }
+}
+
+eval_input_reader: {
+ load_instance_masks: true
+ mask_type: PNG_MASKS
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ shuffle: false
+ num_epochs: 1
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/val2017-?????-of-00032.tfrecord"
+ }
+}
diff --git a/research/object_detection/configs/tf2/center_net_hourglass104_1024x1024_coco17_tpu-32.config b/research/object_detection/configs/tf2/centernet_hourglass104_1024x1024_coco17_tpu-32.config
similarity index 100%
rename from research/object_detection/configs/tf2/center_net_hourglass104_1024x1024_coco17_tpu-32.config
rename to research/object_detection/configs/tf2/centernet_hourglass104_1024x1024_coco17_tpu-32.config
diff --git a/research/object_detection/configs/tf2/centernet_hourglass104_1024x1024_kpts_coco17_tpu-32.config b/research/object_detection/configs/tf2/centernet_hourglass104_1024x1024_kpts_coco17_tpu-32.config
new file mode 100644
index 0000000000000000000000000000000000000000..da7136f15db8a7a6201700ff761b4cab1387fdd2
--- /dev/null
+++ b/research/object_detection/configs/tf2/centernet_hourglass104_1024x1024_kpts_coco17_tpu-32.config
@@ -0,0 +1,374 @@
+# CenterNet meta-architecture from the "Objects as Points" [2] paper with the
+# hourglass[1] backbone. This config achieves an mAP of 42.8/64.5 +/- 0.16 on
+# COCO 17 (averaged over 5 runs). This config is TPU compatible.
+# [1]: https://arxiv.org/abs/1603.06937
+# [2]: https://arxiv.org/abs/1904.07850
+
+model {
+ center_net {
+ num_classes: 90
+ feature_extractor {
+ type: "hourglass_104"
+ channel_means: 104.01361846923828
+ channel_means: 114.03422546386719
+ channel_means: 119.91659545898438
+ channel_stds: 73.60276794433594
+ channel_stds: 69.89082336425781
+ channel_stds: 70.91507720947266
+ bgr_ordering: true
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 1024
+ max_dimension: 1024
+ pad_to_max_dimension: true
+ }
+ }
+ object_detection_task {
+ task_loss_weight: 1.0
+ offset_loss_weight: 1.0
+ scale_loss_weight: 0.10000000149011612
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ object_center_loss_weight: 1.0
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ min_box_overlap_iou: 0.699999988079071
+ max_box_predictions: 100
+ }
+ keypoint_label_map_path: "PATH_TO_BE_CONFIGURED"
+ keypoint_estimation_task {
+ task_name: "human_pose"
+ task_loss_weight: 1.0
+ loss {
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+ keypoint_class_name: "/m/01g317"
+ keypoint_label_to_std {
+ key: "left_ankle"
+ value: 0.8899999856948853
+ }
+ keypoint_label_to_std {
+ key: "left_ear"
+ value: 0.3499999940395355
+ }
+ keypoint_label_to_std {
+ key: "left_elbow"
+ value: 0.7200000286102295
+ }
+ keypoint_label_to_std {
+ key: "left_eye"
+ value: 0.25
+ }
+ keypoint_label_to_std {
+ key: "left_hip"
+ value: 1.0700000524520874
+ }
+ keypoint_label_to_std {
+ key: "left_knee"
+ value: 0.8899999856948853
+ }
+ keypoint_label_to_std {
+ key: "left_shoulder"
+ value: 0.7900000214576721
+ }
+ keypoint_label_to_std {
+ key: "left_wrist"
+ value: 0.6200000047683716
+ }
+ keypoint_label_to_std {
+ key: "nose"
+ value: 0.25999999046325684
+ }
+ keypoint_label_to_std {
+ key: "right_ankle"
+ value: 0.8899999856948853
+ }
+ keypoint_label_to_std {
+ key: "right_ear"
+ value: 0.3499999940395355
+ }
+ keypoint_label_to_std {
+ key: "right_elbow"
+ value: 0.7200000286102295
+ }
+ keypoint_label_to_std {
+ key: "right_eye"
+ value: 0.25
+ }
+ keypoint_label_to_std {
+ key: "right_hip"
+ value: 1.0700000524520874
+ }
+ keypoint_label_to_std {
+ key: "right_knee"
+ value: 0.8899999856948853
+ }
+ keypoint_label_to_std {
+ key: "right_shoulder"
+ value: 0.7900000214576721
+ }
+ keypoint_label_to_std {
+ key: "right_wrist"
+ value: 0.6200000047683716
+ }
+ keypoint_regression_loss_weight: 0.10000000149011612
+ keypoint_heatmap_loss_weight: 1.0
+ keypoint_offset_loss_weight: 1.0
+ offset_peak_radius: 3
+ per_keypoint_offset: true
+ }
+ }
+}
+train_config {
+ batch_size: 128
+ data_augmentation_options {
+ random_horizontal_flip {
+ keypoint_flip_permutation: 0
+ keypoint_flip_permutation: 2
+ keypoint_flip_permutation: 1
+ keypoint_flip_permutation: 4
+ keypoint_flip_permutation: 3
+ keypoint_flip_permutation: 6
+ keypoint_flip_permutation: 5
+ keypoint_flip_permutation: 8
+ keypoint_flip_permutation: 7
+ keypoint_flip_permutation: 10
+ keypoint_flip_permutation: 9
+ keypoint_flip_permutation: 12
+ keypoint_flip_permutation: 11
+ keypoint_flip_permutation: 14
+ keypoint_flip_permutation: 13
+ keypoint_flip_permutation: 16
+ keypoint_flip_permutation: 15
+ }
+ }
+ data_augmentation_options {
+ random_adjust_hue {
+ }
+ }
+ data_augmentation_options {
+ random_adjust_contrast {
+ }
+ }
+ data_augmentation_options {
+ random_adjust_saturation {
+ }
+ }
+ data_augmentation_options {
+ random_adjust_brightness {
+ }
+ }
+ data_augmentation_options {
+ random_square_crop_by_scale {
+ scale_min: 0.6000000238418579
+ scale_max: 1.2999999523162842
+ }
+ }
+ optimizer {
+ adam_optimizer {
+ learning_rate {
+ cosine_decay_learning_rate {
+ learning_rate_base: 0.0010000000474974513
+ total_steps: 250000
+ warmup_learning_rate: 0.0002500000118743628
+ warmup_steps: 5000
+ }
+ }
+ epsilon: 1.0000000116860974e-07
+ }
+ use_moving_average: false
+ }
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED"
+ num_steps: 250000
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+ fine_tune_checkpoint_type: "detection"
+ fine_tune_checkpoint_version: V2
+}
+train_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+ num_keypoints: 17
+}
+eval_config {
+ num_visualizations: 10
+ metrics_set: "coco_detection_metrics"
+ use_moving_averages: false
+ min_score_threshold: 0.20000000298023224
+ max_num_boxes_to_visualize: 20
+ batch_size: 1
+ parameterized_metric {
+ coco_keypoint_metrics {
+ class_label: "person"
+ keypoint_label_to_sigmas {
+ key: "left_ankle"
+ value: 0.08900000154972076
+ }
+ keypoint_label_to_sigmas {
+ key: "left_ear"
+ value: 0.03500000014901161
+ }
+ keypoint_label_to_sigmas {
+ key: "left_elbow"
+ value: 0.07199999690055847
+ }
+ keypoint_label_to_sigmas {
+ key: "left_eye"
+ value: 0.02500000037252903
+ }
+ keypoint_label_to_sigmas {
+ key: "left_hip"
+ value: 0.10700000077486038
+ }
+ keypoint_label_to_sigmas {
+ key: "left_knee"
+ value: 0.08699999749660492
+ }
+ keypoint_label_to_sigmas {
+ key: "left_shoulder"
+ value: 0.07900000363588333
+ }
+ keypoint_label_to_sigmas {
+ key: "left_wrist"
+ value: 0.06199999898672104
+ }
+ keypoint_label_to_sigmas {
+ key: "nose"
+ value: 0.026000000536441803
+ }
+ keypoint_label_to_sigmas {
+ key: "right_ankle"
+ value: 0.08900000154972076
+ }
+ keypoint_label_to_sigmas {
+ key: "right_ear"
+ value: 0.03500000014901161
+ }
+ keypoint_label_to_sigmas {
+ key: "right_elbow"
+ value: 0.07199999690055847
+ }
+ keypoint_label_to_sigmas {
+ key: "right_eye"
+ value: 0.02500000037252903
+ }
+ keypoint_label_to_sigmas {
+ key: "right_hip"
+ value: 0.10700000077486038
+ }
+ keypoint_label_to_sigmas {
+ key: "right_knee"
+ value: 0.08699999749660492
+ }
+ keypoint_label_to_sigmas {
+ key: "right_shoulder"
+ value: 0.07900000363588333
+ }
+ keypoint_label_to_sigmas {
+ key: "right_wrist"
+ value: 0.06199999898672104
+ }
+ }
+ }
+ keypoint_edge {
+ start: 0
+ end: 1
+ }
+ keypoint_edge {
+ start: 0
+ end: 2
+ }
+ keypoint_edge {
+ start: 1
+ end: 3
+ }
+ keypoint_edge {
+ start: 2
+ end: 4
+ }
+ keypoint_edge {
+ start: 0
+ end: 5
+ }
+ keypoint_edge {
+ start: 0
+ end: 6
+ }
+ keypoint_edge {
+ start: 5
+ end: 7
+ }
+ keypoint_edge {
+ start: 7
+ end: 9
+ }
+ keypoint_edge {
+ start: 6
+ end: 8
+ }
+ keypoint_edge {
+ start: 8
+ end: 10
+ }
+ keypoint_edge {
+ start: 5
+ end: 6
+ }
+ keypoint_edge {
+ start: 5
+ end: 11
+ }
+ keypoint_edge {
+ start: 6
+ end: 12
+ }
+ keypoint_edge {
+ start: 11
+ end: 12
+ }
+ keypoint_edge {
+ start: 11
+ end: 13
+ }
+ keypoint_edge {
+ start: 13
+ end: 15
+ }
+ keypoint_edge {
+ start: 12
+ end: 14
+ }
+ keypoint_edge {
+ start: 14
+ end: 16
+ }
+}
+eval_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ shuffle: false
+ num_epochs: 1
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/val2017-?????-of-00032.tfrecord"
+ }
+ num_keypoints: 17
+}
diff --git a/research/object_detection/configs/tf2/center_net_hourglass104_512x512_coco17_tpu-8.config b/research/object_detection/configs/tf2/centernet_hourglass104_512x512_coco17_tpu-8.config
similarity index 100%
rename from research/object_detection/configs/tf2/center_net_hourglass104_512x512_coco17_tpu-8.config
rename to research/object_detection/configs/tf2/centernet_hourglass104_512x512_coco17_tpu-8.config
diff --git a/research/object_detection/configs/tf2/centernet_hourglass104_512x512_kpts_coco17_tpu-32.config b/research/object_detection/configs/tf2/centernet_hourglass104_512x512_kpts_coco17_tpu-32.config
new file mode 100644
index 0000000000000000000000000000000000000000..ce5652895f9331261f28be1e23eca4ccb916d1e1
--- /dev/null
+++ b/research/object_detection/configs/tf2/centernet_hourglass104_512x512_kpts_coco17_tpu-32.config
@@ -0,0 +1,395 @@
+# CenterNet meta-architecture from the "Objects as Points" [2] paper with the
+# hourglass[1] backbone. This config achieves an mAP of 40.0/61.4 +/- 0.16 on
+# COCO 17 (averaged over 5 runs). This config is TPU compatible.
+# [1]: https://arxiv.org/abs/1603.06937
+# [2]: https://arxiv.org/abs/1904.07850
+
+model {
+ center_net {
+ num_classes: 90
+ feature_extractor {
+ type: "hourglass_104"
+ bgr_ordering: true
+ channel_means: [104.01362025, 114.03422265, 119.9165958 ]
+ channel_stds: [73.6027665 , 69.89082075, 70.9150767 ]
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 512
+ max_dimension: 512
+ pad_to_max_dimension: true
+ }
+ }
+ object_detection_task {
+ task_loss_weight: 1.0
+ offset_loss_weight: 1.0
+ scale_loss_weight: 0.1
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ object_center_loss_weight: 1.0
+ min_box_overlap_iou: 0.7
+ max_box_predictions: 100
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+
+ keypoint_label_map_path: "PATH_TO_BE_CONFIGURED"
+ keypoint_estimation_task {
+ task_name: "human_pose"
+ task_loss_weight: 1.0
+ loss {
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+ keypoint_class_name: "/m/01g317"
+ keypoint_label_to_std {
+ key: "left_ankle"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "left_ear"
+ value: 0.35
+ }
+ keypoint_label_to_std {
+ key: "left_elbow"
+ value: 0.72
+ }
+ keypoint_label_to_std {
+ key: "left_eye"
+ value: 0.25
+ }
+ keypoint_label_to_std {
+ key: "left_hip"
+ value: 1.07
+ }
+ keypoint_label_to_std {
+ key: "left_knee"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "left_shoulder"
+ value: 0.79
+ }
+ keypoint_label_to_std {
+ key: "left_wrist"
+ value: 0.62
+ }
+ keypoint_label_to_std {
+ key: "nose"
+ value: 0.26
+ }
+ keypoint_label_to_std {
+ key: "right_ankle"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "right_ear"
+ value: 0.35
+ }
+ keypoint_label_to_std {
+ key: "right_elbow"
+ value: 0.72
+ }
+ keypoint_label_to_std {
+ key: "right_eye"
+ value: 0.25
+ }
+ keypoint_label_to_std {
+ key: "right_hip"
+ value: 1.07
+ }
+ keypoint_label_to_std {
+ key: "right_knee"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "right_shoulder"
+ value: 0.79
+ }
+ keypoint_label_to_std {
+ key: "right_wrist"
+ value: 0.62
+ }
+ keypoint_regression_loss_weight: 0.1
+ keypoint_heatmap_loss_weight: 1.0
+ keypoint_offset_loss_weight: 1.0
+ offset_peak_radius: 3
+ per_keypoint_offset: true
+ }
+ }
+}
+
+train_config: {
+
+ batch_size: 128
+ num_steps: 250000
+
+ data_augmentation_options {
+ random_horizontal_flip {
+ keypoint_flip_permutation: 0
+ keypoint_flip_permutation: 2
+ keypoint_flip_permutation: 1
+ keypoint_flip_permutation: 4
+ keypoint_flip_permutation: 3
+ keypoint_flip_permutation: 6
+ keypoint_flip_permutation: 5
+ keypoint_flip_permutation: 8
+ keypoint_flip_permutation: 7
+ keypoint_flip_permutation: 10
+ keypoint_flip_permutation: 9
+ keypoint_flip_permutation: 12
+ keypoint_flip_permutation: 11
+ keypoint_flip_permutation: 14
+ keypoint_flip_permutation: 13
+ keypoint_flip_permutation: 16
+ keypoint_flip_permutation: 15
+ }
+ }
+
+ data_augmentation_options {
+ random_crop_image {
+ min_aspect_ratio: 0.5
+ max_aspect_ratio: 1.7
+ random_coef: 0.25
+ }
+ }
+
+
+ data_augmentation_options {
+ random_adjust_hue {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_contrast {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_saturation {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_brightness {
+ }
+ }
+
+ data_augmentation_options {
+ random_absolute_pad_image {
+ max_height_padding: 200
+ max_width_padding: 200
+ pad_color: [0, 0, 0]
+ }
+ }
+
+ optimizer {
+ adam_optimizer: {
+ epsilon: 1e-7 # Match tf.keras.optimizers.Adam's default.
+ learning_rate: {
+ cosine_decay_learning_rate {
+ learning_rate_base: 1e-3
+ total_steps: 250000
+ warmup_learning_rate: 2.5e-4
+ warmup_steps: 5000
+ }
+ }
+ }
+ use_moving_average: false
+ }
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+
+ fine_tune_checkpoint_version: V2
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED"
+ fine_tune_checkpoint_type: "detection"
+}
+
+train_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+ num_keypoints: 17
+}
+
+eval_config: {
+ metrics_set: "coco_detection_metrics"
+ use_moving_averages: false
+ num_visualizations: 10
+ max_num_boxes_to_visualize: 20
+ min_score_threshold: 0.2
+ batch_size: 1;
+ parameterized_metric {
+ coco_keypoint_metrics {
+ class_label: "person"
+ keypoint_label_to_sigmas {
+ key: "nose"
+ value: 0.026
+ }
+ keypoint_label_to_sigmas {
+ key: "left_eye"
+ value: 0.025
+ }
+ keypoint_label_to_sigmas {
+ key: "right_eye"
+ value: 0.025
+ }
+ keypoint_label_to_sigmas {
+ key: "left_ear"
+ value: 0.035
+ }
+ keypoint_label_to_sigmas {
+ key: "right_ear"
+ value: 0.035
+ }
+ keypoint_label_to_sigmas {
+ key: "left_shoulder"
+ value: 0.079
+ }
+ keypoint_label_to_sigmas {
+ key: "right_shoulder"
+ value: 0.079
+ }
+ keypoint_label_to_sigmas {
+ key: "left_elbow"
+ value: 0.072
+ }
+ keypoint_label_to_sigmas {
+ key: "right_elbow"
+ value: 0.072
+ }
+ keypoint_label_to_sigmas {
+ key: "left_wrist"
+ value: 0.062
+ }
+ keypoint_label_to_sigmas {
+ key: "right_wrist"
+ value: 0.062
+ }
+ keypoint_label_to_sigmas {
+ key: "left_hip"
+ value: 0.107
+ }
+ keypoint_label_to_sigmas {
+ key: "right_hip"
+ value: 0.107
+ }
+ keypoint_label_to_sigmas {
+ key: "left_knee"
+ value: 0.087
+ }
+ keypoint_label_to_sigmas {
+ key: "right_knee"
+ value: 0.087
+ }
+ keypoint_label_to_sigmas {
+ key: "left_ankle"
+ value: 0.089
+ }
+ keypoint_label_to_sigmas {
+ key: "right_ankle"
+ value: 0.089
+ }
+ }
+ }
+ # Provide the edges to connect the keypoints. The setting is suitable for
+ # COCO's 17 human pose keypoints.
+ keypoint_edge { # nose-left eye
+ start: 0
+ end: 1
+ }
+ keypoint_edge { # nose-right eye
+ start: 0
+ end: 2
+ }
+ keypoint_edge { # left eye-left ear
+ start: 1
+ end: 3
+ }
+ keypoint_edge { # right eye-right ear
+ start: 2
+ end: 4
+ }
+ keypoint_edge { # nose-left shoulder
+ start: 0
+ end: 5
+ }
+ keypoint_edge { # nose-right shoulder
+ start: 0
+ end: 6
+ }
+ keypoint_edge { # left shoulder-left elbow
+ start: 5
+ end: 7
+ }
+ keypoint_edge { # left elbow-left wrist
+ start: 7
+ end: 9
+ }
+ keypoint_edge { # right shoulder-right elbow
+ start: 6
+ end: 8
+ }
+ keypoint_edge { # right elbow-right wrist
+ start: 8
+ end: 10
+ }
+ keypoint_edge { # left shoulder-right shoulder
+ start: 5
+ end: 6
+ }
+ keypoint_edge { # left shoulder-left hip
+ start: 5
+ end: 11
+ }
+ keypoint_edge { # right shoulder-right hip
+ start: 6
+ end: 12
+ }
+ keypoint_edge { # left hip-right hip
+ start: 11
+ end: 12
+ }
+ keypoint_edge { # left hip-left knee
+ start: 11
+ end: 13
+ }
+ keypoint_edge { # left knee-left ankle
+ start: 13
+ end: 15
+ }
+ keypoint_edge { # right hip-right knee
+ start: 12
+ end: 14
+ }
+ keypoint_edge { # right knee-right ankle
+ start: 14
+ end: 16
+ }
+}
+
+eval_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+ num_keypoints: 17
+}
+
diff --git a/research/object_detection/configs/tf2/center_net_resnet101_v1_fpn_512x512_coco17_tpu-8.config b/research/object_detection/configs/tf2/centernet_resnet101_v1_fpn_512x512_coco17_tpu-8.config
similarity index 100%
rename from research/object_detection/configs/tf2/center_net_resnet101_v1_fpn_512x512_coco17_tpu-8.config
rename to research/object_detection/configs/tf2/centernet_resnet101_v1_fpn_512x512_coco17_tpu-8.config
diff --git a/research/object_detection/configs/tf2/centernet_resnet50_v1_fpn_512x512_kpts_coco17_tpu-8.config b/research/object_detection/configs/tf2/centernet_resnet50_v1_fpn_512x512_kpts_coco17_tpu-8.config
new file mode 100644
index 0000000000000000000000000000000000000000..ad25d5c347dafc7f442c62e534e1a7b551c1d728
--- /dev/null
+++ b/research/object_detection/configs/tf2/centernet_resnet50_v1_fpn_512x512_kpts_coco17_tpu-8.config
@@ -0,0 +1,392 @@
+# CenterNet meta-architecture from the "Objects as Points" [1] paper
+# with the ResNet-v1-50 backbone. The ResNet backbone has a few differences
+# as compared to the one mentioned in the paper, hence the performance is
+# slightly worse. This config is TPU comptatible.
+# [1]: https://arxiv.org/abs/1904.07850
+#
+
+model {
+ center_net {
+ num_classes: 90
+ feature_extractor {
+ type: "resnet_v1_50_fpn"
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 512
+ max_dimension: 512
+ pad_to_max_dimension: true
+ }
+ }
+ object_detection_task {
+ task_loss_weight: 1.0
+ offset_loss_weight: 1.0
+ scale_loss_weight: 0.1
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ object_center_loss_weight: 1.0
+ min_box_overlap_iou: 0.7
+ max_box_predictions: 100
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+ keypoint_label_map_path: "PATH_TO_BE_CONFIGURED"
+ keypoint_estimation_task {
+ task_name: "human_pose"
+ task_loss_weight: 1.0
+ loss {
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+ keypoint_class_name: "/m/01g317"
+ keypoint_label_to_std {
+ key: "left_ankle"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "left_ear"
+ value: 0.35
+ }
+ keypoint_label_to_std {
+ key: "left_elbow"
+ value: 0.72
+ }
+ keypoint_label_to_std {
+ key: "left_eye"
+ value: 0.25
+ }
+ keypoint_label_to_std {
+ key: "left_hip"
+ value: 1.07
+ }
+ keypoint_label_to_std {
+ key: "left_knee"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "left_shoulder"
+ value: 0.79
+ }
+ keypoint_label_to_std {
+ key: "left_wrist"
+ value: 0.62
+ }
+ keypoint_label_to_std {
+ key: "nose"
+ value: 0.26
+ }
+ keypoint_label_to_std {
+ key: "right_ankle"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "right_ear"
+ value: 0.35
+ }
+ keypoint_label_to_std {
+ key: "right_elbow"
+ value: 0.72
+ }
+ keypoint_label_to_std {
+ key: "right_eye"
+ value: 0.25
+ }
+ keypoint_label_to_std {
+ key: "right_hip"
+ value: 1.07
+ }
+ keypoint_label_to_std {
+ key: "right_knee"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "right_shoulder"
+ value: 0.79
+ }
+ keypoint_label_to_std {
+ key: "right_wrist"
+ value: 0.62
+ }
+ keypoint_regression_loss_weight: 0.1
+ keypoint_heatmap_loss_weight: 1.0
+ keypoint_offset_loss_weight: 1.0
+ offset_peak_radius: 3
+ per_keypoint_offset: true
+ }
+ }
+}
+
+train_config: {
+
+ batch_size: 128
+ num_steps: 250000
+
+ data_augmentation_options {
+ random_horizontal_flip {
+ keypoint_flip_permutation: 0
+ keypoint_flip_permutation: 2
+ keypoint_flip_permutation: 1
+ keypoint_flip_permutation: 4
+ keypoint_flip_permutation: 3
+ keypoint_flip_permutation: 6
+ keypoint_flip_permutation: 5
+ keypoint_flip_permutation: 8
+ keypoint_flip_permutation: 7
+ keypoint_flip_permutation: 10
+ keypoint_flip_permutation: 9
+ keypoint_flip_permutation: 12
+ keypoint_flip_permutation: 11
+ keypoint_flip_permutation: 14
+ keypoint_flip_permutation: 13
+ keypoint_flip_permutation: 16
+ keypoint_flip_permutation: 15
+ }
+ }
+
+ data_augmentation_options {
+ random_crop_image {
+ min_aspect_ratio: 0.5
+ max_aspect_ratio: 1.7
+ random_coef: 0.25
+ }
+ }
+
+
+ data_augmentation_options {
+ random_adjust_hue {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_contrast {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_saturation {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_brightness {
+ }
+ }
+
+ data_augmentation_options {
+ random_absolute_pad_image {
+ max_height_padding: 200
+ max_width_padding: 200
+ pad_color: [0, 0, 0]
+ }
+ }
+
+ optimizer {
+ adam_optimizer: {
+ epsilon: 1e-7 # Match tf.keras.optimizers.Adam's default.
+ learning_rate: {
+ cosine_decay_learning_rate {
+ learning_rate_base: 1e-3
+ total_steps: 250000
+ warmup_learning_rate: 2.5e-4
+ warmup_steps: 5000
+ }
+ }
+ }
+ use_moving_average: false
+ }
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+
+ fine_tune_checkpoint_version: V2
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED"
+ fine_tune_checkpoint_type: "classification"
+}
+
+train_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+ num_keypoints: 17
+}
+
+eval_config: {
+ metrics_set: "coco_detection_metrics"
+ use_moving_averages: false
+ num_visualizations: 10
+ max_num_boxes_to_visualize: 20
+ min_score_threshold: 0.2
+ batch_size: 1;
+ parameterized_metric {
+ coco_keypoint_metrics {
+ class_label: "person"
+ keypoint_label_to_sigmas {
+ key: "nose"
+ value: 0.026
+ }
+ keypoint_label_to_sigmas {
+ key: "left_eye"
+ value: 0.025
+ }
+ keypoint_label_to_sigmas {
+ key: "right_eye"
+ value: 0.025
+ }
+ keypoint_label_to_sigmas {
+ key: "left_ear"
+ value: 0.035
+ }
+ keypoint_label_to_sigmas {
+ key: "right_ear"
+ value: 0.035
+ }
+ keypoint_label_to_sigmas {
+ key: "left_shoulder"
+ value: 0.079
+ }
+ keypoint_label_to_sigmas {
+ key: "right_shoulder"
+ value: 0.079
+ }
+ keypoint_label_to_sigmas {
+ key: "left_elbow"
+ value: 0.072
+ }
+ keypoint_label_to_sigmas {
+ key: "right_elbow"
+ value: 0.072
+ }
+ keypoint_label_to_sigmas {
+ key: "left_wrist"
+ value: 0.062
+ }
+ keypoint_label_to_sigmas {
+ key: "right_wrist"
+ value: 0.062
+ }
+ keypoint_label_to_sigmas {
+ key: "left_hip"
+ value: 0.107
+ }
+ keypoint_label_to_sigmas {
+ key: "right_hip"
+ value: 0.107
+ }
+ keypoint_label_to_sigmas {
+ key: "left_knee"
+ value: 0.087
+ }
+ keypoint_label_to_sigmas {
+ key: "right_knee"
+ value: 0.087
+ }
+ keypoint_label_to_sigmas {
+ key: "left_ankle"
+ value: 0.089
+ }
+ keypoint_label_to_sigmas {
+ key: "right_ankle"
+ value: 0.089
+ }
+ }
+ }
+ # Provide the edges to connect the keypoints. The setting is suitable for
+ # COCO's 17 human pose keypoints.
+ keypoint_edge { # nose-left eye
+ start: 0
+ end: 1
+ }
+ keypoint_edge { # nose-right eye
+ start: 0
+ end: 2
+ }
+ keypoint_edge { # left eye-left ear
+ start: 1
+ end: 3
+ }
+ keypoint_edge { # right eye-right ear
+ start: 2
+ end: 4
+ }
+ keypoint_edge { # nose-left shoulder
+ start: 0
+ end: 5
+ }
+ keypoint_edge { # nose-right shoulder
+ start: 0
+ end: 6
+ }
+ keypoint_edge { # left shoulder-left elbow
+ start: 5
+ end: 7
+ }
+ keypoint_edge { # left elbow-left wrist
+ start: 7
+ end: 9
+ }
+ keypoint_edge { # right shoulder-right elbow
+ start: 6
+ end: 8
+ }
+ keypoint_edge { # right elbow-right wrist
+ start: 8
+ end: 10
+ }
+ keypoint_edge { # left shoulder-right shoulder
+ start: 5
+ end: 6
+ }
+ keypoint_edge { # left shoulder-left hip
+ start: 5
+ end: 11
+ }
+ keypoint_edge { # right shoulder-right hip
+ start: 6
+ end: 12
+ }
+ keypoint_edge { # left hip-right hip
+ start: 11
+ end: 12
+ }
+ keypoint_edge { # left hip-left knee
+ start: 11
+ end: 13
+ }
+ keypoint_edge { # left knee-left ankle
+ start: 13
+ end: 15
+ }
+ keypoint_edge { # right hip-right knee
+ start: 12
+ end: 14
+ }
+ keypoint_edge { # right knee-right ankle
+ start: 14
+ end: 16
+ }
+}
+eval_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ shuffle: false
+ num_epochs: 1
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/val2017-?????-of-00032.tfrecord"
+ }
+ num_keypoints: 17
+}
diff --git a/research/object_detection/configs/tf2/centernet_resnet50_v2_512x512_kpts_coco17_tpu-8.config b/research/object_detection/configs/tf2/centernet_resnet50_v2_512x512_kpts_coco17_tpu-8.config
new file mode 100644
index 0000000000000000000000000000000000000000..3067ed417b1898b0b2b7839647d138c462c329c9
--- /dev/null
+++ b/research/object_detection/configs/tf2/centernet_resnet50_v2_512x512_kpts_coco17_tpu-8.config
@@ -0,0 +1,393 @@
+# CenterNet meta-architecture from the "Objects as Points" [1] paper
+# with the ResNet-v2-50 backbone. The ResNet backbone has a few differences
+# as compared to the one mentioned in the paper, hence the performance is
+# slightly worse. This config is TPU comptatible.
+# [1]: https://arxiv.org/abs/1904.07850
+
+model {
+ center_net {
+ num_classes: 90
+ feature_extractor {
+ type: "resnet_v2_50"
+ }
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 512
+ max_dimension: 512
+ pad_to_max_dimension: true
+ }
+ }
+ object_detection_task {
+ task_loss_weight: 1.0
+ offset_loss_weight: 1.0
+ scale_loss_weight: 0.1
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ object_center_loss_weight: 1.0
+ min_box_overlap_iou: 0.7
+ max_box_predictions: 100
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+
+ keypoint_label_map_path: "PATH_TO_BE_CONFIGURED"
+ keypoint_estimation_task {
+ task_name: "human_pose"
+ task_loss_weight: 1.0
+ loss {
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ alpha: 2.0
+ beta: 4.0
+ }
+ }
+ }
+ keypoint_class_name: "/m/01g317"
+ keypoint_label_to_std {
+ key: "left_ankle"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "left_ear"
+ value: 0.35
+ }
+ keypoint_label_to_std {
+ key: "left_elbow"
+ value: 0.72
+ }
+ keypoint_label_to_std {
+ key: "left_eye"
+ value: 0.25
+ }
+ keypoint_label_to_std {
+ key: "left_hip"
+ value: 1.07
+ }
+ keypoint_label_to_std {
+ key: "left_knee"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "left_shoulder"
+ value: 0.79
+ }
+ keypoint_label_to_std {
+ key: "left_wrist"
+ value: 0.62
+ }
+ keypoint_label_to_std {
+ key: "nose"
+ value: 0.26
+ }
+ keypoint_label_to_std {
+ key: "right_ankle"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "right_ear"
+ value: 0.35
+ }
+ keypoint_label_to_std {
+ key: "right_elbow"
+ value: 0.72
+ }
+ keypoint_label_to_std {
+ key: "right_eye"
+ value: 0.25
+ }
+ keypoint_label_to_std {
+ key: "right_hip"
+ value: 1.07
+ }
+ keypoint_label_to_std {
+ key: "right_knee"
+ value: 0.89
+ }
+ keypoint_label_to_std {
+ key: "right_shoulder"
+ value: 0.79
+ }
+ keypoint_label_to_std {
+ key: "right_wrist"
+ value: 0.62
+ }
+ keypoint_regression_loss_weight: 0.1
+ keypoint_heatmap_loss_weight: 1.0
+ keypoint_offset_loss_weight: 1.0
+ offset_peak_radius: 3
+ per_keypoint_offset: true
+ }
+ }
+}
+
+train_config: {
+
+ batch_size: 128
+ num_steps: 250000
+
+ data_augmentation_options {
+ random_horizontal_flip {
+ keypoint_flip_permutation: 0
+ keypoint_flip_permutation: 2
+ keypoint_flip_permutation: 1
+ keypoint_flip_permutation: 4
+ keypoint_flip_permutation: 3
+ keypoint_flip_permutation: 6
+ keypoint_flip_permutation: 5
+ keypoint_flip_permutation: 8
+ keypoint_flip_permutation: 7
+ keypoint_flip_permutation: 10
+ keypoint_flip_permutation: 9
+ keypoint_flip_permutation: 12
+ keypoint_flip_permutation: 11
+ keypoint_flip_permutation: 14
+ keypoint_flip_permutation: 13
+ keypoint_flip_permutation: 16
+ keypoint_flip_permutation: 15
+ }
+ }
+
+ data_augmentation_options {
+ random_crop_image {
+ min_aspect_ratio: 0.5
+ max_aspect_ratio: 1.7
+ random_coef: 0.25
+ }
+ }
+
+
+ data_augmentation_options {
+ random_adjust_hue {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_contrast {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_saturation {
+ }
+ }
+
+ data_augmentation_options {
+ random_adjust_brightness {
+ }
+ }
+
+ data_augmentation_options {
+ random_absolute_pad_image {
+ max_height_padding: 200
+ max_width_padding: 200
+ pad_color: [0, 0, 0]
+ }
+ }
+
+ optimizer {
+ adam_optimizer: {
+ epsilon: 1e-7 # Match tf.keras.optimizers.Adam's default.
+ learning_rate: {
+ cosine_decay_learning_rate {
+ learning_rate_base: 1e-3
+ total_steps: 250000
+ warmup_learning_rate: 2.5e-4
+ warmup_steps: 5000
+ }
+ }
+ }
+ use_moving_average: false
+ }
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+
+ fine_tune_checkpoint_version: V2
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED"
+ fine_tune_checkpoint_type: "classification"
+}
+
+train_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+ num_keypoints: 17
+}
+
+eval_config: {
+ metrics_set: "coco_detection_metrics"
+ use_moving_averages: false
+ num_visualizations: 10
+ max_num_boxes_to_visualize: 20
+ min_score_threshold: 0.2
+ batch_size: 1;
+ parameterized_metric {
+ coco_keypoint_metrics {
+ class_label: "person"
+ keypoint_label_to_sigmas {
+ key: "nose"
+ value: 0.026
+ }
+ keypoint_label_to_sigmas {
+ key: "left_eye"
+ value: 0.025
+ }
+ keypoint_label_to_sigmas {
+ key: "right_eye"
+ value: 0.025
+ }
+ keypoint_label_to_sigmas {
+ key: "left_ear"
+ value: 0.035
+ }
+ keypoint_label_to_sigmas {
+ key: "right_ear"
+ value: 0.035
+ }
+ keypoint_label_to_sigmas {
+ key: "left_shoulder"
+ value: 0.079
+ }
+ keypoint_label_to_sigmas {
+ key: "right_shoulder"
+ value: 0.079
+ }
+ keypoint_label_to_sigmas {
+ key: "left_elbow"
+ value: 0.072
+ }
+ keypoint_label_to_sigmas {
+ key: "right_elbow"
+ value: 0.072
+ }
+ keypoint_label_to_sigmas {
+ key: "left_wrist"
+ value: 0.062
+ }
+ keypoint_label_to_sigmas {
+ key: "right_wrist"
+ value: 0.062
+ }
+ keypoint_label_to_sigmas {
+ key: "left_hip"
+ value: 0.107
+ }
+ keypoint_label_to_sigmas {
+ key: "right_hip"
+ value: 0.107
+ }
+ keypoint_label_to_sigmas {
+ key: "left_knee"
+ value: 0.087
+ }
+ keypoint_label_to_sigmas {
+ key: "right_knee"
+ value: 0.087
+ }
+ keypoint_label_to_sigmas {
+ key: "left_ankle"
+ value: 0.089
+ }
+ keypoint_label_to_sigmas {
+ key: "right_ankle"
+ value: 0.089
+ }
+ }
+ }
+ # Provide the edges to connect the keypoints. The setting is suitable for
+ # COCO's 17 human pose keypoints.
+ keypoint_edge { # nose-left eye
+ start: 0
+ end: 1
+ }
+ keypoint_edge { # nose-right eye
+ start: 0
+ end: 2
+ }
+ keypoint_edge { # left eye-left ear
+ start: 1
+ end: 3
+ }
+ keypoint_edge { # right eye-right ear
+ start: 2
+ end: 4
+ }
+ keypoint_edge { # nose-left shoulder
+ start: 0
+ end: 5
+ }
+ keypoint_edge { # nose-right shoulder
+ start: 0
+ end: 6
+ }
+ keypoint_edge { # left shoulder-left elbow
+ start: 5
+ end: 7
+ }
+ keypoint_edge { # left elbow-left wrist
+ start: 7
+ end: 9
+ }
+ keypoint_edge { # right shoulder-right elbow
+ start: 6
+ end: 8
+ }
+ keypoint_edge { # right elbow-right wrist
+ start: 8
+ end: 10
+ }
+ keypoint_edge { # left shoulder-right shoulder
+ start: 5
+ end: 6
+ }
+ keypoint_edge { # left shoulder-left hip
+ start: 5
+ end: 11
+ }
+ keypoint_edge { # right shoulder-right hip
+ start: 6
+ end: 12
+ }
+ keypoint_edge { # left hip-right hip
+ start: 11
+ end: 12
+ }
+ keypoint_edge { # left hip-left knee
+ start: 11
+ end: 13
+ }
+ keypoint_edge { # left knee-left ankle
+ start: 13
+ end: 15
+ }
+ keypoint_edge { # right hip-right knee
+ start: 12
+ end: 14
+ }
+ keypoint_edge { # right knee-right ankle
+ start: 14
+ end: 16
+ }
+}
+
+eval_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ shuffle: false
+ num_epochs: 1
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/val2017-?????-of-00032.tfrecord"
+ }
+ num_keypoints: 17
+}
diff --git a/research/object_detection/configs/tf2/faster_rcnn_resnet50_v1_fpn_640x640_coco17_tpu-8.config b/research/object_detection/configs/tf2/faster_rcnn_resnet50_v1_fpn_640x640_coco17_tpu-8.config
new file mode 100644
index 0000000000000000000000000000000000000000..acb5a91359bd3d0349f628d6f284c19e4dc0e326
--- /dev/null
+++ b/research/object_detection/configs/tf2/faster_rcnn_resnet50_v1_fpn_640x640_coco17_tpu-8.config
@@ -0,0 +1,173 @@
+# Faster RCNN with Resnet 50 v1 FPN feature extractor.
+# See Lin et al, https://arxiv.org/abs/1612.03144
+# Trained on COCO, initialized from Imagenet classification checkpoint
+# Train on TPU-8
+#
+# Achieves 31.4 mAP on COCO17 Val
+
+model {
+ faster_rcnn {
+ num_classes: 90
+ image_resizer {
+ keep_aspect_ratio_resizer {
+ min_dimension: 640
+ max_dimension: 640
+ pad_to_max_dimension: true
+ }
+ }
+ feature_extractor {
+ type: 'faster_rcnn_resnet50_fpn_keras'
+ batch_norm_trainable: true
+ fpn {
+ min_level: 2
+ max_level: 6
+ }
+ conv_hyperparams {
+ activation: RELU_6,
+ regularizer {
+ l2_regularizer {
+ weight: 0.0004
+ }
+ }
+ initializer {
+ truncated_normal_initializer {
+ stddev: 0.03
+ mean: 0.0
+ }
+ }
+ batch_norm {
+ scale: true,
+ decay: 0.997,
+ epsilon: 0.001,
+ }
+ }
+ override_base_feature_extractor_hyperparams: true
+ }
+ first_stage_anchor_generator {
+ multiscale_anchor_generator {
+ min_level: 2
+ max_level: 6
+ # According to the origial paper the value should be 8.0
+ anchor_scale: 4.0
+ aspect_ratios: [1.0, 2.0, 0.5]
+ # According to the original paper the value should be 1
+ scales_per_octave: 2
+ normalize_coordinates: false
+ }
+ }
+ first_stage_box_predictor_conv_hyperparams {
+ op: CONV
+ regularizer {
+ l2_regularizer {
+ weight: 0.0
+ }
+ }
+ initializer {
+ truncated_normal_initializer {
+ stddev: 0.01
+ }
+ }
+ }
+ first_stage_nms_score_threshold: 0.0
+ first_stage_nms_iou_threshold: 0.7
+ first_stage_max_proposals: 300
+ first_stage_localization_loss_weight: 2.0
+ first_stage_objectness_loss_weight: 1.0
+ # According to the origial paper, value should be 7.
+ initial_crop_size: 14
+ maxpool_kernel_size: 2
+ maxpool_stride: 2
+ second_stage_box_predictor {
+ mask_rcnn_box_predictor {
+ use_dropout: false
+ dropout_keep_probability: 1.0
+ fc_hyperparams {
+ op: FC
+ regularizer {
+ l2_regularizer {
+ weight: 0.0
+ }
+ }
+ initializer {
+ variance_scaling_initializer {
+ factor: 1.0
+ uniform: true
+ mode: FAN_AVG
+ }
+ }
+ }
+ }
+ }
+ second_stage_post_processing {
+ batch_non_max_suppression {
+ score_threshold: 0.0
+ iou_threshold: 0.6
+ max_detections_per_class: 100
+ max_total_detections: 300
+ }
+ score_converter: SOFTMAX
+ }
+ second_stage_localization_loss_weight: 2.0
+ second_stage_classification_loss_weight: 1.0
+ use_static_shapes: true
+ use_matmul_crop_and_resize: true
+ clip_anchors_to_image: true
+ use_static_balanced_label_sampler: true
+ use_matmul_gather_in_matcher: true
+ }
+}
+
+train_config: {
+ batch_size: 64
+ sync_replicas: true
+ startup_delay_steps: 0
+ replicas_to_aggregate: 8
+ num_steps: 25000
+ optimizer {
+ momentum_optimizer: {
+ learning_rate: {
+ cosine_decay_learning_rate {
+ learning_rate_base: 0.04
+ total_steps: 25000
+ warmup_learning_rate: .013333
+ warmup_steps: 2000
+ }
+ }
+ momentum_optimizer_value: 0.9
+ }
+ use_moving_average: false
+ }
+ fine_tune_checkpoint_version: V2
+ fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED/resnet50.ckpt-1"
+ fine_tune_checkpoint_type: "classification"
+ data_augmentation_options {
+ random_horizontal_flip {
+ }
+ }
+
+ max_number_of_boxes: 100
+ unpad_groundtruth_tensors: false
+ use_bfloat16: true
+}
+
+train_input_reader: {
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/train2017-?????-of-00256.tfrecord"
+ }
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+}
+
+eval_config: {
+ metrics_set: "coco_detection_metrics"
+ use_moving_averages: false
+ batch_size: 1;
+}
+
+eval_input_reader: {
+ label_map_path: "PATH_TO_BE_CONFIGURED/label_map.txt"
+ shuffle: false
+ num_epochs: 1
+ tf_record_input_reader {
+ input_path: "PATH_TO_BE_CONFIGURED/val2017-?????-of-00032.tfrecord"
+ }
+}
diff --git a/research/object_detection/core/box_list_ops.py b/research/object_detection/core/box_list_ops.py
index 159845b690d5f10ac4e38ca167faa2a6051cc023..cb457b728449cc637de8ab28f5f60711954d3c27 100644
--- a/research/object_detection/core/box_list_ops.py
+++ b/research/object_detection/core/box_list_ops.py
@@ -151,7 +151,10 @@ def clip_to_window(boxlist, window, filter_nonoverlapping=True, scope=None):
with tf.name_scope(scope, 'ClipToWindow'):
y_min, x_min, y_max, x_max = tf.split(
value=boxlist.get(), num_or_size_splits=4, axis=1)
- win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
+ win_y_min = window[0]
+ win_x_min = window[1]
+ win_y_max = window[2]
+ win_x_max = window[3]
y_min_clipped = tf.maximum(tf.minimum(y_min, win_y_max), win_y_min)
y_max_clipped = tf.maximum(tf.minimum(y_max, win_y_max), win_y_min)
x_min_clipped = tf.maximum(tf.minimum(x_min, win_x_max), win_x_min)
@@ -304,6 +307,50 @@ def iou(boxlist1, boxlist2, scope=None):
tf.zeros_like(intersections), tf.truediv(intersections, unions))
+def l1(boxlist1, boxlist2, scope=None):
+ """Computes l1 loss (pairwise) between two boxlists.
+
+ Args:
+ boxlist1: BoxList holding N boxes
+ boxlist2: BoxList holding M boxes
+ scope: name scope.
+
+ Returns:
+ a tensor with shape [N, M] representing the pairwise L1 loss.
+ """
+ with tf.name_scope(scope, 'PairwiseL1'):
+ ycenter1, xcenter1, h1, w1 = boxlist1.get_center_coordinates_and_sizes()
+ ycenter2, xcenter2, h2, w2 = boxlist2.get_center_coordinates_and_sizes()
+ ycenters = tf.abs(tf.expand_dims(ycenter2, axis=0) - tf.expand_dims(
+ tf.transpose(ycenter1), axis=1))
+ xcenters = tf.abs(tf.expand_dims(xcenter2, axis=0) - tf.expand_dims(
+ tf.transpose(xcenter1), axis=1))
+ heights = tf.abs(tf.expand_dims(h2, axis=0) - tf.expand_dims(
+ tf.transpose(h1), axis=1))
+ widths = tf.abs(tf.expand_dims(w2, axis=0) - tf.expand_dims(
+ tf.transpose(w1), axis=1))
+ return ycenters + xcenters + heights + widths
+
+
+def giou(boxlist1, boxlist2, scope=None):
+ """Computes pairwise generalized IOU between two boxlists.
+
+ Args:
+ boxlist1: BoxList holding N boxes
+ boxlist2: BoxList holding M boxes
+ scope: name scope.
+
+ Returns:
+ a tensor with shape [N, M] representing the pairwise GIoU loss.
+ """
+ with tf.name_scope(scope, 'PairwiseGIoU'):
+ n = boxlist1.num_boxes()
+ m = boxlist2.num_boxes()
+ boxes1 = tf.repeat(boxlist1.get(), repeats=m, axis=0)
+ boxes2 = tf.tile(boxlist2.get(), multiples=[n, 1])
+ return tf.reshape(ops.giou(boxes1, boxes2), [n, m])
+
+
def matched_iou(boxlist1, boxlist2, scope=None):
"""Compute intersection-over-union between corresponding boxes in boxlists.
diff --git a/research/object_detection/core/box_list_ops_test.py b/research/object_detection/core/box_list_ops_test.py
index b572dff9e1cdd1b7ae59a63df3ae294d4b01a9a5..767c1899727b9e0280c096dc8e4a440c535d5839 100644
--- a/research/object_detection/core/box_list_ops_test.py
+++ b/research/object_detection/core/box_list_ops_test.py
@@ -229,6 +229,31 @@ class BoxListOpsTest(test_case.TestCase):
iou_output = self.execute(graph_fn, [])
self.assertAllClose(iou_output, exp_output)
+ def test_l1(self):
+ def graph_fn():
+ corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
+ corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
+ [0.0, 0.0, 20.0, 20.0]])
+ boxes1 = box_list.BoxList(corners1)
+ boxes2 = box_list.BoxList(corners2)
+ l1 = box_list_ops.l1(boxes1, boxes2)
+ return l1
+ exp_output = [[5.0, 22.5, 45.5], [8.5, 19.0, 40.0]]
+ l1_output = self.execute(graph_fn, [])
+ self.assertAllClose(l1_output, exp_output)
+
+ def test_giou(self):
+ def graph_fn():
+ corners1 = tf.constant([[5.0, 7.0, 7.0, 9.0]])
+ corners2 = tf.constant([[5.0, 7.0, 7.0, 9.0], [5.0, 11.0, 7.0, 13.0]])
+ boxes1 = box_list.BoxList(corners1)
+ boxes2 = box_list.BoxList(corners2)
+ giou = box_list_ops.giou(boxes1, boxes2)
+ return giou
+ exp_output = [[1.0, -1.0 / 3.0]]
+ giou_output = self.execute(graph_fn, [])
+ self.assertAllClose(giou_output, exp_output)
+
def test_matched_iou(self):
def graph_fn():
corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
diff --git a/research/object_detection/core/densepose_ops.py b/research/object_detection/core/densepose_ops.py
index 0ad852fb4e2de10d3545bf5575050c4f0fc3cefb..8dd8f39bafa357f242170b9ae2ec6c29cd24ef4f 100644
--- a/research/object_detection/core/densepose_ops.py
+++ b/research/object_detection/core/densepose_ops.py
@@ -26,6 +26,7 @@ points for an instance in the example.
"""
import os
+import numpy as np
import scipy.io
import tensorflow.compat.v1 as tf
@@ -278,8 +279,10 @@ class DensePoseHorizontalFlip(object):
for key in ('U_transforms', 'V_transforms'):
uv_symmetry_map_per_part = []
for i in range(data[key].shape[1]):
- # The following tensor has shape [256, 256].
- map_per_part = tf.constant(data[key][0, i], dtype=tf.float32)
+ # The following tensor has shape [256, 256]. The raw data is stored as
+ # uint8 values, so convert to float and scale to the range [0., 1.]
+ data_normalized = data[key][0, i].astype(np.float32) / 255.
+ map_per_part = tf.constant(data_normalized, dtype=tf.float32)
uv_symmetry_map_per_part.append(map_per_part)
uv_symmetry_map[key] = tf.reshape(
tf.stack(uv_symmetry_map_per_part, axis=0), [-1])
diff --git a/research/object_detection/core/freezable_batch_norm.py b/research/object_detection/core/freezable_batch_norm.py
index 7f08fa5df12163e8178f233dbb1d766fe27d8742..295aa7b3a50e3c79e8ece4d1e4d75bcb4d688ed3 100644
--- a/research/object_detection/core/freezable_batch_norm.py
+++ b/research/object_detection/core/freezable_batch_norm.py
@@ -35,7 +35,7 @@ class FreezableBatchNorm(tf.keras.layers.BatchNormalization):
i.e. applies a transformation that maintains the mean activation
close to 0 and the activation standard deviation close to 1.
- Arguments:
+ Args:
training: If False, the layer will normalize using the moving average and
std. dev, without updating the learned avg and std. dev.
If None or True, the layer will follow the keras BatchNormalization layer
diff --git a/research/object_detection/core/freezable_batch_norm_tf2_test.py b/research/object_detection/core/freezable_batch_norm_tf2_test.py
index 4cc42ae3ef7da9b3412d2f461d7f9db62420e603..48b131d6279d34b2c050db46da0112b8e839d1c2 100644
--- a/research/object_detection/core/freezable_batch_norm_tf2_test.py
+++ b/research/object_detection/core/freezable_batch_norm_tf2_test.py
@@ -17,25 +17,40 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
+
import unittest
+
+from absl.testing import parameterized
import numpy as np
from six.moves import zip
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
from object_detection.core import freezable_batch_norm
from object_detection.utils import tf_version
+# pylint: disable=g-import-not-at-top
+if tf_version.is_tf2():
+ from object_detection.core import freezable_sync_batch_norm
+# pylint: enable=g-import-not-at-top
+
@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
-class FreezableBatchNormTest(tf.test.TestCase):
+class FreezableBatchNormTest(tf.test.TestCase, parameterized.TestCase):
"""Tests for FreezableBatchNorm operations."""
- def _build_model(self, training=None):
+ def _build_model(self, use_sync_batch_norm, training=None):
model = tf.keras.models.Sequential()
- norm = freezable_batch_norm.FreezableBatchNorm(training=training,
- input_shape=(10,),
- momentum=0.8)
+ norm = None
+ if use_sync_batch_norm:
+ norm = freezable_sync_batch_norm.FreezableSyncBatchNorm(training=training,
+ input_shape=(10,),
+ momentum=0.8)
+ else:
+ norm = freezable_batch_norm.FreezableBatchNorm(training=training,
+ input_shape=(10,),
+ momentum=0.8)
+
model.add(norm)
return model, norm
@@ -43,8 +58,9 @@ class FreezableBatchNormTest(tf.test.TestCase):
for source, target in zip(source_weights, target_weights):
target.assign(source)
- def _train_freezable_batch_norm(self, training_mean, training_var):
- model, _ = self._build_model()
+ def _train_freezable_batch_norm(self, training_mean, training_var,
+ use_sync_batch_norm):
+ model, _ = self._build_model(use_sync_batch_norm=use_sync_batch_norm)
model.compile(loss='mse', optimizer='sgd')
# centered on training_mean, variance training_var
@@ -72,7 +88,8 @@ class FreezableBatchNormTest(tf.test.TestCase):
np.testing.assert_allclose(out.numpy().mean(), 0.0, atol=1.5e-1)
np.testing.assert_allclose(out.numpy().std(), 1.0, atol=1.5e-1)
- def test_batchnorm_freezing_training_none(self):
+ @parameterized.parameters(True, False)
+ def test_batchnorm_freezing_training_none(self, use_sync_batch_norm):
training_mean = 5.0
training_var = 10.0
@@ -81,12 +98,13 @@ class FreezableBatchNormTest(tf.test.TestCase):
# Initially train the batch norm, and save the weights
trained_weights = self._train_freezable_batch_norm(training_mean,
- training_var)
+ training_var,
+ use_sync_batch_norm)
# Load the batch norm weights, freezing training to True.
# Apply the batch norm layer to testing data and ensure it is normalized
# according to the batch statistics.
- model, norm = self._build_model(training=True)
+ model, norm = self._build_model(use_sync_batch_norm, training=True)
self._copy_weights(trained_weights, model.weights)
# centered on testing_mean, variance testing_var
@@ -136,7 +154,8 @@ class FreezableBatchNormTest(tf.test.TestCase):
testing_mean, testing_var, training_arg,
training_mean, training_var)
- def test_batchnorm_freezing_training_false(self):
+ @parameterized.parameters(True, False)
+ def test_batchnorm_freezing_training_false(self, use_sync_batch_norm):
training_mean = 5.0
training_var = 10.0
@@ -145,12 +164,13 @@ class FreezableBatchNormTest(tf.test.TestCase):
# Initially train the batch norm, and save the weights
trained_weights = self._train_freezable_batch_norm(training_mean,
- training_var)
+ training_var,
+ use_sync_batch_norm)
# Load the batch norm back up, freezing training to False.
# Apply the batch norm layer to testing data and ensure it is normalized
# according to the training data's statistics.
- model, norm = self._build_model(training=False)
+ model, norm = self._build_model(use_sync_batch_norm, training=False)
self._copy_weights(trained_weights, model.weights)
# centered on testing_mean, variance testing_var
diff --git a/research/object_detection/core/freezable_sync_batch_norm.py b/research/object_detection/core/freezable_sync_batch_norm.py
new file mode 100644
index 0000000000000000000000000000000000000000..f95a106498366dd028951686e3cac1b9d5f15802
--- /dev/null
+++ b/research/object_detection/core/freezable_sync_batch_norm.py
@@ -0,0 +1,70 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""A freezable batch norm layer that uses Keras sync batch normalization."""
+import tensorflow as tf
+
+
+class FreezableSyncBatchNorm(tf.keras.layers.experimental.SyncBatchNormalization
+ ):
+ """Sync Batch normalization layer (Ioffe and Szegedy, 2014).
+
+ This is a `freezable` batch norm layer that supports setting the `training`
+ parameter in the __init__ method rather than having to set it either via
+ the Keras learning phase or via the `call` method parameter. This layer will
+ forward all other parameters to the Keras `SyncBatchNormalization` layer
+
+ This is class is necessary because Object Detection model training sometimes
+ requires batch normalization layers to be `frozen` and used as if it was
+ evaluation time, despite still training (and potentially using dropout layers)
+
+ Like the default Keras SyncBatchNormalization layer, this will normalize the
+ activations of the previous layer at each batch,
+ i.e. applies a transformation that maintains the mean activation
+ close to 0 and the activation standard deviation close to 1.
+
+ Input shape:
+ Arbitrary. Use the keyword argument `input_shape`
+ (tuple of integers, does not include the samples axis)
+ when using this layer as the first layer in a model.
+
+ Output shape:
+ Same shape as input.
+
+ References:
+ - [Batch Normalization: Accelerating Deep Network Training by Reducing
+ Internal Covariate Shift](https://arxiv.org/abs/1502.03167)
+ """
+
+ def __init__(self, training=None, **kwargs):
+ """Constructor.
+
+ Args:
+ training: If False, the layer will normalize using the moving average and
+ std. dev, without updating the learned avg and std. dev.
+ If None or True, the layer will follow the keras SyncBatchNormalization
+ layer strategy of checking the Keras learning phase at `call` time to
+ decide what to do.
+ **kwargs: The keyword arguments to forward to the keras
+ SyncBatchNormalization layer constructor.
+ """
+ super(FreezableSyncBatchNorm, self).__init__(**kwargs)
+ self._training = training
+
+ def call(self, inputs, training=None):
+ # Override the call arg only if the batchnorm is frozen. (Ignore None)
+ if self._training is False: # pylint: disable=g-bool-id-comparison
+ training = self._training
+ return super(FreezableSyncBatchNorm, self).call(inputs, training=training)
diff --git a/research/object_detection/core/keypoint_ops.py b/research/object_detection/core/keypoint_ops.py
index 1b0c4ccfed42aae492550331e870173c624f0316..521b9a7e9235040d8f1c6ebbd0ab0e23081ee5a8 100644
--- a/research/object_detection/core/keypoint_ops.py
+++ b/research/object_detection/core/keypoint_ops.py
@@ -56,6 +56,7 @@ def clip_to_window(keypoints, window, scope=None):
Returns:
new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
"""
+ keypoints.get_shape().assert_has_rank(3)
with tf.name_scope(scope, 'ClipToWindow'):
y, x = tf.split(value=keypoints, num_or_size_splits=2, axis=2)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
@@ -81,6 +82,7 @@ def prune_outside_window(keypoints, window, scope=None):
Returns:
new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
"""
+ keypoints.get_shape().assert_has_rank(3)
with tf.name_scope(scope, 'PruneOutsideWindow'):
y, x = tf.split(value=keypoints, num_or_size_splits=2, axis=2)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
@@ -125,22 +127,24 @@ def change_coordinate_frame(keypoints, window, scope=None):
return new_keypoints
-def keypoints_to_enclosing_bounding_boxes(keypoints):
+def keypoints_to_enclosing_bounding_boxes(keypoints, keypoints_axis=1):
"""Creates enclosing bounding boxes from keypoints.
Args:
keypoints: a [num_instances, num_keypoints, 2] float32 tensor with keypoints
in [y, x] format.
+ keypoints_axis: An integer indicating the axis that correspond to the
+ keypoint dimension.
Returns:
A [num_instances, 4] float32 tensor that tightly covers all the keypoints
for each instance.
"""
- ymin = tf.math.reduce_min(keypoints[:, :, 0], axis=1)
- xmin = tf.math.reduce_min(keypoints[:, :, 1], axis=1)
- ymax = tf.math.reduce_max(keypoints[:, :, 0], axis=1)
- xmax = tf.math.reduce_max(keypoints[:, :, 1], axis=1)
- return tf.stack([ymin, xmin, ymax, xmax], axis=1)
+ ymin = tf.math.reduce_min(keypoints[..., 0], axis=keypoints_axis)
+ xmin = tf.math.reduce_min(keypoints[..., 1], axis=keypoints_axis)
+ ymax = tf.math.reduce_max(keypoints[..., 0], axis=keypoints_axis)
+ xmax = tf.math.reduce_max(keypoints[..., 1], axis=keypoints_axis)
+ return tf.stack([ymin, xmin, ymax, xmax], axis=keypoints_axis)
def to_normalized_coordinates(keypoints, height, width,
@@ -240,6 +244,7 @@ def flip_horizontal(keypoints, flip_point, flip_permutation=None, scope=None):
Returns:
new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
"""
+ keypoints.get_shape().assert_has_rank(3)
with tf.name_scope(scope, 'FlipHorizontal'):
keypoints = tf.transpose(keypoints, [1, 0, 2])
if flip_permutation:
@@ -274,6 +279,7 @@ def flip_vertical(keypoints, flip_point, flip_permutation=None, scope=None):
Returns:
new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
"""
+ keypoints.get_shape().assert_has_rank(3)
with tf.name_scope(scope, 'FlipVertical'):
keypoints = tf.transpose(keypoints, [1, 0, 2])
if flip_permutation:
@@ -299,6 +305,7 @@ def rot90(keypoints, rotation_permutation=None, scope=None):
Returns:
new_keypoints: a tensor of shape [num_instances, num_keypoints, 2]
"""
+ keypoints.get_shape().assert_has_rank(3)
with tf.name_scope(scope, 'Rot90'):
keypoints = tf.transpose(keypoints, [1, 0, 2])
if rotation_permutation:
@@ -334,6 +341,7 @@ def keypoint_weights_from_visibilities(keypoint_visibilities,
keypoints deemed visible will have the provided per-keypoint weight, and
all others will be set to zero.
"""
+ keypoint_visibilities.get_shape().assert_has_rank(2)
if per_keypoint_weights is None:
num_keypoints = keypoint_visibilities.shape.as_list()[1]
per_keypoint_weight_mult = tf.ones((1, num_keypoints,), dtype=tf.float32)
@@ -363,6 +371,7 @@ def set_keypoint_visibilities(keypoints, initial_keypoint_visibilities=None):
keypoint_visibilities: a bool tensor of shape [num_instances, num_keypoints]
indicating whether a keypoint is visible or not.
"""
+ keypoints.get_shape().assert_has_rank(3)
if initial_keypoint_visibilities is not None:
keypoint_visibilities = tf.cast(initial_keypoint_visibilities, tf.bool)
else:
diff --git a/research/object_detection/core/keypoint_ops_test.py b/research/object_detection/core/keypoint_ops_test.py
index bbdcf01940dcaf96da283bd6bcf73e91b633f0ee..729bfe8a37797295258abeff627a9c8ef3dc4513 100644
--- a/research/object_detection/core/keypoint_ops_test.py
+++ b/research/object_detection/core/keypoint_ops_test.py
@@ -116,6 +116,35 @@ class KeypointOpsTest(test_case.TestCase):
])
self.assertAllClose(expected_bboxes, output)
+ def test_keypoints_to_enclosing_bounding_boxes_axis2(self):
+ def graph_fn():
+ keypoints = tf.constant(
+ [
+ [ # Instance 0.
+ [5., 10.],
+ [3., 20.],
+ [8., 4.],
+ ],
+ [ # Instance 1.
+ [2., 12.],
+ [0., 3.],
+ [5., 19.],
+ ],
+ ], dtype=tf.float32)
+ keypoints = tf.stack([keypoints, keypoints], axis=0)
+ bboxes = keypoint_ops.keypoints_to_enclosing_bounding_boxes(
+ keypoints, keypoints_axis=2)
+ return bboxes
+ output = self.execute(graph_fn, [])
+
+ expected_bboxes = np.array(
+ [
+ [3., 4., 8., 20.],
+ [0., 3., 5., 19.]
+ ])
+ self.assertAllClose(expected_bboxes, output[0])
+ self.assertAllClose(expected_bboxes, output[1])
+
def test_to_normalized_coordinates(self):
def graph_fn():
keypoints = tf.constant([
diff --git a/research/object_detection/core/losses.py b/research/object_detection/core/losses.py
index c4d499e7e6c4ed5da803c48ff3d8908e713a3c2e..73d26b2123cdeafbf798ec452bfc5be742cca35a 100644
--- a/research/object_detection/core/losses.py
+++ b/research/object_detection/core/losses.py
@@ -36,6 +36,7 @@ import tensorflow.compat.v1 as tf
from object_detection.core import box_list
from object_detection.core import box_list_ops
from object_detection.utils import ops
+from object_detection.utils import shape_utils
class Loss(six.with_metaclass(abc.ABCMeta, object)):
@@ -210,6 +211,38 @@ class WeightedIOULocalizationLoss(Loss):
return tf.reshape(weights, [-1]) * per_anchor_iou_loss
+class WeightedGIOULocalizationLoss(Loss):
+ """GIOU localization loss function.
+
+ Sums the GIOU loss for corresponding pairs of predicted/groundtruth boxes
+ and for each pair assign a loss of 1 - GIOU. We then compute a weighted
+ sum over all pairs which is returned as the total loss.
+ """
+
+ def _compute_loss(self, prediction_tensor, target_tensor, weights):
+ """Compute loss function.
+
+ Args:
+ prediction_tensor: A float tensor of shape [batch_size, num_anchors, 4]
+ representing the decoded predicted boxes
+ target_tensor: A float tensor of shape [batch_size, num_anchors, 4]
+ representing the decoded target boxes
+ weights: a float tensor of shape [batch_size, num_anchors]
+
+ Returns:
+ loss: a float tensor of shape [batch_size, num_anchors] tensor
+ representing the value of the loss function.
+ """
+ batch_size, num_anchors, _ = shape_utils.combined_static_and_dynamic_shape(
+ prediction_tensor)
+ predicted_boxes = tf.reshape(prediction_tensor, [-1, 4])
+ target_boxes = tf.reshape(target_tensor, [-1, 4])
+
+ per_anchor_iou_loss = 1 - ops.giou(predicted_boxes, target_boxes)
+ return tf.reshape(tf.reshape(weights, [-1]) * per_anchor_iou_loss,
+ [batch_size, num_anchors])
+
+
class WeightedSigmoidClassificationLoss(Loss):
"""Sigmoid cross entropy classification loss function."""
@@ -245,6 +278,79 @@ class WeightedSigmoidClassificationLoss(Loss):
return per_entry_cross_ent * weights
+class WeightedDiceClassificationLoss(Loss):
+ """Dice loss for classification [1][2].
+
+ [1]: https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
+ [2]: https://arxiv.org/abs/1606.04797
+
+ """
+
+ def __init__(self, squared_normalization):
+ """Initializes the loss object.
+
+ Args:
+ squared_normalization: boolean, if set, we square the probabilities in the
+ denominator term used for normalization.
+ """
+
+ self._squared_normalization = squared_normalization
+ super(WeightedDiceClassificationLoss, self).__init__()
+
+ def _compute_loss(self,
+ prediction_tensor,
+ target_tensor,
+ weights,
+ class_indices=None):
+ """Computes the loss value.
+
+ Dice loss uses the area of the ground truth and prediction tensors for
+ normalization. We compute area by summing along the anchors (2nd) dimension.
+
+ Args:
+ prediction_tensor: A float tensor of shape [batch_size, num_pixels,
+ num_classes] representing the predicted logits for each class.
+ num_pixels denotes the total number of pixels in the spatial dimensions
+ of the mask after flattening.
+ target_tensor: A float tensor of shape [batch_size, num_pixels,
+ num_classes] representing one-hot encoded classification targets.
+ num_pixels denotes the total number of pixels in the spatial dimensions
+ of the mask after flattening.
+ weights: a float tensor of shape, either [batch_size, num_anchors,
+ num_classes] or [batch_size, num_anchors, 1]. If the shape is
+ [batch_size, num_anchors, 1], all the classses are equally weighted.
+ class_indices: (Optional) A 1-D integer tensor of class indices.
+ If provided, computes loss only for the specified class indices.
+
+ Returns:
+ loss: a float tensor of shape [batch_size, num_classes]
+ representing the value of the loss function.
+ """
+ if class_indices is not None:
+ weights *= tf.reshape(
+ ops.indices_to_dense_vector(class_indices,
+ tf.shape(prediction_tensor)[2]),
+ [1, 1, -1])
+
+ prob_tensor = tf.nn.sigmoid(prediction_tensor)
+
+ if self._squared_normalization:
+ prob_tensor = tf.pow(prob_tensor, 2)
+ target_tensor = tf.pow(target_tensor, 2)
+
+ prob_tensor *= weights
+ target_tensor *= weights
+
+ prediction_area = tf.reduce_sum(prob_tensor, axis=1)
+ gt_area = tf.reduce_sum(target_tensor, axis=1)
+
+ intersection = tf.reduce_sum(prob_tensor * target_tensor, axis=1)
+ dice_coeff = 2 * intersection / tf.maximum(gt_area + prediction_area, 1.0)
+ dice_loss = 1 - dice_coeff
+
+ return dice_loss
+
+
class SigmoidFocalClassificationLoss(Loss):
"""Sigmoid focal cross entropy loss.
diff --git a/research/object_detection/core/losses_test.py b/research/object_detection/core/losses_test.py
index 5957052ee7ed9cc9f682e960afa8514faff30e07..eb2f845f844e0d8354152498895be6eb1c6db1cc 100644
--- a/research/object_detection/core/losses_test.py
+++ b/research/object_detection/core/losses_test.py
@@ -198,6 +198,47 @@ class WeightedIOULocalizationLossTest(test_case.TestCase):
self.assertAllClose(loss_output, exp_loss)
+class WeightedGIOULocalizationLossTest(test_case.TestCase):
+
+ def testReturnsCorrectLoss(self):
+ def graph_fn():
+ prediction_tensor = tf.constant([[[1.5, 0, 2.4, 1],
+ [0, 0, 1, 1],
+ [0, 0, 0, 0]]])
+ target_tensor = tf.constant([[[1.5, 0, 2.4, 1],
+ [0, 0, 1, 1],
+ [5, 5, 10, 10]]])
+ weights = [[1.0, .5, 2.0]]
+ loss_op = losses.WeightedGIOULocalizationLoss()
+ loss = loss_op(prediction_tensor,
+ target_tensor,
+ weights=weights)
+ loss = tf.reduce_sum(loss)
+ return loss
+ exp_loss = 3.5
+ loss_output = self.execute(graph_fn, [])
+ self.assertAllClose(loss_output, exp_loss)
+
+ def testReturnsCorrectLossWithNoLabels(self):
+ def graph_fn():
+ prediction_tensor = tf.constant([[[1.5, 0, 2.4, 1],
+ [0, 0, 1, 1],
+ [0, 0, .5, .25]]])
+ target_tensor = tf.constant([[[1.5, 0, 2.4, 1],
+ [0, 0, 1, 1],
+ [50, 50, 500.5, 100.25]]])
+ weights = [[1.0, .5, 2.0]]
+ losses_mask = tf.constant([False], tf.bool)
+ loss_op = losses.WeightedGIOULocalizationLoss()
+ loss = loss_op(prediction_tensor, target_tensor, weights=weights,
+ losses_mask=losses_mask)
+ loss = tf.reduce_sum(loss)
+ return loss
+ exp_loss = 0.0
+ loss_output = self.execute(graph_fn, [])
+ self.assertAllClose(loss_output, exp_loss)
+
+
class WeightedSigmoidClassificationLossTest(test_case.TestCase):
def testReturnsCorrectLoss(self):
@@ -1406,5 +1447,111 @@ class L1LocalizationLossTest(test_case.TestCase):
self.assertAllClose(computed_value, [[0.8, 0.0], [0.6, 0.1]], rtol=1e-6)
+class WeightedDiceClassificationLoss(test_case.TestCase):
+
+ def test_compute_weights_1(self):
+ def graph_fn():
+ loss = losses.WeightedDiceClassificationLoss(squared_normalization=False)
+ pred = np.zeros((2, 3, 4), dtype=np.float32)
+ target = np.zeros((2, 3, 4), dtype=np.float32)
+
+ pred[0, 1, 0] = _logit(0.9)
+ pred[0, 2, 0] = _logit(0.1)
+ pred[0, 2, 2] = _logit(0.5)
+ pred[0, 1, 3] = _logit(0.1)
+
+ pred[1, 2, 3] = _logit(0.2)
+ pred[1, 1, 1] = _logit(0.3)
+ pred[1, 0, 2] = _logit(0.1)
+
+ target[0, 1, 0] = 1.0
+ target[0, 2, 2] = 1.0
+ target[0, 1, 3] = 1.0
+
+ target[1, 2, 3] = 1.0
+ target[1, 1, 1] = 0.0
+ target[1, 0, 2] = 0.0
+
+ weights = np.ones_like(target)
+ return loss._compute_loss(pred, target, weights)
+
+ dice_coeff = np.zeros((2, 4))
+ dice_coeff[0, 0] = 2 * 0.9 / 2.5
+ dice_coeff[0, 2] = 2 * 0.5 / 2.5
+ dice_coeff[0, 3] = 2 * 0.1 / 2.1
+ dice_coeff[1, 3] = 2 * 0.2 / 2.2
+
+ computed_value = self.execute(graph_fn, [])
+ self.assertAllClose(computed_value, 1 - dice_coeff, rtol=1e-6)
+
+ def test_compute_weights_set(self):
+
+ def graph_fn():
+ loss = losses.WeightedDiceClassificationLoss(squared_normalization=False)
+ pred = np.zeros((2, 3, 4), dtype=np.float32)
+ target = np.zeros((2, 3, 4), dtype=np.float32)
+
+ pred[0, 1, 0] = _logit(0.9)
+ pred[0, 2, 0] = _logit(0.1)
+ pred[0, 2, 2] = _logit(0.5)
+ pred[0, 1, 3] = _logit(0.1)
+
+ pred[1, 2, 3] = _logit(0.2)
+ pred[1, 1, 1] = _logit(0.3)
+ pred[1, 0, 2] = _logit(0.1)
+
+ target[0, 1, 0] = 1.0
+ target[0, 2, 2] = 1.0
+ target[0, 1, 3] = 1.0
+
+ target[1, 2, 3] = 1.0
+ target[1, 1, 1] = 0.0
+ target[1, 0, 2] = 0.0
+
+ weights = np.ones_like(target)
+ weights[:, :, 0] = 0.0
+ return loss._compute_loss(pred, target, weights)
+
+ dice_coeff = np.zeros((2, 4))
+ dice_coeff[0, 2] = 2 * 0.5 / 2.5
+ dice_coeff[0, 3] = 2 * 0.1 / 2.1
+ dice_coeff[1, 3] = 2 * 0.2 / 2.2
+
+ computed_value = self.execute(graph_fn, [])
+ self.assertAllClose(computed_value, 1 - dice_coeff, rtol=1e-6)
+
+ def test_class_indices(self):
+ def graph_fn():
+ loss = losses.WeightedDiceClassificationLoss(squared_normalization=False)
+ pred = np.zeros((2, 3, 4), dtype=np.float32)
+ target = np.zeros((2, 3, 4), dtype=np.float32)
+
+ pred[0, 1, 0] = _logit(0.9)
+ pred[0, 2, 0] = _logit(0.1)
+ pred[0, 2, 2] = _logit(0.5)
+ pred[0, 1, 3] = _logit(0.1)
+
+ pred[1, 2, 3] = _logit(0.2)
+ pred[1, 1, 1] = _logit(0.3)
+ pred[1, 0, 2] = _logit(0.1)
+
+ target[0, 1, 0] = 1.0
+ target[0, 2, 2] = 1.0
+ target[0, 1, 3] = 1.0
+
+ target[1, 2, 3] = 1.0
+ target[1, 1, 1] = 0.0
+ target[1, 0, 2] = 0.0
+
+ weights = np.ones_like(target)
+ return loss._compute_loss(pred, target, weights, class_indices=[0])
+
+ dice_coeff = np.zeros((2, 4))
+ dice_coeff[0, 0] = 2 * 0.9 / 2.5
+
+ computed_value = self.execute(graph_fn, [])
+ self.assertAllClose(computed_value, 1 - dice_coeff, rtol=1e-6)
+
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/core/model.py b/research/object_detection/core/model.py
index f0ace5d050c3df472b6eb53fa5c277523376573b..bb96038dabf53006a21ee19fec1c954f1ab6b035 100644
--- a/research/object_detection/core/model.py
+++ b/research/object_detection/core/model.py
@@ -102,7 +102,8 @@ class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)):
Args:
field: a string key, options are
fields.BoxListFields.{boxes,classes,masks,keypoints,
- keypoint_visibilities, densepose_*}
+ keypoint_visibilities, densepose_*, track_ids,
+ temporal_offsets, track_match_flags}
fields.InputDataFields.is_annotated.
Returns:
@@ -123,7 +124,7 @@ class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)):
Args:
field: a string key, options are
fields.BoxListFields.{boxes,classes,masks,keypoints,
- keypoint_visibilities, densepose_*} or
+ keypoint_visibilities, densepose_*, track_ids} or
fields.InputDataFields.is_annotated.
Returns:
@@ -303,13 +304,20 @@ class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)):
groundtruth_dp_num_points_list=None,
groundtruth_dp_part_ids_list=None,
groundtruth_dp_surface_coords_list=None,
+ groundtruth_track_ids_list=None,
+ groundtruth_temporal_offsets_list=None,
+ groundtruth_track_match_flags_list=None,
groundtruth_weights_list=None,
groundtruth_confidences_list=None,
groundtruth_is_crowd_list=None,
groundtruth_group_of_list=None,
groundtruth_area_list=None,
is_annotated_list=None,
- groundtruth_labeled_classes=None):
+ groundtruth_labeled_classes=None,
+ groundtruth_verified_neg_classes=None,
+ groundtruth_not_exhaustive_classes=None,
+ groundtruth_keypoint_depths_list=None,
+ groundtruth_keypoint_depth_weights_list=None):
"""Provide groundtruth tensors.
Args:
@@ -342,6 +350,14 @@ class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)):
shape [num_boxes, max_sampled_points, 4] containing the DensePose
surface coordinates for each sampled point. Note that there may be
padding.
+ groundtruth_track_ids_list: a list of 1-D tf.int32 tensors of shape
+ [num_boxes] containing the track IDs of groundtruth objects.
+ groundtruth_temporal_offsets_list: a list of 2-D tf.float32 tensors
+ of shape [num_boxes, 2] containing the spatial offsets of objects'
+ centers compared with the previous frame.
+ groundtruth_track_match_flags_list: a list of 1-D tf.float32 tensors
+ of shape [num_boxes] containing 0-1 flags that indicate if an object
+ has existed in the previous frame.
groundtruth_weights_list: A list of 1-D tf.float32 tensors of shape
[num_boxes] containing weights for groundtruth boxes.
groundtruth_confidences_list: A list of 2-D tf.float32 tensors of shape
@@ -359,6 +375,17 @@ class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)):
groundtruth_labeled_classes: A list of 1-D tf.float32 tensors of shape
[num_classes], containing label indices encoded as k-hot of the classes
that are exhaustively annotated.
+ groundtruth_verified_neg_classes: A list of 1-D tf.float32 tensors of
+ shape [num_classes], containing a K-hot representation of classes
+ which were verified as not present in the image.
+ groundtruth_not_exhaustive_classes: A list of 1-D tf.float32 tensors of
+ shape [num_classes], containing a K-hot representation of classes
+ which don't have all of their instances marked exhaustively.
+ groundtruth_keypoint_depths_list: a list of 2-D tf.float32 tensors
+ of shape [num_boxes, num_keypoints] containing keypoint relative depths.
+ groundtruth_keypoint_depth_weights_list: a list of 2-D tf.float32 tensors
+ of shape [num_boxes, num_keypoints] containing the weights of the
+ relative depths.
"""
self._groundtruth_lists[fields.BoxListFields.boxes] = groundtruth_boxes_list
self._groundtruth_lists[
@@ -379,6 +406,14 @@ class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)):
self._groundtruth_lists[
fields.BoxListFields.keypoint_visibilities] = (
groundtruth_keypoint_visibilities_list)
+ if groundtruth_keypoint_depths_list:
+ self._groundtruth_lists[
+ fields.BoxListFields.keypoint_depths] = (
+ groundtruth_keypoint_depths_list)
+ if groundtruth_keypoint_depth_weights_list:
+ self._groundtruth_lists[
+ fields.BoxListFields.keypoint_depth_weights] = (
+ groundtruth_keypoint_depth_weights_list)
if groundtruth_dp_num_points_list:
self._groundtruth_lists[
fields.BoxListFields.densepose_num_points] = (
@@ -391,6 +426,17 @@ class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)):
self._groundtruth_lists[
fields.BoxListFields.densepose_surface_coords] = (
groundtruth_dp_surface_coords_list)
+ if groundtruth_track_ids_list:
+ self._groundtruth_lists[
+ fields.BoxListFields.track_ids] = groundtruth_track_ids_list
+ if groundtruth_temporal_offsets_list:
+ self._groundtruth_lists[
+ fields.BoxListFields.temporal_offsets] = (
+ groundtruth_temporal_offsets_list)
+ if groundtruth_track_match_flags_list:
+ self._groundtruth_lists[
+ fields.BoxListFields.track_match_flags] = (
+ groundtruth_track_match_flags_list)
if groundtruth_is_crowd_list:
self._groundtruth_lists[
fields.BoxListFields.is_crowd] = groundtruth_is_crowd_list
@@ -407,6 +453,15 @@ class DetectionModel(six.with_metaclass(abc.ABCMeta, _BaseClass)):
self._groundtruth_lists[
fields.InputDataFields
.groundtruth_labeled_classes] = groundtruth_labeled_classes
+ if groundtruth_verified_neg_classes:
+ self._groundtruth_lists[
+ fields.InputDataFields
+ .groundtruth_verified_neg_classes] = groundtruth_verified_neg_classes
+ if groundtruth_not_exhaustive_classes:
+ self._groundtruth_lists[
+ fields.InputDataFields
+ .groundtruth_not_exhaustive_classes] = (
+ groundtruth_not_exhaustive_classes)
@abc.abstractmethod
def regularization_losses(self):
diff --git a/research/object_detection/core/post_processing.py b/research/object_detection/core/post_processing.py
index e425cd08a2f11b4ebcc999b01ac4025101a93a8c..c4c42fc9345176ea1ef81d4e4a316cfa1148b0b5 100644
--- a/research/object_detection/core/post_processing.py
+++ b/research/object_detection/core/post_processing.py
@@ -26,6 +26,7 @@ import tensorflow.compat.v1 as tf
from object_detection.core import box_list
from object_detection.core import box_list_ops
+from object_detection.core import keypoint_ops
from object_detection.core import standard_fields as fields
from object_detection.utils import shape_utils
@@ -379,9 +380,23 @@ def _clip_window_prune_boxes(sorted_boxes, clip_window, pad_to_max_output_size,
if change_coordinate_frame:
sorted_boxes = box_list_ops.change_coordinate_frame(sorted_boxes,
clip_window)
+ if sorted_boxes.has_field(fields.BoxListFields.keypoints):
+ sorted_keypoints = sorted_boxes.get_field(fields.BoxListFields.keypoints)
+ sorted_keypoints = keypoint_ops.change_coordinate_frame(sorted_keypoints,
+ clip_window)
+ sorted_boxes.set_field(fields.BoxListFields.keypoints, sorted_keypoints)
return sorted_boxes, num_valid_nms_boxes_cumulative
+class NullContextmanager(object):
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type_arg, value_arg, traceback_arg):
+ return False
+
+
def multiclass_non_max_suppression(boxes,
scores,
score_thresh,
@@ -397,6 +412,7 @@ def multiclass_non_max_suppression(boxes,
additional_fields=None,
soft_nms_sigma=0.0,
use_hard_nms=False,
+ use_cpu_nms=False,
scope=None):
"""Multi-class version of non maximum suppression.
@@ -452,6 +468,7 @@ def multiclass_non_max_suppression(boxes,
NMS. Soft NMS is currently only supported when pad_to_max_output_size is
False.
use_hard_nms: Enforce the usage of hard NMS.
+ use_cpu_nms: Enforce NMS to run on CPU.
scope: name scope.
Returns:
@@ -474,7 +491,8 @@ def multiclass_non_max_suppression(boxes,
raise ValueError('Soft NMS (soft_nms_sigma != 0.0) is currently not '
'supported when pad_to_max_output_size is True.')
- with tf.name_scope(scope, 'MultiClassNonMaxSuppression'):
+ with tf.name_scope(scope, 'MultiClassNonMaxSuppression'), tf.device(
+ 'cpu:0') if use_cpu_nms else NullContextmanager():
num_scores = tf.shape(scores)[0]
num_classes = shape_utils.get_dim_as_int(scores.get_shape()[1])
@@ -855,7 +873,8 @@ def batch_multiclass_non_max_suppression(boxes,
max_classes_per_detection=1,
use_dynamic_map_fn=False,
use_combined_nms=False,
- use_hard_nms=False):
+ use_hard_nms=False,
+ use_cpu_nms=False):
"""Multi-class version of non maximum suppression that operates on a batch.
This op is similar to `multiclass_non_max_suppression` but operates on a batch
@@ -927,6 +946,7 @@ def batch_multiclass_non_max_suppression(boxes,
Masks and additional fields are not supported.
See argument checks in the code below for unsupported arguments.
use_hard_nms: Enforce the usage of hard NMS.
+ use_cpu_nms: Enforce NMS to run on CPU.
Returns:
'nmsed_boxes': A [batch_size, max_detections, 4] float32 tensor
@@ -1162,7 +1182,8 @@ def batch_multiclass_non_max_suppression(boxes,
use_partitioned_nms=use_partitioned_nms,
additional_fields=per_image_additional_fields,
soft_nms_sigma=soft_nms_sigma,
- use_hard_nms=use_hard_nms)
+ use_hard_nms=use_hard_nms,
+ use_cpu_nms=use_cpu_nms)
if not use_static_shapes:
nmsed_boxlist = box_list_ops.pad_or_clip_box_list(
diff --git a/research/object_detection/core/preprocessor.py b/research/object_detection/core/preprocessor.py
index 6cebfd99112cb9542e97626c917778c9be3a7de9..3c98596a24ed0286bb7e369e1c8862ef9133b3bd 100644
--- a/research/object_detection/core/preprocessor.py
+++ b/research/object_detection/core/preprocessor.py
@@ -571,6 +571,8 @@ def random_horizontal_flip(image,
keypoint_visibilities=None,
densepose_part_ids=None,
densepose_surface_coords=None,
+ keypoint_depths=None,
+ keypoint_depth_weights=None,
keypoint_flip_permutation=None,
probability=0.5,
seed=None,
@@ -602,6 +604,12 @@ def random_horizontal_flip(image,
(y, x) are the normalized image coordinates for a
sampled point, and (v, u) is the surface
coordinate for the part.
+ keypoint_depths: (optional) rank 2 float32 tensor with shape [num_instances,
+ num_keypoints] representing the relative depth of the
+ keypoints.
+ keypoint_depth_weights: (optional) rank 2 float32 tensor with shape
+ [num_instances, num_keypoints] representing the
+ weights of the relative depth of the keypoints.
keypoint_flip_permutation: rank 1 int32 tensor containing the keypoint flip
permutation.
probability: the probability of performing this augmentation.
@@ -631,6 +639,10 @@ def random_horizontal_flip(image,
[num_instances, num_points].
densepose_surface_coords: rank 3 float32 tensor with shape
[num_instances, num_points, 4].
+ keypoint_depths: rank 2 float32 tensor with shape [num_instances,
+ num_keypoints]
+ keypoint_depth_weights: rank 2 float32 tensor with shape [num_instances,
+ num_keypoints].
Raises:
ValueError: if keypoints are provided but keypoint_flip_permutation is not.
@@ -708,6 +720,21 @@ def random_horizontal_flip(image,
lambda: (densepose_part_ids, densepose_surface_coords))
result.extend(densepose_tensors)
+ # flip keypoint depths and weights.
+ if (keypoint_depths is not None and
+ keypoint_flip_permutation is not None):
+ kpt_flip_perm = keypoint_flip_permutation
+ keypoint_depths = tf.cond(
+ do_a_flip_random,
+ lambda: tf.gather(keypoint_depths, kpt_flip_perm, axis=1),
+ lambda: keypoint_depths)
+ keypoint_depth_weights = tf.cond(
+ do_a_flip_random,
+ lambda: tf.gather(keypoint_depth_weights, kpt_flip_perm, axis=1),
+ lambda: keypoint_depth_weights)
+ result.append(keypoint_depths)
+ result.append(keypoint_depth_weights)
+
return tuple(result)
@@ -1049,6 +1076,28 @@ def random_rgb_to_gray(image,
return image
+def adjust_gamma(image, gamma=1.0, gain=1.0):
+ """Adjusts the gamma.
+
+ Args:
+ image: rank 3 float32 tensor contains 1 image -> [height, width, channels]
+ with pixel values varying between [0, 255].
+ gamma: the gamma value. Must be a non-negative real number.
+ gain: a constant multiplier.
+
+ Returns:
+ image: image which is the same shape as input image.
+ """
+ with tf.name_scope('AdjustGamma', values=[image]):
+ def _adjust_gamma(image):
+ image = tf.image.adjust_gamma(image / 255, gamma, gain) * 255
+ image = tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=255.0)
+ return image
+
+ image = _augment_only_rgb_channels(image, _adjust_gamma)
+ return image
+
+
def random_adjust_brightness(image,
max_delta=0.2,
seed=None,
@@ -1069,7 +1118,6 @@ def random_adjust_brightness(image,
Returns:
image: image which is the same shape as input image.
- boxes: boxes which is the same shape as input boxes.
"""
with tf.name_scope('RandomAdjustBrightness', values=[image]):
generator_func = functools.partial(tf.random_uniform, [],
@@ -1258,7 +1306,7 @@ def random_distort_color(image, color_ordering=0, preprocess_vars_cache=None):
return image
-def random_jitter_boxes(boxes, ratio=0.05, seed=None):
+def random_jitter_boxes(boxes, ratio=0.05, jitter_mode='random', seed=None):
"""Randomly jitter boxes in image.
Args:
@@ -1269,45 +1317,46 @@ def random_jitter_boxes(boxes, ratio=0.05, seed=None):
ratio: The ratio of the box width and height that the corners can jitter.
For example if the width is 100 pixels and ratio is 0.05,
the corners can jitter up to 5 pixels in the x direction.
+ jitter_mode: One of
+ shrink - Only shrinks boxes.
+ expand - Only expands boxes.
+ default - Randomly and independently perturbs each box boundary.
seed: random seed.
Returns:
boxes: boxes which is the same shape as input boxes.
"""
- def random_jitter_box(box, ratio, seed):
- """Randomly jitter box.
+ with tf.name_scope('RandomJitterBoxes', values=[boxes]):
+ ymin, xmin, ymax, xmax = (boxes[:, i] for i in range(4))
- Args:
- box: bounding box [1, 1, 4].
- ratio: max ratio between jittered box and original box,
- a number between [0, 0.5].
- seed: random seed.
+ height, width = ymax - ymin, xmax - xmin
+ ycenter, xcenter = (ymin + ymax) / 2.0, (xmin + xmax) / 2.0
- Returns:
- jittered_box: jittered box.
- """
- rand_numbers = tf.random_uniform(
- [1, 1, 4], minval=-ratio, maxval=ratio, dtype=tf.float32, seed=seed)
- box_width = tf.subtract(box[0, 0, 3], box[0, 0, 1])
- box_height = tf.subtract(box[0, 0, 2], box[0, 0, 0])
- hw_coefs = tf.stack([box_height, box_width, box_height, box_width])
- hw_rand_coefs = tf.multiply(hw_coefs, rand_numbers)
- jittered_box = tf.add(box, hw_rand_coefs)
- jittered_box = tf.clip_by_value(jittered_box, 0.0, 1.0)
- return jittered_box
+ height = tf.abs(height)
+ width = tf.abs(width)
- with tf.name_scope('RandomJitterBoxes', values=[boxes]):
- # boxes are [N, 4]. Lets first make them [N, 1, 1, 4]
- boxes_shape = tf.shape(boxes)
- boxes = tf.expand_dims(boxes, 1)
- boxes = tf.expand_dims(boxes, 2)
+ if jitter_mode == 'shrink':
+ min_ratio, max_ratio = -ratio, 0
+ elif jitter_mode == 'expand':
+ min_ratio, max_ratio = 0, ratio
+ else:
+ min_ratio, max_ratio = -ratio, ratio
+
+ num_boxes = tf.shape(boxes)[0]
+ distortion = 1.0 + tf.random_uniform(
+ [num_boxes, 4], minval=min_ratio, maxval=max_ratio, dtype=tf.float32,
+ seed=seed)
- distorted_boxes = tf.map_fn(
- lambda x: random_jitter_box(x, ratio, seed), boxes, dtype=tf.float32)
+ ymin_jitter = height * distortion[:, 0]
+ xmin_jitter = width * distortion[:, 1]
+ ymax_jitter = height * distortion[:, 2]
+ xmax_jitter = width * distortion[:, 3]
- distorted_boxes = tf.reshape(distorted_boxes, boxes_shape)
+ ymin, ymax = ycenter - (ymin_jitter / 2.0), ycenter + (ymax_jitter / 2.0)
+ xmin, xmax = xcenter - (xmin_jitter / 2.0), xcenter + (xmax_jitter / 2.0)
- return distorted_boxes
+ boxes = tf.stack([ymin, xmin, ymax, xmax], axis=1)
+ return tf.clip_by_value(boxes, 0.0, 1.0)
def _strict_random_crop_image(image,
@@ -2937,7 +2986,7 @@ def resize_to_range(image,
for i in range(len(channels))
],
axis=2)
- new_image.set_shape([max_dimension, max_dimension, 3])
+ new_image.set_shape([max_dimension, max_dimension, len(channels)])
result = [new_image]
if masks is not None:
@@ -3111,6 +3160,7 @@ def resize_pad_to_multiple(image, masks=None, multiple=1):
image_height, image_width, num_channels = _get_image_info(image)
image = image[tf.newaxis, :, :, :]
image = ops.pad_to_multiple(image, multiple)[0, :, :, :]
+ result = [image]
if masks is not None:
masks = tf.transpose(masks, (1, 2, 0))
@@ -3118,11 +3168,10 @@ def resize_pad_to_multiple(image, masks=None, multiple=1):
masks = ops.pad_to_multiple(masks, multiple)[0, :, :, :]
masks = tf.transpose(masks, (2, 0, 1))
+ result.append(masks)
- if masks is None:
- return image, tf.stack([image_height, image_width, num_channels])
- else:
- return image, masks, tf.stack([image_height, image_width, num_channels])
+ result.append(tf.stack([image_height, image_width, num_channels]))
+ return result
def scale_boxes_to_pixel_coordinates(image, boxes, keypoints=None):
@@ -3971,9 +4020,10 @@ def _get_crop_border(border, size):
def random_square_crop_by_scale(image, boxes, labels, label_weights,
- masks=None, keypoints=None, max_border=128,
- scale_min=0.6, scale_max=1.3, num_scales=8,
- seed=None, preprocess_vars_cache=None):
+ label_confidences=None, masks=None,
+ keypoints=None, max_border=128, scale_min=0.6,
+ scale_max=1.3, num_scales=8, seed=None,
+ preprocess_vars_cache=None):
"""Randomly crop a square in proportion to scale and image size.
Extract a square sized crop from an image whose side length is sampled by
@@ -3993,6 +4043,8 @@ def random_square_crop_by_scale(image, boxes, labels, label_weights,
labels: rank 1 int32 tensor containing the object classes.
label_weights: float32 tensor of shape [num_instances] representing the
weight for each box.
+ label_confidences: (optional) float32 tensor of shape [num_instances]
+ representing the confidence for each box.
masks: (optional) rank 3 float32 tensor with shape
[num_instances, height, width] containing instance masks. The masks
are of the same height, width as the input `image`.
@@ -4021,6 +4073,8 @@ def random_square_crop_by_scale(image, boxes, labels, label_weights,
Boxes are in normalized form.
labels: new labels.
label_weights: rank 1 float32 tensor with shape [num_instances].
+ label_confidences: (optional) float32 tensor of shape [num_instances]
+ representing the confidence for each box.
masks: rank 3 float32 tensor with shape [num_instances, height, width]
containing instance masks.
@@ -4110,6 +4164,9 @@ def random_square_crop_by_scale(image, boxes, labels, label_weights,
tf.gather(labels, indices),
tf.gather(label_weights, indices)]
+ if label_confidences is not None:
+ return_values.append(tf.gather(label_confidences, indices))
+
if masks is not None:
new_masks = tf.expand_dims(masks, -1)
new_masks = new_masks[:, ymin:ymax, xmin:xmax]
@@ -4135,6 +4192,7 @@ def random_scale_crop_and_pad_to_square(
label_weights,
masks=None,
keypoints=None,
+ label_confidences=None,
scale_min=0.1,
scale_max=2.0,
output_size=512,
@@ -4168,6 +4226,8 @@ def random_scale_crop_and_pad_to_square(
as the input `image`.
keypoints: (optional) rank 3 float32 tensor with shape [num_instances,
num_keypoints, 2]. The keypoints are in y-x normalized coordinates.
+ label_confidences: (optional) float32 tensor of shape [num_instance]
+ representing the confidence for each box.
scale_min: float, the minimum value for the random scale factor.
scale_max: float, the maximum value for the random scale factor.
output_size: int, the desired (square) output image size.
@@ -4183,9 +4243,8 @@ def random_scale_crop_and_pad_to_square(
label_weights: rank 1 float32 tensor with shape [num_instances].
masks: rank 3 float32 tensor with shape [num_instances, height, width]
containing instance masks.
-
+ label_confidences: confidences for retained boxes.
"""
-
img_shape = tf.shape(image)
input_height, input_width = img_shape[0], img_shape[1]
random_scale = tf.random_uniform([], scale_min, scale_max, seed=seed)
@@ -4250,6 +4309,9 @@ def random_scale_crop_and_pad_to_square(
keypoints, [0.0, 0.0, 1.0, 1.0])
return_values.append(keypoints)
+ if label_confidences is not None:
+ return_values.append(tf.gather(label_confidences, indices))
+
return return_values
@@ -4259,7 +4321,8 @@ def get_default_func_arg_map(include_label_weights=True,
include_instance_masks=False,
include_keypoints=False,
include_keypoint_visibilities=False,
- include_dense_pose=False):
+ include_dense_pose=False,
+ include_keypoint_depths=False):
"""Returns the default mapping from a preprocessor function to its args.
Args:
@@ -4277,6 +4340,8 @@ def get_default_func_arg_map(include_label_weights=True,
the keypoint visibilities, too.
include_dense_pose: If True, preprocessing functions will modify the
DensePose labels, too.
+ include_keypoint_depths: If True, preprocessing functions will modify the
+ keypoint depth labels, too.
Returns:
A map from preprocessing functions to the arguments they receive.
@@ -4319,6 +4384,13 @@ def get_default_func_arg_map(include_label_weights=True,
fields.InputDataFields.groundtruth_dp_part_ids)
groundtruth_dp_surface_coords = (
fields.InputDataFields.groundtruth_dp_surface_coords)
+ groundtruth_keypoint_depths = None
+ groundtruth_keypoint_depth_weights = None
+ if include_keypoint_depths:
+ groundtruth_keypoint_depths = (
+ fields.InputDataFields.groundtruth_keypoint_depths)
+ groundtruth_keypoint_depth_weights = (
+ fields.InputDataFields.groundtruth_keypoint_depth_weights)
prep_func_arg_map = {
normalize_image: (fields.InputDataFields.image,),
@@ -4330,6 +4402,8 @@ def get_default_func_arg_map(include_label_weights=True,
groundtruth_keypoint_visibilities,
groundtruth_dp_part_ids,
groundtruth_dp_surface_coords,
+ groundtruth_keypoint_depths,
+ groundtruth_keypoint_depth_weights,
),
random_vertical_flip: (
fields.InputDataFields.image,
@@ -4483,14 +4557,15 @@ def get_default_func_arg_map(include_label_weights=True,
(fields.InputDataFields.image,
fields.InputDataFields.groundtruth_boxes,
fields.InputDataFields.groundtruth_classes,
- groundtruth_label_weights, groundtruth_instance_masks,
- groundtruth_keypoints),
+ groundtruth_label_weights, groundtruth_label_confidences,
+ groundtruth_instance_masks, groundtruth_keypoints),
random_scale_crop_and_pad_to_square:
(fields.InputDataFields.image,
fields.InputDataFields.groundtruth_boxes,
fields.InputDataFields.groundtruth_classes,
groundtruth_label_weights, groundtruth_instance_masks,
- groundtruth_keypoints),
+ groundtruth_keypoints, groundtruth_label_confidences),
+ adjust_gamma: (fields.InputDataFields.image,),
}
return prep_func_arg_map
@@ -4541,7 +4616,6 @@ def preprocess(tensor_dict,
"""
if func_arg_map is None:
func_arg_map = get_default_func_arg_map()
-
# changes the images to image (rank 4 to rank 3) since the functions
# receive rank 3 tensor for image
if fields.InputDataFields.image in tensor_dict:
diff --git a/research/object_detection/core/preprocessor_test.py b/research/object_detection/core/preprocessor_test.py
index 396ff96da95948d0adebac14634e8f6ecbfcd7fc..91c6922f196e26bfcc332430a6857011a3bebdfd 100644
--- a/research/object_detection/core/preprocessor_test.py
+++ b/research/object_detection/core/preprocessor_test.py
@@ -105,6 +105,17 @@ class PreprocessorTest(test_case.TestCase, parameterized.TestCase):
])
return keypoints, keypoint_visibilities
+ def createTestKeypointDepths(self):
+ keypoint_depths = tf.constant([
+ [1.0, 0.9, 0.8],
+ [0.7, 0.6, 0.5]
+ ], dtype=tf.float32)
+ keypoint_depth_weights = tf.constant([
+ [0.5, 0.6, 0.7],
+ [0.8, 0.9, 1.0]
+ ], dtype=tf.float32)
+ return keypoint_depths, keypoint_depth_weights
+
def createTestKeypointsInsideCrop(self):
keypoints = np.array([
[[0.4, 0.4], [0.5, 0.5], [0.6, 0.6]],
@@ -713,6 +724,59 @@ class PreprocessorTest(test_case.TestCase, parameterized.TestCase):
test_keypoints=True)
+ def testRunRandomHorizontalFlipWithKeypointDepth(self):
+
+ def graph_fn():
+ preprocess_options = [(preprocessor.random_horizontal_flip, {})]
+ image_height = 3
+ image_width = 3
+ images = tf.random_uniform([1, image_height, image_width, 3])
+ boxes = self.createTestBoxes()
+ masks = self.createTestMasks()
+ keypoints, keypoint_visibilities = self.createTestKeypoints()
+ keypoint_depths, keypoint_depth_weights = self.createTestKeypointDepths()
+ keypoint_flip_permutation = self.createKeypointFlipPermutation()
+ tensor_dict = {
+ fields.InputDataFields.image:
+ images,
+ fields.InputDataFields.groundtruth_boxes:
+ boxes,
+ fields.InputDataFields.groundtruth_instance_masks:
+ masks,
+ fields.InputDataFields.groundtruth_keypoints:
+ keypoints,
+ fields.InputDataFields.groundtruth_keypoint_visibilities:
+ keypoint_visibilities,
+ fields.InputDataFields.groundtruth_keypoint_depths:
+ keypoint_depths,
+ fields.InputDataFields.groundtruth_keypoint_depth_weights:
+ keypoint_depth_weights,
+ }
+ preprocess_options = [(preprocessor.random_horizontal_flip, {
+ 'keypoint_flip_permutation': keypoint_flip_permutation,
+ 'probability': 1.0
+ })]
+ preprocessor_arg_map = preprocessor.get_default_func_arg_map(
+ include_instance_masks=True,
+ include_keypoints=True,
+ include_keypoint_visibilities=True,
+ include_dense_pose=False,
+ include_keypoint_depths=True)
+ tensor_dict = preprocessor.preprocess(
+ tensor_dict, preprocess_options, func_arg_map=preprocessor_arg_map)
+ keypoint_depths = tensor_dict[
+ fields.InputDataFields.groundtruth_keypoint_depths]
+ keypoint_depth_weights = tensor_dict[
+ fields.InputDataFields.groundtruth_keypoint_depth_weights]
+ output_tensors = [keypoint_depths, keypoint_depth_weights]
+ return output_tensors
+
+ output_tensors = self.execute_cpu(graph_fn, [])
+ expected_keypoint_depths = [[1.0, 0.8, 0.9], [0.7, 0.5, 0.6]]
+ expected_keypoint_depth_weights = [[0.5, 0.7, 0.6], [0.8, 1.0, 0.9]]
+ self.assertAllClose(expected_keypoint_depths, output_tensors[0])
+ self.assertAllClose(expected_keypoint_depth_weights, output_tensors[1])
+
def testRandomVerticalFlip(self):
def graph_fn():
@@ -1199,6 +1263,67 @@ class PreprocessorTest(test_case.TestCase, parameterized.TestCase):
(boxes_shape_, distorted_boxes_shape_) = self.execute_cpu(graph_fn, [])
self.assertAllEqual(boxes_shape_, distorted_boxes_shape_)
+ def testRandomJitterBoxesZeroRatio(self):
+
+ def graph_fn():
+ preprocessing_options = []
+ preprocessing_options.append((preprocessor.random_jitter_boxes,
+ {'ratio': 0.0}))
+ boxes = self.createTestBoxes()
+ tensor_dict = {fields.InputDataFields.groundtruth_boxes: boxes}
+ tensor_dict = preprocessor.preprocess(tensor_dict, preprocessing_options)
+ distorted_boxes = tensor_dict[fields.InputDataFields.groundtruth_boxes]
+ return [boxes, distorted_boxes]
+
+ (boxes, distorted_boxes) = self.execute_cpu(graph_fn, [])
+ self.assertAllEqual(boxes, distorted_boxes)
+
+ def testRandomJitterBoxesExpand(self):
+
+ def graph_fn():
+ preprocessing_options = []
+ preprocessing_options.append((preprocessor.random_jitter_boxes,
+ {'jitter_mode': 'expand'}))
+ boxes = self.createTestBoxes()
+ tensor_dict = {fields.InputDataFields.groundtruth_boxes: boxes}
+ tensor_dict = preprocessor.preprocess(tensor_dict, preprocessing_options)
+ distorted_boxes = tensor_dict[fields.InputDataFields.groundtruth_boxes]
+ return [boxes, distorted_boxes]
+
+ boxes, distorted_boxes = self.execute_cpu(graph_fn, [])
+ ymin, xmin, ymax, xmax = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
+ distorted_ymin, distorted_xmin, distorted_ymax, distorted_xmax = (
+ distorted_boxes[:, 0], distorted_boxes[:, 1], distorted_boxes[0, 2],
+ distorted_boxes[:, 3])
+
+ self.assertTrue(np.all(distorted_ymin <= ymin))
+ self.assertTrue(np.all(distorted_xmin <= xmin))
+ self.assertTrue(np.all(distorted_ymax >= ymax))
+ self.assertTrue(np.all(distorted_xmax >= xmax))
+
+ def testRandomJitterBoxesShrink(self):
+
+ def graph_fn():
+ preprocessing_options = []
+ preprocessing_options.append((preprocessor.random_jitter_boxes,
+ {'jitter_mode': 'shrink'}))
+ boxes = self.createTestBoxes()
+ tensor_dict = {fields.InputDataFields.groundtruth_boxes: boxes}
+ tensor_dict = preprocessor.preprocess(tensor_dict, preprocessing_options)
+ distorted_boxes = tensor_dict[fields.InputDataFields.groundtruth_boxes]
+ return [boxes, distorted_boxes]
+
+ boxes, distorted_boxes = self.execute_cpu(graph_fn, [])
+ ymin, xmin, ymax, xmax = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3]
+ distorted_ymin, distorted_xmin, distorted_ymax, distorted_xmax = (
+ distorted_boxes[:, 0], distorted_boxes[:, 1], distorted_boxes[0, 2],
+ distorted_boxes[:, 3])
+
+ self.assertTrue(np.all(distorted_ymin >= ymin))
+ self.assertTrue(np.all(distorted_xmin >= xmin))
+ self.assertTrue(np.all(distorted_ymax <= ymax))
+ self.assertTrue(np.all(distorted_xmax <= xmax))
+
def testRandomCropImage(self):
def graph_fn():
@@ -3814,21 +3939,23 @@ class PreprocessorTest(test_case.TestCase, parameterized.TestCase):
boxes = tf.constant([[0.25, .25, .75, .75]])
labels = tf.constant([[1]])
+ label_confidences = tf.constant([0.75])
label_weights = tf.constant([[1.]])
- (new_image, new_boxes, _, _, new_masks,
+ (new_image, new_boxes, _, _, new_confidences, new_masks,
new_keypoints) = preprocessor.random_square_crop_by_scale(
image,
boxes,
labels,
label_weights,
+ label_confidences,
masks=masks,
keypoints=keypoints,
max_border=256,
scale_min=scale,
scale_max=scale)
- return new_image, new_boxes, new_masks, new_keypoints
- image, boxes, masks, keypoints = self.execute_cpu(graph_fn, [])
+ return new_image, new_boxes, new_confidences, new_masks, new_keypoints
+ image, boxes, confidences, masks, keypoints = self.execute_cpu(graph_fn, [])
ymin, xmin, ymax, xmax = boxes[0]
self.assertAlmostEqual(ymax - ymin, 0.5 / scale)
self.assertAlmostEqual(xmax - xmin, 0.5 / scale)
@@ -3842,6 +3969,7 @@ class PreprocessorTest(test_case.TestCase, parameterized.TestCase):
self.assertAlmostEqual(scale * 256.0, size)
self.assertAllClose(image[:, :, 0], masks[0, :, :])
+ self.assertAllClose(confidences, [0.75])
@parameterized.named_parameters(('scale_0_1', 0.1), ('scale_1_0', 1.0),
('scale_2_0', 2.0))
@@ -3928,6 +4056,54 @@ class PreprocessorTest(test_case.TestCase, parameterized.TestCase):
self.assertAllClose(image[:, :, 0],
masks[0, :, :])
+ def test_random_scale_crop_and_pad_to_square_handles_confidences(self):
+
+ def graph_fn():
+ image = tf.zeros([10, 10, 1])
+ boxes = tf.constant([[0, 0, 0.5, 0.5], [0.5, 0.5, 0.75, 0.75]])
+ label_weights = tf.constant([1.0, 1.0])
+ box_labels = tf.constant([0, 1])
+ box_confidences = tf.constant([-1.0, 1.0])
+
+ (_, new_boxes, _, _,
+ new_confidences) = preprocessor.random_scale_crop_and_pad_to_square(
+ image,
+ boxes,
+ box_labels,
+ label_weights,
+ label_confidences=box_confidences,
+ scale_min=0.8,
+ scale_max=0.9,
+ output_size=10)
+ return new_boxes, new_confidences
+
+ boxes, confidences = self.execute_cpu(graph_fn, [])
+
+ self.assertLen(boxes, 2)
+ self.assertAllEqual(confidences, [-1.0, 1.0])
+
+ def testAdjustGamma(self):
+
+ def graph_fn():
+ preprocessing_options = []
+ preprocessing_options.append((preprocessor.normalize_image, {
+ 'original_minval': 0,
+ 'original_maxval': 255,
+ 'target_minval': 0,
+ 'target_maxval': 1
+ }))
+ preprocessing_options.append((preprocessor.adjust_gamma, {}))
+ images_original = self.createTestImages()
+ tensor_dict = {fields.InputDataFields.image: images_original}
+ tensor_dict = preprocessor.preprocess(tensor_dict, preprocessing_options)
+ images_gamma = tensor_dict[fields.InputDataFields.image]
+ image_original_shape = tf.shape(images_original)
+ image_gamma_shape = tf.shape(images_gamma)
+ return [image_original_shape, image_gamma_shape]
+
+ (image_original_shape_, image_gamma_shape_) = self.execute_cpu(graph_fn, [])
+ self.assertAllEqual(image_original_shape_, image_gamma_shape_)
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/core/region_similarity_calculator.py b/research/object_detection/core/region_similarity_calculator.py
index fd75d52f9f7b2d79e846f733fc312ec24176bf95..fcaba76104fbf6d706c838aa93a1c2b8db9886fe 100644
--- a/research/object_detection/core/region_similarity_calculator.py
+++ b/research/object_detection/core/region_similarity_calculator.py
@@ -1,3 +1,4 @@
+# Lint as: python3
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -79,6 +80,39 @@ class IouSimilarity(RegionSimilarityCalculator):
return box_list_ops.iou(boxlist1, boxlist2)
+class DETRSimilarity(RegionSimilarityCalculator):
+ """Class to compute similarity for the Detection Transformer model.
+
+ This class computes pairwise DETR similarity between two BoxLists using a
+ weighted combination of GIOU, classification scores, and the L1 loss.
+ """
+
+ def __init__(self, l1_weight=5, giou_weight=2):
+ super().__init__()
+ self.l1_weight = l1_weight
+ self.giou_weight = giou_weight
+
+ def _compare(self, boxlist1, boxlist2):
+ """Compute pairwise DETR similarity between the two BoxLists.
+
+ Args:
+ boxlist1: BoxList holding N groundtruth boxes.
+ boxlist2: BoxList holding M predicted boxes.
+
+ Returns:
+ A tensor with shape [N, M] representing pairwise DETR similarity scores.
+ """
+ groundtruth_labels = boxlist1.get_field(fields.BoxListFields.classes)
+ predicted_labels = boxlist2.get_field(fields.BoxListFields.classes)
+ classification_scores = tf.matmul(groundtruth_labels,
+ predicted_labels,
+ transpose_b=True)
+ loss = self.l1_weight * box_list_ops.l1(
+ boxlist1, boxlist2) + self.giou_weight * (1 - box_list_ops.giou(
+ boxlist1, boxlist2)) - classification_scores
+ return -loss
+
+
class NegSqDistSimilarity(RegionSimilarityCalculator):
"""Class to compute similarity based on the squared distance metric.
diff --git a/research/object_detection/core/region_similarity_calculator_test.py b/research/object_detection/core/region_similarity_calculator_test.py
index 9f9a10b637fa127dfd83637f43508f9dbdb80c18..ec1de45be14772a9f56e32eec67632095c629235 100644
--- a/research/object_detection/core/region_similarity_calculator_test.py
+++ b/research/object_detection/core/region_similarity_calculator_test.py
@@ -93,6 +93,25 @@ class RegionSimilarityCalculatorTest(test_case.TestCase):
iou_output = self.execute(graph_fn, [])
self.assertAllClose(iou_output, exp_output)
+ def test_detr_similarity(self):
+ def graph_fn():
+ corners1 = tf.constant([[5.0, 7.0, 7.0, 9.0]])
+ corners2 = tf.constant([[5.0, 7.0, 7.0, 9.0], [5.0, 11.0, 7.0, 13.0]])
+ groundtruth_labels = tf.constant([[1.0, 0.0]])
+ predicted_labels = tf.constant([[0.0, 1000.0], [1000.0, 0.0]])
+ boxes1 = box_list.BoxList(corners1)
+ boxes2 = box_list.BoxList(corners2)
+ boxes1.add_field(fields.BoxListFields.classes, groundtruth_labels)
+ boxes2.add_field(fields.BoxListFields.classes, predicted_labels)
+ detr_similarity_calculator = \
+ region_similarity_calculator.DETRSimilarity()
+ detr_similarity = detr_similarity_calculator.compare(
+ boxes1, boxes2, None)
+ return detr_similarity
+ exp_output = [[0.0, -20 - 8.0/3.0 + 1000.0]]
+ sim_output = self.execute(graph_fn, [])
+ self.assertAllClose(sim_output, exp_output)
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/core/standard_fields.py b/research/object_detection/core/standard_fields.py
index ddb4a842db2a70c8d9497b1032249785ae539eef..1925c550615fd9c48eab538f1e042a9ceff82a1d 100644
--- a/research/object_detection/core/standard_fields.py
+++ b/research/object_detection/core/standard_fields.py
@@ -46,6 +46,11 @@ class InputDataFields(object):
classes for which an image has been labeled.
groundtruth_boxes: coordinates of the ground truth boxes in the image.
groundtruth_classes: box-level class labels.
+ groundtruth_track_ids: box-level track ID labels.
+ groundtruth_temporal_offset: box-level temporal offsets, i.e.,
+ movement of the box center in adjacent frames.
+ groundtruth_track_match_flags: box-level flags indicating if objects
+ exist in the previous frame.
groundtruth_confidences: box-level class confidences. The shape should be
the same as the shape of groundtruth_classes.
groundtruth_label_types: box-level label types (e.g. explicit negative).
@@ -62,9 +67,15 @@ class InputDataFields(object):
groundtruth_instance_boundaries: ground truth instance boundaries.
groundtruth_instance_classes: instance mask-level class labels.
groundtruth_keypoints: ground truth keypoints.
+ groundtruth_keypoint_depths: Relative depth of the keypoints.
+ groundtruth_keypoint_depth_weights: Weights of the relative depth of the
+ keypoints.
groundtruth_keypoint_visibilities: ground truth keypoint visibilities.
groundtruth_keypoint_weights: groundtruth weight factor for keypoints.
groundtruth_label_weights: groundtruth label weights.
+ groundtruth_verified_negative_classes: groundtruth verified negative classes
+ groundtruth_not_exhaustive_classes: groundtruth not-exhaustively labeled
+ classes.
groundtruth_weights: groundtruth weight factor for bounding boxes.
groundtruth_dp_num_points: The number of DensePose sampled points for each
instance.
@@ -81,6 +92,8 @@ class InputDataFields(object):
context_features, used for reshaping.
valid_context_size: the valid context size, used in filtering the padded
context features.
+ context_features_image_id_list: the list of image source ids corresponding
+ to the features in context_features
image_format: format for the images, used to decode
image_height: height of images, used to decode
image_width: width of images, used to decode
@@ -97,6 +110,9 @@ class InputDataFields(object):
groundtruth_labeled_classes = 'groundtruth_labeled_classes'
groundtruth_boxes = 'groundtruth_boxes'
groundtruth_classes = 'groundtruth_classes'
+ groundtruth_track_ids = 'groundtruth_track_ids'
+ groundtruth_temporal_offset = 'groundtruth_temporal_offset'
+ groundtruth_track_match_flags = 'groundtruth_track_match_flags'
groundtruth_confidences = 'groundtruth_confidences'
groundtruth_label_types = 'groundtruth_label_types'
groundtruth_is_crowd = 'groundtruth_is_crowd'
@@ -109,9 +125,13 @@ class InputDataFields(object):
groundtruth_instance_boundaries = 'groundtruth_instance_boundaries'
groundtruth_instance_classes = 'groundtruth_instance_classes'
groundtruth_keypoints = 'groundtruth_keypoints'
+ groundtruth_keypoint_depths = 'groundtruth_keypoint_depths'
+ groundtruth_keypoint_depth_weights = 'groundtruth_keypoint_depth_weights'
groundtruth_keypoint_visibilities = 'groundtruth_keypoint_visibilities'
groundtruth_keypoint_weights = 'groundtruth_keypoint_weights'
groundtruth_label_weights = 'groundtruth_label_weights'
+ groundtruth_verified_neg_classes = 'groundtruth_verified_neg_classes'
+ groundtruth_not_exhaustive_classes = 'groundtruth_not_exhaustive_classes'
groundtruth_weights = 'groundtruth_weights'
groundtruth_dp_num_points = 'groundtruth_dp_num_points'
groundtruth_dp_part_ids = 'groundtruth_dp_part_ids'
@@ -123,6 +143,7 @@ class InputDataFields(object):
context_features = 'context_features'
context_feature_length = 'context_feature_length'
valid_context_size = 'valid_context_size'
+ context_features_image_id_list = 'context_features_image_id_list'
image_timestamps = 'image_timestamps'
image_format = 'image_format'
image_height = 'image_height'
@@ -146,6 +167,7 @@ class DetectionResultFields(object):
detection_boundaries: contains an object boundary for each detection box.
detection_keypoints: contains detection keypoints for each detection box.
detection_keypoint_scores: contains detection keypoint scores.
+ detection_keypoint_depths: contains detection keypoint depths.
num_detections: number of detections in the batch.
raw_detection_boxes: contains decoded detection boxes without Non-Max
suppression.
@@ -167,6 +189,9 @@ class DetectionResultFields(object):
detection_boundaries = 'detection_boundaries'
detection_keypoints = 'detection_keypoints'
detection_keypoint_scores = 'detection_keypoint_scores'
+ detection_keypoint_depths = 'detection_keypoint_depths'
+ detection_embeddings = 'detection_embeddings'
+ detection_offsets = 'detection_temporal_offsets'
num_detections = 'num_detections'
raw_detection_boxes = 'raw_detection_boxes'
raw_detection_scores = 'raw_detection_scores'
@@ -187,10 +212,14 @@ class BoxListFields(object):
keypoints: keypoints per bounding box.
keypoint_visibilities: keypoint visibilities per bounding box.
keypoint_heatmaps: keypoint heatmaps per bounding box.
+ keypoint_depths: keypoint depths per bounding box.
+ keypoint_depth_weights: keypoint depth weights per bounding box.
densepose_num_points: number of DensePose points per bounding box.
densepose_part_ids: DensePose part ids per bounding box.
densepose_surface_coords: DensePose surface coordinates per bounding box.
is_crowd: is_crowd annotation per bounding box.
+ temporal_offsets: temporal center offsets per bounding box.
+ track_match_flags: match flags per bounding box.
"""
boxes = 'boxes'
classes = 'classes'
@@ -203,11 +232,16 @@ class BoxListFields(object):
keypoints = 'keypoints'
keypoint_visibilities = 'keypoint_visibilities'
keypoint_heatmaps = 'keypoint_heatmaps'
+ keypoint_depths = 'keypoint_depths'
+ keypoint_depth_weights = 'keypoint_depth_weights'
densepose_num_points = 'densepose_num_points'
densepose_part_ids = 'densepose_part_ids'
densepose_surface_coords = 'densepose_surface_coords'
is_crowd = 'is_crowd'
group_of = 'group_of'
+ track_ids = 'track_ids'
+ temporal_offsets = 'temporal_offsets'
+ track_match_flags = 'track_match_flags'
class PredictionFields(object):
diff --git a/research/object_detection/core/target_assigner.py b/research/object_detection/core/target_assigner.py
index 4a450c5c076f7197d097211af95a361388a4d717..e491bfcfb5978e9db8e333e21461f4cd5dafd508 100644
--- a/research/object_detection/core/target_assigner.py
+++ b/research/object_detection/core/target_assigner.py
@@ -51,6 +51,7 @@ from object_detection.core import matcher as mat
from object_detection.core import region_similarity_calculator as sim_calc
from object_detection.core import standard_fields as fields
from object_detection.matchers import argmax_matcher
+from object_detection.matchers import hungarian_matcher
from object_detection.utils import shape_utils
from object_detection.utils import target_assigner_utils as ta_utils
from object_detection.utils import tf_version
@@ -510,7 +511,8 @@ def batch_assign(target_assigner,
anchors_batch, gt_box_batch, gt_class_targets_batch, gt_weights_batch):
(cls_targets, cls_weights,
reg_targets, reg_weights, match) = target_assigner.assign(
- anchors, gt_boxes, gt_class_targets, unmatched_class_label, gt_weights)
+ anchors, gt_boxes, gt_class_targets, unmatched_class_label,
+ gt_weights)
cls_targets_list.append(cls_targets)
cls_weights_list.append(cls_weights)
reg_targets_list.append(reg_targets)
@@ -810,7 +812,20 @@ def get_batch_predictions_from_indices(batch_predictions, indices):
values: A tensor of shape [num_instances, channels] holding the predicted
values at the given indices.
"""
- return tf.gather_nd(batch_predictions, indices)
+ # Note, gather_nd (and its gradient scatter_nd) runs significantly slower (on
+ # TPU) than gather with flattened inputs, so reshape the tensor, flatten the
+ # indices, and run gather.
+ shape = shape_utils.combined_static_and_dynamic_shape(batch_predictions)
+
+ # [B, H, W, C] -> [H*W, W, 1] or [B, H, W, N, C] -> [H*W*N, W*N, N, 1]
+ rev_cum_interior_indices = tf.reverse(tf.math.cumprod(shape[-2:0:-1]), [0])
+ rev_cum_interior_indices = tf.concat([rev_cum_interior_indices, [1]], axis=0)
+
+ # Compute flattened indices and gather.
+ flattened_inds = tf.linalg.matmul(
+ indices, rev_cum_interior_indices[:, tf.newaxis])[:, 0]
+ batch_predictions_2d = tf.reshape(batch_predictions, [-1, shape[-1]])
+ return tf.gather(batch_predictions_2d, flattened_inds, axis=0)
def _compute_std_dev_from_box_size(boxes_height, boxes_width, min_overlap):
@@ -835,20 +850,111 @@ def _compute_std_dev_from_box_size(boxes_height, boxes_width, min_overlap):
return sigma
+def _preprocess_keypoints_and_weights(out_height, out_width, keypoints,
+ class_onehot, class_weights,
+ keypoint_weights, class_id,
+ keypoint_indices):
+ """Preprocesses the keypoints and the corresponding keypoint weights.
+
+ This function performs several common steps to preprocess the keypoints and
+ keypoint weights features, including:
+ 1) Select the subset of keypoints based on the keypoint indices, fill the
+ keypoint NaN values with zeros and convert to absolute coordinates.
+ 2) Generate the weights of the keypoint using the following information:
+ a. The class of the instance.
+ b. The NaN value of the keypoint coordinates.
+ c. The provided keypoint weights.
+
+ Args:
+ out_height: An integer or an integer tensor indicating the output height
+ of the model.
+ out_width: An integer or an integer tensor indicating the output width of
+ the model.
+ keypoints: A float tensor of shape [num_instances, num_total_keypoints, 2]
+ representing the original keypoint grountruth coordinates.
+ class_onehot: A float tensor of shape [num_instances, num_classes]
+ containing the class targets with the 0th index assumed to map to the
+ first non-background class.
+ class_weights: A float tensor of shape [num_instances] containing weights
+ for groundtruth instances.
+ keypoint_weights: A float tensor of shape
+ [num_instances, num_total_keypoints] representing the weights of each
+ keypoints.
+ class_id: int, the ID of the class (0-indexed) that contains the target
+ keypoints to consider in this task.
+ keypoint_indices: A list of integers representing the indices of the
+ keypoints to be considered in this task. This is used to retrieve the
+ subset of the keypoints that should be considered in this task.
+
+ Returns:
+ A tuple of two tensors:
+ keypoint_absolute: A float tensor of shape
+ [num_instances, num_keypoints, 2] which is the selected and updated
+ keypoint coordinates.
+ keypoint_weights: A float tensor of shape [num_instances, num_keypoints]
+ representing the updated weight of each keypoint.
+ """
+ # Select the targets keypoints by their type ids and generate the mask
+ # of valid elements.
+ valid_mask, keypoints = ta_utils.get_valid_keypoint_mask_for_class(
+ keypoint_coordinates=keypoints,
+ class_id=class_id,
+ class_onehot=class_onehot,
+ class_weights=class_weights,
+ keypoint_indices=keypoint_indices)
+ # Keypoint coordinates in absolute coordinate system.
+ # The shape of the tensors: [num_instances, num_keypoints, 2].
+ keypoints_absolute = keypoint_ops.to_absolute_coordinates(
+ keypoints, out_height, out_width)
+ # Assign default weights for the keypoints.
+ if keypoint_weights is None:
+ keypoint_weights = tf.ones_like(keypoints[:, :, 0])
+ else:
+ keypoint_weights = tf.gather(
+ keypoint_weights, indices=keypoint_indices, axis=1)
+ keypoint_weights = keypoint_weights * valid_mask
+ return keypoints_absolute, keypoint_weights
+
+
class CenterNetCenterHeatmapTargetAssigner(object):
"""Wrapper to compute the object center heatmap."""
- def __init__(self, stride, min_overlap=0.7):
+ def __init__(self,
+ stride,
+ min_overlap=0.7,
+ compute_heatmap_sparse=False,
+ keypoint_class_id=None,
+ keypoint_indices=None,
+ keypoint_weights_for_center=None):
"""Initializes the target assigner.
Args:
stride: int, the stride of the network in output pixels.
min_overlap: The minimum IOU overlap that boxes need to have to not be
penalized.
+ compute_heatmap_sparse: bool, indicating whether or not to use the sparse
+ version of the Op that computes the heatmap. The sparse version scales
+ better with number of classes, but in some cases is known to cause
+ OOM error. See (b/170989061).
+ keypoint_class_id: int, the ID of the class (0-indexed) that contains the
+ target keypoints to consider in this task.
+ keypoint_indices: A list of integers representing the indices of the
+ keypoints to be considered in this task. This is used to retrieve the
+ subset of the keypoints from gt_keypoints that should be considered in
+ this task.
+ keypoint_weights_for_center: The keypoint weights used for calculating the
+ location of object center. The number of weights need to be the same as
+ the number of keypoints. The object center is calculated by the weighted
+ mean of the keypoint locations. If not provided, the object center is
+ determined by the center of the bounding box (default behavior).
"""
self._stride = stride
self._min_overlap = min_overlap
+ self._compute_heatmap_sparse = compute_heatmap_sparse
+ self._keypoint_class_id = keypoint_class_id
+ self._keypoint_indices = keypoint_indices
+ self._keypoint_weights_for_center = keypoint_weights_for_center
def assign_center_targets_from_boxes(self,
height,
@@ -879,8 +985,8 @@ class CenterNetCenterHeatmapTargetAssigner(object):
the stride specified during initialization.
"""
- out_height = tf.cast(height // self._stride, tf.float32)
- out_width = tf.cast(width // self._stride, tf.float32)
+ out_height = tf.cast(tf.maximum(height // self._stride, 1), tf.float32)
+ out_width = tf.cast(tf.maximum(width // self._stride, 1), tf.float32)
# Compute the yx-grid to be used to generate the heatmap. Each returned
# tensor has shape of [out_height, out_width]
(y_grid, x_grid) = ta_utils.image_shape_to_grids(out_height, out_width)
@@ -893,9 +999,10 @@ class CenterNetCenterHeatmapTargetAssigner(object):
gt_weights_list):
boxes = box_list.BoxList(boxes)
# Convert the box coordinates to absolute output image dimension space.
- boxes = box_list_ops.to_absolute_coordinates(boxes,
- height // self._stride,
- width // self._stride)
+ boxes = box_list_ops.to_absolute_coordinates(
+ boxes,
+ tf.maximum(height // self._stride, 1),
+ tf.maximum(width // self._stride, 1))
# Get the box center coordinates. Each returned tensors have the shape of
# [num_instances]
(y_center, x_center, boxes_height,
@@ -913,7 +1020,147 @@ class CenterNetCenterHeatmapTargetAssigner(object):
x_coordinates=x_center,
sigma=sigma,
channel_onehot=class_targets,
- channel_weights=weights)
+ channel_weights=weights,
+ sparse=self._compute_heatmap_sparse)
+ heatmaps.append(heatmap)
+
+ # Return the stacked heatmaps over the batch.
+ return tf.stack(heatmaps, axis=0)
+
+ def assign_center_targets_from_keypoints(self,
+ height,
+ width,
+ gt_classes_list,
+ gt_keypoints_list,
+ gt_weights_list=None,
+ gt_keypoints_weights_list=None):
+ """Computes the object center heatmap target using keypoint locations.
+
+ Args:
+ height: int, height of input to the model. This is used to
+ determine the height of the output.
+ width: int, width of the input to the model. This is used to
+ determine the width of the output.
+ gt_classes_list: A list of float tensors with shape [num_boxes,
+ num_classes] representing the one-hot encoded class labels for each box
+ in the gt_boxes_list.
+ gt_keypoints_list: A list of float tensors with shape [num_boxes, 4]
+ representing the groundtruth detection bounding boxes for each sample in
+ the batch. The box coordinates are expected in normalized coordinates.
+ gt_weights_list: A list of float tensors with shape [num_boxes]
+ representing the weight of each groundtruth detection box.
+ gt_keypoints_weights_list: [Optional] a list of 3D tf.float32 tensors of
+ shape [num_instances, num_total_keypoints] representing the weights of
+ each keypoints. If not provided, then all not NaN keypoints will be
+ equally weighted.
+
+ Returns:
+ heatmap: A Tensor of size [batch_size, output_height, output_width,
+ num_classes] representing the per class center heatmap. output_height
+ and output_width are computed by dividing the input height and width by
+ the stride specified during initialization.
+ """
+ assert (self._keypoint_weights_for_center is not None and
+ self._keypoint_class_id is not None and
+ self._keypoint_indices is not None)
+ out_height = tf.cast(tf.maximum(height // self._stride, 1), tf.float32)
+ out_width = tf.cast(tf.maximum(width // self._stride, 1), tf.float32)
+ # Compute the yx-grid to be used to generate the heatmap. Each returned
+ # tensor has shape of [out_height, out_width]
+ (y_grid, x_grid) = ta_utils.image_shape_to_grids(out_height, out_width)
+
+ heatmaps = []
+ if gt_weights_list is None:
+ gt_weights_list = [None] * len(gt_classes_list)
+ if gt_keypoints_weights_list is None:
+ gt_keypoints_weights_list = [None] * len(gt_keypoints_list)
+
+ for keypoints, classes, kp_weights, weights in zip(
+ gt_keypoints_list, gt_classes_list, gt_keypoints_weights_list,
+ gt_weights_list):
+
+ keypoints_absolute, kp_weights = _preprocess_keypoints_and_weights(
+ out_height=out_height,
+ out_width=out_width,
+ keypoints=keypoints,
+ class_onehot=classes,
+ class_weights=weights,
+ keypoint_weights=kp_weights,
+ class_id=self._keypoint_class_id,
+ keypoint_indices=self._keypoint_indices)
+ # _, num_keypoints, _ = (
+ # shape_utils.combined_static_and_dynamic_shape(keypoints_absolute))
+
+ # Update the keypoint weights by the specified keypoints weights.
+ kp_loc_weights = tf.constant(
+ self._keypoint_weights_for_center, dtype=tf.float32)
+ updated_kp_weights = kp_weights * kp_loc_weights[tf.newaxis, :]
+
+ # Obtain the sum of the weights for each instance.
+ # instance_weight_sum has shape: [num_instance].
+ instance_weight_sum = tf.reduce_sum(updated_kp_weights, axis=1)
+
+ # Weight the keypoint coordinates by updated_kp_weights.
+ # weighted_keypoints has shape: [num_instance, num_keypoints, 2]
+ weighted_keypoints = keypoints_absolute * tf.expand_dims(
+ updated_kp_weights, axis=2)
+
+ # Compute the mean of the keypoint coordinates over the weighted
+ # keypoints.
+ # keypoint_mean has shape: [num_instance, 2]
+ keypoint_mean = tf.math.divide(
+ tf.reduce_sum(weighted_keypoints, axis=1),
+ tf.expand_dims(instance_weight_sum, axis=-1))
+
+ # Replace the NaN values (due to divided by zeros in the above operation)
+ # by 0.0 where the sum of instance weight is zero.
+ # keypoint_mean has shape: [num_instance, 2]
+ keypoint_mean = tf.where(
+ tf.stack([instance_weight_sum, instance_weight_sum], axis=1) > 0.0,
+ keypoint_mean, tf.zeros_like(keypoint_mean))
+
+ # Compute the distance from each keypoint to the mean location using
+ # broadcasting and weighted by updated_kp_weights.
+ # keypoint_dist has shape: [num_instance, num_keypoints]
+ keypoint_mean = tf.expand_dims(keypoint_mean, axis=1)
+ keypoint_dist = tf.math.sqrt(
+ tf.reduce_sum(
+ tf.math.square(keypoints_absolute - keypoint_mean), axis=2))
+ keypoint_dist = keypoint_dist * updated_kp_weights
+
+ # Compute the average of the distances from each keypoint to the mean
+ # location and update the average value by zero when the instance weight
+ # is zero.
+ # avg_radius has shape: [num_instance]
+ avg_radius = tf.math.divide(
+ tf.reduce_sum(keypoint_dist, axis=1), instance_weight_sum)
+ avg_radius = tf.where(
+ instance_weight_sum > 0.0, avg_radius, tf.zeros_like(avg_radius))
+
+ # Update the class instance weight. If the instance doesn't contain enough
+ # valid keypoint values (i.e. instance_weight_sum == 0.0), then set the
+ # instance weight to zero.
+ # updated_class_weights has shape: [num_instance]
+ updated_class_weights = tf.where(
+ instance_weight_sum > 0.0, weights, tf.zeros_like(weights))
+
+ # Compute the sigma from average distance. We use 2 * average distance to
+ # to approximate the width/height of the bounding box.
+ # sigma has shape: [num_instances].
+ sigma = _compute_std_dev_from_box_size(2 * avg_radius, 2 * avg_radius,
+ self._min_overlap)
+
+ # Apply the Gaussian kernel to the center coordinates. Returned heatmap
+ # has shape of [out_height, out_width, num_classes]
+ heatmap = ta_utils.coordinates_to_heatmap(
+ y_grid=y_grid,
+ x_grid=x_grid,
+ y_coordinates=keypoint_mean[:, 0, 0],
+ x_coordinates=keypoint_mean[:, 0, 1],
+ sigma=sigma,
+ channel_onehot=classes,
+ channel_weights=updated_class_weights,
+ sparse=self._compute_heatmap_sparse)
heatmaps.append(heatmap)
# Return the stacked heatmaps over the batch.
@@ -984,9 +1231,10 @@ class CenterNetBoxTargetAssigner(object):
for i, (boxes, weights) in enumerate(zip(gt_boxes_list, gt_weights_list)):
boxes = box_list.BoxList(boxes)
- boxes = box_list_ops.to_absolute_coordinates(boxes,
- height // self._stride,
- width // self._stride)
+ boxes = box_list_ops.to_absolute_coordinates(
+ boxes,
+ tf.maximum(height // self._stride, 1),
+ tf.maximum(width // self._stride, 1))
# Get the box center coordinates. Each returned tensors have the shape of
# [num_boxes]
(y_center, x_center, boxes_height,
@@ -1071,7 +1319,9 @@ class CenterNetKeypointTargetAssigner(object):
keypoint_indices,
keypoint_std_dev=None,
per_keypoint_offset=False,
- peak_radius=0):
+ peak_radius=0,
+ compute_heatmap_sparse=False,
+ per_keypoint_depth=False):
"""Initializes a CenterNet keypoints target assigner.
Args:
@@ -1098,13 +1348,22 @@ class CenterNetKeypointTargetAssigner(object):
out_width, 2 * num_keypoints].
peak_radius: int, the radius (in the unit of output pixel) around heatmap
peak to assign the offset targets.
+ compute_heatmap_sparse: bool, indicating whether or not to use the sparse
+ version of the Op that computes the heatmap. The sparse version scales
+ better with number of keypoint types, but in some cases is known to
+ cause an OOM error. See (b/170989061).
+ per_keypoint_depth: A bool indicates whether the model predicts the depth
+ of each keypoints in independent channels. Similar to
+ per_keypoint_offset but for the keypoint depth.
"""
self._stride = stride
self._class_id = class_id
self._keypoint_indices = keypoint_indices
self._per_keypoint_offset = per_keypoint_offset
+ self._per_keypoint_depth = per_keypoint_depth
self._peak_radius = peak_radius
+ self._compute_heatmap_sparse = compute_heatmap_sparse
if keypoint_std_dev is None:
self._keypoint_std_dev = ([_DEFAULT_KEYPOINT_OFFSET_STD_DEV] *
len(keypoint_indices))
@@ -1112,65 +1371,6 @@ class CenterNetKeypointTargetAssigner(object):
assert len(keypoint_indices) == len(keypoint_std_dev)
self._keypoint_std_dev = keypoint_std_dev
- def _preprocess_keypoints_and_weights(self, out_height, out_width, keypoints,
- class_onehot, class_weights,
- keypoint_weights):
- """Preprocesses the keypoints and the corresponding keypoint weights.
-
- This function performs several common steps to preprocess the keypoints and
- keypoint weights features, including:
- 1) Select the subset of keypoints based on the keypoint indices, fill the
- keypoint NaN values with zeros and convert to absoluate coordinates.
- 2) Generate the weights of the keypoint using the following information:
- a. The class of the instance.
- b. The NaN value of the keypoint coordinates.
- c. The provided keypoint weights.
-
- Args:
- out_height: An integer or an interger tensor indicating the output height
- of the model.
- out_width: An integer or an interger tensor indicating the output width of
- the model.
- keypoints: A float tensor of shape [num_instances, num_total_keypoints, 2]
- representing the original keypoint grountruth coordinates.
- class_onehot: A float tensor of shape [num_instances, num_classes]
- containing the class targets with the 0th index assumed to map to the
- first non-background class.
- class_weights: A float tensor of shape [num_instances] containing weights
- for groundtruth instances.
- keypoint_weights: A float tensor of shape
- [num_instances, num_total_keypoints] representing the weights of each
- keypoints.
-
- Returns:
- A tuple of two tensors:
- keypoint_absolute: A float tensor of shape
- [num_instances, num_keypoints, 2] which is the selected and updated
- keypoint coordinates.
- keypoint_weights: A float tensor of shape [num_instances, num_keypoints]
- representing the updated weight of each keypoint.
- """
- # Select the targets keypoints by their type ids and generate the mask
- # of valid elements.
- valid_mask, keypoints = ta_utils.get_valid_keypoint_mask_for_class(
- keypoint_coordinates=keypoints,
- class_id=self._class_id,
- class_onehot=class_onehot,
- class_weights=class_weights,
- keypoint_indices=self._keypoint_indices)
- # Keypoint coordinates in absolute coordinate system.
- # The shape of the tensors: [num_instances, num_keypoints, 2].
- keypoints_absolute = keypoint_ops.to_absolute_coordinates(
- keypoints, out_height, out_width)
- # Assign default weights for the keypoints.
- if keypoint_weights is None:
- keypoint_weights = tf.ones_like(keypoints[:, :, 0])
- else:
- keypoint_weights = tf.gather(
- keypoint_weights, indices=self._keypoint_indices, axis=1)
- keypoint_weights = keypoint_weights * valid_mask
- return keypoints_absolute, keypoint_weights
-
def assign_keypoint_heatmap_targets(self,
height,
width,
@@ -1212,8 +1412,8 @@ class CenterNetKeypointTargetAssigner(object):
output_width] where all values within the regions of the blackout boxes
are 0.0 and 1.0 else where.
"""
- out_width = tf.cast(width // self._stride, tf.float32)
- out_height = tf.cast(height // self._stride, tf.float32)
+ out_width = tf.cast(tf.maximum(width // self._stride, 1), tf.float32)
+ out_height = tf.cast(tf.maximum(height // self._stride, 1), tf.float32)
# Compute the yx-grid to be used to generate the heatmap. Each returned
# tensor has shape of [out_height, out_width]
y_grid, x_grid = ta_utils.image_shape_to_grids(out_height, out_width)
@@ -1231,13 +1431,15 @@ class CenterNetKeypointTargetAssigner(object):
for keypoints, classes, kp_weights, weights, boxes in zip(
gt_keypoints_list, gt_classes_list, gt_keypoints_weights_list,
gt_weights_list, gt_boxes_list):
- keypoints_absolute, kp_weights = self._preprocess_keypoints_and_weights(
+ keypoints_absolute, kp_weights = _preprocess_keypoints_and_weights(
out_height=out_height,
out_width=out_width,
keypoints=keypoints,
class_onehot=classes,
class_weights=weights,
- keypoint_weights=kp_weights)
+ keypoint_weights=kp_weights,
+ class_id=self._class_id,
+ keypoint_indices=self._keypoint_indices)
num_instances, num_keypoints, _ = (
shape_utils.combined_static_and_dynamic_shape(keypoints_absolute))
@@ -1264,9 +1466,10 @@ class CenterNetKeypointTargetAssigner(object):
if boxes is not None:
boxes = box_list.BoxList(boxes)
# Convert the box coordinates to absolute output image dimension space.
- boxes = box_list_ops.to_absolute_coordinates(boxes,
- height // self._stride,
- width // self._stride)
+ boxes = box_list_ops.to_absolute_coordinates(
+ boxes,
+ tf.maximum(height // self._stride, 1),
+ tf.maximum(width // self._stride, 1))
# Get the box height and width. Each returned tensors have the shape
# of [num_instances]
(_, _, boxes_height,
@@ -1385,13 +1588,15 @@ class CenterNetKeypointTargetAssigner(object):
for i, (keypoints, classes, kp_weights, weights) in enumerate(
zip(gt_keypoints_list, gt_classes_list, gt_keypoints_weights_list,
gt_weights_list)):
- keypoints_absolute, kp_weights = self._preprocess_keypoints_and_weights(
- out_height=height // self._stride,
- out_width=width // self._stride,
+ keypoints_absolute, kp_weights = _preprocess_keypoints_and_weights(
+ out_height=tf.maximum(height // self._stride, 1),
+ out_width=tf.maximum(width // self._stride, 1),
keypoints=keypoints,
class_onehot=classes,
class_weights=weights,
- keypoint_weights=kp_weights)
+ keypoint_weights=kp_weights,
+ class_id=self._class_id,
+ keypoint_indices=self._keypoint_indices)
num_instances, num_keypoints, _ = (
shape_utils.combined_static_and_dynamic_shape(keypoints_absolute))
@@ -1402,10 +1607,11 @@ class CenterNetKeypointTargetAssigner(object):
# All keypoint coordinates and their neighbors:
# [num_instance * num_keypoints, num_neighbors]
(y_source_neighbors, x_source_neighbors,
- valid_sources) = ta_utils.get_surrounding_grids(height // self._stride,
- width // self._stride,
- y_source, x_source,
- self._peak_radius)
+ valid_sources) = ta_utils.get_surrounding_grids(
+ tf.cast(tf.maximum(height // self._stride, 1), tf.float32),
+ tf.cast(tf.maximum(width // self._stride, 1), tf.float32),
+ y_source, x_source,
+ self._peak_radius)
_, num_neighbors = shape_utils.combined_static_and_dynamic_shape(
y_source_neighbors)
@@ -1454,6 +1660,179 @@ class CenterNetKeypointTargetAssigner(object):
batch_offsets = tf.concat(batch_offsets, axis=0)
return (batch_indices, batch_offsets, batch_weights)
+ def assign_keypoints_depth_targets(self,
+ height,
+ width,
+ gt_keypoints_list,
+ gt_classes_list,
+ gt_keypoint_depths_list,
+ gt_keypoint_depth_weights_list,
+ gt_keypoints_weights_list=None,
+ gt_weights_list=None):
+ """Returns the target depths of the keypoints.
+
+ The returned values are the relative depth information of each keypoints.
+
+ Args:
+ height: int, height of input to the CenterNet model. This is used to
+ determine the height of the output.
+ width: int, width of the input to the CenterNet model. This is used to
+ determine the width of the output.
+ gt_keypoints_list: A list of tensors with shape [num_instances,
+ num_total_keypoints, 2]. See class-level description for more detail.
+ gt_classes_list: A list of tensors with shape [num_instances,
+ num_classes]. See class-level description for more detail.
+ gt_keypoint_depths_list: A list of tensors with shape [num_instances,
+ num_total_keypoints] corresponding to the relative depth of the
+ keypoints.
+ gt_keypoint_depth_weights_list: A list of tensors with shape
+ [num_instances, num_total_keypoints] corresponding to the weights of
+ the relative depth.
+ gt_keypoints_weights_list: A list of tensors with shape [num_instances,
+ num_total_keypoints] corresponding to the weight of each keypoint.
+ gt_weights_list: A list of float tensors with shape [num_instances]. See
+ class-level description for more detail.
+
+ Returns:
+ batch_indices: an integer tensor of shape [num_total_instances, 3] (or
+ [num_total_instances, 4] if 'per_keypoint_depth' is set True) holding
+ the indices inside the predicted tensor which should be penalized. The
+ first column indicates the index along the batch dimension and the
+ second and third columns indicate the index along the y and x
+ dimensions respectively. The fourth column corresponds to the channel
+ dimension (if 'per_keypoint_offset' is set True).
+ batch_depths: a float tensor of shape [num_total_instances, 1] (or
+ [num_total_instances, num_keypoints] if per_keypoint_depth is set True)
+ indicating the target depth of each keypoint.
+ batch_weights: a float tensor of shape [num_total_instances] indicating
+ the weight of each prediction.
+ Note that num_total_instances = batch_size * num_instances *
+ num_keypoints * num_neighbors
+ """
+
+ batch_indices = []
+ batch_weights = []
+ batch_depths = []
+
+ if gt_keypoints_weights_list is None:
+ gt_keypoints_weights_list = [None] * len(gt_keypoints_list)
+ if gt_weights_list is None:
+ gt_weights_list = [None] * len(gt_classes_list)
+ if gt_keypoint_depths_list is None:
+ gt_keypoint_depths_list = [None] * len(gt_classes_list)
+ for i, (keypoints, classes, kp_weights, weights,
+ keypoint_depths, keypoint_depth_weights) in enumerate(
+ zip(gt_keypoints_list, gt_classes_list,
+ gt_keypoints_weights_list, gt_weights_list,
+ gt_keypoint_depths_list, gt_keypoint_depth_weights_list)):
+ keypoints_absolute, kp_weights = _preprocess_keypoints_and_weights(
+ out_height=tf.maximum(height // self._stride, 1),
+ out_width=tf.maximum(width // self._stride, 1),
+ keypoints=keypoints,
+ class_onehot=classes,
+ class_weights=weights,
+ keypoint_weights=kp_weights,
+ class_id=self._class_id,
+ keypoint_indices=self._keypoint_indices)
+ num_instances, num_keypoints, _ = (
+ shape_utils.combined_static_and_dynamic_shape(keypoints_absolute))
+
+ # [num_instances * num_keypoints]
+ y_source = tf.keras.backend.flatten(keypoints_absolute[:, :, 0])
+ x_source = tf.keras.backend.flatten(keypoints_absolute[:, :, 1])
+
+ # All keypoint coordinates and their neighbors:
+ # [num_instance * num_keypoints, num_neighbors]
+ (y_source_neighbors, x_source_neighbors,
+ valid_sources) = ta_utils.get_surrounding_grids(
+ tf.cast(tf.maximum(height // self._stride, 1), tf.float32),
+ tf.cast(tf.maximum(width // self._stride, 1), tf.float32),
+ y_source, x_source,
+ self._peak_radius)
+ _, num_neighbors = shape_utils.combined_static_and_dynamic_shape(
+ y_source_neighbors)
+
+ # Update the valid keypoint weights.
+ # [num_instance * num_keypoints, num_neighbors]
+ valid_keypoints = tf.cast(
+ valid_sources, dtype=tf.float32) * tf.stack(
+ [tf.keras.backend.flatten(kp_weights)] * num_neighbors, axis=-1)
+
+ # Compute the offsets and indices of the box centers. Shape:
+ # indices: [num_instances * num_keypoints, num_neighbors, 2]
+ _, indices = ta_utils.compute_floor_offsets_with_indices(
+ y_source=y_source_neighbors,
+ x_source=x_source_neighbors,
+ y_target=y_source,
+ x_target=x_source)
+ # Reshape to:
+ # indices: [num_instances * num_keypoints * num_neighbors, 2]
+ indices = tf.reshape(indices, [-1, 2])
+
+ # Gather the keypoint depth from corresponding keypoint indices:
+ # [num_instances, num_keypoints]
+ keypoint_depths = tf.gather(
+ keypoint_depths, self._keypoint_indices, axis=1)
+ # Tile the depth target to surrounding pixels.
+ # [num_instances, num_keypoints, num_neighbors]
+ tiled_keypoint_depths = tf.tile(
+ tf.expand_dims(keypoint_depths, axis=-1),
+ multiples=[1, 1, num_neighbors])
+
+ # [num_instances, num_keypoints]
+ keypoint_depth_weights = tf.gather(
+ keypoint_depth_weights, self._keypoint_indices, axis=1)
+ # [num_instances, num_keypoints, num_neighbors]
+ keypoint_depth_weights = tf.tile(
+ tf.expand_dims(keypoint_depth_weights, axis=-1),
+ multiples=[1, 1, num_neighbors])
+ # Update the weights of keypoint depth by the weights of the keypoints.
+ # A keypoint depth target is valid only if its corresponding keypoint
+ # target is also valid.
+ # [num_instances, num_keypoints, num_neighbors]
+ tiled_depth_weights = (
+ tf.reshape(valid_keypoints,
+ [num_instances, num_keypoints, num_neighbors]) *
+ keypoint_depth_weights)
+ invalid_depths = tf.logical_or(
+ tf.math.is_nan(tiled_depth_weights),
+ tf.math.is_nan(tiled_keypoint_depths))
+ # Assign zero values and weights to NaN values.
+ final_keypoint_depths = tf.where(invalid_depths,
+ tf.zeros_like(tiled_keypoint_depths),
+ tiled_keypoint_depths)
+ final_keypoint_depth_weights = tf.where(
+ invalid_depths,
+ tf.zeros_like(tiled_depth_weights),
+ tiled_depth_weights)
+ # [num_instances * num_keypoints * num_neighbors, 1]
+ batch_depths.append(tf.reshape(final_keypoint_depths, [-1, 1]))
+
+ # Prepare the batch indices to be prepended.
+ batch_index = tf.fill(
+ [num_instances * num_keypoints * num_neighbors, 1], i)
+ if self._per_keypoint_depth:
+ tiled_keypoint_types = self._get_keypoint_types(
+ num_instances, num_keypoints, num_neighbors)
+ batch_indices.append(
+ tf.concat([batch_index, indices,
+ tf.reshape(tiled_keypoint_types, [-1, 1])], axis=1))
+ else:
+ batch_indices.append(tf.concat([batch_index, indices], axis=1))
+ batch_weights.append(
+ tf.keras.backend.flatten(final_keypoint_depth_weights))
+
+ # Concatenate the tensors in the batch in the first dimension:
+ # shape: [batch_size * num_instances * num_keypoints * num_neighbors, 3] or
+ # [batch_size * num_instances * num_keypoints * num_neighbors, 4] if
+ # 'per_keypoint_offset' is set to True.
+ batch_indices = tf.concat(batch_indices, axis=0)
+ # shape: [batch_size * num_instances * num_keypoints * num_neighbors]
+ batch_weights = tf.concat(batch_weights, axis=0)
+ # shape: [batch_size * num_instances * num_keypoints * num_neighbors, 1]
+ batch_depths = tf.concat(batch_depths, axis=0)
+ return (batch_indices, batch_depths, batch_weights)
+
def assign_joint_regression_targets(self,
height,
width,
@@ -1519,13 +1898,15 @@ class CenterNetKeypointTargetAssigner(object):
for i, (keypoints, classes, boxes, kp_weights, weights) in enumerate(
zip(gt_keypoints_list, gt_classes_list,
gt_boxes_list, gt_keypoints_weights_list, gt_weights_list)):
- keypoints_absolute, kp_weights = self._preprocess_keypoints_and_weights(
- out_height=height // self._stride,
- out_width=width // self._stride,
+ keypoints_absolute, kp_weights = _preprocess_keypoints_and_weights(
+ out_height=tf.maximum(height // self._stride, 1),
+ out_width=tf.maximum(width // self._stride, 1),
keypoints=keypoints,
class_onehot=classes,
class_weights=weights,
- keypoint_weights=kp_weights)
+ keypoint_weights=kp_weights,
+ class_id=self._class_id,
+ keypoint_indices=self._keypoint_indices)
num_instances, num_keypoints, _ = (
shape_utils.combined_static_and_dynamic_shape(keypoints_absolute))
@@ -1533,9 +1914,10 @@ class CenterNetKeypointTargetAssigner(object):
if boxes is not None:
# Compute joint center from boxes.
boxes = box_list.BoxList(boxes)
- boxes = box_list_ops.to_absolute_coordinates(boxes,
- height // self._stride,
- width // self._stride)
+ boxes = box_list_ops.to_absolute_coordinates(
+ boxes,
+ tf.maximum(height // self._stride, 1),
+ tf.maximum(width // self._stride, 1))
y_center, x_center, _, _ = boxes.get_center_coordinates_and_sizes()
else:
# TODO(yuhuic): Add the logic to generate object centers from keypoints.
@@ -1554,7 +1936,8 @@ class CenterNetKeypointTargetAssigner(object):
# [num_instance * num_keypoints, num_neighbors]
(y_source_neighbors, x_source_neighbors,
valid_sources) = ta_utils.get_surrounding_grids(
- height // self._stride, width // self._stride,
+ tf.cast(tf.maximum(height // self._stride, 1), tf.float32),
+ tf.cast(tf.maximum(width // self._stride, 1), tf.float32),
tf.keras.backend.flatten(y_center_tiled),
tf.keras.backend.flatten(x_center_tiled), self._peak_radius)
@@ -1600,6 +1983,17 @@ class CenterNetKeypointTargetAssigner(object):
return (batch_indices, batch_offsets, batch_weights)
+def _resize_masks(masks, height, width, method):
+ # Resize segmentation masks to conform to output dimensions. Use TF2
+ # image resize because TF1's version is buggy:
+ # https://yaqs.corp.google.com/eng/q/4970450458378240
+ masks = tf2.image.resize(
+ masks[:, :, :, tf.newaxis],
+ size=(height, width),
+ method=method)
+ return masks[:, :, :, 0]
+
+
class CenterNetMaskTargetAssigner(object):
"""Wrapper to compute targets for segmentation masks."""
@@ -1636,22 +2030,20 @@ class CenterNetMaskTargetAssigner(object):
_, input_height, input_width = (
shape_utils.combined_static_and_dynamic_shape(gt_masks_list[0]))
- output_height = input_height // self._stride
- output_width = input_width // self._stride
+ output_height = tf.maximum(input_height // self._stride, 1)
+ output_width = tf.maximum(input_width // self._stride, 1)
segmentation_targets_list = []
for gt_masks, gt_classes in zip(gt_masks_list, gt_classes_list):
- # Resize segmentation masks to conform to output dimensions. Use TF2
- # image resize because TF1's version is buggy:
- # https://yaqs.corp.google.com/eng/q/4970450458378240
- gt_masks = tf2.image.resize(
- gt_masks[:, :, :, tf.newaxis],
- size=(output_height, output_width),
- method=mask_resize_method)
+ gt_masks = _resize_masks(gt_masks, output_height, output_width,
+ mask_resize_method)
+ gt_masks = gt_masks[:, :, :, tf.newaxis]
gt_classes_reshaped = tf.reshape(gt_classes, [-1, 1, 1, num_classes])
# Shape: [h, w, num_classes].
segmentations_for_image = tf.reduce_max(
gt_masks * gt_classes_reshaped, axis=0)
+ # Avoid the case where max of an empty array is -inf.
+ segmentations_for_image = tf.maximum(segmentations_for_image, 0.0)
segmentation_targets_list.append(segmentations_for_image)
segmentation_target = tf.stack(segmentation_targets_list, axis=0)
@@ -1729,7 +2121,9 @@ class CenterNetDensePoseTargetAssigner(object):
part_ids_one_hot = tf.one_hot(part_ids_flattened, depth=self._num_parts)
# Get DensePose coordinates in the output space.
surface_coords_abs = densepose_ops.to_absolute_coordinates(
- surface_coords, height // self._stride, width // self._stride)
+ surface_coords,
+ tf.maximum(height // self._stride, 1),
+ tf.maximum(width // self._stride, 1))
surface_coords_abs = tf.reshape(surface_coords_abs, [-1, 4])
# Each tensor has shape [num_boxes * max_sampled_points].
yabs, xabs, v, u = tf.unstack(surface_coords_abs, axis=-1)
@@ -1771,3 +2165,495 @@ class CenterNetDensePoseTargetAssigner(object):
batch_surface_coords = tf.concat(batch_surface_coords, axis=0)
batch_weights = tf.concat(batch_weights, axis=0)
return batch_indices, batch_part_ids, batch_surface_coords, batch_weights
+
+
+class CenterNetTrackTargetAssigner(object):
+ """Wrapper to compute targets for tracking task.
+
+ Reference paper: A Simple Baseline for Multi-Object Tracking [1]
+ [1]: https://arxiv.org/abs/2004.01888
+ """
+
+ def __init__(self, stride, num_track_ids):
+ self._stride = stride
+ self._num_track_ids = num_track_ids
+
+ def assign_track_targets(self,
+ height,
+ width,
+ gt_track_ids_list,
+ gt_boxes_list,
+ gt_weights_list=None):
+ """Computes the track ID targets.
+
+ Args:
+ height: int, height of input to the model. This is used to determine the
+ height of the output.
+ width: int, width of the input to the model. This is used to determine the
+ width of the output.
+ gt_track_ids_list: A list of 1-D tensors with shape [num_boxes]
+ corresponding to the track ID of each groundtruth detection box.
+ gt_boxes_list: A list of float tensors with shape [num_boxes, 4]
+ representing the groundtruth detection bounding boxes for each sample in
+ the batch. The coordinates are expected in normalized coordinates.
+ gt_weights_list: A list of 1-D tensors with shape [num_boxes]
+ corresponding to the weight of each groundtruth detection box.
+
+ Returns:
+ batch_indices: an integer tensor of shape [batch_size, num_boxes, 3]
+ holding the indices inside the predicted tensor which should be
+ penalized. The first column indicates the index along the batch
+ dimension and the second and third columns indicate the index
+ along the y and x dimensions respectively.
+ batch_weights: a float tensor of shape [batch_size, num_boxes] indicating
+ the weight of each prediction.
+ track_id_targets: An int32 tensor of size [batch_size, num_boxes,
+ num_track_ids] containing the one-hot track ID vector of each
+ groundtruth detection box.
+ """
+ track_id_targets = tf.one_hot(
+ gt_track_ids_list, depth=self._num_track_ids, axis=-1)
+
+ if gt_weights_list is None:
+ gt_weights_list = [None] * len(gt_boxes_list)
+
+ batch_indices = []
+ batch_weights = []
+
+ for i, (boxes, weights) in enumerate(zip(gt_boxes_list, gt_weights_list)):
+ boxes = box_list.BoxList(boxes)
+ boxes = box_list_ops.to_absolute_coordinates(
+ boxes,
+ tf.maximum(height // self._stride, 1),
+ tf.maximum(width // self._stride, 1))
+ # Get the box center coordinates. Each returned tensors have the shape of
+ # [num_boxes]
+ (y_center, x_center, _, _) = boxes.get_center_coordinates_and_sizes()
+ num_boxes = tf.shape(x_center)
+
+ # Compute the indices of the box centers. Shape:
+ # indices: [num_boxes, 2]
+ (_, indices) = ta_utils.compute_floor_offsets_with_indices(
+ y_source=y_center, x_source=x_center)
+
+ # Assign ones if weights are not provided.
+ if weights is None:
+ weights = tf.ones(num_boxes, dtype=tf.float32)
+
+ # Shape of [num_boxes, 1] integer tensor filled with current batch index.
+ batch_index = i * tf.ones_like(indices[:, 0:1], dtype=tf.int32)
+ batch_indices.append(tf.concat([batch_index, indices], axis=1))
+ batch_weights.append(weights)
+
+ batch_indices = tf.stack(batch_indices, axis=0)
+ batch_weights = tf.stack(batch_weights, axis=0)
+
+ return batch_indices, batch_weights, track_id_targets
+
+
+def filter_mask_overlap_min_area(masks):
+ """If a pixel belongs to 2 instances, remove it from the larger instance."""
+
+ num_instances = tf.shape(masks)[0]
+ def _filter_min_area():
+ """Helper function to filter non empty masks."""
+ areas = tf.reduce_sum(masks, axis=[1, 2], keepdims=True)
+ per_pixel_area = masks * areas
+ # Make sure background is ignored in argmin.
+ per_pixel_area = (masks * per_pixel_area +
+ (1 - masks) * per_pixel_area.dtype.max)
+ min_index = tf.cast(tf.argmin(per_pixel_area, axis=0), tf.int32)
+
+ filtered_masks = (
+ tf.range(num_instances)[:, tf.newaxis, tf.newaxis]
+ ==
+ min_index[tf.newaxis, :, :]
+ )
+
+ return tf.cast(filtered_masks, tf.float32) * masks
+
+ return tf.cond(num_instances > 0, _filter_min_area,
+ lambda: masks)
+
+
+def filter_mask_overlap(masks, method='min_area'):
+
+ if method == 'min_area':
+ return filter_mask_overlap_min_area(masks)
+ else:
+ raise ValueError('Unknown mask overlap filter type - {}'.format(method))
+
+
+class CenterNetCornerOffsetTargetAssigner(object):
+ """Wrapper to compute corner offsets for boxes using masks."""
+
+ def __init__(self, stride, overlap_resolution='min_area'):
+ """Initializes the corner offset target assigner.
+
+ Args:
+ stride: int, the stride of the network in output pixels.
+ overlap_resolution: string, specifies how we handle overlapping
+ instance masks. Currently only 'min_area' is supported which assigns
+ overlapping pixels to the instance with the minimum area.
+ """
+
+ self._stride = stride
+ self._overlap_resolution = overlap_resolution
+
+ def assign_corner_offset_targets(
+ self, gt_boxes_list, gt_masks_list):
+ """Computes the corner offset targets and foreground map.
+
+ For each pixel that is part of any object's foreground, this function
+ computes the relative offsets to the top-left and bottom-right corners of
+ that instance's bounding box. It also returns a foreground map to indicate
+ which pixels contain valid corner offsets.
+
+ Args:
+ gt_boxes_list: A list of float tensors with shape [num_boxes, 4]
+ representing the groundtruth detection bounding boxes for each sample in
+ the batch. The coordinates are expected in normalized coordinates.
+ gt_masks_list: A list of float tensors with shape [num_boxes,
+ input_height, input_width] with values in {0, 1} representing instance
+ masks for each object.
+
+ Returns:
+ corner_offsets: A float tensor of shape [batch_size, height, width, 4]
+ containing, in order, the (y, x) offsets to the top left corner and
+ the (y, x) offsets to the bottom right corner for each foregroung pixel
+ foreground: A float tensor of shape [batch_size, height, width] in which
+ each pixel is set to 1 if it is a part of any instance's foreground
+ (and thus contains valid corner offsets) and 0 otherwise.
+
+ """
+ _, input_height, input_width = (
+ shape_utils.combined_static_and_dynamic_shape(gt_masks_list[0]))
+ output_height = tf.maximum(input_height // self._stride, 1)
+ output_width = tf.maximum(input_width // self._stride, 1)
+ y_grid, x_grid = tf.meshgrid(
+ tf.range(output_height), tf.range(output_width),
+ indexing='ij')
+ y_grid, x_grid = tf.cast(y_grid, tf.float32), tf.cast(x_grid, tf.float32)
+
+ corner_targets = []
+ foreground_targets = []
+ for gt_masks, gt_boxes in zip(gt_masks_list, gt_boxes_list):
+ gt_masks = _resize_masks(gt_masks, output_height, output_width,
+ method=ResizeMethod.NEAREST_NEIGHBOR)
+ gt_masks = filter_mask_overlap(gt_masks, self._overlap_resolution)
+
+ output_height = tf.cast(output_height, tf.float32)
+ output_width = tf.cast(output_width, tf.float32)
+ ymin, xmin, ymax, xmax = tf.unstack(gt_boxes, axis=1)
+ ymin, ymax = ymin * output_height, ymax * output_height
+ xmin, xmax = xmin * output_width, xmax * output_width
+
+ top_y = ymin[:, tf.newaxis, tf.newaxis] - y_grid[tf.newaxis]
+ left_x = xmin[:, tf.newaxis, tf.newaxis] - x_grid[tf.newaxis]
+ bottom_y = ymax[:, tf.newaxis, tf.newaxis] - y_grid[tf.newaxis]
+ right_x = xmax[:, tf.newaxis, tf.newaxis] - x_grid[tf.newaxis]
+
+ foreground_target = tf.cast(tf.reduce_sum(gt_masks, axis=0) > 0.5,
+ tf.float32)
+ foreground_targets.append(foreground_target)
+
+ corner_target = tf.stack([
+ tf.reduce_sum(top_y * gt_masks, axis=0),
+ tf.reduce_sum(left_x * gt_masks, axis=0),
+ tf.reduce_sum(bottom_y * gt_masks, axis=0),
+ tf.reduce_sum(right_x * gt_masks, axis=0),
+ ], axis=2)
+
+ corner_targets.append(corner_target)
+
+ return (tf.stack(corner_targets, axis=0),
+ tf.stack(foreground_targets, axis=0))
+
+
+class CenterNetTemporalOffsetTargetAssigner(object):
+ """Wrapper to compute target tensors for the temporal offset task.
+
+ This class has methods that take as input a batch of ground truth tensors
+ (in the form of a list) and returns the targets required to train the
+ temporal offset task.
+ """
+
+ def __init__(self, stride):
+ """Initializes the target assigner.
+
+ Args:
+ stride: int, the stride of the network in output pixels.
+ """
+
+ self._stride = stride
+
+ def assign_temporal_offset_targets(self,
+ height,
+ width,
+ gt_boxes_list,
+ gt_offsets_list,
+ gt_match_list,
+ gt_weights_list=None):
+ """Returns the temporal offset targets and their indices.
+
+ For each ground truth box, this function assigns it the corresponding
+ temporal offset to train the model.
+
+ Args:
+ height: int, height of input to the model. This is used to determine the
+ height of the output.
+ width: int, width of the input to the model. This is used to determine the
+ width of the output.
+ gt_boxes_list: A list of float tensors with shape [num_boxes, 4]
+ representing the groundtruth detection bounding boxes for each sample in
+ the batch. The coordinates are expected in normalized coordinates.
+ gt_offsets_list: A list of 2-D tf.float32 tensors of shape [num_boxes, 2]
+ containing the spatial offsets of objects' centers compared with the
+ previous frame.
+ gt_match_list: A list of 1-D tf.float32 tensors of shape [num_boxes]
+ containing flags that indicate if an object has existed in the
+ previous frame.
+ gt_weights_list: A list of tensors with shape [num_boxes] corresponding to
+ the weight of each groundtruth detection box.
+
+ Returns:
+ batch_indices: an integer tensor of shape [num_boxes, 3] holding the
+ indices inside the predicted tensor which should be penalized. The
+ first column indicates the index along the batch dimension and the
+ second and third columns indicate the index along the y and x
+ dimensions respectively.
+ batch_temporal_offsets: a float tensor of shape [num_boxes, 2] of the
+ expected y and x temporal offset of each object center in the
+ output space.
+ batch_weights: a float tensor of shape [num_boxes] indicating the
+ weight of each prediction.
+ """
+
+ if gt_weights_list is None:
+ gt_weights_list = [None] * len(gt_boxes_list)
+
+ batch_indices = []
+ batch_weights = []
+ batch_temporal_offsets = []
+
+ for i, (boxes, offsets, match_flags, weights) in enumerate(zip(
+ gt_boxes_list, gt_offsets_list, gt_match_list, gt_weights_list)):
+ boxes = box_list.BoxList(boxes)
+ boxes = box_list_ops.to_absolute_coordinates(
+ boxes,
+ tf.maximum(height // self._stride, 1),
+ tf.maximum(width // self._stride, 1))
+ # Get the box center coordinates. Each returned tensors have the shape of
+ # [num_boxes]
+ (y_center, x_center, _, _) = boxes.get_center_coordinates_and_sizes()
+ num_boxes = tf.shape(x_center)
+
+ # Compute the offsets and indices of the box centers. Shape:
+ # offsets: [num_boxes, 2]
+ # indices: [num_boxes, 2]
+ (_, indices) = ta_utils.compute_floor_offsets_with_indices(
+ y_source=y_center, x_source=x_center)
+
+ # Assign ones if weights are not provided.
+ # if an object is not matched, its weight becomes zero.
+ if weights is None:
+ weights = tf.ones(num_boxes, dtype=tf.float32)
+ weights *= match_flags
+
+ # Shape of [num_boxes, 1] integer tensor filled with current batch index.
+ batch_index = i * tf.ones_like(indices[:, 0:1], dtype=tf.int32)
+ batch_indices.append(tf.concat([batch_index, indices], axis=1))
+ batch_weights.append(weights)
+ batch_temporal_offsets.append(offsets)
+
+ batch_indices = tf.concat(batch_indices, axis=0)
+ batch_weights = tf.concat(batch_weights, axis=0)
+ batch_temporal_offsets = tf.concat(batch_temporal_offsets, axis=0)
+ return (batch_indices, batch_temporal_offsets, batch_weights)
+
+
+class DETRTargetAssigner(object):
+ """Target assigner for DETR (https://arxiv.org/abs/2005.12872).
+
+ Detection Transformer (DETR) matches predicted boxes to groundtruth directly
+ to determine targets instead of matching anchors to groundtruth. Hence, the
+ new target assigner.
+ """
+
+ def __init__(self):
+ """Construct Object Detection Target Assigner."""
+ self._similarity_calc = sim_calc.DETRSimilarity()
+ self._matcher = hungarian_matcher.HungarianBipartiteMatcher()
+
+ def batch_assign(self,
+ pred_box_batch,
+ gt_box_batch,
+ pred_class_batch,
+ gt_class_targets_batch,
+ gt_weights_batch=None,
+ unmatched_class_label_batch=None):
+ """Batched assignment of classification and regression targets.
+
+ Args:
+ pred_box_batch: a tensor of shape [batch_size, num_queries, 4]
+ representing predicted bounding boxes.
+ gt_box_batch: a tensor of shape [batch_size, num_queries, 4]
+ representing groundtruth bounding boxes.
+ pred_class_batch: A list of tensors with length batch_size, where each
+ each tensor has shape [num_queries, num_classes] to be used
+ by certain similarity calculators.
+ gt_class_targets_batch: a list of tensors with length batch_size, where
+ each tensor has shape [num_gt_boxes_i, num_classes] and
+ num_gt_boxes_i is the number of boxes in the ith boxlist of
+ gt_box_batch.
+ gt_weights_batch: A list of 1-D tf.float32 tensors of shape
+ [num_boxes] containing weights for groundtruth boxes.
+ unmatched_class_label_batch: a float32 tensor with shape
+ [d_1, d_2, ..., d_k] which is consistent with the classification target
+ for each anchor (and can be empty for scalar targets). This shape must
+ thus be compatible with the `gt_class_targets_batch`.
+
+ Returns:
+ batch_cls_targets: a tensor with shape [batch_size, num_pred_boxes,
+ num_classes],
+ batch_cls_weights: a tensor with shape [batch_size, num_pred_boxes,
+ num_classes],
+ batch_reg_targets: a tensor with shape [batch_size, num_pred_boxes,
+ box_code_dimension]
+ batch_reg_weights: a tensor with shape [batch_size, num_pred_boxes].
+ """
+ pred_box_batch = [
+ box_list.BoxList(pred_box)
+ for pred_box in tf.unstack(pred_box_batch)]
+ gt_box_batch = [
+ box_list.BoxList(gt_box)
+ for gt_box in tf.unstack(gt_box_batch)]
+
+ cls_targets_list = []
+ cls_weights_list = []
+ reg_targets_list = []
+ reg_weights_list = []
+ if gt_weights_batch is None:
+ gt_weights_batch = [None] * len(gt_class_targets_batch)
+ if unmatched_class_label_batch is None:
+ unmatched_class_label_batch = [None] * len(gt_class_targets_batch)
+ pred_class_batch = tf.unstack(pred_class_batch)
+ for (pred_boxes, gt_boxes, pred_class_batch, gt_class_targets, gt_weights,
+ unmatched_class_label) in zip(pred_box_batch, gt_box_batch,
+ pred_class_batch, gt_class_targets_batch,
+ gt_weights_batch,
+ unmatched_class_label_batch):
+ (cls_targets, cls_weights, reg_targets,
+ reg_weights) = self.assign(pred_boxes, gt_boxes, pred_class_batch,
+ gt_class_targets, gt_weights,
+ unmatched_class_label)
+ cls_targets_list.append(cls_targets)
+ cls_weights_list.append(cls_weights)
+ reg_targets_list.append(reg_targets)
+ reg_weights_list.append(reg_weights)
+ batch_cls_targets = tf.stack(cls_targets_list)
+ batch_cls_weights = tf.stack(cls_weights_list)
+ batch_reg_targets = tf.stack(reg_targets_list)
+ batch_reg_weights = tf.stack(reg_weights_list)
+ return (batch_cls_targets, batch_cls_weights, batch_reg_targets,
+ batch_reg_weights)
+
+ def assign(self,
+ pred_boxes,
+ gt_boxes,
+ pred_classes,
+ gt_labels,
+ gt_weights=None,
+ unmatched_class_label=None):
+ """Assign classification and regression targets to each box_pred.
+
+ For a given set of pred_boxes and groundtruth detections, match pred_boxes
+ to gt_boxes and assign classification and regression targets to
+ each box_pred as well as weights based on the resulting match (specifying,
+ e.g., which pred_boxes should not contribute to training loss).
+
+ pred_boxes that are not matched to anything are given a classification
+ target of `unmatched_cls_target`.
+
+ Args:
+ pred_boxes: a BoxList representing N pred_boxes
+ gt_boxes: a BoxList representing M groundtruth boxes
+ pred_classes: A tensor with shape [max_num_boxes, num_classes]
+ to be used by certain similarity calculators.
+ gt_labels: a tensor of shape [M, num_classes]
+ with labels for each of the ground_truth boxes. The subshape
+ [num_classes] can be empty (corresponding to scalar inputs). When set
+ to None, gt_labels assumes a binary problem where all
+ ground_truth boxes get a positive label (of 1).
+ gt_weights: a float tensor of shape [M] indicating the weight to
+ assign to all pred_boxes match to a particular groundtruth box. The
+ weights must be in [0., 1.]. If None, all weights are set to 1.
+ Generally no groundtruth boxes with zero weight match to any pred_boxes
+ as matchers are aware of groundtruth weights. Additionally,
+ `cls_weights` and `reg_weights` are calculated using groundtruth
+ weights as an added safety.
+ unmatched_class_label: a float32 tensor with shape [d_1, d_2, ..., d_k]
+ which is consistent with the classification target for each
+ anchor (and can be empty for scalar targets). This shape must thus be
+ compatible with the groundtruth labels that are passed to the "assign"
+ function (which have shape [num_gt_boxes, d_1, d_2, ..., d_k]).
+
+ Returns:
+ cls_targets: a float32 tensor with shape [num_pred_boxes, num_classes],
+ where the subshape [num_classes] is compatible with gt_labels
+ which has shape [num_gt_boxes, num_classes].
+ cls_weights: a float32 tensor with shape [num_pred_boxes, num_classes],
+ representing weights for each element in cls_targets.
+ reg_targets: a float32 tensor with shape [num_pred_boxes,
+ box_code_dimension]
+ reg_weights: a float32 tensor with shape [num_pred_boxes]
+
+ """
+ if not unmatched_class_label:
+ unmatched_class_label = tf.constant(
+ [1] + [0] * (gt_labels.shape[1] - 1), tf.float32)
+
+ if gt_weights is None:
+ num_gt_boxes = gt_boxes.num_boxes_static()
+ if not num_gt_boxes:
+ num_gt_boxes = gt_boxes.num_boxes()
+ gt_weights = tf.ones([num_gt_boxes], dtype=tf.float32)
+
+ gt_boxes.add_field(fields.BoxListFields.classes, gt_labels)
+ pred_boxes.add_field(fields.BoxListFields.classes, pred_classes)
+
+ match_quality_matrix = self._similarity_calc.compare(
+ gt_boxes,
+ pred_boxes)
+ match = self._matcher.match(match_quality_matrix,
+ valid_rows=tf.greater(gt_weights, 0))
+
+ matched_gt_boxes = match.gather_based_on_match(
+ gt_boxes.get(),
+ unmatched_value=tf.zeros(4),
+ ignored_value=tf.zeros(4))
+ matched_gt_boxlist = box_list.BoxList(matched_gt_boxes)
+ ty, tx, th, tw = matched_gt_boxlist.get_center_coordinates_and_sizes()
+ reg_targets = tf.transpose(tf.stack([ty, tx, th, tw]))
+ cls_targets = match.gather_based_on_match(
+ gt_labels,
+ unmatched_value=unmatched_class_label,
+ ignored_value=unmatched_class_label)
+ reg_weights = match.gather_based_on_match(
+ gt_weights,
+ ignored_value=0.,
+ unmatched_value=0.)
+ cls_weights = match.gather_based_on_match(
+ gt_weights,
+ ignored_value=0.,
+ unmatched_value=1)
+
+ # convert cls_weights from per-box_pred to per-class.
+ class_label_shape = tf.shape(cls_targets)[1:]
+ weights_multiple = tf.concat(
+ [tf.constant([1]), class_label_shape],
+ axis=0)
+ cls_weights = tf.expand_dims(cls_weights, -1)
+ cls_weights = tf.tile(cls_weights, weights_multiple)
+
+ return (cls_targets, cls_weights, reg_targets, reg_weights)
diff --git a/research/object_detection/core/target_assigner_test.py b/research/object_detection/core/target_assigner_test.py
index 4dec1eb778c901a2d6ec5ed108db58d8b399a1cd..ad0eaa82006e4f97987fe14fae9a1e0473f83cc6 100644
--- a/research/object_detection/core/target_assigner_test.py
+++ b/research/object_detection/core/target_assigner_test.py
@@ -14,6 +14,7 @@
# ==============================================================================
"""Tests for object_detection.core.target_assigner."""
+from absl.testing import parameterized
import numpy as np
import tensorflow.compat.v1 as tf
@@ -115,6 +116,7 @@ class TargetAssignerTest(test_case.TestCase):
self.assertEqual(reg_weights_out.dtype, np.float32)
def test_assign_agnostic_with_keypoints(self):
+
def graph_fn(anchor_means, groundtruth_box_corners,
groundtruth_keypoints):
similarity_calc = region_similarity_calculator.IouSimilarity()
@@ -1234,7 +1236,8 @@ def _array_argmax(array):
return np.unravel_index(np.argmax(array), array.shape)
-class CenterNetCenterHeatmapTargetAssignerTest(test_case.TestCase):
+class CenterNetCenterHeatmapTargetAssignerTest(test_case.TestCase,
+ parameterized.TestCase):
def setUp(self):
super(CenterNetCenterHeatmapTargetAssignerTest, self).setUp()
@@ -1262,6 +1265,66 @@ class CenterNetCenterHeatmapTargetAssignerTest(test_case.TestCase):
self.assertEqual((15, 5), _array_argmax(targets[0, :, :, 1]))
self.assertAlmostEqual(1.0, targets[0, 15, 5, 1])
+ @parameterized.parameters(
+ {'keypoint_weights_for_center': [1.0, 1.0, 1.0, 1.0]},
+ {'keypoint_weights_for_center': [0.0, 0.0, 1.0, 1.0]},
+ )
+ def test_center_location_by_keypoints(self, keypoint_weights_for_center):
+ """Test that the centers are at the correct location."""
+ kpts_y = [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.0, 0.0, 0.0, 0.0]]
+ kpts_x = [[0.5, 0.6, 0.7, 0.8], [0.1, 0.2, 0.3, 0.4], [0.0, 0.0, 0.0, 0.0]]
+ gt_keypoints_list = [
+ tf.stack([tf.constant(kpts_y), tf.constant(kpts_x)], axis=2)
+ ]
+ kpts_weight = [[1.0, 1.0, 1.0, 1.0], [1.0, 0.0, 1.0, 0.0],
+ [1.0, 0.0, 1.0, 0.0]]
+ gt_keypoints_weights_list = [tf.constant(kpts_weight)]
+ gt_classes_list = [
+ tf.one_hot([0, 0, 0], depth=1),
+ ]
+ gt_weights_list = [tf.constant([1.0, 1.0, 0.0])]
+
+ def graph_fn():
+ assigner = targetassigner.CenterNetCenterHeatmapTargetAssigner(
+ 4,
+ keypoint_class_id=0,
+ keypoint_indices=[0, 1, 2, 3],
+ keypoint_weights_for_center=keypoint_weights_for_center)
+ targets = assigner.assign_center_targets_from_keypoints(
+ 80,
+ 80,
+ gt_classes_list=gt_classes_list,
+ gt_keypoints_list=gt_keypoints_list,
+ gt_weights_list=gt_weights_list,
+ gt_keypoints_weights_list=gt_keypoints_weights_list)
+ return targets
+
+ targets = self.execute(graph_fn, [])
+
+ if sum(keypoint_weights_for_center) == 4.0:
+ # There should be two peaks at location (5, 13), and (12, 4).
+ # (5, 13) = ((0.1 + 0.2 + 0.3 + 0.4) / 4 * 80 / 4,
+ # (0.5 + 0.6 + 0.7 + 0.8) / 4 * 80 / 4)
+ # (12, 4) = ((0.5 + 0.7) / 2 * 80 / 4,
+ # (0.1 + 0.3) / 2 * 80 / 4)
+ self.assertEqual((5, 13), _array_argmax(targets[0, :, :, 0]))
+ self.assertAlmostEqual(1.0, targets[0, 5, 13, 0])
+ self.assertEqual((1, 20, 20, 1), targets.shape)
+ targets[0, 5, 13, 0] = 0.0
+ self.assertEqual((12, 4), _array_argmax(targets[0, :, :, 0]))
+ self.assertAlmostEqual(1.0, targets[0, 12, 4, 0])
+ else:
+ # There should be two peaks at location (5, 13), and (12, 4).
+ # (7, 15) = ((0.3 + 0.4) / 2 * 80 / 4,
+ # (0.7 + 0.8) / 2 * 80 / 4)
+ # (14, 6) = (0.7 * 80 / 4, 0.3 * 80 / 4)
+ self.assertEqual((7, 15), _array_argmax(targets[0, :, :, 0]))
+ self.assertAlmostEqual(1.0, targets[0, 7, 15, 0])
+ self.assertEqual((1, 20, 20, 1), targets.shape)
+ targets[0, 7, 15, 0] = 0.0
+ self.assertEqual((14, 6), _array_argmax(targets[0, :, :, 0]))
+ self.assertAlmostEqual(1.0, targets[0, 14, 6, 0])
+
def test_center_batch_shape(self):
"""Test that the shape of the target for a batch is correct."""
def graph_fn():
@@ -1565,22 +1628,48 @@ class CenterNetBoxTargetAssignerTest(test_case.TestCase):
"""
def graph_fn():
- box_batch = [
- tf.constant([self._box_center, self._box_lower_left]),
- tf.constant([self._box_center_small, self._box_odd_coordinates]),
- ]
-
pred_array = np.ones((2, 40, 20, 2), dtype=np.int32) * -1000
pred_array[0, 20, 10] = [1, 2]
pred_array[0, 30, 5] = [3, 4]
pred_array[1, 20, 10] = [5, 6]
pred_array[1, 14, 11] = [7, 8]
+ pred_tensor = tf.constant(pred_array)
+
+ indices = tf.constant([
+ [0, 20, 10],
+ [0, 30, 5],
+ [1, 20, 10],
+ [1, 14, 11]
+ ], dtype=tf.int32)
+
+ preds = targetassigner.get_batch_predictions_from_indices(
+ pred_tensor, indices)
+ return preds
+ preds = self.execute(graph_fn, [])
+ np.testing.assert_array_equal(preds, [[1, 2], [3, 4], [5, 6], [7, 8]])
+
+ def test_get_batch_predictions_from_indices_with_class(self):
+ """Test the get_batch_predictions_from_indices function with class axis.
+
+ This test verifies that the indices returned by
+ assign_size_and_offset_targets function work as expected with a predicted
+ tensor.
+ """
+ def graph_fn():
+ pred_array = np.ones((2, 40, 20, 5, 2), dtype=np.int32) * -1000
+ pred_array[0, 20, 10, 0] = [1, 2]
+ pred_array[0, 30, 5, 2] = [3, 4]
+ pred_array[1, 20, 10, 1] = [5, 6]
+ pred_array[1, 14, 11, 4] = [7, 8]
pred_tensor = tf.constant(pred_array)
- cn_assigner = targetassigner.CenterNetBoxTargetAssigner(4)
- indices, _, _, _ = cn_assigner.assign_size_and_offset_targets(
- 160, 80, box_batch)
+ indices = tf.constant([
+ [0, 20, 10, 0],
+ [0, 30, 5, 2],
+ [1, 20, 10, 1],
+ [1, 14, 11, 4]
+ ], dtype=tf.int32)
preds = targetassigner.get_batch_predictions_from_indices(
pred_tensor, indices)
@@ -1682,6 +1771,121 @@ class CenterNetKeypointTargetAssignerTest(test_case.TestCase):
np.testing.assert_array_equal([0, 3, 2], indices[7, :])
np.testing.assert_array_almost_equal([0.6, 0.4], offsets[7, :])
+ def test_assign_keypoint_depths_target(self):
+ def graph_fn():
+ gt_classes_list = [
+ tf.one_hot([0, 1, 0, 1], depth=4),
+ ]
+ coordinates = tf.expand_dims(
+ tf.constant(
+ np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
+ [float('nan'), 0.7, 0.7, 0.9, 0.4],
+ [0.4, 0.1, 0.4, 0.2, 0.0],
+ [float('nan'), 0.0, 0.12, 0.7, 0.4]]),
+ dtype=tf.float32),
+ axis=2)
+ gt_keypoints_list = [tf.concat([coordinates, coordinates], axis=2)]
+ depths = tf.constant(
+ np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
+ [float('nan'), 0.7, float('nan'), 0.9, 0.4],
+ [0.4, 0.1, 0.4, 0.2, 0.0],
+ [0.5, 0.0, 7.0, 0.7, 0.4]]),
+ dtype=tf.float32)
+ gt_keypoint_depths_list = [depths]
+
+ gt_keypoint_depth_weights = tf.constant(
+ np.array([[1.0, 1.0, 1.0, 1.0, 1.0],
+ [float('nan'), 0.0, 1.0, 0.0, 0.0],
+ [1.0, 1.0, 1.0, 1.0, 1.0],
+ [1.0, 1.0, 0.5, 1.0, 1.0]]),
+ dtype=tf.float32)
+ gt_keypoint_depth_weights_list = [gt_keypoint_depth_weights]
+
+ cn_assigner = targetassigner.CenterNetKeypointTargetAssigner(
+ stride=4,
+ class_id=1,
+ keypoint_indices=[0, 2],
+ peak_radius=1)
+ (indices, depths, weights) = cn_assigner.assign_keypoints_depth_targets(
+ height=120,
+ width=80,
+ gt_keypoints_list=gt_keypoints_list,
+ gt_classes_list=gt_classes_list,
+ gt_keypoint_depths_list=gt_keypoint_depths_list,
+ gt_keypoint_depth_weights_list=gt_keypoint_depth_weights_list)
+ return indices, depths, weights
+ indices, depths, weights = self.execute(graph_fn, [])
+
+ # Only the last 5 elements has positive weight.
+ np.testing.assert_array_almost_equal([
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.5, 0.5, 0.5, 0.5
+ ], weights)
+ # Validate the last 5 elements' depth value.
+ np.testing.assert_array_almost_equal(
+ [7.0, 7.0, 7.0, 7.0, 7.0], depths[35:, 0])
+ self.assertEqual((40, 3), indices.shape)
+ np.testing.assert_array_equal([0, 2, 2], indices[35, :])
+
+ def test_assign_keypoint_depths_per_keypoints(self):
+ def graph_fn():
+ gt_classes_list = [
+ tf.one_hot([0, 1, 0, 1], depth=4),
+ ]
+ coordinates = tf.expand_dims(
+ tf.constant(
+ np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
+ [float('nan'), 0.7, 0.7, 0.9, 0.4],
+ [0.4, 0.1, 0.4, 0.2, 0.0],
+ [float('nan'), 0.0, 0.12, 0.7, 0.4]]),
+ dtype=tf.float32),
+ axis=2)
+ gt_keypoints_list = [tf.concat([coordinates, coordinates], axis=2)]
+ depths = tf.constant(
+ np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
+ [float('nan'), 0.7, float('nan'), 0.9, 0.4],
+ [0.4, 0.1, 0.4, 0.2, 0.0],
+ [0.5, 0.0, 7.0, 0.7, 0.4]]),
+ dtype=tf.float32)
+ gt_keypoint_depths_list = [depths]
+
+ gt_keypoint_depth_weights = tf.constant(
+ np.array([[1.0, 1.0, 1.0, 1.0, 1.0],
+ [float('nan'), 0.0, 1.0, 0.0, 0.0],
+ [1.0, 1.0, 1.0, 1.0, 1.0],
+ [1.0, 1.0, 0.5, 1.0, 1.0]]),
+ dtype=tf.float32)
+ gt_keypoint_depth_weights_list = [gt_keypoint_depth_weights]
+
+ cn_assigner = targetassigner.CenterNetKeypointTargetAssigner(
+ stride=4,
+ class_id=1,
+ keypoint_indices=[0, 2],
+ peak_radius=1,
+ per_keypoint_depth=True)
+ (indices, depths, weights) = cn_assigner.assign_keypoints_depth_targets(
+ height=120,
+ width=80,
+ gt_keypoints_list=gt_keypoints_list,
+ gt_classes_list=gt_classes_list,
+ gt_keypoint_depths_list=gt_keypoint_depths_list,
+ gt_keypoint_depth_weights_list=gt_keypoint_depth_weights_list)
+ return indices, depths, weights
+ indices, depths, weights = self.execute(graph_fn, [])
+
+ # Only the last 5 elements has positive weight.
+ np.testing.assert_array_almost_equal([
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.5, 0.5, 0.5, 0.5
+ ], weights)
+ # Validate the last 5 elements' depth value.
+ np.testing.assert_array_almost_equal(
+ [7.0, 7.0, 7.0, 7.0, 7.0], depths[35:, 0])
+ self.assertEqual((40, 4), indices.shape)
+ np.testing.assert_array_equal([0, 2, 2, 1], indices[35, :])
+
def test_assign_keypoints_offset_targets_radius(self):
def graph_fn():
gt_classes_list = [
@@ -1905,6 +2109,22 @@ class CenterNetMaskTargetAssignerTest(test_case.TestCase):
np.testing.assert_array_almost_equal(
expected_seg_target, segmentation_target)
+ def test_assign_segmentation_targets_no_objects(self):
+ def graph_fn():
+ gt_masks_list = [tf.zeros((0, 5, 5))]
+ gt_classes_list = [tf.zeros((0, 10))]
+ cn_assigner = targetassigner.CenterNetMaskTargetAssigner(stride=1)
+ segmentation_target = cn_assigner.assign_segmentation_targets(
+ gt_masks_list=gt_masks_list,
+ gt_classes_list=gt_classes_list,
+ mask_resize_method=targetassigner.ResizeMethod.NEAREST_NEIGHBOR)
+ return segmentation_target
+
+ segmentation_target = self.execute(graph_fn, [])
+ expected_seg_target = np.zeros((1, 5, 5, 10))
+ np.testing.assert_array_almost_equal(
+ expected_seg_target, segmentation_target)
+
class CenterNetDensePoseTargetAssignerTest(test_case.TestCase):
@@ -1999,6 +2219,490 @@ class CenterNetDensePoseTargetAssignerTest(test_case.TestCase):
self.assertAllClose(expected_batch_weights, batch_weights)
+class CenterNetTrackTargetAssignerTest(test_case.TestCase):
+
+ def setUp(self):
+ super(CenterNetTrackTargetAssignerTest, self).setUp()
+ self._box_center = [0.0, 0.0, 1.0, 1.0]
+ self._box_center_small = [0.25, 0.25, 0.75, 0.75]
+ self._box_lower_left = [0.5, 0.0, 1.0, 0.5]
+ self._box_center_offset = [0.1, 0.05, 1.0, 1.0]
+ self._box_odd_coordinates = [0.1625, 0.2125, 0.5625, 0.9625]
+
+ def test_assign_track_targets(self):
+ """Test the assign_track_targets function."""
+ def graph_fn():
+ box_batch = [
+ tf.constant([self._box_center, self._box_lower_left]),
+ tf.constant([self._box_lower_left, self._box_center_small]),
+ tf.constant([self._box_center_small, self._box_odd_coordinates]),
+ ]
+ track_id_batch = [
+ tf.constant([0, 1]),
+ tf.constant([1, 0]),
+ tf.constant([0, 2]),
+ ]
+
+ assigner = targetassigner.CenterNetTrackTargetAssigner(
+ stride=4, num_track_ids=3)
+
+ (batch_indices, batch_weights,
+ track_targets) = assigner.assign_track_targets(
+ height=80,
+ width=80,
+ gt_track_ids_list=track_id_batch,
+ gt_boxes_list=box_batch)
+ return batch_indices, batch_weights, track_targets
+
+ indices, weights, track_ids = self.execute(graph_fn, [])
+
+ self.assertEqual(indices.shape, (3, 2, 3))
+ self.assertEqual(track_ids.shape, (3, 2, 3))
+ self.assertEqual(weights.shape, (3, 2))
+
+ np.testing.assert_array_equal(indices,
+ [[[0, 10, 10], [0, 15, 5]],
+ [[1, 15, 5], [1, 10, 10]],
+ [[2, 10, 10], [2, 7, 11]]])
+ np.testing.assert_array_equal(track_ids,
+ [[[1, 0, 0], [0, 1, 0]],
+ [[0, 1, 0], [1, 0, 0]],
+ [[1, 0, 0], [0, 0, 1]]])
+ np.testing.assert_array_equal(weights, [[1, 1], [1, 1], [1, 1]])
+
+ def test_assign_track_targets_weights(self):
+ """Test the assign_track_targets function with box weights."""
+ def graph_fn():
+ box_batch = [
+ tf.constant([self._box_center, self._box_lower_left]),
+ tf.constant([self._box_lower_left, self._box_center_small]),
+ tf.constant([self._box_center_small, self._box_odd_coordinates]),
+ ]
+ track_id_batch = [
+ tf.constant([0, 1]),
+ tf.constant([1, 0]),
+ tf.constant([0, 2]),
+ ]
+ weights_batch = [
+ tf.constant([0.0, 1.0]),
+ tf.constant([1.0, 1.0]),
+ tf.constant([0.0, 0.0])
+ ]
+
+ assigner = targetassigner.CenterNetTrackTargetAssigner(
+ stride=4, num_track_ids=3)
+
+ (batch_indices, batch_weights,
+ track_targets) = assigner.assign_track_targets(
+ height=80,
+ width=80,
+ gt_track_ids_list=track_id_batch,
+ gt_boxes_list=box_batch,
+ gt_weights_list=weights_batch)
+ return batch_indices, batch_weights, track_targets
+
+ indices, weights, track_ids = self.execute(graph_fn, [])
+
+ self.assertEqual(indices.shape, (3, 2, 3))
+ self.assertEqual(track_ids.shape, (3, 2, 3))
+ self.assertEqual(weights.shape, (3, 2))
+
+ np.testing.assert_array_equal(indices,
+ [[[0, 10, 10], [0, 15, 5]],
+ [[1, 15, 5], [1, 10, 10]],
+ [[2, 10, 10], [2, 7, 11]]])
+ np.testing.assert_array_equal(track_ids,
+ [[[1, 0, 0], [0, 1, 0]],
+ [[0, 1, 0], [1, 0, 0]],
+ [[1, 0, 0], [0, 0, 1]]])
+ np.testing.assert_array_equal(weights, [[0, 1], [1, 1], [0, 0]])
+ # TODO(xwwang): Add a test for the case when no objects are detected.
+
+
+class CornerOffsetTargetAssignerTest(test_case.TestCase):
+
+ def test_filter_overlap_min_area_empty(self):
+ """Test that empty masks work on CPU."""
+ def graph_fn(masks):
+ return targetassigner.filter_mask_overlap_min_area(masks)
+
+ masks = self.execute_cpu(graph_fn, [np.zeros((0, 5, 5), dtype=np.float32)])
+ self.assertEqual(masks.shape, (0, 5, 5))
+
+ def test_filter_overlap_min_area(self):
+ """Test the object with min. area is selected instead of overlap."""
+ def graph_fn(masks):
+ return targetassigner.filter_mask_overlap_min_area(masks)
+
+ masks = np.zeros((3, 4, 4), dtype=np.float32)
+ masks[0, :2, :2] = 1.0
+ masks[1, :3, :3] = 1.0
+ masks[2, 3, 3] = 1.0
+
+ masks = self.execute(graph_fn, [masks])
+
+ self.assertAllClose(masks[0],
+ [[1, 1, 0, 0],
+ [1, 1, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 0]])
+ self.assertAllClose(masks[1],
+ [[0, 0, 1, 0],
+ [0, 0, 1, 0],
+ [1, 1, 1, 0],
+ [0, 0, 0, 0]])
+
+ self.assertAllClose(masks[2],
+ [[0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 1]])
+
+ def test_assign_corner_offset_single_object(self):
+ """Test that corner offsets are correct with a single object."""
+ assigner = targetassigner.CenterNetCornerOffsetTargetAssigner(stride=1)
+
+ def graph_fn():
+ boxes = [
+ tf.constant([[0., 0., 1., 1.]])
+ ]
+ mask = np.zeros((1, 4, 4), dtype=np.float32)
+ mask[0, 1:3, 1:3] = 1.0
+
+ masks = [tf.constant(mask)]
+ return assigner.assign_corner_offset_targets(boxes, masks)
+
+ corner_offsets, foreground = self.execute(graph_fn, [])
+ self.assertAllClose(foreground[0],
+ [[0, 0, 0, 0],
+ [0, 1, 1, 0],
+ [0, 1, 1, 0],
+ [0, 0, 0, 0]])
+
+ self.assertAllClose(corner_offsets[0, :, :, 0],
+ [[0, 0, 0, 0],
+ [0, -1, -1, 0],
+ [0, -2, -2, 0],
+ [0, 0, 0, 0]])
+ self.assertAllClose(corner_offsets[0, :, :, 1],
+ [[0, 0, 0, 0],
+ [0, -1, -2, 0],
+ [0, -1, -2, 0],
+ [0, 0, 0, 0]])
+ self.assertAllClose(corner_offsets[0, :, :, 2],
+ [[0, 0, 0, 0],
+ [0, 3, 3, 0],
+ [0, 2, 2, 0],
+ [0, 0, 0, 0]])
+ self.assertAllClose(corner_offsets[0, :, :, 3],
+ [[0, 0, 0, 0],
+ [0, 3, 2, 0],
+ [0, 3, 2, 0],
+ [0, 0, 0, 0]])
+
+ def test_assign_corner_offset_multiple_objects(self):
+ """Test corner offsets are correct with multiple objects."""
+ assigner = targetassigner.CenterNetCornerOffsetTargetAssigner(stride=1)
+
+ def graph_fn():
+ boxes = [
+ tf.constant([[0., 0., 1., 1.], [0., 0., 0., 0.]]),
+ tf.constant([[0., 0., .25, .25], [.25, .25, 1., 1.]])
+ ]
+ mask1 = np.zeros((2, 4, 4), dtype=np.float32)
+ mask1[0, 0, 0] = 1.0
+ mask1[0, 3, 3] = 1.0
+
+ mask2 = np.zeros((2, 4, 4), dtype=np.float32)
+ mask2[0, :2, :2] = 1.0
+ mask2[1, 1:, 1:] = 1.0
+
+ masks = [tf.constant(mask1), tf.constant(mask2)]
+ return assigner.assign_corner_offset_targets(boxes, masks)
+
+ corner_offsets, foreground = self.execute(graph_fn, [])
+ self.assertEqual(corner_offsets.shape, (2, 4, 4, 4))
+ self.assertEqual(foreground.shape, (2, 4, 4))
+
+ self.assertAllClose(foreground[0],
+ [[1, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 1]])
+
+ self.assertAllClose(corner_offsets[0, :, :, 0],
+ [[0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, -3]])
+ self.assertAllClose(corner_offsets[0, :, :, 1],
+ [[0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, -3]])
+ self.assertAllClose(corner_offsets[0, :, :, 2],
+ [[4, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 1]])
+ self.assertAllClose(corner_offsets[0, :, :, 3],
+ [[4, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 0],
+ [0, 0, 0, 1]])
+
+ self.assertAllClose(foreground[1],
+ [[1, 1, 0, 0],
+ [1, 1, 1, 1],
+ [0, 1, 1, 1],
+ [0, 1, 1, 1]])
+
+ self.assertAllClose(corner_offsets[1, :, :, 0],
+ [[0, 0, 0, 0],
+ [-1, -1, 0, 0],
+ [0, -1, -1, -1],
+ [0, -2, -2, -2]])
+ self.assertAllClose(corner_offsets[1, :, :, 1],
+ [[0, -1, 0, 0],
+ [0, -1, -1, -2],
+ [0, 0, -1, -2],
+ [0, 0, -1, -2]])
+ self.assertAllClose(corner_offsets[1, :, :, 2],
+ [[1, 1, 0, 0],
+ [0, 0, 3, 3],
+ [0, 2, 2, 2],
+ [0, 1, 1, 1]])
+ self.assertAllClose(corner_offsets[1, :, :, 3],
+ [[1, 0, 0, 0],
+ [1, 0, 2, 1],
+ [0, 3, 2, 1],
+ [0, 3, 2, 1]])
+
+ def test_assign_corner_offsets_no_objects(self):
+ """Test assignment works with empty input on cpu."""
+ assigner = targetassigner.CenterNetCornerOffsetTargetAssigner(stride=1)
+
+ def graph_fn():
+ boxes = [
+ tf.zeros((0, 4), dtype=tf.float32)
+ ]
+ masks = [tf.zeros((0, 5, 5), dtype=tf.float32)]
+ return assigner.assign_corner_offset_targets(boxes, masks)
+
+ corner_offsets, foreground = self.execute_cpu(graph_fn, [])
+ self.assertAllClose(corner_offsets, np.zeros((1, 5, 5, 4)))
+ self.assertAllClose(foreground, np.zeros((1, 5, 5)))
+
+
+class CenterNetTemporalOffsetTargetAssigner(test_case.TestCase):
+
+ def setUp(self):
+ super(CenterNetTemporalOffsetTargetAssigner, self).setUp()
+ self._box_center = [0.0, 0.0, 1.0, 1.0]
+ self._box_center_small = [0.25, 0.25, 0.75, 0.75]
+ self._box_lower_left = [0.5, 0.0, 1.0, 0.5]
+ self._box_center_offset = [0.1, 0.05, 1.0, 1.0]
+ self._box_odd_coordinates = [0.1625, 0.2125, 0.5625, 0.9625]
+ self._offset_center = [0.5, 0.4]
+ self._offset_center_small = [0.1, 0.1]
+ self._offset_lower_left = [-0.1, 0.1]
+ self._offset_center_offset = [0.4, 0.3]
+ self._offset_odd_coord = [0.125, -0.125]
+
+ def test_assign_empty_groundtruths(self):
+ """Tests the assign_offset_targets function with empty inputs."""
+ def graph_fn():
+ box_batch = [
+ tf.zeros((0, 4), dtype=tf.float32),
+ ]
+
+ offset_batch = [
+ tf.zeros((0, 2), dtype=tf.float32),
+ ]
+
+ match_flag_batch = [
+ tf.zeros((0), dtype=tf.float32),
+ ]
+
+ assigner = targetassigner.CenterNetTemporalOffsetTargetAssigner(4)
+ indices, temporal_offset, weights = assigner.assign_temporal_offset_targets(
+ 80, 80, box_batch, offset_batch, match_flag_batch)
+ return indices, temporal_offset, weights
+ indices, temporal_offset, weights = self.execute(graph_fn, [])
+ self.assertEqual(indices.shape, (0, 3))
+ self.assertEqual(temporal_offset.shape, (0, 2))
+ self.assertEqual(weights.shape, (0,))
+
+ def test_assign_offset_targets(self):
+ """Tests the assign_offset_targets function."""
+ def graph_fn():
+ box_batch = [
+ tf.constant([self._box_center, self._box_lower_left]),
+ tf.constant([self._box_center_offset]),
+ tf.constant([self._box_center_small, self._box_odd_coordinates]),
+ ]
+
+ offset_batch = [
+ tf.constant([self._offset_center, self._offset_lower_left]),
+ tf.constant([self._offset_center_offset]),
+ tf.constant([self._offset_center_small, self._offset_odd_coord]),
+ ]
+
+ match_flag_batch = [
+ tf.constant([1.0, 1.0]),
+ tf.constant([1.0]),
+ tf.constant([1.0, 1.0]),
+ ]
+
+ assigner = targetassigner.CenterNetTemporalOffsetTargetAssigner(4)
+ indices, temporal_offset, weights = assigner.assign_temporal_offset_targets(
+ 80, 80, box_batch, offset_batch, match_flag_batch)
+ return indices, temporal_offset, weights
+ indices, temporal_offset, weights = self.execute(graph_fn, [])
+ self.assertEqual(indices.shape, (5, 3))
+ self.assertEqual(temporal_offset.shape, (5, 2))
+ self.assertEqual(weights.shape, (5,))
+ np.testing.assert_array_equal(
+ indices,
+ [[0, 10, 10], [0, 15, 5], [1, 11, 10], [2, 10, 10], [2, 7, 11]])
+ np.testing.assert_array_almost_equal(
+ temporal_offset,
+ [[0.5, 0.4], [-0.1, 0.1], [0.4, 0.3], [0.1, 0.1], [0.125, -0.125]])
+ np.testing.assert_array_equal(weights, 1)
+
+ def test_assign_offset_targets_with_match_flags(self):
+ """Tests the assign_offset_targets function with match flags."""
+ def graph_fn():
+ box_batch = [
+ tf.constant([self._box_center, self._box_lower_left]),
+ tf.constant([self._box_center_offset]),
+ tf.constant([self._box_center_small, self._box_odd_coordinates]),
+ ]
+
+ offset_batch = [
+ tf.constant([self._offset_center, self._offset_lower_left]),
+ tf.constant([self._offset_center_offset]),
+ tf.constant([self._offset_center_small, self._offset_odd_coord]),
+ ]
+
+ match_flag_batch = [
+ tf.constant([0.0, 1.0]),
+ tf.constant([1.0]),
+ tf.constant([1.0, 1.0]),
+ ]
+
+ cn_assigner = targetassigner.CenterNetTemporalOffsetTargetAssigner(4)
+ weights_batch = [
+ tf.constant([1.0, 0.0]),
+ tf.constant([1.0]),
+ tf.constant([1.0, 1.0])
+ ]
+ indices, temporal_offset, weights = cn_assigner.assign_temporal_offset_targets(
+ 80, 80, box_batch, offset_batch, match_flag_batch, weights_batch)
+ return indices, temporal_offset, weights
+ indices, temporal_offset, weights = self.execute(graph_fn, [])
+ self.assertEqual(indices.shape, (5, 3))
+ self.assertEqual(temporal_offset.shape, (5, 2))
+ self.assertEqual(weights.shape, (5,))
+
+ np.testing.assert_array_equal(
+ indices,
+ [[0, 10, 10], [0, 15, 5], [1, 11, 10], [2, 10, 10], [2, 7, 11]])
+ np.testing.assert_array_almost_equal(
+ temporal_offset,
+ [[0.5, 0.4], [-0.1, 0.1], [0.4, 0.3], [0.1, 0.1], [0.125, -0.125]])
+ np.testing.assert_array_equal(weights, [0, 0, 1, 1, 1])
+
+
+class DETRTargetAssignerTest(test_case.TestCase):
+
+ def test_assign_detr(self):
+ def graph_fn(pred_corners, groundtruth_box_corners,
+ groundtruth_labels, predicted_labels):
+ detr_target_assigner = targetassigner.DETRTargetAssigner()
+ pred_boxlist = box_list.BoxList(pred_corners)
+ groundtruth_boxlist = box_list.BoxList(groundtruth_box_corners)
+ result = detr_target_assigner.assign(
+ pred_boxlist, groundtruth_boxlist,
+ predicted_labels, groundtruth_labels)
+ (cls_targets, cls_weights, reg_targets, reg_weights) = result
+ return (cls_targets, cls_weights, reg_targets, reg_weights)
+
+ pred_corners = np.array([[0.25, 0.25, 0.4, 0.2],
+ [0.5, 0.8, 1.0, 0.8],
+ [0.9, 0.5, 0.1, 1.0]], dtype=np.float32)
+ groundtruth_box_corners = np.array([[0.0, 0.0, 0.5, 0.5],
+ [0.5, 0.5, 0.9, 0.9]],
+ dtype=np.float32)
+ predicted_labels = np.array([[-3.0, 3.0], [2.0, 9.4], [5.0, 1.0]],
+ dtype=np.float32)
+ groundtruth_labels = np.array([[0.0, 1.0], [0.0, 1.0]],
+ dtype=np.float32)
+
+ exp_cls_targets = [[0, 1], [0, 1], [1, 0]]
+ exp_cls_weights = [[1, 1], [1, 1], [1, 1]]
+ exp_reg_targets = [[0.25, 0.25, 0.5, 0.5],
+ [0.7, 0.7, 0.4, 0.4],
+ [0, 0, 0, 0]]
+ exp_reg_weights = [1, 1, 0]
+
+ (cls_targets_out,
+ cls_weights_out, reg_targets_out, reg_weights_out) = self.execute_cpu(
+ graph_fn, [pred_corners, groundtruth_box_corners,
+ groundtruth_labels, predicted_labels])
+
+ self.assertAllClose(cls_targets_out, exp_cls_targets)
+ self.assertAllClose(cls_weights_out, exp_cls_weights)
+ self.assertAllClose(reg_targets_out, exp_reg_targets)
+ self.assertAllClose(reg_weights_out, exp_reg_weights)
+ self.assertEqual(cls_targets_out.dtype, np.float32)
+ self.assertEqual(cls_weights_out.dtype, np.float32)
+ self.assertEqual(reg_targets_out.dtype, np.float32)
+ self.assertEqual(reg_weights_out.dtype, np.float32)
+
+ def test_batch_assign_detr(self):
+ def graph_fn(pred_corners, groundtruth_box_corners,
+ groundtruth_labels, predicted_labels):
+ detr_target_assigner = targetassigner.DETRTargetAssigner()
+ result = detr_target_assigner.batch_assign(
+ pred_corners, groundtruth_box_corners,
+ [predicted_labels], [groundtruth_labels])
+ (cls_targets, cls_weights, reg_targets, reg_weights) = result
+ return (cls_targets, cls_weights, reg_targets, reg_weights)
+
+ pred_corners = np.array([[[0.25, 0.25, 0.4, 0.2],
+ [0.5, 0.8, 1.0, 0.8],
+ [0.9, 0.5, 0.1, 1.0]]], dtype=np.float32)
+ groundtruth_box_corners = np.array([[[0.0, 0.0, 0.5, 0.5],
+ [0.5, 0.5, 0.9, 0.9]]],
+ dtype=np.float32)
+ predicted_labels = np.array([[-3.0, 3.0], [2.0, 9.4], [5.0, 1.0]],
+ dtype=np.float32)
+ groundtruth_labels = np.array([[0.0, 1.0], [0.0, 1.0]],
+ dtype=np.float32)
+
+ exp_cls_targets = [[[0, 1], [0, 1], [1, 0]]]
+ exp_cls_weights = [[[1, 1], [1, 1], [1, 1]]]
+ exp_reg_targets = [[[0.25, 0.25, 0.5, 0.5],
+ [0.7, 0.7, 0.4, 0.4],
+ [0, 0, 0, 0]]]
+ exp_reg_weights = [[1, 1, 0]]
+
+ (cls_targets_out,
+ cls_weights_out, reg_targets_out, reg_weights_out) = self.execute_cpu(
+ graph_fn, [pred_corners, groundtruth_box_corners,
+ groundtruth_labels, predicted_labels])
+
+ self.assertAllClose(cls_targets_out, exp_cls_targets)
+ self.assertAllClose(cls_weights_out, exp_cls_weights)
+ self.assertAllClose(reg_targets_out, exp_reg_targets)
+ self.assertAllClose(reg_weights_out, exp_reg_weights)
+ self.assertEqual(cls_targets_out.dtype, np.float32)
+ self.assertEqual(cls_weights_out.dtype, np.float32)
+ self.assertEqual(reg_targets_out.dtype, np.float32)
+ self.assertEqual(reg_weights_out.dtype, np.float32)
+
+
if __name__ == '__main__':
tf.enable_v2_behavior()
tf.test.main()
diff --git a/research/object_detection/data_decoders/tf_example_decoder.py b/research/object_detection/data_decoders/tf_example_decoder.py
index 04cc4db59988161345c5cacd2e6f513b2707b0a1..acd48750fd9b390b84c6df2e8ad80ced19adc928 100644
--- a/research/object_detection/data_decoders/tf_example_decoder.py
+++ b/research/object_detection/data_decoders/tf_example_decoder.py
@@ -124,40 +124,6 @@ class _ClassTensorHandler(slim_example_decoder.Tensor):
self._display_name_to_id_table.lookup(unmapped_tensor))
-class _BackupHandler(slim_example_decoder.ItemHandler):
- """An ItemHandler that tries two ItemHandlers in order."""
-
- def __init__(self, handler, backup):
- """Initializes the BackupHandler handler.
-
- If the first Handler's tensors_to_item returns a Tensor with no elements,
- the second Handler is used.
-
- Args:
- handler: The primary ItemHandler.
- backup: The backup ItemHandler.
-
- Raises:
- ValueError: if either is not an ItemHandler.
- """
- if not isinstance(handler, slim_example_decoder.ItemHandler):
- raise ValueError('Primary handler is of type %s instead of ItemHandler' %
- type(handler))
- if not isinstance(backup, slim_example_decoder.ItemHandler):
- raise ValueError(
- 'Backup handler is of type %s instead of ItemHandler' % type(backup))
- self._handler = handler
- self._backup = backup
- super(_BackupHandler, self).__init__(handler.keys + backup.keys)
-
- def tensors_to_item(self, keys_to_tensors):
- item = self._handler.tensors_to_item(keys_to_tensors)
- return tf.cond(
- pred=tf.equal(tf.reduce_prod(tf.shape(item)), 0),
- true_fn=lambda: self._backup.tensors_to_item(keys_to_tensors),
- false_fn=lambda: item)
-
-
class TfExampleDecoder(data_decoder.DataDecoder):
"""Tensorflow Example proto decoder."""
@@ -172,7 +138,9 @@ class TfExampleDecoder(data_decoder.DataDecoder):
load_multiclass_scores=False,
load_context_features=False,
expand_hierarchy_labels=False,
- load_dense_pose=False):
+ load_dense_pose=False,
+ load_track_id=False,
+ load_keypoint_depth_features=False):
"""Constructor sets keys_to_features and items_to_handlers.
Args:
@@ -204,6 +172,11 @@ class TfExampleDecoder(data_decoder.DataDecoder):
classes, the labels are extended to ancestor. For negative classes,
the labels are expanded to descendants.
load_dense_pose: Whether to load DensePose annotations.
+ load_track_id: Whether to load tracking annotations.
+ load_keypoint_depth_features: Whether to load the keypoint depth features
+ including keypoint relative depths and weights. If this field is set to
+ True but no keypoint depth features are in the input tf.Example, then
+ default values will be populated.
Raises:
ValueError: If `instance_mask_type` option is not one of
@@ -212,6 +185,7 @@ class TfExampleDecoder(data_decoder.DataDecoder):
ValueError: If `expand_labels_hierarchy` is True, but the
`label_map_proto_file` is not provided.
"""
+
# TODO(rathodv): delete unused `use_display_name` argument once we change
# other decoders to handle label maps similarly.
del use_display_name
@@ -235,6 +209,10 @@ class TfExampleDecoder(data_decoder.DataDecoder):
tf.VarLenFeature(tf.string),
'image/class/label':
tf.VarLenFeature(tf.int64),
+ 'image/neg_category_ids':
+ tf.VarLenFeature(tf.int64),
+ 'image/not_exhaustive_category_ids':
+ tf.VarLenFeature(tf.int64),
'image/class/confidence':
tf.VarLenFeature(tf.float32),
# Object boxes and classes.
@@ -296,6 +274,10 @@ class TfExampleDecoder(data_decoder.DataDecoder):
# Image-level labels.
fields.InputDataFields.groundtruth_image_confidences: (
slim_example_decoder.Tensor('image/class/confidence')),
+ fields.InputDataFields.groundtruth_verified_neg_classes: (
+ slim_example_decoder.Tensor('image/neg_category_ids')),
+ fields.InputDataFields.groundtruth_not_exhaustive_classes: (
+ slim_example_decoder.Tensor('image/not_exhaustive_category_ids')),
# Object boxes and classes.
fields.InputDataFields.groundtruth_boxes: (
slim_example_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
@@ -355,6 +337,23 @@ class TfExampleDecoder(data_decoder.DataDecoder):
slim_example_decoder.ItemHandlerCallback(
['image/object/keypoint/x', 'image/object/keypoint/visibility'],
self._reshape_keypoint_visibilities))
+ if load_keypoint_depth_features:
+ self.keys_to_features['image/object/keypoint/z'] = (
+ tf.VarLenFeature(tf.float32))
+ self.keys_to_features['image/object/keypoint/z/weights'] = (
+ tf.VarLenFeature(tf.float32))
+ self.items_to_handlers[
+ fields.InputDataFields.groundtruth_keypoint_depths] = (
+ slim_example_decoder.ItemHandlerCallback(
+ ['image/object/keypoint/x', 'image/object/keypoint/z'],
+ self._reshape_keypoint_depths))
+ self.items_to_handlers[
+ fields.InputDataFields.groundtruth_keypoint_depth_weights] = (
+ slim_example_decoder.ItemHandlerCallback(
+ ['image/object/keypoint/x',
+ 'image/object/keypoint/z/weights'],
+ self._reshape_keypoint_depth_weights))
+
if load_instance_masks:
if instance_mask_type in (input_reader_pb2.DEFAULT,
input_reader_pb2.NUMERICAL_MASKS):
@@ -401,16 +400,22 @@ class TfExampleDecoder(data_decoder.DataDecoder):
'image/object/densepose/u', 'image/object/densepose/v',
'image/object/densepose/num'],
self._dense_pose_surface_coordinates))
+ if load_track_id:
+ self.keys_to_features['image/object/track/label'] = (
+ tf.VarLenFeature(tf.int64))
+ self.items_to_handlers[
+ fields.InputDataFields.groundtruth_track_ids] = (
+ slim_example_decoder.Tensor('image/object/track/label'))
if label_map_proto_file:
# If the label_map_proto is provided, try to use it in conjunction with
# the class text, and fall back to a materialized ID.
- label_handler = _BackupHandler(
+ label_handler = slim_example_decoder.BackupHandler(
_ClassTensorHandler(
'image/object/class/text', label_map_proto_file,
default_value=''),
slim_example_decoder.Tensor('image/object/class/label'))
- image_label_handler = _BackupHandler(
+ image_label_handler = slim_example_decoder.BackupHandler(
_ClassTensorHandler(
fields.TfExampleFields.image_class_text,
label_map_proto_file,
@@ -586,6 +591,11 @@ class TfExampleDecoder(data_decoder.DataDecoder):
tensor_dict[fields.InputDataFields.groundtruth_dp_part_ids],
dtype=tf.int32)
+ if fields.InputDataFields.groundtruth_track_ids in tensor_dict:
+ tensor_dict[fields.InputDataFields.groundtruth_track_ids] = tf.cast(
+ tensor_dict[fields.InputDataFields.groundtruth_track_ids],
+ dtype=tf.int32)
+
return tensor_dict
def _reshape_keypoints(self, keys_to_tensors):
@@ -614,6 +624,73 @@ class TfExampleDecoder(data_decoder.DataDecoder):
keypoints = tf.reshape(keypoints, [-1, self._num_keypoints, 2])
return keypoints
+ def _reshape_keypoint_depths(self, keys_to_tensors):
+ """Reshape keypoint depths.
+
+ The keypoint depths are reshaped to [num_instances, num_keypoints]. The
+ keypoint depth tensor is expected to have the same shape as the keypoint x
+ (or y) tensors. If not (usually because the example does not have the depth
+ groundtruth), then default depth values (zero) are provided.
+
+ Args:
+ keys_to_tensors: a dictionary from keys to tensors. Expected keys are:
+ 'image/object/keypoint/x'
+ 'image/object/keypoint/z'
+
+ Returns:
+ A 2-D float tensor of shape [num_instances, num_keypoints] with values
+ representing the keypoint depths.
+ """
+ x = keys_to_tensors['image/object/keypoint/x']
+ z = keys_to_tensors['image/object/keypoint/z']
+ if isinstance(z, tf.SparseTensor):
+ z = tf.sparse_tensor_to_dense(z)
+ if isinstance(x, tf.SparseTensor):
+ x = tf.sparse_tensor_to_dense(x)
+
+ default_z = tf.zeros_like(x)
+ # Use keypoint depth groundtruth if provided, otherwise use the default
+ # depth value.
+ z = tf.cond(tf.equal(tf.size(x), tf.size(z)),
+ true_fn=lambda: z,
+ false_fn=lambda: default_z)
+ z = tf.reshape(z, [-1, self._num_keypoints])
+ return z
+
+ def _reshape_keypoint_depth_weights(self, keys_to_tensors):
+ """Reshape keypoint depth weights.
+
+ The keypoint depth weights are reshaped to [num_instances, num_keypoints].
+ The keypoint depth weights tensor is expected to have the same shape as the
+ keypoint x (or y) tensors. If not (usually because the example does not have
+ the depth weights groundtruth), then default weight values (zero) are
+ provided.
+
+ Args:
+ keys_to_tensors: a dictionary from keys to tensors. Expected keys are:
+ 'image/object/keypoint/x'
+ 'image/object/keypoint/z/weights'
+
+ Returns:
+ A 2-D float tensor of shape [num_instances, num_keypoints] with values
+ representing the keypoint depth weights.
+ """
+ x = keys_to_tensors['image/object/keypoint/x']
+ z = keys_to_tensors['image/object/keypoint/z/weights']
+ if isinstance(z, tf.SparseTensor):
+ z = tf.sparse_tensor_to_dense(z)
+ if isinstance(x, tf.SparseTensor):
+ x = tf.sparse_tensor_to_dense(x)
+
+ default_z = tf.zeros_like(x)
+ # Use keypoint depth weights if provided, otherwise use the default
+ # values.
+ z = tf.cond(tf.equal(tf.size(x), tf.size(z)),
+ true_fn=lambda: z,
+ false_fn=lambda: default_z)
+ z = tf.reshape(z, [-1, self._num_keypoints])
+ return z
+
def _reshape_keypoint_visibilities(self, keys_to_tensors):
"""Reshape keypoint visibilities.
diff --git a/research/object_detection/data_decoders/tf_example_decoder_test.py b/research/object_detection/data_decoders/tf_example_decoder_test.py
index 81ed9258e650d7534bd9e3ae76aa574bc2a06b61..5311bdf4dfe6e2813dcf2c28b40dad10195c1693 100644
--- a/research/object_detection/data_decoders/tf_example_decoder_test.py
+++ b/research/object_detection/data_decoders/tf_example_decoder_test.py
@@ -275,6 +275,124 @@ class TfExampleDecoderTest(test_case.TestCase):
self.assertAllEqual(expected_boxes,
tensor_dict[fields.InputDataFields.groundtruth_boxes])
+ def testDecodeKeypointDepth(self):
+ image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
+ encoded_jpeg, _ = self._create_encoded_and_decoded_data(
+ image_tensor, 'jpeg')
+ bbox_ymins = [0.0, 4.0]
+ bbox_xmins = [1.0, 5.0]
+ bbox_ymaxs = [2.0, 6.0]
+ bbox_xmaxs = [3.0, 7.0]
+ keypoint_ys = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
+ keypoint_xs = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
+ keypoint_visibility = [1, 2, 0, 1, 0, 2]
+ keypoint_depths = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]
+ keypoint_depth_weights = [1.0, 0.9, 0.8, 0.7, 0.6, 0.5]
+
+ def graph_fn():
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ dataset_util.bytes_feature(encoded_jpeg),
+ 'image/format':
+ dataset_util.bytes_feature(six.b('jpeg')),
+ 'image/object/bbox/ymin':
+ dataset_util.float_list_feature(bbox_ymins),
+ 'image/object/bbox/xmin':
+ dataset_util.float_list_feature(bbox_xmins),
+ 'image/object/bbox/ymax':
+ dataset_util.float_list_feature(bbox_ymaxs),
+ 'image/object/bbox/xmax':
+ dataset_util.float_list_feature(bbox_xmaxs),
+ 'image/object/keypoint/y':
+ dataset_util.float_list_feature(keypoint_ys),
+ 'image/object/keypoint/x':
+ dataset_util.float_list_feature(keypoint_xs),
+ 'image/object/keypoint/z':
+ dataset_util.float_list_feature(keypoint_depths),
+ 'image/object/keypoint/z/weights':
+ dataset_util.float_list_feature(keypoint_depth_weights),
+ 'image/object/keypoint/visibility':
+ dataset_util.int64_list_feature(keypoint_visibility),
+ })).SerializeToString()
+
+ example_decoder = tf_example_decoder.TfExampleDecoder(
+ num_keypoints=3, load_keypoint_depth_features=True)
+ output = example_decoder.decode(tf.convert_to_tensor(example))
+
+ self.assertAllEqual(
+ (output[fields.InputDataFields.groundtruth_keypoint_depths].get_shape(
+ ).as_list()), [2, 3])
+ self.assertAllEqual(
+ (output[fields.InputDataFields.groundtruth_keypoint_depth_weights]
+ .get_shape().as_list()), [2, 3])
+ return output
+
+ tensor_dict = self.execute_cpu(graph_fn, [])
+
+ expected_keypoint_depths = [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]
+ self.assertAllClose(
+ expected_keypoint_depths,
+ tensor_dict[fields.InputDataFields.groundtruth_keypoint_depths])
+
+ expected_keypoint_depth_weights = [[1.0, 0.9, 0.8], [0.7, 0.6, 0.5]]
+ self.assertAllClose(
+ expected_keypoint_depth_weights,
+ tensor_dict[fields.InputDataFields.groundtruth_keypoint_depth_weights])
+
+ def testDecodeKeypointDepthNoDepth(self):
+ image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
+ encoded_jpeg, _ = self._create_encoded_and_decoded_data(
+ image_tensor, 'jpeg')
+ bbox_ymins = [0.0, 4.0]
+ bbox_xmins = [1.0, 5.0]
+ bbox_ymaxs = [2.0, 6.0]
+ bbox_xmaxs = [3.0, 7.0]
+ keypoint_ys = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
+ keypoint_xs = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
+ keypoint_visibility = [1, 2, 0, 1, 0, 2]
+
+ def graph_fn():
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ dataset_util.bytes_feature(encoded_jpeg),
+ 'image/format':
+ dataset_util.bytes_feature(six.b('jpeg')),
+ 'image/object/bbox/ymin':
+ dataset_util.float_list_feature(bbox_ymins),
+ 'image/object/bbox/xmin':
+ dataset_util.float_list_feature(bbox_xmins),
+ 'image/object/bbox/ymax':
+ dataset_util.float_list_feature(bbox_ymaxs),
+ 'image/object/bbox/xmax':
+ dataset_util.float_list_feature(bbox_xmaxs),
+ 'image/object/keypoint/y':
+ dataset_util.float_list_feature(keypoint_ys),
+ 'image/object/keypoint/x':
+ dataset_util.float_list_feature(keypoint_xs),
+ 'image/object/keypoint/visibility':
+ dataset_util.int64_list_feature(keypoint_visibility),
+ })).SerializeToString()
+
+ example_decoder = tf_example_decoder.TfExampleDecoder(
+ num_keypoints=3, load_keypoint_depth_features=True)
+ output = example_decoder.decode(tf.convert_to_tensor(example))
+
+ return output
+
+ tensor_dict = self.execute_cpu(graph_fn, [])
+
+ expected_keypoints_depth_default = [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]
+ self.assertAllClose(
+ expected_keypoints_depth_default,
+ tensor_dict[fields.InputDataFields.groundtruth_keypoint_depths])
+ self.assertAllClose(
+ expected_keypoints_depth_default,
+ tensor_dict[fields.InputDataFields.groundtruth_keypoint_depth_weights])
+
def testDecodeKeypoint(self):
image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
encoded_jpeg, _ = self._create_encoded_and_decoded_data(
@@ -841,6 +959,61 @@ class TfExampleDecoderTest(test_case.TestCase):
self.assertAllEqual(object_area,
tensor_dict[fields.InputDataFields.groundtruth_area])
+ def testDecodeVerifiedNegClasses(self):
+ image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
+ encoded_jpeg, _ = self._create_encoded_and_decoded_data(
+ image_tensor, 'jpeg')
+ neg_category_ids = [0, 5, 8]
+
+ def graph_fn():
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ dataset_util.bytes_feature(encoded_jpeg),
+ 'image/format':
+ dataset_util.bytes_feature(six.b('jpeg')),
+ 'image/neg_category_ids':
+ dataset_util.int64_list_feature(neg_category_ids),
+ })).SerializeToString()
+
+ example_decoder = tf_example_decoder.TfExampleDecoder()
+ output = example_decoder.decode(tf.convert_to_tensor(example))
+ return output
+
+ tensor_dict = self.execute_cpu(graph_fn, [])
+ self.assertAllEqual(
+ neg_category_ids,
+ tensor_dict[fields.InputDataFields.groundtruth_verified_neg_classes])
+
+ def testDecodeNotExhaustiveClasses(self):
+ image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
+ encoded_jpeg, _ = self._create_encoded_and_decoded_data(
+ image_tensor, 'jpeg')
+ not_exhaustive_category_ids = [0, 5, 8]
+
+ def graph_fn():
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ dataset_util.bytes_feature(encoded_jpeg),
+ 'image/format':
+ dataset_util.bytes_feature(six.b('jpeg')),
+ 'image/not_exhaustive_category_ids':
+ dataset_util.int64_list_feature(
+ not_exhaustive_category_ids),
+ })).SerializeToString()
+
+ example_decoder = tf_example_decoder.TfExampleDecoder()
+ output = example_decoder.decode(tf.convert_to_tensor(example))
+ return output
+
+ tensor_dict = self.execute_cpu(graph_fn, [])
+ self.assertAllEqual(
+ not_exhaustive_category_ids,
+ tensor_dict[fields.InputDataFields.groundtruth_not_exhaustive_classes])
+
def testDecodeObjectIsCrowd(self):
image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
encoded_jpeg, _ = self._create_encoded_and_decoded_data(
@@ -1430,6 +1603,48 @@ class TfExampleDecoderTest(test_case.TestCase):
self.assertAllEqual(dp_part_ids, expected_dp_part_ids)
self.assertAllClose(dp_surface_coords, expected_dp_surface_coords)
+ def testDecodeTrack(self):
+ image_tensor = np.random.randint(256, size=(4, 5, 3)).astype(np.uint8)
+ encoded_jpeg, _ = self._create_encoded_and_decoded_data(
+ image_tensor, 'jpeg')
+ bbox_ymins = [0.0, 4.0, 2.0]
+ bbox_xmins = [1.0, 5.0, 8.0]
+ bbox_ymaxs = [2.0, 6.0, 1.0]
+ bbox_xmaxs = [3.0, 7.0, 3.3]
+ track_labels = [0, 1, 2]
+
+ def graph_fn():
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ dataset_util.bytes_feature(encoded_jpeg),
+ 'image/format':
+ dataset_util.bytes_feature(six.b('jpeg')),
+ 'image/object/bbox/ymin':
+ dataset_util.float_list_feature(bbox_ymins),
+ 'image/object/bbox/xmin':
+ dataset_util.float_list_feature(bbox_xmins),
+ 'image/object/bbox/ymax':
+ dataset_util.float_list_feature(bbox_ymaxs),
+ 'image/object/bbox/xmax':
+ dataset_util.float_list_feature(bbox_xmaxs),
+ 'image/object/track/label':
+ dataset_util.int64_list_feature(track_labels),
+ })).SerializeToString()
+
+ example_decoder = tf_example_decoder.TfExampleDecoder(
+ load_track_id=True)
+ output = example_decoder.decode(tf.convert_to_tensor(example))
+ track_ids = output[fields.InputDataFields.groundtruth_track_ids]
+ return track_ids
+
+ track_ids = self.execute_cpu(graph_fn, [])
+
+ expected_track_labels = [0, 1, 2]
+
+ self.assertAllEqual(track_ids, expected_track_labels)
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/data_decoders/tf_sequence_example_decoder.py b/research/object_detection/data_decoders/tf_sequence_example_decoder.py
index 1565a910eb1726ce0846e9c78488a7e8d4f97fdf..9bed15970353255c0293057db3a5933d3c3a961f 100644
--- a/research/object_detection/data_decoders/tf_sequence_example_decoder.py
+++ b/research/object_detection/data_decoders/tf_sequence_example_decoder.py
@@ -117,11 +117,13 @@ class TfSequenceExampleDecoder(data_decoder.DataDecoder):
Context R-CNN (see https://arxiv.org/abs/1912.03538):
'image/context_features'
'image/context_feature_length'
+ 'image/context_features_image_id_list'
"""
def __init__(self,
label_map_proto_file,
load_context_features=False,
+ load_context_image_ids=False,
use_display_name=False,
fully_annotated=False):
"""Constructs `TfSequenceExampleDecoder` object.
@@ -134,6 +136,8 @@ class TfSequenceExampleDecoder(data_decoder.DataDecoder):
load_context_features: Whether to load information from context_features,
to provide additional context to a detection model for training and/or
inference
+ load_context_image_ids: Whether to load the corresponding image ids for
+ the context_features in order to visualize attention.
use_display_name: whether or not to use the `display_name` for label
mapping (instead of `name`). Only used if label_map_proto_file is
provided.
@@ -207,6 +211,16 @@ class TfSequenceExampleDecoder(data_decoder.DataDecoder):
tf.FixedLenFeature((), tf.int64))
self._items_to_handlers[fields.InputDataFields.context_feature_length] = (
slim_example_decoder.Tensor('image/context_feature_length'))
+
+ if load_context_image_ids:
+ self._context_keys_to_features['image/context_features_image_id_list'] = (
+ tf.VarLenFeature(dtype=tf.string))
+ self._items_to_handlers[
+ fields.InputDataFields.context_features_image_id_list] = (
+ slim_example_decoder.Tensor(
+ 'image/context_features_image_id_list',
+ default_value=''))
+
self._fully_annotated = fully_annotated
def decode(self, tf_seq_example_string_tensor):
@@ -239,6 +253,8 @@ class TfSequenceExampleDecoder(data_decoder.DataDecoder):
the length of each feature in context_features
fields.InputDataFields.image: a [num_frames] string tensor with
the encoded images.
+ fields.inputDataFields.context_features_image_id_list: a 1D vector
+ of shape [num_context_features] containing string tensors.
"""
serialized_example = tf.reshape(tf_seq_example_string_tensor, shape=[])
decoder = slim_example_decoder.TFSequenceExampleDecoder(
diff --git a/research/object_detection/data_decoders/tf_sequence_example_decoder_test.py b/research/object_detection/data_decoders/tf_sequence_example_decoder_test.py
index 2ea1c6163454cf2d05065713b2e0657f24af5e64..4aa3afbe073a8df2f221ef741cdfb4adc4207cf4 100644
--- a/research/object_detection/data_decoders/tf_sequence_example_decoder_test.py
+++ b/research/object_detection/data_decoders/tf_sequence_example_decoder_test.py
@@ -120,6 +120,145 @@ class TfSequenceExampleDecoderTest(test_case.TestCase):
self.assertAllEqual(expected_groundtruth_classes,
tensor_dict_out[flds.groundtruth_classes])
+ def test_decode_sequence_example_context(self):
+ num_frames = 4
+ image_height = 20
+ image_width = 30
+
+ expected_groundtruth_boxes = [
+ [[0.0, 0.0, 1.0, 1.0], [0.0, 0.0, 0.0, 0.0]],
+ [[0.2, 0.2, 1.0, 1.0], [0.0, 0.0, 1.0, 1.0]],
+ [[0.0, 0.0, 1.0, 1.0], [0.1, 0.1, 0.2, 0.2]],
+ [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]
+ ]
+ expected_groundtruth_classes = [
+ [-1, -1],
+ [-1, 1],
+ [1, 2],
+ [-1, -1]
+ ]
+
+ expected_context_features = np.array(
+ [[0.0, 0.1, 0.2], [0.3, 0.4, 0.5]], dtype=np.float32)
+
+ flds = fields.InputDataFields
+ encoded_images = self._make_random_serialized_jpeg_images(
+ num_frames, image_height, image_width)
+
+ def graph_fn():
+ label_map_proto_file = os.path.join(self.get_temp_dir(), 'labelmap.pbtxt')
+ self._create_label_map(label_map_proto_file)
+ decoder = tf_sequence_example_decoder.TfSequenceExampleDecoder(
+ label_map_proto_file=label_map_proto_file,
+ load_context_features=True)
+ sequence_example_serialized = seq_example_util.make_sequence_example(
+ dataset_name='video_dataset',
+ video_id='video',
+ encoded_images=encoded_images,
+ image_height=image_height,
+ image_width=image_width,
+ image_format='JPEG',
+ image_source_ids=[str(i) for i in range(num_frames)],
+ is_annotated=[[1], [1], [1], [1]],
+ bboxes=[
+ [[0., 0., 1., 1.]], # Frame 0.
+ [[0.2, 0.2, 1., 1.],
+ [0., 0., 1., 1.]], # Frame 1.
+ [[0., 0., 1., 1.], # Frame 2.
+ [0.1, 0.1, 0.2, 0.2]],
+ [[]], # Frame 3.
+ ],
+ label_strings=[
+ ['fox'], # Frame 0. Fox will be filtered out.
+ ['fox', 'dog'], # Frame 1. Fox will be filtered out.
+ ['dog', 'cat'], # Frame 2.
+ [], # Frame 3
+ ],
+ context_features=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
+ context_feature_length=[3],
+ context_features_image_id_list=[b'im_1', b'im_2']
+ ).SerializeToString()
+
+ example_string_tensor = tf.convert_to_tensor(sequence_example_serialized)
+ return decoder.decode(example_string_tensor)
+
+ tensor_dict_out = self.execute(graph_fn, [])
+ self.assertAllClose(expected_groundtruth_boxes,
+ tensor_dict_out[flds.groundtruth_boxes])
+ self.assertAllEqual(expected_groundtruth_classes,
+ tensor_dict_out[flds.groundtruth_classes])
+ self.assertAllClose(expected_context_features,
+ tensor_dict_out[flds.context_features])
+
+ def test_decode_sequence_example_context_image_id_list(self):
+ num_frames = 4
+ image_height = 20
+ image_width = 30
+
+ expected_groundtruth_boxes = [
+ [[0.0, 0.0, 1.0, 1.0], [0.0, 0.0, 0.0, 0.0]],
+ [[0.2, 0.2, 1.0, 1.0], [0.0, 0.0, 1.0, 1.0]],
+ [[0.0, 0.0, 1.0, 1.0], [0.1, 0.1, 0.2, 0.2]],
+ [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]
+ ]
+ expected_groundtruth_classes = [
+ [-1, -1],
+ [-1, 1],
+ [1, 2],
+ [-1, -1]
+ ]
+
+ expected_context_image_ids = [b'im_1', b'im_2']
+
+ flds = fields.InputDataFields
+ encoded_images = self._make_random_serialized_jpeg_images(
+ num_frames, image_height, image_width)
+
+ def graph_fn():
+ label_map_proto_file = os.path.join(self.get_temp_dir(), 'labelmap.pbtxt')
+ self._create_label_map(label_map_proto_file)
+ decoder = tf_sequence_example_decoder.TfSequenceExampleDecoder(
+ label_map_proto_file=label_map_proto_file,
+ load_context_image_ids=True)
+ sequence_example_serialized = seq_example_util.make_sequence_example(
+ dataset_name='video_dataset',
+ video_id='video',
+ encoded_images=encoded_images,
+ image_height=image_height,
+ image_width=image_width,
+ image_format='JPEG',
+ image_source_ids=[str(i) for i in range(num_frames)],
+ is_annotated=[[1], [1], [1], [1]],
+ bboxes=[
+ [[0., 0., 1., 1.]], # Frame 0.
+ [[0.2, 0.2, 1., 1.],
+ [0., 0., 1., 1.]], # Frame 1.
+ [[0., 0., 1., 1.], # Frame 2.
+ [0.1, 0.1, 0.2, 0.2]],
+ [[]], # Frame 3.
+ ],
+ label_strings=[
+ ['fox'], # Frame 0. Fox will be filtered out.
+ ['fox', 'dog'], # Frame 1. Fox will be filtered out.
+ ['dog', 'cat'], # Frame 2.
+ [], # Frame 3
+ ],
+ context_features=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5],
+ context_feature_length=[3],
+ context_features_image_id_list=[b'im_1', b'im_2']
+ ).SerializeToString()
+
+ example_string_tensor = tf.convert_to_tensor(sequence_example_serialized)
+ return decoder.decode(example_string_tensor)
+
+ tensor_dict_out = self.execute(graph_fn, [])
+ self.assertAllClose(expected_groundtruth_boxes,
+ tensor_dict_out[flds.groundtruth_boxes])
+ self.assertAllEqual(expected_groundtruth_classes,
+ tensor_dict_out[flds.groundtruth_classes])
+ self.assertAllEqual(expected_context_image_ids,
+ tensor_dict_out[flds.context_features_image_id_list])
+
def test_decode_sequence_example_negative_clip(self):
num_frames = 4
image_height = 20
diff --git a/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples.py b/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples.py
index 89f89467c5e887f1410a6c706e10c2ab9002c48d..21890aa9a02362316a9b9a2377b005fc783ee0fe 100644
--- a/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples.py
+++ b/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples.py
@@ -53,7 +53,7 @@ import os
import numpy as np
import PIL.Image
import six
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
try:
import apache_beam as beam # pylint:disable=g-import-not-at-top
@@ -294,20 +294,46 @@ class SortGroupedDataFn(beam.DoFn):
sorted_example_list = sorted(example_list, key=sorting_fn)
+ num_embeddings = 0
+ for example in sorted_example_list:
+ num_embeddings += example.features.feature[
+ 'image/embedding_count'].int64_list.value[0]
+
self._num_examples_processed.inc(1)
- if len(sorted_example_list) > self._max_num_elements_in_context_features:
+ # To handle cases where there are more context embeddings within
+ # the time horizon than the specified maximum, we split the context group
+ # into subsets sequentially in time, with each subset having the maximum
+ # number of context embeddings except the final one, which holds the
+ # remainder.
+ if num_embeddings > self._max_num_elements_in_context_features:
leftovers = sorted_example_list
output_list = []
count = 0
self._too_many_elements.inc(1)
- while len(leftovers) > self._max_num_elements_in_context_features:
+ num_embeddings = 0
+ max_idx = 0
+ for idx, example in enumerate(leftovers):
+ num_embeddings += example.features.feature[
+ 'image/embedding_count'].int64_list.value[0]
+ if num_embeddings <= self._max_num_elements_in_context_features:
+ max_idx = idx
+ while num_embeddings > self._max_num_elements_in_context_features:
self._split_elements.inc(1)
new_key = key + six.ensure_binary('_' + str(count))
- new_list = leftovers[:self._max_num_elements_in_context_features]
+ new_list = leftovers[:max_idx]
output_list.append((new_key, new_list))
- leftovers = leftovers[:self._max_num_elements_in_context_features]
+ leftovers = leftovers[max_idx:]
count += 1
+ num_embeddings = 0
+ max_idx = 0
+ for idx, example in enumerate(leftovers):
+ num_embeddings += example.features.feature[
+ 'image/embedding_count'].int64_list.value[0]
+ if num_embeddings <= self._max_num_elements_in_context_features:
+ max_idx = idx
+ new_key = key + six.ensure_binary('_' + str(count))
+ output_list.append((new_key, leftovers))
else:
output_list = [(key, sorted_example_list)]
@@ -454,12 +480,15 @@ class GenerateContextFn(beam.DoFn):
example_embedding = list(example.features.feature[
'image/embedding'].float_list.value)
context_features.extend(example_embedding)
- example.features.feature[
- 'context_features_idx'].int64_list.value.append(count)
- count += 1
+ num_embeddings = example.features.feature[
+ 'image/embedding_count'].int64_list.value[0]
example_image_id = example.features.feature[
'image/source_id'].bytes_list.value[0]
- context_features_image_id_list.append(example_image_id)
+ for _ in range(num_embeddings):
+ example.features.feature[
+ 'context_features_idx'].int64_list.value.append(count)
+ count += 1
+ context_features_image_id_list.append(example_image_id)
if not example_embedding:
example_embedding.append(np.zeros(self._context_feature_length))
@@ -926,6 +955,7 @@ def main(argv=None, save_main_session=True):
args.context_features_score_threshold,
args.keep_only_positives_gt,
args.max_num_elements_in_context_features,
+ args.num_shards,
args.output_type,
args.max_clip_length,
args.context_feature_length)
diff --git a/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples_tf1_test.py b/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples_tf1_test.py
deleted file mode 100644
index 42f970c226dce687f6d99b30912e5673208d15dd..0000000000000000000000000000000000000000
--- a/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples_tf1_test.py
+++ /dev/null
@@ -1,396 +0,0 @@
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for add_context_to_examples."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-import contextlib
-import datetime
-import os
-import tempfile
-import unittest
-
-import numpy as np
-import six
-import tensorflow.compat.v1 as tf
-
-from object_detection.dataset_tools.context_rcnn import add_context_to_examples
-from object_detection.utils import tf_version
-
-
-try:
- import apache_beam as beam # pylint:disable=g-import-not-at-top
-except ModuleNotFoundError:
- pass
-
-
-@contextlib.contextmanager
-def InMemoryTFRecord(entries):
- temp = tempfile.NamedTemporaryFile(delete=False)
- filename = temp.name
- try:
- with tf.python_io.TFRecordWriter(filename) as writer:
- for value in entries:
- writer.write(value)
- yield filename
- finally:
- os.unlink(temp.name)
-
-
-def BytesFeature(value):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
-
-def BytesListFeature(value):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
-
-
-def Int64Feature(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-
-
-def Int64ListFeature(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
-
-
-def FloatListFeature(value):
- return tf.train.Feature(float_list=tf.train.FloatList(value=value))
-
-
-@unittest.skipIf(tf_version.is_tf2(), 'Skipping TF1.X only test.')
-class GenerateContextDataTest(tf.test.TestCase):
-
- def _create_first_tf_example(self):
- with self.test_session():
- encoded_image = tf.image.encode_jpeg(
- tf.constant(np.ones((4, 4, 3)).astype(np.uint8))).eval()
-
- example = tf.train.Example(features=tf.train.Features(feature={
- 'image/encoded': BytesFeature(encoded_image),
- 'image/source_id': BytesFeature(six.ensure_binary('image_id_1')),
- 'image/height': Int64Feature(4),
- 'image/width': Int64Feature(4),
- 'image/object/class/label': Int64ListFeature([5, 5]),
- 'image/object/class/text': BytesListFeature([six.ensure_binary('hyena'),
- six.ensure_binary('hyena')
- ]),
- 'image/object/bbox/xmin': FloatListFeature([0.0, 0.1]),
- 'image/object/bbox/xmax': FloatListFeature([0.2, 0.3]),
- 'image/object/bbox/ymin': FloatListFeature([0.4, 0.5]),
- 'image/object/bbox/ymax': FloatListFeature([0.6, 0.7]),
- 'image/seq_id': BytesFeature(six.ensure_binary('01')),
- 'image/seq_num_frames': Int64Feature(2),
- 'image/seq_frame_num': Int64Feature(0),
- 'image/date_captured': BytesFeature(
- six.ensure_binary(str(datetime.datetime(2020, 1, 1, 1, 0, 0)))),
- 'image/embedding': FloatListFeature([0.1, 0.2, 0.3]),
- 'image/embedding_score': FloatListFeature([0.9]),
- 'image/embedding_length': Int64Feature(3)
-
- }))
-
- return example.SerializeToString()
-
- def _create_second_tf_example(self):
- with self.test_session():
- encoded_image = tf.image.encode_jpeg(
- tf.constant(np.ones((4, 4, 3)).astype(np.uint8))).eval()
-
- example = tf.train.Example(features=tf.train.Features(feature={
- 'image/encoded': BytesFeature(encoded_image),
- 'image/source_id': BytesFeature(six.ensure_binary('image_id_2')),
- 'image/height': Int64Feature(4),
- 'image/width': Int64Feature(4),
- 'image/object/class/label': Int64ListFeature([5]),
- 'image/object/class/text': BytesListFeature([six.ensure_binary('hyena')
- ]),
- 'image/object/bbox/xmin': FloatListFeature([0.0]),
- 'image/object/bbox/xmax': FloatListFeature([0.1]),
- 'image/object/bbox/ymin': FloatListFeature([0.2]),
- 'image/object/bbox/ymax': FloatListFeature([0.3]),
- 'image/seq_id': BytesFeature(six.ensure_binary('01')),
- 'image/seq_num_frames': Int64Feature(2),
- 'image/seq_frame_num': Int64Feature(1),
- 'image/date_captured': BytesFeature(
- six.ensure_binary(str(datetime.datetime(2020, 1, 1, 1, 1, 0)))),
- 'image/embedding': FloatListFeature([0.4, 0.5, 0.6]),
- 'image/embedding_score': FloatListFeature([0.9]),
- 'image/embedding_length': Int64Feature(3)
- }))
-
- return example.SerializeToString()
-
- def assert_expected_examples(self, tf_example_list):
- self.assertAllEqual(
- {tf_example.features.feature['image/source_id'].bytes_list.value[0]
- for tf_example in tf_example_list},
- {six.ensure_binary('image_id_1'), six.ensure_binary('image_id_2')})
- self.assertAllClose(
- tf_example_list[0].features.feature[
- 'image/context_features'].float_list.value,
- [0.1, 0.2, 0.3, 0.4, 0.5, 0.6])
- self.assertAllClose(
- tf_example_list[1].features.feature[
- 'image/context_features'].float_list.value,
- [0.1, 0.2, 0.3, 0.4, 0.5, 0.6])
-
- def assert_expected_sequence_example(self, tf_sequence_example_list):
- tf_sequence_example = tf_sequence_example_list[0]
- num_frames = 2
-
- self.assertAllEqual(
- tf_sequence_example.context.feature[
- 'clip/media_id'].bytes_list.value[0], six.ensure_binary(
- '01_0'))
- self.assertAllClose(
- tf_sequence_example.context.feature[
- 'image/context_features'].float_list.value,
- [0.1, 0.2, 0.3, 0.4, 0.5, 0.6])
-
- seq_feature_dict = tf_sequence_example.feature_lists.feature_list
-
- self.assertLen(
- seq_feature_dict['image/encoded'].feature[:],
- num_frames)
- actual_timestamps = [
- feature.int64_list.value[0] for feature
- in seq_feature_dict['image/timestamp'].feature]
- timestamps = [0, 1]
- self.assertAllEqual(timestamps, actual_timestamps)
-
- # First image.
- self.assertAllClose(
- [0.4, 0.5],
- seq_feature_dict['region/bbox/ymin'].feature[0].float_list.value[:])
- self.assertAllClose(
- [0.0, 0.1],
- seq_feature_dict['region/bbox/xmin'].feature[0].float_list.value[:])
- self.assertAllClose(
- [0.6, 0.7],
- seq_feature_dict['region/bbox/ymax'].feature[0].float_list.value[:])
- self.assertAllClose(
- [0.2, 0.3],
- seq_feature_dict['region/bbox/xmax'].feature[0].float_list.value[:])
- self.assertAllEqual(
- [six.ensure_binary('hyena'), six.ensure_binary('hyena')],
- seq_feature_dict['region/label/string'].feature[0].bytes_list.value[:])
-
- # Second example.
- self.assertAllClose(
- [0.2],
- seq_feature_dict['region/bbox/ymin'].feature[1].float_list.value[:])
- self.assertAllClose(
- [0.0],
- seq_feature_dict['region/bbox/xmin'].feature[1].float_list.value[:])
- self.assertAllClose(
- [0.3],
- seq_feature_dict['region/bbox/ymax'].feature[1].float_list.value[:])
- self.assertAllClose(
- [0.1],
- seq_feature_dict['region/bbox/xmax'].feature[1].float_list.value[:])
- self.assertAllEqual(
- [six.ensure_binary('hyena')],
- seq_feature_dict['region/label/string'].feature[1].bytes_list.value[:])
-
- def assert_expected_key(self, key):
- self.assertAllEqual(key, b'01')
-
- def assert_sorted(self, example_collection):
- example_list = list(example_collection)
- counter = 0
- for example in example_list:
- frame_num = example.features.feature[
- 'image/seq_frame_num'].int64_list.value[0]
- self.assertGreaterEqual(frame_num, counter)
- counter = frame_num
-
- def assert_context(self, example_collection):
- example_list = list(example_collection)
- for example in example_list:
- context = example.features.feature[
- 'image/context_features'].float_list.value
- self.assertAllClose([0.1, 0.2, 0.3, 0.4, 0.5, 0.6], context)
-
- def assert_resized(self, example):
- width = example.features.feature['image/width'].int64_list.value[0]
- self.assertAllEqual(width, 2)
- height = example.features.feature['image/height'].int64_list.value[0]
- self.assertAllEqual(height, 2)
-
- def assert_size(self, example):
- width = example.features.feature['image/width'].int64_list.value[0]
- self.assertAllEqual(width, 4)
- height = example.features.feature['image/height'].int64_list.value[0]
- self.assertAllEqual(height, 4)
-
- def test_sliding_window(self):
- example_list = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
- max_clip_length = 3
- stride_length = 3
- out_list = [list(i) for i in add_context_to_examples.get_sliding_window(
- example_list, max_clip_length, stride_length)]
- self.assertAllEqual(out_list, [['a', 'b', 'c'],
- ['d', 'e', 'f'],
- ['g']])
-
- def test_rekey_data_fn(self):
- sequence_key = 'image/seq_id'
- time_horizon = None
- reduce_image_size = False
- max_dim = None
-
- rekey_fn = add_context_to_examples.ReKeyDataFn(
- sequence_key, time_horizon,
- reduce_image_size, max_dim)
- output = rekey_fn.process(self._create_first_tf_example())
-
- self.assert_expected_key(output[0][0])
- self.assert_size(output[0][1])
-
- def test_rekey_data_fn_w_resize(self):
- sequence_key = 'image/seq_id'
- time_horizon = None
- reduce_image_size = True
- max_dim = 2
-
- rekey_fn = add_context_to_examples.ReKeyDataFn(
- sequence_key, time_horizon,
- reduce_image_size, max_dim)
- output = rekey_fn.process(self._create_first_tf_example())
-
- self.assert_expected_key(output[0][0])
- self.assert_resized(output[0][1])
-
- def test_sort_fn(self):
- sequence_key = 'image/seq_id'
- sorted_image_ids = False
- max_num_elements_in_context_features = 10
- sort_fn = add_context_to_examples.SortGroupedDataFn(
- sequence_key, sorted_image_ids, max_num_elements_in_context_features)
- output = sort_fn.process(
- ('dummy_key', [tf.train.Example.FromString(
- self._create_second_tf_example()),
- tf.train.Example.FromString(
- self._create_first_tf_example())]))
-
- self.assert_sorted(output[0][1])
-
- def test_add_context_fn(self):
- sequence_key = 'image/seq_id'
- add_context_features = True
- image_ids_to_keep = 'All'
- context_fn = add_context_to_examples.GenerateContextFn(
- sequence_key, add_context_features, image_ids_to_keep)
- output = context_fn.process(
- ('dummy_key', [tf.train.Example.FromString(
- self._create_first_tf_example()),
- tf.train.Example.FromString(
- self._create_second_tf_example())]))
-
- self.assertEqual(len(output), 2)
- self.assert_context(output)
-
- def test_add_context_fn_output_sequence_example(self):
- sequence_key = 'image/seq_id'
- add_context_features = True
- image_ids_to_keep = 'All'
- context_fn = add_context_to_examples.GenerateContextFn(
- sequence_key, add_context_features, image_ids_to_keep,
- output_type='tf_sequence_example')
- output = context_fn.process(
- ('01',
- [tf.train.Example.FromString(self._create_first_tf_example()),
- tf.train.Example.FromString(self._create_second_tf_example())]))
-
- self.assertEqual(len(output), 1)
- self.assert_expected_sequence_example(output)
-
- def test_add_context_fn_output_sequence_example_cliplen(self):
- sequence_key = 'image/seq_id'
- add_context_features = True
- image_ids_to_keep = 'All'
- context_fn = add_context_to_examples.GenerateContextFn(
- sequence_key, add_context_features, image_ids_to_keep,
- output_type='tf_sequence_example', max_clip_length=1)
- output = context_fn.process(
- ('01',
- [tf.train.Example.FromString(self._create_first_tf_example()),
- tf.train.Example.FromString(self._create_second_tf_example())]))
- self.assertEqual(len(output), 2)
-
- def test_beam_pipeline(self):
- with InMemoryTFRecord(
- [self._create_first_tf_example(),
- self._create_second_tf_example()]) as input_tfrecord:
- temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
- output_tfrecord = os.path.join(temp_dir, 'output_tfrecord')
- sequence_key = six.ensure_binary('image/seq_id')
- max_num_elements = 10
- num_shards = 1
- pipeline_options = beam.options.pipeline_options.PipelineOptions(
- runner='DirectRunner')
- p = beam.Pipeline(options=pipeline_options)
- add_context_to_examples.construct_pipeline(
- p,
- input_tfrecord,
- output_tfrecord,
- sequence_key,
- max_num_elements_in_context_features=max_num_elements,
- num_shards=num_shards)
- p.run()
- filenames = tf.io.gfile.glob(output_tfrecord + '-?????-of-?????')
- actual_output = []
- record_iterator = tf.python_io.tf_record_iterator(path=filenames[0])
- for record in record_iterator:
- actual_output.append(record)
- self.assertEqual(len(actual_output), 2)
- self.assert_expected_examples([tf.train.Example.FromString(
- tf_example) for tf_example in actual_output])
-
- def test_beam_pipeline_sequence_example(self):
- with InMemoryTFRecord(
- [self._create_first_tf_example(),
- self._create_second_tf_example()]) as input_tfrecord:
- temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
- output_tfrecord = os.path.join(temp_dir, 'output_tfrecord')
- sequence_key = six.ensure_binary('image/seq_id')
- max_num_elements = 10
- num_shards = 1
- pipeline_options = beam.options.pipeline_options.PipelineOptions(
- runner='DirectRunner')
- p = beam.Pipeline(options=pipeline_options)
- add_context_to_examples.construct_pipeline(
- p,
- input_tfrecord,
- output_tfrecord,
- sequence_key,
- max_num_elements_in_context_features=max_num_elements,
- num_shards=num_shards,
- output_type='tf_sequence_example')
- p.run()
- filenames = tf.io.gfile.glob(output_tfrecord + '-?????-of-?????')
- actual_output = []
- record_iterator = tf.python_io.tf_record_iterator(
- path=filenames[0])
- for record in record_iterator:
- actual_output.append(record)
- self.assertEqual(len(actual_output), 1)
- self.assert_expected_sequence_example(
- [tf.train.SequenceExample.FromString(
- tf_example) for tf_example in actual_output])
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples_tf2_test.py b/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..61020b008bcf464630a55a29d56928e6b8cd41cf
--- /dev/null
+++ b/research/object_detection/dataset_tools/context_rcnn/add_context_to_examples_tf2_test.py
@@ -0,0 +1,398 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for add_context_to_examples."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import contextlib
+import datetime
+import os
+import tempfile
+import unittest
+
+import numpy as np
+import six
+import tensorflow as tf
+
+from object_detection.utils import tf_version
+
+if tf_version.is_tf2():
+ from object_detection.dataset_tools.context_rcnn import add_context_to_examples # pylint:disable=g-import-not-at-top
+
+try:
+ import apache_beam as beam # pylint:disable=g-import-not-at-top
+except ModuleNotFoundError:
+ pass
+
+
+@contextlib.contextmanager
+def InMemoryTFRecord(entries):
+ temp = tempfile.NamedTemporaryFile(delete=False)
+ filename = temp.name
+ try:
+ with tf.io.TFRecordWriter(filename) as writer:
+ for value in entries:
+ writer.write(value)
+ yield filename
+ finally:
+ os.unlink(temp.name)
+
+
+def BytesFeature(value):
+ return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+
+def BytesListFeature(value):
+ return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
+
+
+def Int64Feature(value):
+ return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+
+def Int64ListFeature(value):
+ return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
+
+
+def FloatListFeature(value):
+ return tf.train.Feature(float_list=tf.train.FloatList(value=value))
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class GenerateContextDataTest(tf.test.TestCase):
+
+ def _create_first_tf_example(self):
+ encoded_image = tf.io.encode_jpeg(
+ tf.constant(np.ones((4, 4, 3)).astype(np.uint8))).numpy()
+
+ example = tf.train.Example(features=tf.train.Features(feature={
+ 'image/encoded': BytesFeature(encoded_image),
+ 'image/source_id': BytesFeature(six.ensure_binary('image_id_1')),
+ 'image/height': Int64Feature(4),
+ 'image/width': Int64Feature(4),
+ 'image/object/class/label': Int64ListFeature([5, 5]),
+ 'image/object/class/text': BytesListFeature([six.ensure_binary('hyena'),
+ six.ensure_binary('hyena')
+ ]),
+ 'image/object/bbox/xmin': FloatListFeature([0.0, 0.1]),
+ 'image/object/bbox/xmax': FloatListFeature([0.2, 0.3]),
+ 'image/object/bbox/ymin': FloatListFeature([0.4, 0.5]),
+ 'image/object/bbox/ymax': FloatListFeature([0.6, 0.7]),
+ 'image/seq_id': BytesFeature(six.ensure_binary('01')),
+ 'image/seq_num_frames': Int64Feature(2),
+ 'image/seq_frame_num': Int64Feature(0),
+ 'image/date_captured': BytesFeature(
+ six.ensure_binary(str(datetime.datetime(2020, 1, 1, 1, 0, 0)))),
+ 'image/embedding': FloatListFeature([0.1, 0.2, 0.3]),
+ 'image/embedding_score': FloatListFeature([0.9]),
+ 'image/embedding_length': Int64Feature(3),
+ 'image/embedding_count': Int64Feature(1)
+
+ }))
+
+ return example.SerializeToString()
+
+ def _create_second_tf_example(self):
+ encoded_image = tf.io.encode_jpeg(
+ tf.constant(np.ones((4, 4, 3)).astype(np.uint8))).numpy()
+
+ example = tf.train.Example(features=tf.train.Features(feature={
+ 'image/encoded': BytesFeature(encoded_image),
+ 'image/source_id': BytesFeature(six.ensure_binary('image_id_2')),
+ 'image/height': Int64Feature(4),
+ 'image/width': Int64Feature(4),
+ 'image/object/class/label': Int64ListFeature([5]),
+ 'image/object/class/text': BytesListFeature([six.ensure_binary('hyena')
+ ]),
+ 'image/object/bbox/xmin': FloatListFeature([0.0]),
+ 'image/object/bbox/xmax': FloatListFeature([0.1]),
+ 'image/object/bbox/ymin': FloatListFeature([0.2]),
+ 'image/object/bbox/ymax': FloatListFeature([0.3]),
+ 'image/seq_id': BytesFeature(six.ensure_binary('01')),
+ 'image/seq_num_frames': Int64Feature(2),
+ 'image/seq_frame_num': Int64Feature(1),
+ 'image/date_captured': BytesFeature(
+ six.ensure_binary(str(datetime.datetime(2020, 1, 1, 1, 1, 0)))),
+ 'image/embedding': FloatListFeature([0.4, 0.5, 0.6]),
+ 'image/embedding_score': FloatListFeature([0.9]),
+ 'image/embedding_length': Int64Feature(3),
+ 'image/embedding_count': Int64Feature(1)
+ }))
+
+ return example.SerializeToString()
+
+ def assert_expected_examples(self, tf_example_list):
+ self.assertAllEqual(
+ {tf_example.features.feature['image/source_id'].bytes_list.value[0]
+ for tf_example in tf_example_list},
+ {six.ensure_binary('image_id_1'), six.ensure_binary('image_id_2')})
+ self.assertAllClose(
+ tf_example_list[0].features.feature[
+ 'image/context_features'].float_list.value,
+ [0.1, 0.2, 0.3, 0.4, 0.5, 0.6])
+ self.assertAllClose(
+ tf_example_list[1].features.feature[
+ 'image/context_features'].float_list.value,
+ [0.1, 0.2, 0.3, 0.4, 0.5, 0.6])
+
+ def assert_expected_sequence_example(self, tf_sequence_example_list):
+ tf_sequence_example = tf_sequence_example_list[0]
+ num_frames = 2
+
+ self.assertAllEqual(
+ tf_sequence_example.context.feature[
+ 'clip/media_id'].bytes_list.value[0], six.ensure_binary(
+ '01_0'))
+ self.assertAllClose(
+ tf_sequence_example.context.feature[
+ 'image/context_features'].float_list.value,
+ [0.1, 0.2, 0.3, 0.4, 0.5, 0.6])
+
+ seq_feature_dict = tf_sequence_example.feature_lists.feature_list
+
+ self.assertLen(
+ seq_feature_dict['image/encoded'].feature[:],
+ num_frames)
+ actual_timestamps = [
+ feature.int64_list.value[0] for feature
+ in seq_feature_dict['image/timestamp'].feature]
+ timestamps = [0, 1]
+ self.assertAllEqual(timestamps, actual_timestamps)
+
+ # First image.
+ self.assertAllClose(
+ [0.4, 0.5],
+ seq_feature_dict['region/bbox/ymin'].feature[0].float_list.value[:])
+ self.assertAllClose(
+ [0.0, 0.1],
+ seq_feature_dict['region/bbox/xmin'].feature[0].float_list.value[:])
+ self.assertAllClose(
+ [0.6, 0.7],
+ seq_feature_dict['region/bbox/ymax'].feature[0].float_list.value[:])
+ self.assertAllClose(
+ [0.2, 0.3],
+ seq_feature_dict['region/bbox/xmax'].feature[0].float_list.value[:])
+ self.assertAllEqual(
+ [six.ensure_binary('hyena'), six.ensure_binary('hyena')],
+ seq_feature_dict['region/label/string'].feature[0].bytes_list.value[:])
+
+ # Second example.
+ self.assertAllClose(
+ [0.2],
+ seq_feature_dict['region/bbox/ymin'].feature[1].float_list.value[:])
+ self.assertAllClose(
+ [0.0],
+ seq_feature_dict['region/bbox/xmin'].feature[1].float_list.value[:])
+ self.assertAllClose(
+ [0.3],
+ seq_feature_dict['region/bbox/ymax'].feature[1].float_list.value[:])
+ self.assertAllClose(
+ [0.1],
+ seq_feature_dict['region/bbox/xmax'].feature[1].float_list.value[:])
+ self.assertAllEqual(
+ [six.ensure_binary('hyena')],
+ seq_feature_dict['region/label/string'].feature[1].bytes_list.value[:])
+
+ def assert_expected_key(self, key):
+ self.assertAllEqual(key, b'01')
+
+ def assert_sorted(self, example_collection):
+ example_list = list(example_collection)
+ counter = 0
+ for example in example_list:
+ frame_num = example.features.feature[
+ 'image/seq_frame_num'].int64_list.value[0]
+ self.assertGreaterEqual(frame_num, counter)
+ counter = frame_num
+
+ def assert_context(self, example_collection):
+ example_list = list(example_collection)
+ for example in example_list:
+ context = example.features.feature[
+ 'image/context_features'].float_list.value
+ self.assertAllClose([0.1, 0.2, 0.3, 0.4, 0.5, 0.6], context)
+
+ def assert_resized(self, example):
+ width = example.features.feature['image/width'].int64_list.value[0]
+ self.assertAllEqual(width, 2)
+ height = example.features.feature['image/height'].int64_list.value[0]
+ self.assertAllEqual(height, 2)
+
+ def assert_size(self, example):
+ width = example.features.feature['image/width'].int64_list.value[0]
+ self.assertAllEqual(width, 4)
+ height = example.features.feature['image/height'].int64_list.value[0]
+ self.assertAllEqual(height, 4)
+
+ def test_sliding_window(self):
+ example_list = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
+ max_clip_length = 3
+ stride_length = 3
+ out_list = [list(i) for i in add_context_to_examples.get_sliding_window(
+ example_list, max_clip_length, stride_length)]
+ self.assertAllEqual(out_list, [['a', 'b', 'c'],
+ ['d', 'e', 'f'],
+ ['g']])
+
+ def test_rekey_data_fn(self):
+ sequence_key = 'image/seq_id'
+ time_horizon = None
+ reduce_image_size = False
+ max_dim = None
+
+ rekey_fn = add_context_to_examples.ReKeyDataFn(
+ sequence_key, time_horizon,
+ reduce_image_size, max_dim)
+ output = rekey_fn.process(self._create_first_tf_example())
+
+ self.assert_expected_key(output[0][0])
+ self.assert_size(output[0][1])
+
+ def test_rekey_data_fn_w_resize(self):
+ sequence_key = 'image/seq_id'
+ time_horizon = None
+ reduce_image_size = True
+ max_dim = 2
+
+ rekey_fn = add_context_to_examples.ReKeyDataFn(
+ sequence_key, time_horizon,
+ reduce_image_size, max_dim)
+ output = rekey_fn.process(self._create_first_tf_example())
+
+ self.assert_expected_key(output[0][0])
+ self.assert_resized(output[0][1])
+
+ def test_sort_fn(self):
+ sequence_key = 'image/seq_id'
+ sorted_image_ids = False
+ max_num_elements_in_context_features = 10
+ sort_fn = add_context_to_examples.SortGroupedDataFn(
+ sequence_key, sorted_image_ids, max_num_elements_in_context_features)
+ output = sort_fn.process(
+ ('dummy_key', [tf.train.Example.FromString(
+ self._create_second_tf_example()),
+ tf.train.Example.FromString(
+ self._create_first_tf_example())]))
+
+ self.assert_sorted(output[0][1])
+
+ def test_add_context_fn(self):
+ sequence_key = 'image/seq_id'
+ add_context_features = True
+ image_ids_to_keep = 'All'
+ context_fn = add_context_to_examples.GenerateContextFn(
+ sequence_key, add_context_features, image_ids_to_keep)
+ output = context_fn.process(
+ ('dummy_key', [tf.train.Example.FromString(
+ self._create_first_tf_example()),
+ tf.train.Example.FromString(
+ self._create_second_tf_example())]))
+
+ self.assertEqual(len(output), 2)
+ self.assert_context(output)
+
+ def test_add_context_fn_output_sequence_example(self):
+ sequence_key = 'image/seq_id'
+ add_context_features = True
+ image_ids_to_keep = 'All'
+ context_fn = add_context_to_examples.GenerateContextFn(
+ sequence_key, add_context_features, image_ids_to_keep,
+ output_type='tf_sequence_example')
+ output = context_fn.process(
+ ('01',
+ [tf.train.Example.FromString(self._create_first_tf_example()),
+ tf.train.Example.FromString(self._create_second_tf_example())]))
+
+ self.assertEqual(len(output), 1)
+ self.assert_expected_sequence_example(output)
+
+ def test_add_context_fn_output_sequence_example_cliplen(self):
+ sequence_key = 'image/seq_id'
+ add_context_features = True
+ image_ids_to_keep = 'All'
+ context_fn = add_context_to_examples.GenerateContextFn(
+ sequence_key, add_context_features, image_ids_to_keep,
+ output_type='tf_sequence_example', max_clip_length=1)
+ output = context_fn.process(
+ ('01',
+ [tf.train.Example.FromString(self._create_first_tf_example()),
+ tf.train.Example.FromString(self._create_second_tf_example())]))
+ self.assertEqual(len(output), 2)
+
+ def test_beam_pipeline(self):
+ with InMemoryTFRecord(
+ [self._create_first_tf_example(),
+ self._create_second_tf_example()]) as input_tfrecord:
+ temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
+ output_tfrecord = os.path.join(temp_dir, 'output_tfrecord')
+ sequence_key = six.ensure_binary('image/seq_id')
+ max_num_elements = 10
+ num_shards = 1
+ pipeline_options = beam.options.pipeline_options.PipelineOptions(
+ runner='DirectRunner')
+ p = beam.Pipeline(options=pipeline_options)
+ add_context_to_examples.construct_pipeline(
+ p,
+ input_tfrecord,
+ output_tfrecord,
+ sequence_key,
+ max_num_elements_in_context_features=max_num_elements,
+ num_shards=num_shards)
+ p.run()
+ filenames = tf.io.gfile.glob(output_tfrecord + '-?????-of-?????')
+ actual_output = []
+ record_iterator = tf.data.TFRecordDataset(
+ tf.convert_to_tensor(filenames)).as_numpy_iterator()
+ for record in record_iterator:
+ actual_output.append(record)
+ self.assertEqual(len(actual_output), 2)
+ self.assert_expected_examples([tf.train.Example.FromString(
+ tf_example) for tf_example in actual_output])
+
+ def test_beam_pipeline_sequence_example(self):
+ with InMemoryTFRecord(
+ [self._create_first_tf_example(),
+ self._create_second_tf_example()]) as input_tfrecord:
+ temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
+ output_tfrecord = os.path.join(temp_dir, 'output_tfrecord')
+ sequence_key = six.ensure_binary('image/seq_id')
+ max_num_elements = 10
+ num_shards = 1
+ pipeline_options = beam.options.pipeline_options.PipelineOptions(
+ runner='DirectRunner')
+ p = beam.Pipeline(options=pipeline_options)
+ add_context_to_examples.construct_pipeline(
+ p,
+ input_tfrecord,
+ output_tfrecord,
+ sequence_key,
+ max_num_elements_in_context_features=max_num_elements,
+ num_shards=num_shards,
+ output_type='tf_sequence_example')
+ p.run()
+ filenames = tf.io.gfile.glob(output_tfrecord + '-?????-of-?????')
+ actual_output = []
+ record_iterator = tf.data.TFRecordDataset(
+ tf.convert_to_tensor(filenames)).as_numpy_iterator()
+ for record in record_iterator:
+ actual_output.append(record)
+ self.assertEqual(len(actual_output), 1)
+ self.assert_expected_sequence_example(
+ [tf.train.SequenceExample.FromString(
+ tf_example) for tf_example in actual_output])
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_main.py b/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_main.py
index bafc406be12ad3f7a8e4d83cde3707bc0b48b23e..dbf3cad0eacaa4883aba340e34bb623a96d3af50 100644
--- a/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_main.py
+++ b/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_main.py
@@ -37,11 +37,10 @@ import argparse
import hashlib
import io
import json
-import logging
import os
import numpy as np
import PIL.Image
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
from object_detection.utils import dataset_util
try:
@@ -110,16 +109,9 @@ class ParseImage(beam.DoFn):
encoded_jpg = fid.read()
encoded_jpg_io = io.BytesIO(encoded_jpg)
image = PIL.Image.open(encoded_jpg_io)
- # Ensure the image can be read by tf
- with tf.Graph().as_default():
- image = tf.image.decode_jpeg(encoded_jpg, channels=3)
- init_op = tf.initialize_all_tables()
- with tf.Session() as sess:
- sess.run(init_op)
- sess.run(image)
- except Exception as e: # pylint: disable=broad-except
+ image = tf.io.decode_jpeg(encoded_jpg, channels=3)
+ except Exception: # pylint: disable=broad-except
# The image file is missing or corrupt
- tf.logging.error(str(e))
return []
key = hashlib.sha256(encoded_jpg).hexdigest()
@@ -257,8 +249,6 @@ def create_pipeline(pipeline,
keep_bboxes: Whether to keep any bounding boxes that exist in the json file
"""
- logging.info('Reading data from COCO-CameraTraps Dataset.')
-
data = load_json_data(input_annotations_file)
num_shards = int(np.ceil(float(len(data['images']))/num_images_per_shard))
diff --git a/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_tf1_test.py b/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_tf1_test.py
deleted file mode 100644
index 19018a3a1158d3fdebd9b81f5af3e24b6327cd74..0000000000000000000000000000000000000000
--- a/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_tf1_test.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for create_cococameratraps_tfexample_main."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-import datetime
-import json
-import os
-import tempfile
-import unittest
-
-import numpy as np
-
-from PIL import Image
-import tensorflow.compat.v1 as tf
-from object_detection.dataset_tools.context_rcnn import create_cococameratraps_tfexample_main
-from object_detection.utils import tf_version
-
-try:
- import apache_beam as beam # pylint:disable=g-import-not-at-top
-except ModuleNotFoundError:
- pass
-
-
-@unittest.skipIf(tf_version.is_tf2(), 'Skipping TF1.X only test.')
-class CreateCOCOCameraTrapsTfexampleTest(tf.test.TestCase):
-
- IMAGE_HEIGHT = 360
- IMAGE_WIDTH = 480
-
- def _write_random_images_to_directory(self, directory, num_frames):
- for frame_num in range(num_frames):
- img = np.random.randint(0, high=256,
- size=(self.IMAGE_HEIGHT, self.IMAGE_WIDTH, 3),
- dtype=np.uint8)
- pil_image = Image.fromarray(img)
- fname = 'im_' + str(frame_num) + '.jpg'
- pil_image.save(os.path.join(directory, fname), 'JPEG')
-
- def _create_json_file(self, directory, num_frames, keep_bboxes=False):
- json_dict = {'images': [], 'annotations': []}
- json_dict['categories'] = [{'id': 0, 'name': 'empty'},
- {'id': 1, 'name': 'animal'}]
- for idx in range(num_frames):
- im = {'id': 'im_' + str(idx),
- 'file_name': 'im_' + str(idx) + '.jpg',
- 'height': self.IMAGE_HEIGHT,
- 'width': self.IMAGE_WIDTH,
- 'seq_id': 'seq_1',
- 'seq_num_frames': num_frames,
- 'frame_num': idx,
- 'location': 'loc_' + str(idx),
- 'date_captured': str(datetime.datetime.now())
- }
- json_dict['images'].append(im)
- ann = {'id': 'ann' + str(idx),
- 'image_id': 'im_' + str(idx),
- 'category_id': 1,
- }
- if keep_bboxes:
- ann['bbox'] = [0.0 * self.IMAGE_WIDTH,
- 0.1 * self.IMAGE_HEIGHT,
- 0.5 * self.IMAGE_WIDTH,
- 0.5 * self.IMAGE_HEIGHT]
- json_dict['annotations'].append(ann)
-
- json_path = os.path.join(directory, 'test_file.json')
- with tf.io.gfile.GFile(json_path, 'w') as f:
- json.dump(json_dict, f)
- return json_path
-
- def assert_expected_example_bbox(self, example):
- self.assertAllClose(
- example.features.feature['image/object/bbox/ymin'].float_list.value,
- [0.1])
- self.assertAllClose(
- example.features.feature['image/object/bbox/xmin'].float_list.value,
- [0.0])
- self.assertAllClose(
- example.features.feature['image/object/bbox/ymax'].float_list.value,
- [0.6])
- self.assertAllClose(
- example.features.feature['image/object/bbox/xmax'].float_list.value,
- [0.5])
- self.assertAllClose(
- example.features.feature['image/object/class/label']
- .int64_list.value, [1])
- self.assertAllEqual(
- example.features.feature['image/object/class/text']
- .bytes_list.value, [b'animal'])
- self.assertAllClose(
- example.features.feature['image/class/label']
- .int64_list.value, [1])
- self.assertAllEqual(
- example.features.feature['image/class/text']
- .bytes_list.value, [b'animal'])
-
- # Check other essential attributes.
- self.assertAllEqual(
- example.features.feature['image/height'].int64_list.value,
- [self.IMAGE_HEIGHT])
- self.assertAllEqual(
- example.features.feature['image/width'].int64_list.value,
- [self.IMAGE_WIDTH])
- self.assertAllEqual(
- example.features.feature['image/source_id'].bytes_list.value,
- [b'im_0'])
- self.assertTrue(
- example.features.feature['image/encoded'].bytes_list.value)
-
- def assert_expected_example(self, example):
- self.assertAllClose(
- example.features.feature['image/object/bbox/ymin'].float_list.value,
- [])
- self.assertAllClose(
- example.features.feature['image/object/bbox/xmin'].float_list.value,
- [])
- self.assertAllClose(
- example.features.feature['image/object/bbox/ymax'].float_list.value,
- [])
- self.assertAllClose(
- example.features.feature['image/object/bbox/xmax'].float_list.value,
- [])
- self.assertAllClose(
- example.features.feature['image/object/class/label']
- .int64_list.value, [1])
- self.assertAllEqual(
- example.features.feature['image/object/class/text']
- .bytes_list.value, [b'animal'])
- self.assertAllClose(
- example.features.feature['image/class/label']
- .int64_list.value, [1])
- self.assertAllEqual(
- example.features.feature['image/class/text']
- .bytes_list.value, [b'animal'])
-
- # Check other essential attributes.
- self.assertAllEqual(
- example.features.feature['image/height'].int64_list.value,
- [self.IMAGE_HEIGHT])
- self.assertAllEqual(
- example.features.feature['image/width'].int64_list.value,
- [self.IMAGE_WIDTH])
- self.assertAllEqual(
- example.features.feature['image/source_id'].bytes_list.value,
- [b'im_0'])
- self.assertTrue(
- example.features.feature['image/encoded'].bytes_list.value)
-
- def test_beam_pipeline(self):
- num_frames = 1
- temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
- json_path = self._create_json_file(temp_dir, num_frames)
- output_tfrecord = temp_dir+'/output'
- self._write_random_images_to_directory(temp_dir, num_frames)
- pipeline_options = beam.options.pipeline_options.PipelineOptions(
- runner='DirectRunner')
- p = beam.Pipeline(options=pipeline_options)
- create_cococameratraps_tfexample_main.create_pipeline(
- p, temp_dir, json_path,
- output_tfrecord_prefix=output_tfrecord)
- p.run()
- filenames = tf.io.gfile.glob(output_tfrecord + '-?????-of-?????')
- actual_output = []
- record_iterator = tf.python_io.tf_record_iterator(path=filenames[0])
- for record in record_iterator:
- actual_output.append(record)
- self.assertEqual(len(actual_output), num_frames)
- self.assert_expected_example(tf.train.Example.FromString(
- actual_output[0]))
-
- def test_beam_pipeline_bbox(self):
- num_frames = 1
- temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
- json_path = self._create_json_file(temp_dir, num_frames, keep_bboxes=True)
- output_tfrecord = temp_dir+'/output'
- self._write_random_images_to_directory(temp_dir, num_frames)
- pipeline_options = beam.options.pipeline_options.PipelineOptions(
- runner='DirectRunner')
- p = beam.Pipeline(options=pipeline_options)
- create_cococameratraps_tfexample_main.create_pipeline(
- p, temp_dir, json_path,
- output_tfrecord_prefix=output_tfrecord,
- keep_bboxes=True)
- p.run()
- filenames = tf.io.gfile.glob(output_tfrecord+'-?????-of-?????')
- actual_output = []
- record_iterator = tf.python_io.tf_record_iterator(path=filenames[0])
- for record in record_iterator:
- actual_output.append(record)
- self.assertEqual(len(actual_output), num_frames)
- self.assert_expected_example_bbox(tf.train.Example.FromString(
- actual_output[0]))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_tf2_test.py b/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a1ac203f334574a3b09654fd736047b8236fa38
--- /dev/null
+++ b/research/object_detection/dataset_tools/context_rcnn/create_cococameratraps_tfexample_tf2_test.py
@@ -0,0 +1,214 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for create_cococameratraps_tfexample_main."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import datetime
+import json
+import os
+import tempfile
+import unittest
+
+import numpy as np
+
+from PIL import Image
+import tensorflow as tf
+from object_detection.utils import tf_version
+
+if tf_version.is_tf2():
+ from object_detection.dataset_tools.context_rcnn import create_cococameratraps_tfexample_main # pylint:disable=g-import-not-at-top
+
+try:
+ import apache_beam as beam # pylint:disable=g-import-not-at-top
+except ModuleNotFoundError:
+ pass
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class CreateCOCOCameraTrapsTfexampleTest(tf.test.TestCase):
+
+ IMAGE_HEIGHT = 360
+ IMAGE_WIDTH = 480
+
+ def _write_random_images_to_directory(self, directory, num_frames):
+ for frame_num in range(num_frames):
+ img = np.random.randint(0, high=256,
+ size=(self.IMAGE_HEIGHT, self.IMAGE_WIDTH, 3),
+ dtype=np.uint8)
+ pil_image = Image.fromarray(img)
+ fname = 'im_' + str(frame_num) + '.jpg'
+ pil_image.save(os.path.join(directory, fname), 'JPEG')
+
+ def _create_json_file(self, directory, num_frames, keep_bboxes=False):
+ json_dict = {'images': [], 'annotations': []}
+ json_dict['categories'] = [{'id': 0, 'name': 'empty'},
+ {'id': 1, 'name': 'animal'}]
+ for idx in range(num_frames):
+ im = {'id': 'im_' + str(idx),
+ 'file_name': 'im_' + str(idx) + '.jpg',
+ 'height': self.IMAGE_HEIGHT,
+ 'width': self.IMAGE_WIDTH,
+ 'seq_id': 'seq_1',
+ 'seq_num_frames': num_frames,
+ 'frame_num': idx,
+ 'location': 'loc_' + str(idx),
+ 'date_captured': str(datetime.datetime.now())
+ }
+ json_dict['images'].append(im)
+ ann = {'id': 'ann' + str(idx),
+ 'image_id': 'im_' + str(idx),
+ 'category_id': 1,
+ }
+ if keep_bboxes:
+ ann['bbox'] = [0.0 * self.IMAGE_WIDTH,
+ 0.1 * self.IMAGE_HEIGHT,
+ 0.5 * self.IMAGE_WIDTH,
+ 0.5 * self.IMAGE_HEIGHT]
+ json_dict['annotations'].append(ann)
+
+ json_path = os.path.join(directory, 'test_file.json')
+ with tf.io.gfile.GFile(json_path, 'w') as f:
+ json.dump(json_dict, f)
+ return json_path
+
+ def assert_expected_example_bbox(self, example):
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/ymin'].float_list.value,
+ [0.1])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/xmin'].float_list.value,
+ [0.0])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/ymax'].float_list.value,
+ [0.6])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/xmax'].float_list.value,
+ [0.5])
+ self.assertAllClose(
+ example.features.feature['image/object/class/label']
+ .int64_list.value, [1])
+ self.assertAllEqual(
+ example.features.feature['image/object/class/text']
+ .bytes_list.value, [b'animal'])
+ self.assertAllClose(
+ example.features.feature['image/class/label']
+ .int64_list.value, [1])
+ self.assertAllEqual(
+ example.features.feature['image/class/text']
+ .bytes_list.value, [b'animal'])
+
+ # Check other essential attributes.
+ self.assertAllEqual(
+ example.features.feature['image/height'].int64_list.value,
+ [self.IMAGE_HEIGHT])
+ self.assertAllEqual(
+ example.features.feature['image/width'].int64_list.value,
+ [self.IMAGE_WIDTH])
+ self.assertAllEqual(
+ example.features.feature['image/source_id'].bytes_list.value,
+ [b'im_0'])
+ self.assertTrue(
+ example.features.feature['image/encoded'].bytes_list.value)
+
+ def assert_expected_example(self, example):
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/ymin'].float_list.value,
+ [])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/xmin'].float_list.value,
+ [])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/ymax'].float_list.value,
+ [])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/xmax'].float_list.value,
+ [])
+ self.assertAllClose(
+ example.features.feature['image/object/class/label']
+ .int64_list.value, [1])
+ self.assertAllEqual(
+ example.features.feature['image/object/class/text']
+ .bytes_list.value, [b'animal'])
+ self.assertAllClose(
+ example.features.feature['image/class/label']
+ .int64_list.value, [1])
+ self.assertAllEqual(
+ example.features.feature['image/class/text']
+ .bytes_list.value, [b'animal'])
+
+ # Check other essential attributes.
+ self.assertAllEqual(
+ example.features.feature['image/height'].int64_list.value,
+ [self.IMAGE_HEIGHT])
+ self.assertAllEqual(
+ example.features.feature['image/width'].int64_list.value,
+ [self.IMAGE_WIDTH])
+ self.assertAllEqual(
+ example.features.feature['image/source_id'].bytes_list.value,
+ [b'im_0'])
+ self.assertTrue(
+ example.features.feature['image/encoded'].bytes_list.value)
+
+ def test_beam_pipeline(self):
+ num_frames = 1
+ temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
+ json_path = self._create_json_file(temp_dir, num_frames)
+ output_tfrecord = temp_dir+'/output'
+ self._write_random_images_to_directory(temp_dir, num_frames)
+ pipeline_options = beam.options.pipeline_options.PipelineOptions(
+ runner='DirectRunner')
+ p = beam.Pipeline(options=pipeline_options)
+ create_cococameratraps_tfexample_main.create_pipeline(
+ p, temp_dir, json_path,
+ output_tfrecord_prefix=output_tfrecord)
+ p.run()
+ filenames = tf.io.gfile.glob(output_tfrecord + '-?????-of-?????')
+ actual_output = []
+ record_iterator = tf.data.TFRecordDataset(
+ tf.convert_to_tensor(filenames)).as_numpy_iterator()
+ for record in record_iterator:
+ actual_output.append(record)
+ self.assertEqual(len(actual_output), num_frames)
+ self.assert_expected_example(tf.train.Example.FromString(
+ actual_output[0]))
+
+ def test_beam_pipeline_bbox(self):
+ num_frames = 1
+ temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
+ json_path = self._create_json_file(temp_dir, num_frames, keep_bboxes=True)
+ output_tfrecord = temp_dir+'/output'
+ self._write_random_images_to_directory(temp_dir, num_frames)
+ pipeline_options = beam.options.pipeline_options.PipelineOptions(
+ runner='DirectRunner')
+ p = beam.Pipeline(options=pipeline_options)
+ create_cococameratraps_tfexample_main.create_pipeline(
+ p, temp_dir, json_path,
+ output_tfrecord_prefix=output_tfrecord,
+ keep_bboxes=True)
+ p.run()
+ filenames = tf.io.gfile.glob(output_tfrecord+'-?????-of-?????')
+ actual_output = []
+ record_iterator = tf.data.TFRecordDataset(
+ tf.convert_to_tensor(filenames)).as_numpy_iterator()
+ for record in record_iterator:
+ actual_output.append(record)
+ self.assertEqual(len(actual_output), num_frames)
+ self.assert_expected_example_bbox(tf.train.Example.FromString(
+ actual_output[0]))
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/dataset_tools/context_rcnn/generate_detection_data.py b/research/object_detection/dataset_tools/context_rcnn/generate_detection_data.py
index aafac9edf8388e33e95ebe9e61c1fa424930676a..c826873802f09ffbc48788576eb9c02038ceeb65 100644
--- a/research/object_detection/dataset_tools/context_rcnn/generate_detection_data.py
+++ b/research/object_detection/dataset_tools/context_rcnn/generate_detection_data.py
@@ -48,7 +48,8 @@ from __future__ import print_function
import argparse
import os
import threading
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
+
try:
import apache_beam as beam # pylint:disable=g-import-not-at-top
except ModuleNotFoundError:
@@ -77,7 +78,7 @@ class GenerateDetectionDataFn(beam.DoFn):
self._num_examples_processed = beam.metrics.Metrics.counter(
'detection_data_generation', 'num_tf_examples_processed')
- def start_bundle(self):
+ def setup(self):
self._load_inference_model()
def _load_inference_model(self):
@@ -85,22 +86,7 @@ class GenerateDetectionDataFn(beam.DoFn):
# one instance across all threads in the worker. This is possible since
# tf.Session.run() is thread safe.
with self.session_lock:
- if self._session is None:
- graph = tf.Graph()
- self._session = tf.Session(graph=graph)
- with graph.as_default():
- meta_graph = tf.saved_model.loader.load(
- self._session, [tf.saved_model.tag_constants.SERVING],
- self._model_dir)
- signature = meta_graph.signature_def['serving_default']
- input_tensor_name = signature.inputs['inputs'].name
- self._input = graph.get_tensor_by_name(input_tensor_name)
- self._boxes_node = graph.get_tensor_by_name(
- signature.outputs['detection_boxes'].name)
- self._scores_node = graph.get_tensor_by_name(
- signature.outputs['detection_scores'].name)
- self._num_detections_node = graph.get_tensor_by_name(
- signature.outputs['num_detections'].name)
+ self._detect_fn = tf.saved_model.load(self._model_dir)
def process(self, tfrecord_entry):
return self._run_inference_and_generate_detections(tfrecord_entry)
@@ -112,9 +98,11 @@ class GenerateDetectionDataFn(beam.DoFn):
# There are already ground truth boxes for this image, just keep them.
return [input_example]
- detection_boxes, detection_scores, num_detections = self._session.run(
- [self._boxes_node, self._scores_node, self._num_detections_node],
- feed_dict={self._input: [tfrecord_entry]})
+ detections = self._detect_fn.signatures['serving_default'](
+ (tf.expand_dims(tf.convert_to_tensor(tfrecord_entry), 0)))
+ detection_boxes = detections['detection_boxes']
+ num_detections = detections['num_detections']
+ detection_scores = detections['detection_scores']
example = tf.train.Example()
diff --git a/research/object_detection/dataset_tools/context_rcnn/generate_detection_data_tf1_test.py b/research/object_detection/dataset_tools/context_rcnn/generate_detection_data_tf1_test.py
deleted file mode 100644
index 545e832338f156d7cc6426bdc92e36350be1c24c..0000000000000000000000000000000000000000
--- a/research/object_detection/dataset_tools/context_rcnn/generate_detection_data_tf1_test.py
+++ /dev/null
@@ -1,276 +0,0 @@
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for generate_detection_data."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import contextlib
-import os
-import tempfile
-import unittest
-import numpy as np
-import six
-import tensorflow.compat.v1 as tf
-
-from object_detection import exporter
-from object_detection.builders import model_builder
-from object_detection.core import model
-from object_detection.dataset_tools.context_rcnn import generate_detection_data
-from object_detection.protos import pipeline_pb2
-from object_detection.utils import tf_version
-
-if six.PY2:
- import mock # pylint: disable=g-import-not-at-top
-else:
- mock = unittest.mock
-
-try:
- import apache_beam as beam # pylint:disable=g-import-not-at-top
-except ModuleNotFoundError:
- pass
-
-
-class FakeModel(model.DetectionModel):
- """A Fake Detection model with expected output nodes from post-processing."""
-
- def preprocess(self, inputs):
- true_image_shapes = [] # Doesn't matter for the fake model.
- return tf.identity(inputs), true_image_shapes
-
- def predict(self, preprocessed_inputs, true_image_shapes):
- return {'image': tf.layers.conv2d(preprocessed_inputs, 3, 1)}
-
- def postprocess(self, prediction_dict, true_image_shapes):
- with tf.control_dependencies(prediction_dict.values()):
- postprocessed_tensors = {
- 'detection_boxes': tf.constant([[[0.0, 0.1, 0.5, 0.6],
- [0.5, 0.5, 0.8, 0.8]]], tf.float32),
- 'detection_scores': tf.constant([[0.95, 0.6]], tf.float32),
- 'detection_multiclass_scores': tf.constant([[[0.1, 0.7, 0.2],
- [0.3, 0.1, 0.6]]],
- tf.float32),
- 'detection_classes': tf.constant([[0, 1]], tf.float32),
- 'num_detections': tf.constant([2], tf.float32)
- }
- return postprocessed_tensors
-
- def restore_map(self, checkpoint_path, fine_tune_checkpoint_type):
- pass
-
- def restore_from_objects(self, fine_tune_checkpoint_type):
- pass
-
- def loss(self, prediction_dict, true_image_shapes):
- pass
-
- def regularization_losses(self):
- pass
-
- def updates(self):
- pass
-
-
-@contextlib.contextmanager
-def InMemoryTFRecord(entries):
- temp = tempfile.NamedTemporaryFile(delete=False)
- filename = temp.name
- try:
- with tf.python_io.TFRecordWriter(filename) as writer:
- for value in entries:
- writer.write(value)
- yield filename
- finally:
- os.unlink(filename)
-
-
-@unittest.skipIf(tf_version.is_tf2(), 'Skipping TF1.X only test.')
-class GenerateDetectionDataTest(tf.test.TestCase):
-
- def _save_checkpoint_from_mock_model(self, checkpoint_path):
- """A function to save checkpoint from a fake Detection Model.
-
- Args:
- checkpoint_path: Path to save checkpoint from Fake model.
- """
- g = tf.Graph()
- with g.as_default():
- mock_model = FakeModel(num_classes=5)
- preprocessed_inputs, true_image_shapes = mock_model.preprocess(
- tf.placeholder(tf.float32, shape=[None, None, None, 3]))
- predictions = mock_model.predict(preprocessed_inputs, true_image_shapes)
- mock_model.postprocess(predictions, true_image_shapes)
- tf.train.get_or_create_global_step()
- saver = tf.train.Saver()
- init = tf.global_variables_initializer()
- with self.test_session(graph=g) as sess:
- sess.run(init)
- saver.save(sess, checkpoint_path)
-
- def _export_saved_model(self):
- tmp_dir = self.get_temp_dir()
- checkpoint_path = os.path.join(tmp_dir, 'model.ckpt')
- self._save_checkpoint_from_mock_model(checkpoint_path)
- output_directory = os.path.join(tmp_dir, 'output')
- saved_model_path = os.path.join(output_directory, 'saved_model')
- tf.io.gfile.makedirs(output_directory)
- with mock.patch.object(
- model_builder, 'build', autospec=True) as mock_builder:
- mock_builder.return_value = FakeModel(num_classes=5)
- pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
- pipeline_config.eval_config.use_moving_averages = False
- detection_model = model_builder.build(pipeline_config.model,
- is_training=False)
- outputs, placeholder_tensor = exporter.build_detection_graph(
- input_type='tf_example',
- detection_model=detection_model,
- input_shape=None,
- output_collection_name='inference_op',
- graph_hook_fn=None)
- output_node_names = ','.join(outputs.keys())
- saver = tf.train.Saver()
- input_saver_def = saver.as_saver_def()
- frozen_graph_def = exporter.freeze_graph_with_def_protos(
- input_graph_def=tf.get_default_graph().as_graph_def(),
- input_saver_def=input_saver_def,
- input_checkpoint=checkpoint_path,
- output_node_names=output_node_names,
- restore_op_name='save/restore_all',
- filename_tensor_name='save/Const:0',
- output_graph='',
- clear_devices=True,
- initializer_nodes='')
- exporter.write_saved_model(
- saved_model_path=saved_model_path,
- frozen_graph_def=frozen_graph_def,
- inputs=placeholder_tensor,
- outputs=outputs)
- return saved_model_path
-
- def _create_tf_example(self):
- with self.test_session():
- encoded_image = tf.image.encode_jpeg(
- tf.constant(np.ones((4, 6, 3)).astype(np.uint8))).eval()
-
- def BytesFeature(value):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
- def Int64Feature(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-
- example = tf.train.Example(features=tf.train.Features(feature={
- 'image/encoded': BytesFeature(encoded_image),
- 'image/source_id': BytesFeature(b'image_id'),
- 'image/height': Int64Feature(4),
- 'image/width': Int64Feature(6),
- 'image/object/class/label': Int64Feature(5),
- 'image/object/class/text': BytesFeature(b'hyena'),
- 'image/class/label': Int64Feature(5),
- 'image/class/text': BytesFeature(b'hyena'),
- }))
-
- return example.SerializeToString()
-
- def assert_expected_example(self, example):
- self.assertAllClose(
- example.features.feature['image/object/bbox/ymin'].float_list.value,
- [0.0])
- self.assertAllClose(
- example.features.feature['image/object/bbox/xmin'].float_list.value,
- [0.1])
- self.assertAllClose(
- example.features.feature['image/object/bbox/ymax'].float_list.value,
- [0.5])
- self.assertAllClose(
- example.features.feature['image/object/bbox/xmax'].float_list.value,
- [0.6])
- self.assertAllClose(
- example.features.feature['image/object/class/score']
- .float_list.value, [0.95])
- self.assertAllClose(
- example.features.feature['image/object/class/label']
- .int64_list.value, [5])
- self.assertAllEqual(
- example.features.feature['image/object/class/text']
- .bytes_list.value, [b'hyena'])
- self.assertAllClose(
- example.features.feature['image/class/label']
- .int64_list.value, [5])
- self.assertAllEqual(
- example.features.feature['image/class/text']
- .bytes_list.value, [b'hyena'])
-
- # Check other essential attributes.
- self.assertAllEqual(
- example.features.feature['image/height'].int64_list.value, [4])
- self.assertAllEqual(
- example.features.feature['image/width'].int64_list.value, [6])
- self.assertAllEqual(
- example.features.feature['image/source_id'].bytes_list.value,
- [b'image_id'])
- self.assertTrue(
- example.features.feature['image/encoded'].bytes_list.value)
-
- def test_generate_detection_data_fn(self):
- saved_model_path = self._export_saved_model()
- confidence_threshold = 0.8
- inference_fn = generate_detection_data.GenerateDetectionDataFn(
- saved_model_path, confidence_threshold)
- inference_fn.start_bundle()
- generated_example = self._create_tf_example()
- self.assertAllEqual(tf.train.Example.FromString(
- generated_example).features.feature['image/object/class/label']
- .int64_list.value, [5])
- self.assertAllEqual(tf.train.Example.FromString(
- generated_example).features.feature['image/object/class/text']
- .bytes_list.value, [b'hyena'])
- output = inference_fn.process(generated_example)
- output_example = output[0]
-
- self.assertAllEqual(
- output_example.features.feature['image/object/class/label']
- .int64_list.value, [5])
- self.assertAllEqual(output_example.features.feature['image/width']
- .int64_list.value, [6])
-
- self.assert_expected_example(output_example)
-
- def test_beam_pipeline(self):
- with InMemoryTFRecord([self._create_tf_example()]) as input_tfrecord:
- temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
- output_tfrecord = os.path.join(temp_dir, 'output_tfrecord')
- saved_model_path = self._export_saved_model()
- confidence_threshold = 0.8
- num_shards = 1
- pipeline_options = beam.options.pipeline_options.PipelineOptions(
- runner='DirectRunner')
- p = beam.Pipeline(options=pipeline_options)
- generate_detection_data.construct_pipeline(
- p, input_tfrecord, output_tfrecord, saved_model_path,
- confidence_threshold, num_shards)
- p.run()
- filenames = tf.io.gfile.glob(output_tfrecord + '-?????-of-?????')
- actual_output = []
- record_iterator = tf.python_io.tf_record_iterator(path=filenames[0])
- for record in record_iterator:
- actual_output.append(record)
- self.assertEqual(len(actual_output), 1)
- self.assert_expected_example(tf.train.Example.FromString(
- actual_output[0]))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/object_detection/dataset_tools/context_rcnn/generate_detection_data_tf2_test.py b/research/object_detection/dataset_tools/context_rcnn/generate_detection_data_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..3350eb2df7ff51147434b019658d8588ab1521d5
--- /dev/null
+++ b/research/object_detection/dataset_tools/context_rcnn/generate_detection_data_tf2_test.py
@@ -0,0 +1,260 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for generate_detection_data."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import contextlib
+import os
+import tempfile
+import unittest
+import numpy as np
+import six
+import tensorflow as tf
+
+from object_detection import exporter_lib_v2
+from object_detection.builders import model_builder
+from object_detection.core import model
+from object_detection.protos import pipeline_pb2
+from object_detection.utils import tf_version
+
+if tf_version.is_tf2():
+ from object_detection.dataset_tools.context_rcnn import generate_detection_data # pylint:disable=g-import-not-at-top
+
+if six.PY2:
+ import mock # pylint: disable=g-import-not-at-top
+else:
+ mock = unittest.mock
+
+try:
+ import apache_beam as beam # pylint:disable=g-import-not-at-top
+except ModuleNotFoundError:
+ pass
+
+
+class FakeModel(model.DetectionModel):
+
+ def __init__(self, conv_weight_scalar=1.0):
+ super(FakeModel, self).__init__(num_classes=5)
+ self._conv = tf.keras.layers.Conv2D(
+ filters=1, kernel_size=1, strides=(1, 1), padding='valid',
+ kernel_initializer=tf.keras.initializers.Constant(
+ value=conv_weight_scalar))
+
+ def preprocess(self, inputs):
+ return tf.identity(inputs), exporter_lib_v2.get_true_shapes(inputs)
+
+ def predict(self, preprocessed_inputs, true_image_shapes):
+ return {'image': self._conv(preprocessed_inputs)}
+
+ def postprocess(self, prediction_dict, true_image_shapes):
+ with tf.control_dependencies(list(prediction_dict.values())):
+ postprocessed_tensors = {
+ 'detection_boxes': tf.constant([[[0.0, 0.1, 0.5, 0.6],
+ [0.5, 0.5, 0.8, 0.8]]], tf.float32),
+ 'detection_scores': tf.constant([[0.95, 0.6]], tf.float32),
+ 'detection_multiclass_scores': tf.constant([[[0.1, 0.7, 0.2],
+ [0.3, 0.1, 0.6]]],
+ tf.float32),
+ 'detection_classes': tf.constant([[0, 1]], tf.float32),
+ 'num_detections': tf.constant([2], tf.float32)
+ }
+ return postprocessed_tensors
+
+ def restore_map(self, checkpoint_path, fine_tune_checkpoint_type):
+ pass
+
+ def restore_from_objects(self, fine_tune_checkpoint_type):
+ pass
+
+ def loss(self, prediction_dict, true_image_shapes):
+ pass
+
+ def regularization_losses(self):
+ pass
+
+ def updates(self):
+ pass
+
+
+@contextlib.contextmanager
+def InMemoryTFRecord(entries):
+ temp = tempfile.NamedTemporaryFile(delete=False)
+ filename = temp.name
+ try:
+ with tf.io.TFRecordWriter(filename) as writer:
+ for value in entries:
+ writer.write(value)
+ yield filename
+ finally:
+ os.unlink(filename)
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class GenerateDetectionDataTest(tf.test.TestCase):
+
+ def _save_checkpoint_from_mock_model(self, checkpoint_path):
+ """A function to save checkpoint from a fake Detection Model.
+
+ Args:
+ checkpoint_path: Path to save checkpoint from Fake model.
+ """
+ mock_model = FakeModel()
+ fake_image = tf.zeros(shape=[1, 10, 10, 3], dtype=tf.float32)
+ preprocessed_inputs, true_image_shapes = mock_model.preprocess(fake_image)
+ predictions = mock_model.predict(preprocessed_inputs, true_image_shapes)
+ mock_model.postprocess(predictions, true_image_shapes)
+ ckpt = tf.train.Checkpoint(model=mock_model)
+ exported_checkpoint_manager = tf.train.CheckpointManager(
+ ckpt, checkpoint_path, max_to_keep=1)
+ exported_checkpoint_manager.save(checkpoint_number=0)
+
+ def _export_saved_model(self):
+ tmp_dir = self.get_temp_dir()
+ self._save_checkpoint_from_mock_model(tmp_dir)
+ output_directory = os.path.join(tmp_dir, 'output')
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ tf.io.gfile.makedirs(output_directory)
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ exporter_lib_v2.INPUT_BUILDER_UTIL_MAP['model_build'] = mock_builder
+ output_directory = os.path.join(tmp_dir, 'output')
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ exporter_lib_v2.export_inference_graph(
+ input_type='tf_example',
+ pipeline_config=pipeline_config,
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory)
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ return saved_model_path
+
+ def _create_tf_example(self):
+ with self.test_session():
+ encoded_image = tf.io.encode_jpeg(
+ tf.constant(np.ones((4, 6, 3)).astype(np.uint8))).numpy()
+
+ def BytesFeature(value):
+ return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+ def Int64Feature(value):
+ return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+ example = tf.train.Example(features=tf.train.Features(feature={
+ 'image/encoded': BytesFeature(encoded_image),
+ 'image/source_id': BytesFeature(b'image_id'),
+ 'image/height': Int64Feature(4),
+ 'image/width': Int64Feature(6),
+ 'image/object/class/label': Int64Feature(5),
+ 'image/object/class/text': BytesFeature(b'hyena'),
+ 'image/class/label': Int64Feature(5),
+ 'image/class/text': BytesFeature(b'hyena'),
+ }))
+
+ return example.SerializeToString()
+
+ def assert_expected_example(self, example):
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/ymin'].float_list.value,
+ [0.0])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/xmin'].float_list.value,
+ [0.1])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/ymax'].float_list.value,
+ [0.5])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/xmax'].float_list.value,
+ [0.6])
+ self.assertAllClose(
+ example.features.feature['image/object/class/score']
+ .float_list.value, [0.95])
+ self.assertAllClose(
+ example.features.feature['image/object/class/label']
+ .int64_list.value, [5])
+ self.assertAllEqual(
+ example.features.feature['image/object/class/text']
+ .bytes_list.value, [b'hyena'])
+ self.assertAllClose(
+ example.features.feature['image/class/label']
+ .int64_list.value, [5])
+ self.assertAllEqual(
+ example.features.feature['image/class/text']
+ .bytes_list.value, [b'hyena'])
+
+ # Check other essential attributes.
+ self.assertAllEqual(
+ example.features.feature['image/height'].int64_list.value, [4])
+ self.assertAllEqual(
+ example.features.feature['image/width'].int64_list.value, [6])
+ self.assertAllEqual(
+ example.features.feature['image/source_id'].bytes_list.value,
+ [b'image_id'])
+ self.assertTrue(
+ example.features.feature['image/encoded'].bytes_list.value)
+
+ def test_generate_detection_data_fn(self):
+ saved_model_path = self._export_saved_model()
+ confidence_threshold = 0.8
+ inference_fn = generate_detection_data.GenerateDetectionDataFn(
+ saved_model_path, confidence_threshold)
+ inference_fn.setup()
+ generated_example = self._create_tf_example()
+ self.assertAllEqual(tf.train.Example.FromString(
+ generated_example).features.feature['image/object/class/label']
+ .int64_list.value, [5])
+ self.assertAllEqual(tf.train.Example.FromString(
+ generated_example).features.feature['image/object/class/text']
+ .bytes_list.value, [b'hyena'])
+ output = inference_fn.process(generated_example)
+ output_example = output[0]
+
+ self.assertAllEqual(
+ output_example.features.feature['image/object/class/label']
+ .int64_list.value, [5])
+ self.assertAllEqual(output_example.features.feature['image/width']
+ .int64_list.value, [6])
+
+ self.assert_expected_example(output_example)
+
+ def test_beam_pipeline(self):
+ with InMemoryTFRecord([self._create_tf_example()]) as input_tfrecord:
+ temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
+ output_tfrecord = os.path.join(temp_dir, 'output_tfrecord')
+ saved_model_path = self._export_saved_model()
+ confidence_threshold = 0.8
+ num_shards = 1
+ pipeline_options = beam.options.pipeline_options.PipelineOptions(
+ runner='DirectRunner')
+ p = beam.Pipeline(options=pipeline_options)
+ generate_detection_data.construct_pipeline(
+ p, input_tfrecord, output_tfrecord, saved_model_path,
+ confidence_threshold, num_shards)
+ p.run()
+ filenames = tf.io.gfile.glob(output_tfrecord + '-?????-of-?????')
+ actual_output = []
+ record_iterator = tf.data.TFRecordDataset(
+ tf.convert_to_tensor(filenames)).as_numpy_iterator()
+ for record in record_iterator:
+ actual_output.append(record)
+ self.assertEqual(len(actual_output), 1)
+ self.assert_expected_example(tf.train.Example.FromString(
+ actual_output[0]))
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data.py b/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data.py
index 74d15901dec3706d39e6feaaad3d5ad99d9f45d7..dac1168c14a469a72fc2ad796caaf62c06f5b5fc 100644
--- a/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data.py
+++ b/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data.py
@@ -55,7 +55,7 @@ import threading
import numpy as np
import six
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
try:
import apache_beam as beam # pylint:disable=g-import-not-at-top
@@ -63,6 +63,76 @@ except ModuleNotFoundError:
pass
+def add_keys(serialized_example):
+ key = hash(serialized_example)
+ return key, serialized_example
+
+
+def drop_keys(key_value_tuple):
+ return key_value_tuple[1]
+
+
+def get_date_captured(example):
+ date_captured = datetime.datetime.strptime(
+ six.ensure_str(
+ example.features.feature['image/date_captured'].bytes_list.value[0]),
+ '%Y-%m-%d %H:%M:%S')
+ return date_captured
+
+
+def embed_date_captured(date_captured):
+ """Encodes the datetime of the image."""
+ embedded_date_captured = []
+ month_max = 12.0
+ day_max = 31.0
+ hour_max = 24.0
+ minute_max = 60.0
+ min_year = 1990.0
+ max_year = 2030.0
+
+ year = (date_captured.year - min_year) / float(max_year - min_year)
+ embedded_date_captured.append(year)
+
+ month = (date_captured.month - 1) / month_max
+ embedded_date_captured.append(month)
+
+ day = (date_captured.day - 1) / day_max
+ embedded_date_captured.append(day)
+
+ hour = date_captured.hour / hour_max
+ embedded_date_captured.append(hour)
+
+ minute = date_captured.minute / minute_max
+ embedded_date_captured.append(minute)
+
+ return np.asarray(embedded_date_captured)
+
+
+def embed_position_and_size(box):
+ """Encodes the bounding box of the object of interest."""
+ ymin = box[0]
+ xmin = box[1]
+ ymax = box[2]
+ xmax = box[3]
+ w = xmax - xmin
+ h = ymax - ymin
+ x = xmin + w / 2.0
+ y = ymin + h / 2.0
+ return np.asarray([x, y, w, h])
+
+
+def get_bb_embedding(detection_features, detection_boxes, detection_scores,
+ index):
+ embedding = detection_features[0][index]
+ pooled_embedding = np.mean(np.mean(embedding, axis=1), axis=0)
+
+ box = detection_boxes[0][index]
+ position_embedding = embed_position_and_size(box)
+
+ score = detection_scores[0][index]
+ return np.concatenate((pooled_embedding, position_embedding)), score
+
+
class GenerateEmbeddingDataFn(beam.DoFn):
"""Generates embedding data for camera trap images.
@@ -72,13 +142,14 @@ class GenerateEmbeddingDataFn(beam.DoFn):
session_lock = threading.Lock()
def __init__(self, model_dir, top_k_embedding_count,
- bottom_k_embedding_count):
+ bottom_k_embedding_count, embedding_type='final_box_features'):
"""Initialization function.
Args:
model_dir: A directory containing saved model.
top_k_embedding_count: the number of high-confidence embeddings to store
bottom_k_embedding_count: the number of low-confidence embeddings to store
+ embedding_type: One of 'final_box_features', 'rpn_box_features'
"""
self._model_dir = model_dir
self._session = None
@@ -86,8 +157,9 @@ class GenerateEmbeddingDataFn(beam.DoFn):
'embedding_data_generation', 'num_tf_examples_processed')
self._top_k_embedding_count = top_k_embedding_count
self._bottom_k_embedding_count = bottom_k_embedding_count
+ self._embedding_type = embedding_type
- def start_bundle(self):
+ def setup(self):
self._load_inference_model()
def _load_inference_model(self):
@@ -95,102 +167,38 @@ class GenerateEmbeddingDataFn(beam.DoFn):
# one instance across all threads in the worker. This is possible since
# tf.Session.run() is thread safe.
with self.session_lock:
- if self._session is None:
- graph = tf.Graph()
- self._session = tf.Session(graph=graph)
- with graph.as_default():
- meta_graph = tf.saved_model.loader.load(
- self._session, [tf.saved_model.tag_constants.SERVING],
- self._model_dir)
- signature = meta_graph.signature_def['serving_default']
- input_tensor_name = signature.inputs['inputs'].name
- detection_features_name = signature.outputs['detection_features'].name
- detection_boxes_name = signature.outputs['detection_boxes'].name
- num_detections_name = signature.outputs['num_detections'].name
- self._input = graph.get_tensor_by_name(input_tensor_name)
- self._embedding_node = graph.get_tensor_by_name(detection_features_name)
- self._box_node = graph.get_tensor_by_name(detection_boxes_name)
- self._scores_node = graph.get_tensor_by_name(
- signature.outputs['detection_scores'].name)
- self._num_detections = graph.get_tensor_by_name(num_detections_name)
- tf.logging.info(signature.outputs['detection_features'].name)
- tf.logging.info(signature.outputs['detection_boxes'].name)
- tf.logging.info(signature.outputs['num_detections'].name)
-
- def process(self, tfrecord_entry):
- return self._run_inference_and_generate_embedding(tfrecord_entry)
-
- def _run_inference_and_generate_embedding(self, tfrecord_entry):
- input_example = tf.train.Example.FromString(tfrecord_entry)
- # Convert date_captured datetime string to unix time integer and store
-
- def get_date_captured(example):
- date_captured = datetime.datetime.strptime(
- six.ensure_str(
- example.features.feature[
- 'image/date_captured'].bytes_list.value[0]),
- '%Y-%m-%d %H:%M:%S')
- return date_captured
+ self._detect_fn = tf.saved_model.load(self._model_dir)
- try:
- date_captured = get_date_captured(input_example)
- except Exception: # pylint: disable=broad-except
- # we require date_captured to be available for all images
- return []
-
- def embed_date_captured(date_captured):
- """Encodes the datetime of the image."""
- embedded_date_captured = []
- month_max = 12.0
- day_max = 31.0
- hour_max = 24.0
- minute_max = 60.0
- min_year = 1990.0
- max_year = 2030.0
-
- year = (date_captured.year-min_year)/float(max_year-min_year)
- embedded_date_captured.append(year)
-
- month = (date_captured.month-1)/month_max
- embedded_date_captured.append(month)
-
- day = (date_captured.day-1)/day_max
- embedded_date_captured.append(day)
-
- hour = date_captured.hour/hour_max
- embedded_date_captured.append(hour)
-
- minute = date_captured.minute/minute_max
- embedded_date_captured.append(minute)
-
- return np.asarray(embedded_date_captured)
-
- def embed_position_and_size(box):
- """Encodes the bounding box of the object of interest."""
- ymin = box[0]
- xmin = box[1]
- ymax = box[2]
- xmax = box[3]
- w = xmax - xmin
- h = ymax - ymin
- x = xmin + w / 2.0
- y = ymin + h / 2.0
- return np.asarray([x, y, w, h])
-
- unix_time = (
- (date_captured - datetime.datetime.fromtimestamp(0)).total_seconds())
+ def process(self, tfexample_key_value):
+ return self._run_inference_and_generate_embedding(tfexample_key_value)
+ def _run_inference_and_generate_embedding(self, tfexample_key_value):
+ key, tfexample = tfexample_key_value
+ input_example = tf.train.Example.FromString(tfexample)
example = tf.train.Example()
- example.features.feature['image/unix_time'].float_list.value.extend(
- [unix_time])
+ example.CopyFrom(input_example)
- (detection_features, detection_boxes, num_detections,
- detection_scores) = self._session.run(
- [
- self._embedding_node, self._box_node, self._num_detections[0],
- self._scores_node
- ],
- feed_dict={self._input: [tfrecord_entry]})
+ try:
+ date_captured = get_date_captured(input_example)
+ unix_time = ((date_captured -
+ datetime.datetime.fromtimestamp(0)).total_seconds())
+ example.features.feature['image/unix_time'].float_list.value.extend(
+ [unix_time])
+ temporal_embedding = embed_date_captured(date_captured)
+ except Exception: # pylint: disable=broad-except
+ temporal_embedding = None
+
+ detections = self._detect_fn.signatures['serving_default'](
+ (tf.expand_dims(tf.convert_to_tensor(tfexample), 0)))
+ if self._embedding_type == 'final_box_features':
+ detection_features = detections['detection_features']
+ elif self._embedding_type == 'rpn_box_features':
+ detection_features = detections['cropped_rpn_box_features']
+ else:
+ raise ValueError('embedding type not supported')
+ detection_boxes = detections['detection_boxes']
+ num_detections = detections['num_detections']
+ detection_scores = detections['detection_scores']
num_detections = int(num_detections)
embed_all = []
@@ -198,25 +206,12 @@ class GenerateEmbeddingDataFn(beam.DoFn):
detection_features = np.asarray(detection_features)
- def get_bb_embedding(detection_features, detection_boxes, detection_scores,
- index):
- embedding = detection_features[0][index]
- pooled_embedding = np.mean(np.mean(embedding, axis=1), axis=0)
-
- box = detection_boxes[0][index]
- position_embedding = embed_position_and_size(box)
-
- score = detection_scores[0][index]
- return np.concatenate((pooled_embedding, position_embedding)), score
-
- temporal_embedding = embed_date_captured(date_captured)
-
embedding_count = 0
for index in range(min(num_detections, self._top_k_embedding_count)):
bb_embedding, score = get_bb_embedding(
detection_features, detection_boxes, detection_scores, index)
embed_all.extend(bb_embedding)
- embed_all.extend(temporal_embedding)
+ if temporal_embedding is not None: embed_all.extend(temporal_embedding)
score_all.append(score)
embedding_count += 1
@@ -226,7 +221,7 @@ class GenerateEmbeddingDataFn(beam.DoFn):
bb_embedding, score = get_bb_embedding(
detection_features, detection_boxes, detection_scores, index)
embed_all.extend(bb_embedding)
- embed_all.extend(temporal_embedding)
+ if temporal_embedding is not None: embed_all.extend(temporal_embedding)
score_all.append(score)
embedding_count += 1
@@ -234,7 +229,7 @@ class GenerateEmbeddingDataFn(beam.DoFn):
bb_embedding, score = get_bb_embedding(
detection_features, detection_boxes, detection_scores, 0)
embed_all.extend(bb_embedding)
- embed_all.extend(temporal_embedding)
+ if temporal_embedding is not None: embed_all.extend(temporal_embedding)
score_all.append(score)
# Takes max in case embedding_count is 0.
@@ -251,65 +246,13 @@ class GenerateEmbeddingDataFn(beam.DoFn):
example.features.feature['image/embedding_count'].int64_list.value.append(
embedding_count)
- # Add other essential example attributes
- example.features.feature['image/encoded'].bytes_list.value.extend(
- input_example.features.feature['image/encoded'].bytes_list.value)
- example.features.feature['image/height'].int64_list.value.extend(
- input_example.features.feature['image/height'].int64_list.value)
- example.features.feature['image/width'].int64_list.value.extend(
- input_example.features.feature['image/width'].int64_list.value)
- example.features.feature['image/source_id'].bytes_list.value.extend(
- input_example.features.feature['image/source_id'].bytes_list.value)
- example.features.feature['image/location'].bytes_list.value.extend(
- input_example.features.feature['image/location'].bytes_list.value)
-
- example.features.feature['image/date_captured'].bytes_list.value.extend(
- input_example.features.feature['image/date_captured'].bytes_list.value)
-
- example.features.feature['image/class/text'].bytes_list.value.extend(
- input_example.features.feature['image/class/text'].bytes_list.value)
- example.features.feature['image/class/label'].int64_list.value.extend(
- input_example.features.feature['image/class/label'].int64_list.value)
-
- example.features.feature['image/seq_id'].bytes_list.value.extend(
- input_example.features.feature['image/seq_id'].bytes_list.value)
- example.features.feature['image/seq_num_frames'].int64_list.value.extend(
- input_example.features.feature['image/seq_num_frames'].int64_list.value)
- example.features.feature['image/seq_frame_num'].int64_list.value.extend(
- input_example.features.feature['image/seq_frame_num'].int64_list.value)
-
- example.features.feature['image/object/bbox/ymax'].float_list.value.extend(
- input_example.features.feature[
- 'image/object/bbox/ymax'].float_list.value)
- example.features.feature['image/object/bbox/ymin'].float_list.value.extend(
- input_example.features.feature[
- 'image/object/bbox/ymin'].float_list.value)
- example.features.feature['image/object/bbox/xmax'].float_list.value.extend(
- input_example.features.feature[
- 'image/object/bbox/xmax'].float_list.value)
- example.features.feature['image/object/bbox/xmin'].float_list.value.extend(
- input_example.features.feature[
- 'image/object/bbox/xmin'].float_list.value)
- example.features.feature[
- 'image/object/class/score'].float_list.value.extend(
- input_example.features.feature[
- 'image/object/class/score'].float_list.value)
- example.features.feature[
- 'image/object/class/label'].int64_list.value.extend(
- input_example.features.feature[
- 'image/object/class/label'].int64_list.value)
- example.features.feature[
- 'image/object/class/text'].bytes_list.value.extend(
- input_example.features.feature[
- 'image/object/class/text'].bytes_list.value)
-
self._num_examples_processed.inc(1)
- return [example]
+ return [(key, example)]
def construct_pipeline(pipeline, input_tfrecord, output_tfrecord, model_dir,
top_k_embedding_count, bottom_k_embedding_count,
- num_shards):
+ num_shards, embedding_type):
"""Returns a beam pipeline to run object detection inference.
Args:
@@ -321,19 +264,21 @@ def construct_pipeline(pipeline, input_tfrecord, output_tfrecord, model_dir,
top_k_embedding_count: The number of high-confidence embeddings to store.
bottom_k_embedding_count: The number of low-confidence embeddings to store.
num_shards: The number of output shards.
+ embedding_type: Which features to embed.
"""
input_collection = (
pipeline | 'ReadInputTFRecord' >> beam.io.tfrecordio.ReadFromTFRecord(
- input_tfrecord,
- coder=beam.coders.BytesCoder()))
+ input_tfrecord, coder=beam.coders.BytesCoder())
+ | 'AddKeys' >> beam.Map(add_keys))
output_collection = input_collection | 'ExtractEmbedding' >> beam.ParDo(
GenerateEmbeddingDataFn(model_dir, top_k_embedding_count,
- bottom_k_embedding_count))
+ bottom_k_embedding_count, embedding_type))
output_collection = output_collection | 'Reshuffle' >> beam.Reshuffle()
- _ = output_collection | 'WritetoDisk' >> beam.io.tfrecordio.WriteToTFRecord(
- output_tfrecord,
- num_shards=num_shards,
- coder=beam.coders.ProtoCoder(tf.train.Example))
+ _ = output_collection | 'DropKeys' >> beam.Map(
+ drop_keys) | 'WritetoDisk' >> beam.io.tfrecordio.WriteToTFRecord(
+ output_tfrecord,
+ num_shards=num_shards,
+ coder=beam.coders.ProtoCoder(tf.train.Example))
def parse_args(argv):
@@ -378,6 +323,12 @@ def parse_args(argv):
dest='num_shards',
default=0,
help='Number of output shards.')
+ parser.add_argument(
+ '--embedding_type',
+ dest='embedding_type',
+ default='final_box_features',
+ help='What features to embed, supports `final_box_features`, '
+ '`rpn_box_features`.')
beam_args, pipeline_args = parser.parse_known_args(argv)
return beam_args, pipeline_args
@@ -409,11 +360,11 @@ def main(argv=None, save_main_session=True):
args.embedding_model_dir,
args.top_k_embedding_count,
args.bottom_k_embedding_count,
- args.num_shards)
+ args.num_shards,
+ args.embedding_type)
p.run()
if __name__ == '__main__':
main()
-
diff --git a/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data_tf1_test.py b/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data_tf1_test.py
deleted file mode 100644
index 71d1d600d8fa933751c615db3c06dfadfe0b28f4..0000000000000000000000000000000000000000
--- a/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data_tf1_test.py
+++ /dev/null
@@ -1,347 +0,0 @@
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for generate_embedding_data."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-import contextlib
-import os
-import tempfile
-import unittest
-import numpy as np
-import six
-import tensorflow.compat.v1 as tf
-from object_detection import exporter
-from object_detection.builders import model_builder
-from object_detection.core import model
-from object_detection.dataset_tools.context_rcnn import generate_embedding_data
-from object_detection.protos import pipeline_pb2
-from object_detection.utils import tf_version
-
-
-if six.PY2:
- import mock # pylint: disable=g-import-not-at-top
-else:
- mock = unittest.mock
-
-try:
- import apache_beam as beam # pylint:disable=g-import-not-at-top
-except ModuleNotFoundError:
- pass
-
-
-class FakeModel(model.DetectionModel):
- """A Fake Detection model with expected output nodes from post-processing."""
-
- def preprocess(self, inputs):
- true_image_shapes = [] # Doesn't matter for the fake model.
- return tf.identity(inputs), true_image_shapes
-
- def predict(self, preprocessed_inputs, true_image_shapes):
- return {'image': tf.layers.conv2d(preprocessed_inputs, 3, 1)}
-
- def postprocess(self, prediction_dict, true_image_shapes):
- with tf.control_dependencies(prediction_dict.values()):
- num_features = 100
- feature_dims = 10
- classifier_feature = np.ones(
- (2, feature_dims, feature_dims, num_features),
- dtype=np.float32).tolist()
- postprocessed_tensors = {
- 'detection_boxes': tf.constant([[[0.0, 0.1, 0.5, 0.6],
- [0.5, 0.5, 0.8, 0.8]]], tf.float32),
- 'detection_scores': tf.constant([[0.95, 0.6]], tf.float32),
- 'detection_multiclass_scores': tf.constant([[[0.1, 0.7, 0.2],
- [0.3, 0.1, 0.6]]],
- tf.float32),
- 'detection_classes': tf.constant([[0, 1]], tf.float32),
- 'num_detections': tf.constant([2], tf.float32),
- 'detection_features':
- tf.constant([classifier_feature],
- tf.float32)
- }
- return postprocessed_tensors
-
- def restore_map(self, checkpoint_path, fine_tune_checkpoint_type):
- pass
-
- def restore_from_objects(self, fine_tune_checkpoint_type):
- pass
-
- def loss(self, prediction_dict, true_image_shapes):
- pass
-
- def regularization_losses(self):
- pass
-
- def updates(self):
- pass
-
-
-@contextlib.contextmanager
-def InMemoryTFRecord(entries):
- temp = tempfile.NamedTemporaryFile(delete=False)
- filename = temp.name
- try:
- with tf.python_io.TFRecordWriter(filename) as writer:
- for value in entries:
- writer.write(value)
- yield filename
- finally:
- os.unlink(temp.name)
-
-
-@unittest.skipIf(tf_version.is_tf2(), 'Skipping TF1.X only test.')
-class GenerateEmbeddingData(tf.test.TestCase):
-
- def _save_checkpoint_from_mock_model(self, checkpoint_path):
- """A function to save checkpoint from a fake Detection Model.
-
- Args:
- checkpoint_path: Path to save checkpoint from Fake model.
- """
- g = tf.Graph()
- with g.as_default():
- mock_model = FakeModel(num_classes=5)
- preprocessed_inputs, true_image_shapes = mock_model.preprocess(
- tf.placeholder(tf.float32, shape=[None, None, None, 3]))
- predictions = mock_model.predict(preprocessed_inputs, true_image_shapes)
- mock_model.postprocess(predictions, true_image_shapes)
- tf.train.get_or_create_global_step()
- saver = tf.train.Saver()
- init = tf.global_variables_initializer()
- with self.test_session(graph=g) as sess:
- sess.run(init)
- saver.save(sess, checkpoint_path)
-
- def _export_saved_model(self):
- tmp_dir = self.get_temp_dir()
- checkpoint_path = os.path.join(tmp_dir, 'model.ckpt')
- self._save_checkpoint_from_mock_model(checkpoint_path)
- output_directory = os.path.join(tmp_dir, 'output')
- saved_model_path = os.path.join(output_directory, 'saved_model')
- tf.io.gfile.makedirs(output_directory)
- with mock.patch.object(
- model_builder, 'build', autospec=True) as mock_builder:
- mock_builder.return_value = FakeModel(num_classes=5)
- pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
- pipeline_config.eval_config.use_moving_averages = False
- detection_model = model_builder.build(pipeline_config.model,
- is_training=False)
- outputs, placeholder_tensor = exporter.build_detection_graph(
- input_type='tf_example',
- detection_model=detection_model,
- input_shape=None,
- output_collection_name='inference_op',
- graph_hook_fn=None)
- output_node_names = ','.join(outputs.keys())
- saver = tf.train.Saver()
- input_saver_def = saver.as_saver_def()
- frozen_graph_def = exporter.freeze_graph_with_def_protos(
- input_graph_def=tf.get_default_graph().as_graph_def(),
- input_saver_def=input_saver_def,
- input_checkpoint=checkpoint_path,
- output_node_names=output_node_names,
- restore_op_name='save/restore_all',
- filename_tensor_name='save/Const:0',
- output_graph='',
- clear_devices=True,
- initializer_nodes='')
- exporter.write_saved_model(
- saved_model_path=saved_model_path,
- frozen_graph_def=frozen_graph_def,
- inputs=placeholder_tensor,
- outputs=outputs)
- return saved_model_path
-
- def _create_tf_example(self):
- with self.test_session():
- encoded_image = tf.image.encode_jpeg(
- tf.constant(np.ones((4, 4, 3)).astype(np.uint8))).eval()
-
- def BytesFeature(value):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
- def Int64Feature(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-
- def FloatFeature(value):
- return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
-
- example = tf.train.Example(
- features=tf.train.Features(
- feature={
- 'image/encoded': BytesFeature(encoded_image),
- 'image/source_id': BytesFeature(b'image_id'),
- 'image/height': Int64Feature(400),
- 'image/width': Int64Feature(600),
- 'image/class/label': Int64Feature(5),
- 'image/class/text': BytesFeature(b'hyena'),
- 'image/object/bbox/xmin': FloatFeature(0.1),
- 'image/object/bbox/xmax': FloatFeature(0.6),
- 'image/object/bbox/ymin': FloatFeature(0.0),
- 'image/object/bbox/ymax': FloatFeature(0.5),
- 'image/object/class/score': FloatFeature(0.95),
- 'image/object/class/label': Int64Feature(5),
- 'image/object/class/text': BytesFeature(b'hyena'),
- 'image/date_captured': BytesFeature(b'2019-10-20 12:12:12')
- }))
-
- return example.SerializeToString()
-
- def assert_expected_example(self, example, topk=False, botk=False):
- # Check embeddings
- if topk or botk:
- self.assertEqual(len(
- example.features.feature['image/embedding'].float_list.value),
- 218)
- self.assertAllEqual(
- example.features.feature['image/embedding_count'].int64_list.value,
- [2])
- else:
- self.assertEqual(len(
- example.features.feature['image/embedding'].float_list.value),
- 109)
- self.assertAllEqual(
- example.features.feature['image/embedding_count'].int64_list.value,
- [1])
-
- self.assertAllEqual(
- example.features.feature['image/embedding_length'].int64_list.value,
- [109])
-
- # Check annotations
- self.assertAllClose(
- example.features.feature['image/object/bbox/ymin'].float_list.value,
- [0.0])
- self.assertAllClose(
- example.features.feature['image/object/bbox/xmin'].float_list.value,
- [0.1])
- self.assertAllClose(
- example.features.feature['image/object/bbox/ymax'].float_list.value,
- [0.5])
- self.assertAllClose(
- example.features.feature['image/object/bbox/xmax'].float_list.value,
- [0.6])
- self.assertAllClose(
- example.features.feature['image/object/class/score']
- .float_list.value, [0.95])
- self.assertAllClose(
- example.features.feature['image/object/class/label']
- .int64_list.value, [5])
- self.assertAllEqual(
- example.features.feature['image/object/class/text']
- .bytes_list.value, [b'hyena'])
- self.assertAllClose(
- example.features.feature['image/class/label']
- .int64_list.value, [5])
- self.assertAllEqual(
- example.features.feature['image/class/text']
- .bytes_list.value, [b'hyena'])
-
- # Check other essential attributes.
- self.assertAllEqual(
- example.features.feature['image/height'].int64_list.value, [400])
- self.assertAllEqual(
- example.features.feature['image/width'].int64_list.value, [600])
- self.assertAllEqual(
- example.features.feature['image/source_id'].bytes_list.value,
- [b'image_id'])
- self.assertTrue(
- example.features.feature['image/encoded'].bytes_list.value)
-
- def test_generate_embedding_data_fn(self):
- saved_model_path = self._export_saved_model()
- top_k_embedding_count = 1
- bottom_k_embedding_count = 0
- inference_fn = generate_embedding_data.GenerateEmbeddingDataFn(
- saved_model_path, top_k_embedding_count, bottom_k_embedding_count)
- inference_fn.start_bundle()
- generated_example = self._create_tf_example()
- self.assertAllEqual(tf.train.Example.FromString(
- generated_example).features.feature['image/object/class/label']
- .int64_list.value, [5])
- self.assertAllEqual(tf.train.Example.FromString(
- generated_example).features.feature['image/object/class/text']
- .bytes_list.value, [b'hyena'])
- output = inference_fn.process(generated_example)
- output_example = output[0]
- self.assert_expected_example(output_example)
-
- def test_generate_embedding_data_with_top_k_boxes(self):
- saved_model_path = self._export_saved_model()
- top_k_embedding_count = 2
- bottom_k_embedding_count = 0
- inference_fn = generate_embedding_data.GenerateEmbeddingDataFn(
- saved_model_path, top_k_embedding_count, bottom_k_embedding_count)
- inference_fn.start_bundle()
- generated_example = self._create_tf_example()
- self.assertAllEqual(
- tf.train.Example.FromString(generated_example).features
- .feature['image/object/class/label'].int64_list.value, [5])
- self.assertAllEqual(
- tf.train.Example.FromString(generated_example).features
- .feature['image/object/class/text'].bytes_list.value, [b'hyena'])
- output = inference_fn.process(generated_example)
- output_example = output[0]
- self.assert_expected_example(output_example, topk=True)
-
- def test_generate_embedding_data_with_bottom_k_boxes(self):
- saved_model_path = self._export_saved_model()
- top_k_embedding_count = 0
- bottom_k_embedding_count = 2
- inference_fn = generate_embedding_data.GenerateEmbeddingDataFn(
- saved_model_path, top_k_embedding_count, bottom_k_embedding_count)
- inference_fn.start_bundle()
- generated_example = self._create_tf_example()
- self.assertAllEqual(
- tf.train.Example.FromString(generated_example).features
- .feature['image/object/class/label'].int64_list.value, [5])
- self.assertAllEqual(
- tf.train.Example.FromString(generated_example).features
- .feature['image/object/class/text'].bytes_list.value, [b'hyena'])
- output = inference_fn.process(generated_example)
- output_example = output[0]
- self.assert_expected_example(output_example, botk=True)
-
- def test_beam_pipeline(self):
- with InMemoryTFRecord([self._create_tf_example()]) as input_tfrecord:
- temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
- output_tfrecord = os.path.join(temp_dir, 'output_tfrecord')
- saved_model_path = self._export_saved_model()
- top_k_embedding_count = 1
- bottom_k_embedding_count = 0
- num_shards = 1
- pipeline_options = beam.options.pipeline_options.PipelineOptions(
- runner='DirectRunner')
- p = beam.Pipeline(options=pipeline_options)
- generate_embedding_data.construct_pipeline(
- p, input_tfrecord, output_tfrecord, saved_model_path,
- top_k_embedding_count, bottom_k_embedding_count, num_shards)
- p.run()
- filenames = tf.io.gfile.glob(
- output_tfrecord + '-?????-of-?????')
- actual_output = []
- record_iterator = tf.python_io.tf_record_iterator(path=filenames[0])
- for record in record_iterator:
- actual_output.append(record)
- self.assertEqual(len(actual_output), 1)
- self.assert_expected_example(tf.train.Example.FromString(
- actual_output[0]))
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data_tf2_test.py b/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..156e283eff5936b3e0d909c7fb9a2b940a21f7c2
--- /dev/null
+++ b/research/object_detection/dataset_tools/context_rcnn/generate_embedding_data_tf2_test.py
@@ -0,0 +1,331 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for generate_embedding_data."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import contextlib
+import os
+import tempfile
+import unittest
+import numpy as np
+import six
+import tensorflow as tf
+from object_detection import exporter_lib_v2
+from object_detection.builders import model_builder
+from object_detection.core import model
+from object_detection.protos import pipeline_pb2
+from object_detection.utils import tf_version
+
+if tf_version.is_tf2():
+ from object_detection.dataset_tools.context_rcnn import generate_embedding_data # pylint:disable=g-import-not-at-top
+
+if six.PY2:
+ import mock # pylint: disable=g-import-not-at-top
+else:
+ mock = unittest.mock
+
+try:
+ import apache_beam as beam # pylint:disable=g-import-not-at-top
+except ModuleNotFoundError:
+ pass
+
+
+class FakeModel(model.DetectionModel):
+
+ def __init__(self, conv_weight_scalar=1.0):
+ super(FakeModel, self).__init__(num_classes=5)
+ self._conv = tf.keras.layers.Conv2D(
+ filters=1, kernel_size=1, strides=(1, 1), padding='valid',
+ kernel_initializer=tf.keras.initializers.Constant(
+ value=conv_weight_scalar))
+
+ def preprocess(self, inputs):
+ return tf.identity(inputs), exporter_lib_v2.get_true_shapes(inputs)
+
+ def predict(self, preprocessed_inputs, true_image_shapes):
+ return {'image': self._conv(preprocessed_inputs)}
+
+ def postprocess(self, prediction_dict, true_image_shapes):
+ with tf.control_dependencies(prediction_dict.values()):
+ num_features = 100
+ feature_dims = 10
+ classifier_feature = np.ones(
+ (2, feature_dims, feature_dims, num_features),
+ dtype=np.float32).tolist()
+ postprocessed_tensors = {
+ 'detection_boxes': tf.constant([[[0.0, 0.1, 0.5, 0.6],
+ [0.5, 0.5, 0.8, 0.8]]], tf.float32),
+ 'detection_scores': tf.constant([[0.95, 0.6]], tf.float32),
+ 'detection_multiclass_scores': tf.constant([[[0.1, 0.7, 0.2],
+ [0.3, 0.1, 0.6]]],
+ tf.float32),
+ 'detection_classes': tf.constant([[0, 1]], tf.float32),
+ 'num_detections': tf.constant([2], tf.float32),
+ 'detection_features':
+ tf.constant([classifier_feature],
+ tf.float32)
+ }
+ return postprocessed_tensors
+
+ def restore_map(self, checkpoint_path, fine_tune_checkpoint_type):
+ pass
+
+ def restore_from_objects(self, fine_tune_checkpoint_type):
+ pass
+
+ def loss(self, prediction_dict, true_image_shapes):
+ pass
+
+ def regularization_losses(self):
+ pass
+
+ def updates(self):
+ pass
+
+
+@contextlib.contextmanager
+def InMemoryTFRecord(entries):
+ temp = tempfile.NamedTemporaryFile(delete=False)
+ filename = temp.name
+ try:
+ with tf.io.TFRecordWriter(filename) as writer:
+ for value in entries:
+ writer.write(value)
+ yield filename
+ finally:
+ os.unlink(temp.name)
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class GenerateEmbeddingData(tf.test.TestCase):
+
+ def _save_checkpoint_from_mock_model(self, checkpoint_path):
+ """A function to save checkpoint from a fake Detection Model.
+
+ Args:
+ checkpoint_path: Path to save checkpoint from Fake model.
+ """
+ mock_model = FakeModel()
+ fake_image = tf.zeros(shape=[1, 10, 10, 3], dtype=tf.float32)
+ preprocessed_inputs, true_image_shapes = mock_model.preprocess(fake_image)
+ predictions = mock_model.predict(preprocessed_inputs, true_image_shapes)
+ mock_model.postprocess(predictions, true_image_shapes)
+ ckpt = tf.train.Checkpoint(model=mock_model)
+ exported_checkpoint_manager = tf.train.CheckpointManager(
+ ckpt, checkpoint_path, max_to_keep=1)
+ exported_checkpoint_manager.save(checkpoint_number=0)
+
+ def _export_saved_model(self):
+ tmp_dir = self.get_temp_dir()
+ self._save_checkpoint_from_mock_model(tmp_dir)
+ output_directory = os.path.join(tmp_dir, 'output')
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ tf.io.gfile.makedirs(output_directory)
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ exporter_lib_v2.INPUT_BUILDER_UTIL_MAP['model_build'] = mock_builder
+ output_directory = os.path.join(tmp_dir, 'output')
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ exporter_lib_v2.export_inference_graph(
+ input_type='tf_example',
+ pipeline_config=pipeline_config,
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory)
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ return saved_model_path
+
+ def _create_tf_example(self):
+ encoded_image = tf.io.encode_jpeg(
+ tf.constant(np.ones((4, 4, 3)).astype(np.uint8))).numpy()
+
+ def BytesFeature(value):
+ return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+ def Int64Feature(value):
+ return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+ def FloatFeature(value):
+ return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
+
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded': BytesFeature(encoded_image),
+ 'image/source_id': BytesFeature(b'image_id'),
+ 'image/height': Int64Feature(400),
+ 'image/width': Int64Feature(600),
+ 'image/class/label': Int64Feature(5),
+ 'image/class/text': BytesFeature(b'hyena'),
+ 'image/object/bbox/xmin': FloatFeature(0.1),
+ 'image/object/bbox/xmax': FloatFeature(0.6),
+ 'image/object/bbox/ymin': FloatFeature(0.0),
+ 'image/object/bbox/ymax': FloatFeature(0.5),
+ 'image/object/class/score': FloatFeature(0.95),
+ 'image/object/class/label': Int64Feature(5),
+ 'image/object/class/text': BytesFeature(b'hyena'),
+ 'image/date_captured': BytesFeature(b'2019-10-20 12:12:12')
+ }))
+
+ return example.SerializeToString()
+
+ def assert_expected_example(self, example, topk=False, botk=False):
+ # Check embeddings
+ if topk or botk:
+ self.assertEqual(len(
+ example.features.feature['image/embedding'].float_list.value),
+ 218)
+ self.assertAllEqual(
+ example.features.feature['image/embedding_count'].int64_list.value,
+ [2])
+ else:
+ self.assertEqual(len(
+ example.features.feature['image/embedding'].float_list.value),
+ 109)
+ self.assertAllEqual(
+ example.features.feature['image/embedding_count'].int64_list.value,
+ [1])
+
+ self.assertAllEqual(
+ example.features.feature['image/embedding_length'].int64_list.value,
+ [109])
+
+ # Check annotations
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/ymin'].float_list.value,
+ [0.0])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/xmin'].float_list.value,
+ [0.1])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/ymax'].float_list.value,
+ [0.5])
+ self.assertAllClose(
+ example.features.feature['image/object/bbox/xmax'].float_list.value,
+ [0.6])
+ self.assertAllClose(
+ example.features.feature['image/object/class/score']
+ .float_list.value, [0.95])
+ self.assertAllClose(
+ example.features.feature['image/object/class/label']
+ .int64_list.value, [5])
+ self.assertAllEqual(
+ example.features.feature['image/object/class/text']
+ .bytes_list.value, [b'hyena'])
+ self.assertAllClose(
+ example.features.feature['image/class/label']
+ .int64_list.value, [5])
+ self.assertAllEqual(
+ example.features.feature['image/class/text']
+ .bytes_list.value, [b'hyena'])
+
+ # Check other essential attributes.
+ self.assertAllEqual(
+ example.features.feature['image/height'].int64_list.value, [400])
+ self.assertAllEqual(
+ example.features.feature['image/width'].int64_list.value, [600])
+ self.assertAllEqual(
+ example.features.feature['image/source_id'].bytes_list.value,
+ [b'image_id'])
+ self.assertTrue(
+ example.features.feature['image/encoded'].bytes_list.value)
+
+ def test_generate_embedding_data_fn(self):
+ saved_model_path = self._export_saved_model()
+ top_k_embedding_count = 1
+ bottom_k_embedding_count = 0
+ inference_fn = generate_embedding_data.GenerateEmbeddingDataFn(
+ saved_model_path, top_k_embedding_count, bottom_k_embedding_count)
+ inference_fn.setup()
+ generated_example = self._create_tf_example()
+ self.assertAllEqual(tf.train.Example.FromString(
+ generated_example).features.feature['image/object/class/label']
+ .int64_list.value, [5])
+ self.assertAllEqual(tf.train.Example.FromString(
+ generated_example).features.feature['image/object/class/text']
+ .bytes_list.value, [b'hyena'])
+ output = inference_fn.process(('dummy_key', generated_example))
+ output_example = output[0][1]
+ self.assert_expected_example(output_example)
+
+ def test_generate_embedding_data_with_top_k_boxes(self):
+ saved_model_path = self._export_saved_model()
+ top_k_embedding_count = 2
+ bottom_k_embedding_count = 0
+ inference_fn = generate_embedding_data.GenerateEmbeddingDataFn(
+ saved_model_path, top_k_embedding_count, bottom_k_embedding_count)
+ inference_fn.setup()
+ generated_example = self._create_tf_example()
+ self.assertAllEqual(
+ tf.train.Example.FromString(generated_example).features
+ .feature['image/object/class/label'].int64_list.value, [5])
+ self.assertAllEqual(
+ tf.train.Example.FromString(generated_example).features
+ .feature['image/object/class/text'].bytes_list.value, [b'hyena'])
+ output = inference_fn.process(('dummy_key', generated_example))
+ output_example = output[0][1]
+ self.assert_expected_example(output_example, topk=True)
+
+ def test_generate_embedding_data_with_bottom_k_boxes(self):
+ saved_model_path = self._export_saved_model()
+ top_k_embedding_count = 0
+ bottom_k_embedding_count = 2
+ inference_fn = generate_embedding_data.GenerateEmbeddingDataFn(
+ saved_model_path, top_k_embedding_count, bottom_k_embedding_count)
+ inference_fn.setup()
+ generated_example = self._create_tf_example()
+ self.assertAllEqual(
+ tf.train.Example.FromString(generated_example).features
+ .feature['image/object/class/label'].int64_list.value, [5])
+ self.assertAllEqual(
+ tf.train.Example.FromString(generated_example).features
+ .feature['image/object/class/text'].bytes_list.value, [b'hyena'])
+ output = inference_fn.process(('dummy_key', generated_example))
+ output_example = output[0][1]
+ self.assert_expected_example(output_example, botk=True)
+
+ def test_beam_pipeline(self):
+ with InMemoryTFRecord([self._create_tf_example()]) as input_tfrecord:
+ temp_dir = tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))
+ output_tfrecord = os.path.join(temp_dir, 'output_tfrecord')
+ saved_model_path = self._export_saved_model()
+ top_k_embedding_count = 1
+ bottom_k_embedding_count = 0
+ num_shards = 1
+ embedding_type = 'final_box_features'
+ pipeline_options = beam.options.pipeline_options.PipelineOptions(
+ runner='DirectRunner')
+ p = beam.Pipeline(options=pipeline_options)
+ generate_embedding_data.construct_pipeline(
+ p, input_tfrecord, output_tfrecord, saved_model_path,
+ top_k_embedding_count, bottom_k_embedding_count, num_shards,
+ embedding_type)
+ p.run()
+ filenames = tf.io.gfile.glob(
+ output_tfrecord + '-?????-of-?????')
+ actual_output = []
+ record_iterator = tf.data.TFRecordDataset(
+ tf.convert_to_tensor(filenames)).as_numpy_iterator()
+ for record in record_iterator:
+ actual_output.append(record)
+ self.assertEqual(len(actual_output), 1)
+ self.assert_expected_example(tf.train.Example.FromString(
+ actual_output[0]))
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/dataset_tools/create_ava_actions_tf_record.py b/research/object_detection/dataset_tools/create_ava_actions_tf_record.py
new file mode 100644
index 0000000000000000000000000000000000000000..a27001d879c48e1e10194015f20eecb0724dfdf9
--- /dev/null
+++ b/research/object_detection/dataset_tools/create_ava_actions_tf_record.py
@@ -0,0 +1,540 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""Code to download and parse the AVA Actions dataset for TensorFlow models.
+
+The [AVA Actions data set](
+https://research.google.com/ava/index.html)
+is a dataset for human action recognition.
+
+This script downloads the annotations and prepares data from similar annotations
+if local video files are available. The video files can be downloaded
+from the following website:
+https://github.com/cvdfoundation/ava-dataset
+
+Prior to running this script, please run download_and_preprocess_ava.sh to
+download input videos.
+
+Running this code as a module generates the data set on disk. First, the
+required files are downloaded (_download_data) which enables constructing the
+label map. Then (in generate_examples), for each split in the data set, the
+metadata and image frames are generated from the annotations for each sequence
+example (_generate_examples). The data set is written to disk as a set of
+numbered TFRecord files.
+
+Generating the data on disk can take considerable time and disk space.
+(Image compression quality is the primary determiner of disk usage.
+
+If using the Tensorflow Object Detection API, set the input_type field
+in the input_reader to TF_SEQUENCE_EXAMPLE. If using this script to generate
+data for Context R-CNN scripts, the --examples_for_context flag should be
+set to true, so that properly-formatted tf.example objects are written to disk.
+
+This data is structured for per-clip action classification where images is
+the sequence of images and labels are a one-hot encoded value. See
+as_dataset() for more details.
+
+Note that the number of videos changes in the data set over time, so it will
+likely be necessary to change the expected number of examples.
+
+The argument video_path_format_string expects a value as such:
+ '/path/to/videos/{0}'
+
+"""
+import collections
+import contextlib
+import csv
+import glob
+import hashlib
+import os
+import random
+import sys
+import zipfile
+
+from absl import app
+from absl import flags
+from absl import logging
+import cv2
+from six.moves import range
+from six.moves import urllib
+import tensorflow.compat.v1 as tf
+
+from object_detection.dataset_tools import seq_example_util
+from object_detection.utils import dataset_util
+from object_detection.utils import label_map_util
+
+
+POSSIBLE_TIMESTAMPS = range(902, 1798)
+ANNOTATION_URL = 'https://research.google.com/ava/download/ava_v2.2.zip'
+SECONDS_TO_MILLI = 1000
+FILEPATTERN = 'ava_actions_%s_1fps_rgb'
+SPLITS = {
+ 'train': {
+ 'shards': 1000,
+ 'examples': 862663,
+ 'csv': '',
+ 'excluded-csv': ''
+ },
+ 'val': {
+ 'shards': 100,
+ 'examples': 243029,
+ 'csv': '',
+ 'excluded-csv': ''
+ },
+ # Test doesn't have ground truth, so TF Records can't be created
+ 'test': {
+ 'shards': 100,
+ 'examples': 0,
+ 'csv': '',
+ 'excluded-csv': ''
+ }
+}
+
+NUM_CLASSES = 80
+
+
+def feature_list_feature(value):
+ return tf.train.FeatureList(feature=value)
+
+
+class Ava(object):
+ """Generates and loads the AVA Actions 2.2 data set."""
+
+ def __init__(self, path_to_output_dir, path_to_data_download):
+ if not path_to_output_dir:
+ raise ValueError('You must supply the path to the data directory.')
+ self.path_to_data_download = path_to_data_download
+ self.path_to_output_dir = path_to_output_dir
+
+ def generate_and_write_records(self,
+ splits_to_process='train,val,test',
+ video_path_format_string=None,
+ seconds_per_sequence=10,
+ hop_between_sequences=10,
+ examples_for_context=False):
+ """Downloads data and generates sharded TFRecords.
+
+ Downloads the data files, generates metadata, and processes the metadata
+ with MediaPipe to produce tf.SequenceExamples for training. The resulting
+ files can be read with as_dataset(). After running this function the
+ original data files can be deleted.
+
+ Args:
+ splits_to_process: csv string of which splits to process. Allows
+ providing a custom CSV with the CSV flag. The original data is still
+ downloaded to generate the label_map.
+ video_path_format_string: The format string for the path to local files.
+ seconds_per_sequence: The length of each sequence, in seconds.
+ hop_between_sequences: The gap between the centers of
+ successive sequences.
+ examples_for_context: Whether to generate sequence examples with context
+ for context R-CNN.
+ """
+ example_function = self._generate_sequence_examples
+ if examples_for_context:
+ example_function = self._generate_examples
+
+ logging.info('Downloading data.')
+ download_output = self._download_data()
+ for key in splits_to_process.split(','):
+ logging.info('Generating examples for split: %s', key)
+ all_metadata = list(example_function(
+ download_output[0][key][0], download_output[0][key][1],
+ download_output[1], seconds_per_sequence, hop_between_sequences,
+ video_path_format_string))
+ logging.info('An example of the metadata: ')
+ logging.info(all_metadata[0])
+ random.seed(47)
+ random.shuffle(all_metadata)
+ shards = SPLITS[key]['shards']
+ shard_names = [os.path.join(
+ self.path_to_output_dir, FILEPATTERN % key + '-%05d-of-%05d' % (
+ i, shards)) for i in range(shards)]
+ writers = [tf.io.TFRecordWriter(shard) for shard in shard_names]
+ with _close_on_exit(writers) as writers:
+ for i, seq_ex in enumerate(all_metadata):
+ writers[i % len(writers)].write(seq_ex.SerializeToString())
+ logging.info('Data extraction complete.')
+
+ def _generate_sequence_examples(self, annotation_file, excluded_file,
+ label_map, seconds_per_sequence,
+ hop_between_sequences,
+ video_path_format_string):
+ """For each row in the annotation CSV, generates corresponding examples.
+
+ When iterating through frames for a single sequence example, skips over
+ excluded frames. When moving to the next sequence example, also skips over
+ excluded frames as if they don't exist. Generates equal-length sequence
+ examples, each with length seconds_per_sequence (1 fps) and gaps of
+ hop_between_sequences frames (and seconds) between them, possible greater
+ due to excluded frames.
+
+ Args:
+ annotation_file: path to the file of AVA CSV annotations.
+ excluded_file: path to a CSV file of excluded timestamps for each video.
+ label_map: an {int: string} label map.
+ seconds_per_sequence: The number of seconds per example in each example.
+ hop_between_sequences: The hop between sequences. If less than
+ seconds_per_sequence, will overlap.
+ video_path_format_string: File path format to glob video files.
+
+ Yields:
+ Each prepared tf.SequenceExample of metadata also containing video frames
+ """
+ fieldnames = ['id', 'timestamp_seconds', 'xmin', 'ymin', 'xmax', 'ymax',
+ 'action_label']
+ frame_excluded = {}
+ # create a sparse, nested map of videos and frame indices.
+ with open(excluded_file, 'r') as excluded:
+ reader = csv.reader(excluded)
+ for row in reader:
+ frame_excluded[(row[0], int(float(row[1])))] = True
+ with open(annotation_file, 'r') as annotations:
+ reader = csv.DictReader(annotations, fieldnames)
+ frame_annotations = collections.defaultdict(list)
+ ids = set()
+ # aggreggate by video and timestamp:
+ for row in reader:
+ ids.add(row['id'])
+ key = (row['id'], int(float(row['timestamp_seconds'])))
+ frame_annotations[key].append(row)
+ # for each video, find aggregates near each sampled frame.:
+ logging.info('Generating metadata...')
+ media_num = 1
+ for media_id in ids:
+ logging.info('%d/%d, ignore warnings.\n', media_num, len(ids))
+ media_num += 1
+
+ filepath = glob.glob(
+ video_path_format_string.format(media_id) + '*')[0]
+ cur_vid = cv2.VideoCapture(filepath)
+ width = cur_vid.get(cv2.CAP_PROP_FRAME_WIDTH)
+ height = cur_vid.get(cv2.CAP_PROP_FRAME_HEIGHT)
+ middle_frame_time = POSSIBLE_TIMESTAMPS[0]
+ while middle_frame_time < POSSIBLE_TIMESTAMPS[-1]:
+ start_time = middle_frame_time - seconds_per_sequence // 2 - (
+ 0 if seconds_per_sequence % 2 == 0 else 1)
+ end_time = middle_frame_time + (seconds_per_sequence // 2)
+
+ total_boxes = []
+ total_labels = []
+ total_label_strings = []
+ total_images = []
+ total_source_ids = []
+ total_confidences = []
+ total_is_annotated = []
+ windowed_timestamp = start_time
+
+ while windowed_timestamp < end_time:
+ if (media_id, windowed_timestamp) in frame_excluded:
+ end_time += 1
+ windowed_timestamp += 1
+ logging.info('Ignoring and skipping excluded frame.')
+ continue
+
+ cur_vid.set(cv2.CAP_PROP_POS_MSEC,
+ (windowed_timestamp) * SECONDS_TO_MILLI)
+ _, image = cur_vid.read()
+ _, buffer = cv2.imencode('.jpg', image)
+
+ bufstring = buffer.tostring()
+ total_images.append(bufstring)
+ source_id = str(windowed_timestamp) + '_' + media_id
+ total_source_ids.append(source_id)
+ total_is_annotated.append(1)
+
+ boxes = []
+ labels = []
+ label_strings = []
+ confidences = []
+ for row in frame_annotations[(media_id, windowed_timestamp)]:
+ if len(row) > 2 and int(row['action_label']) in label_map:
+ boxes.append([float(row['ymin']), float(row['xmin']),
+ float(row['ymax']), float(row['xmax'])])
+ labels.append(int(row['action_label']))
+ label_strings.append(label_map[int(row['action_label'])])
+ confidences.append(1)
+ else:
+ logging.warning('Unknown label: %s', row['action_label'])
+
+ total_boxes.append(boxes)
+ total_labels.append(labels)
+ total_label_strings.append(label_strings)
+ total_confidences.append(confidences)
+ windowed_timestamp += 1
+
+ if total_boxes:
+ yield seq_example_util.make_sequence_example(
+ 'AVA', media_id, total_images, int(height), int(width), 'jpeg',
+ total_source_ids, None, total_is_annotated, total_boxes,
+ total_label_strings, use_strs_for_source_id=True)
+
+ # Move middle_time_frame, skipping excluded frames
+ frames_mv = 0
+ frames_excluded_count = 0
+ while (frames_mv < hop_between_sequences + frames_excluded_count
+ and middle_frame_time + frames_mv < POSSIBLE_TIMESTAMPS[-1]):
+ frames_mv += 1
+ if (media_id, windowed_timestamp + frames_mv) in frame_excluded:
+ frames_excluded_count += 1
+ middle_frame_time += frames_mv
+
+ cur_vid.release()
+
+ def _generate_examples(self, annotation_file, excluded_file, label_map,
+ seconds_per_sequence, hop_between_sequences,
+ video_path_format_string):
+ """For each row in the annotation CSV, generates examples.
+
+ When iterating through frames for a single example, skips
+ over excluded frames. Generates equal-length sequence examples, each with
+ length seconds_per_sequence (1 fps) and gaps of hop_between_sequences
+ frames (and seconds) between them, possible greater due to excluded frames.
+
+ Args:
+ annotation_file: path to the file of AVA CSV annotations.
+ excluded_file: path to a CSV file of excluded timestamps for each video.
+ label_map: an {int: string} label map.
+ seconds_per_sequence: The number of seconds per example in each example.
+ hop_between_sequences: The hop between sequences. If less than
+ seconds_per_sequence, will overlap.
+ video_path_format_string: File path format to glob video files.
+
+ Yields:
+ Each prepared tf.Example of metadata also containing video frames
+ """
+ del seconds_per_sequence
+ del hop_between_sequences
+ fieldnames = ['id', 'timestamp_seconds', 'xmin', 'ymin', 'xmax', 'ymax',
+ 'action_label']
+ frame_excluded = {}
+ # create a sparse, nested map of videos and frame indices.
+ with open(excluded_file, 'r') as excluded:
+ reader = csv.reader(excluded)
+ for row in reader:
+ frame_excluded[(row[0], int(float(row[1])))] = True
+ with open(annotation_file, 'r') as annotations:
+ reader = csv.DictReader(annotations, fieldnames)
+ frame_annotations = collections.defaultdict(list)
+ ids = set()
+ # aggreggate by video and timestamp:
+ for row in reader:
+ ids.add(row['id'])
+ key = (row['id'], int(float(row['timestamp_seconds'])))
+ frame_annotations[key].append(row)
+ # for each video, find aggreggates near each sampled frame.:
+ logging.info('Generating metadata...')
+ media_num = 1
+ for media_id in ids:
+ logging.info('%d/%d, ignore warnings.\n', media_num, len(ids))
+ media_num += 1
+
+ filepath = glob.glob(
+ video_path_format_string.format(media_id) + '*')[0]
+ cur_vid = cv2.VideoCapture(filepath)
+ width = cur_vid.get(cv2.CAP_PROP_FRAME_WIDTH)
+ height = cur_vid.get(cv2.CAP_PROP_FRAME_HEIGHT)
+ middle_frame_time = POSSIBLE_TIMESTAMPS[0]
+ total_non_excluded = 0
+ while middle_frame_time < POSSIBLE_TIMESTAMPS[-1]:
+ if (media_id, middle_frame_time) not in frame_excluded:
+ total_non_excluded += 1
+ middle_frame_time += 1
+
+ middle_frame_time = POSSIBLE_TIMESTAMPS[0]
+ cur_frame_num = 0
+ while middle_frame_time < POSSIBLE_TIMESTAMPS[-1]:
+ cur_vid.set(cv2.CAP_PROP_POS_MSEC,
+ middle_frame_time * SECONDS_TO_MILLI)
+ _, image = cur_vid.read()
+ _, buffer = cv2.imencode('.jpg', image)
+
+ bufstring = buffer.tostring()
+
+ if (media_id, middle_frame_time) in frame_excluded:
+ middle_frame_time += 1
+ logging.info('Ignoring and skipping excluded frame.')
+ continue
+
+ cur_frame_num += 1
+ source_id = str(middle_frame_time) + '_' + media_id
+
+ xmins = []
+ xmaxs = []
+ ymins = []
+ ymaxs = []
+ areas = []
+ labels = []
+ label_strings = []
+ confidences = []
+ for row in frame_annotations[(media_id, middle_frame_time)]:
+ if len(row) > 2 and int(row['action_label']) in label_map:
+ xmins.append(float(row['xmin']))
+ xmaxs.append(float(row['xmax']))
+ ymins.append(float(row['ymin']))
+ ymaxs.append(float(row['ymax']))
+ areas.append(float((xmaxs[-1] - xmins[-1]) *
+ (ymaxs[-1] - ymins[-1])) / 2)
+ labels.append(int(row['action_label']))
+ label_strings.append(label_map[int(row['action_label'])])
+ confidences.append(1)
+ else:
+ logging.warning('Unknown label: %s', row['action_label'])
+
+ middle_frame_time += 1/3
+ if abs(middle_frame_time - round(middle_frame_time) < 0.0001):
+ middle_frame_time = round(middle_frame_time)
+
+ key = hashlib.sha256(bufstring).hexdigest()
+ date_captured_feature = (
+ '2020-06-17 00:%02d:%02d' % ((middle_frame_time - 900)*3 // 60,
+ (middle_frame_time - 900)*3 % 60))
+ context_feature_dict = {
+ 'image/height':
+ dataset_util.int64_feature(int(height)),
+ 'image/width':
+ dataset_util.int64_feature(int(width)),
+ 'image/format':
+ dataset_util.bytes_feature('jpeg'.encode('utf8')),
+ 'image/source_id':
+ dataset_util.bytes_feature(source_id.encode('utf8')),
+ 'image/filename':
+ dataset_util.bytes_feature(source_id.encode('utf8')),
+ 'image/encoded':
+ dataset_util.bytes_feature(bufstring),
+ 'image/key/sha256':
+ dataset_util.bytes_feature(key.encode('utf8')),
+ 'image/object/bbox/xmin':
+ dataset_util.float_list_feature(xmins),
+ 'image/object/bbox/xmax':
+ dataset_util.float_list_feature(xmaxs),
+ 'image/object/bbox/ymin':
+ dataset_util.float_list_feature(ymins),
+ 'image/object/bbox/ymax':
+ dataset_util.float_list_feature(ymaxs),
+ 'image/object/area':
+ dataset_util.float_list_feature(areas),
+ 'image/object/class/label':
+ dataset_util.int64_list_feature(labels),
+ 'image/object/class/text':
+ dataset_util.bytes_list_feature(label_strings),
+ 'image/location':
+ dataset_util.bytes_feature(media_id.encode('utf8')),
+ 'image/date_captured':
+ dataset_util.bytes_feature(
+ date_captured_feature.encode('utf8')),
+ 'image/seq_num_frames':
+ dataset_util.int64_feature(total_non_excluded),
+ 'image/seq_frame_num':
+ dataset_util.int64_feature(cur_frame_num),
+ 'image/seq_id':
+ dataset_util.bytes_feature(media_id.encode('utf8')),
+ }
+
+ yield tf.train.Example(
+ features=tf.train.Features(feature=context_feature_dict))
+
+ cur_vid.release()
+
+ def _download_data(self):
+ """Downloads and extracts data if not already available."""
+ if sys.version_info >= (3, 0):
+ urlretrieve = urllib.request.urlretrieve
+ else:
+ urlretrieve = urllib.request.urlretrieve
+ logging.info('Creating data directory.')
+ tf.io.gfile.makedirs(self.path_to_data_download)
+ logging.info('Downloading annotations.')
+ paths = {}
+
+ zip_path = os.path.join(self.path_to_data_download,
+ ANNOTATION_URL.split('/')[-1])
+ urlretrieve(ANNOTATION_URL, zip_path)
+ with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+ zip_ref.extractall(self.path_to_data_download)
+ for split in ['train', 'test', 'val']:
+ csv_path = os.path.join(self.path_to_data_download,
+ 'ava_%s_v2.2.csv' % split)
+ excl_name = 'ava_%s_excluded_timestamps_v2.2.csv' % split
+ excluded_csv_path = os.path.join(self.path_to_data_download, excl_name)
+ SPLITS[split]['csv'] = csv_path
+ SPLITS[split]['excluded-csv'] = excluded_csv_path
+ paths[split] = (csv_path, excluded_csv_path)
+
+ label_map = self.get_label_map(os.path.join(
+ self.path_to_data_download,
+ 'ava_action_list_v2.2_for_activitynet_2019.pbtxt'))
+ return paths, label_map
+
+ def get_label_map(self, path):
+ """Parses a label map into {integer:string} format."""
+ label_map_dict = label_map_util.get_label_map_dict(path)
+ label_map_dict = {v: bytes(k, 'utf8') for k, v in label_map_dict.items()}
+ logging.info(label_map_dict)
+ return label_map_dict
+
+
+@contextlib.contextmanager
+def _close_on_exit(writers):
+ """Call close on all writers on exit."""
+ try:
+ yield writers
+ finally:
+ for writer in writers:
+ writer.close()
+
+
+def main(argv):
+ if len(argv) > 1:
+ raise app.UsageError('Too many command-line arguments.')
+ Ava(flags.FLAGS.path_to_output_dir,
+ flags.FLAGS.path_to_download_data).generate_and_write_records(
+ flags.FLAGS.splits_to_process,
+ flags.FLAGS.video_path_format_string,
+ flags.FLAGS.seconds_per_sequence,
+ flags.FLAGS.hop_between_sequences,
+ flags.FLAGS.examples_for_context)
+
+if __name__ == '__main__':
+ flags.DEFINE_string('path_to_download_data',
+ '',
+ 'Path to directory to download data to.')
+ flags.DEFINE_string('path_to_output_dir',
+ '',
+ 'Path to directory to write data to.')
+ flags.DEFINE_string('splits_to_process',
+ 'train,val',
+ 'Process these splits. Useful for custom data splits.')
+ flags.DEFINE_string('video_path_format_string',
+ None,
+ 'The format string for the path to local video files. '
+ 'Uses the Python string.format() syntax with possible '
+ 'arguments of {video}, {start}, {end}, {label_name}, and '
+ '{split}, corresponding to columns of the data csvs.')
+ flags.DEFINE_integer('seconds_per_sequence',
+ 10,
+ 'The number of seconds per example in each example.'
+ 'Always 1 when examples_for_context is True.')
+ flags.DEFINE_integer('hop_between_sequences',
+ 10,
+ 'The hop between sequences. If less than '
+ 'seconds_per_sequence, will overlap. Always 1 when '
+ 'examples_for_context is True.')
+ flags.DEFINE_boolean('examples_for_context',
+ False,
+ 'Whether to generate examples instead of sequence '
+ 'examples. If true, will generate tf.Example objects '
+ 'for use in Context R-CNN.')
+ app.run(main)
diff --git a/research/object_detection/dataset_tools/densepose/UV_symmetry_transforms.mat b/research/object_detection/dataset_tools/densepose/UV_symmetry_transforms.mat
index d09d70fb1264efe1d151c6c7e557c020ae569af1..2836cac4d6b37a16fbff8ac6efda1d7ecb88a711 100644
Binary files a/research/object_detection/dataset_tools/densepose/UV_symmetry_transforms.mat and b/research/object_detection/dataset_tools/densepose/UV_symmetry_transforms.mat differ
diff --git a/research/object_detection/dataset_tools/download_and_preprocess_ava.sh b/research/object_detection/dataset_tools/download_and_preprocess_ava.sh
new file mode 100755
index 0000000000000000000000000000000000000000..723f6a7fcf5421e4bbbd015b8579b14cf3b0d61f
--- /dev/null
+++ b/research/object_detection/dataset_tools/download_and_preprocess_ava.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# This script downloads the videos for the AVA dataset. There are no arguments.
+# Copy this script into the desired parent directory of the ava_vids_raw/
+# directory created in this script to store the raw videos.
+
+mkdir ava_vids_raw
+cd ava_vids_raw
+
+curl -O s3.amazonaws.com/ava-dataset/annotations/ava_file_names_trainval_v2.1.txt
+
+echo "Downloading all videos."
+
+cat "ava_file_names_trainval_v2.1.txt" | while read line
+do
+ curl -O s3.amazonaws.com/ava-dataset/trainval/$line
+ echo "Downloaded " $line
+done
+
+rm "ava_file_names_trainval_v2.1.txt"
+cd ..
+
+# Trimming causes issues with frame seeking in the python script, so it is best left out.
+# If included, need to modify the python script to subtract 900 seconds wheen seeking.
+
+# echo "Trimming all videos."
+
+# mkdir ava_vids_trimmed
+# for filename in ava_vids_raw/*; do
+# ffmpeg -ss 900 -to 1800 -i $filename -c copy ava_vids_trimmed/${filename##*/}
+# done
diff --git a/research/object_detection/dataset_tools/seq_example_util.py b/research/object_detection/dataset_tools/seq_example_util.py
index 84573ec7eff5c2217693bd777386533c2c164af0..d4e160a29d7f27b03cccedfcdce0414980915091 100644
--- a/research/object_detection/dataset_tools/seq_example_util.py
+++ b/research/object_detection/dataset_tools/seq_example_util.py
@@ -1,4 +1,3 @@
-# Lint as: python2, python3
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -123,6 +122,15 @@ def sequence_bytes_feature(ndarray):
return feature_list
+def sequence_strings_feature(strings):
+ new_str_arr = []
+ for single_str in strings:
+ new_str_arr.append(tf.train.Feature(
+ bytes_list=tf.train.BytesList(
+ value=[single_str.encode('utf8')])))
+ return tf.train.FeatureList(feature=new_str_arr)
+
+
def boxes_to_box_components(bboxes):
"""Converts a list of numpy arrays (boxes) to box components.
@@ -137,8 +145,11 @@ def boxes_to_box_components(bboxes):
ymax_list = []
xmax_list = []
for bbox in bboxes:
- bbox = np.array(bbox).astype(np.float32)
- ymin, xmin, ymax, xmax = np.split(bbox, 4, axis=1)
+ if bbox != []: # pylint: disable=g-explicit-bool-comparison
+ bbox = np.array(bbox).astype(np.float32)
+ ymin, xmin, ymax, xmax = np.split(bbox, 4, axis=1)
+ else:
+ ymin, xmin, ymax, xmax = [], [], [], []
ymin_list.append(np.reshape(ymin, [-1]))
xmin_list.append(np.reshape(xmin, [-1]))
ymax_list.append(np.reshape(ymax, [-1]))
@@ -159,7 +170,11 @@ def make_sequence_example(dataset_name,
label_strings=None,
detection_bboxes=None,
detection_classes=None,
- detection_scores=None):
+ detection_scores=None,
+ use_strs_for_source_id=False,
+ context_features=None,
+ context_feature_length=None,
+ context_features_image_id_list=None):
"""Constructs tf.SequenceExamples.
Args:
@@ -189,6 +204,14 @@ def make_sequence_example(dataset_name,
detection_scores: (Optional) A list (with num_frames_elements) of
[num_boxes_i] numpy float32 arrays holding predicted object scores for
each frame.
+ use_strs_for_source_id: (Optional) Whether to write the source IDs as
+ strings rather than byte lists of characters.
+ context_features: (Optional) A list or numpy array of features to use in
+ Context R-CNN, of length num_context_features * context_feature_length.
+ context_feature_length: (Optional) The length of each context feature, used
+ for reshaping.
+ context_features_image_id_list: (Optional) A list of image ids of length
+ num_context_features corresponding to the context features.
Returns:
A tf.train.SequenceExample.
@@ -221,7 +244,11 @@ def make_sequence_example(dataset_name,
if image_format is not None:
context_dict['image/format'] = context_bytes_feature([image_format])
if image_source_ids is not None:
- feature_list['image/source_id'] = sequence_bytes_feature(image_source_ids)
+ if use_strs_for_source_id:
+ feature_list['image/source_id'] = sequence_strings_feature(
+ image_source_ids)
+ else:
+ feature_list['image/source_id'] = sequence_bytes_feature(image_source_ids)
if bboxes is not None:
bbox_ymin, bbox_xmin, bbox_ymax, bbox_xmax = boxes_to_box_components(bboxes)
feature_list['region/bbox/xmin'] = sequence_float_feature(bbox_xmin)
@@ -255,6 +282,16 @@ def make_sequence_example(dataset_name,
feature_list['predicted/region/label/confidence'] = sequence_float_feature(
detection_scores)
+ if context_features is not None:
+ context_dict['image/context_features'] = context_float_feature(
+ context_features)
+ if context_feature_length is not None:
+ context_dict['image/context_feature_length'] = context_int64_feature(
+ context_feature_length)
+ if context_features_image_id_list is not None:
+ context_dict['image/context_features_image_id_list'] = (
+ context_bytes_feature(context_features_image_id_list))
+
context = tf.train.Features(feature=context_dict)
feature_lists = tf.train.FeatureLists(feature_list=feature_list)
diff --git a/research/object_detection/dataset_tools/seq_example_util_test.py b/research/object_detection/dataset_tools/seq_example_util_test.py
index fd721954be896b4044735dd67928044e413422e7..cb36f7753d110b9b1fad75e13a4311cf259e1f40 100644
--- a/research/object_detection/dataset_tools/seq_example_util_test.py
+++ b/research/object_detection/dataset_tools/seq_example_util_test.py
@@ -104,6 +104,107 @@ class SeqExampleUtilTest(tf.test.TestCase):
source_ids)
def test_make_labeled_example(self):
+ num_frames = 3
+ image_height = 100
+ image_width = 200
+ dataset_name = b'unlabeled_dataset'
+ video_id = b'video_000'
+ labels = [b'dog', b'cat', b'wolf']
+ images = tf.cast(tf.random.uniform(
+ [num_frames, image_height, image_width, 3],
+ maxval=256,
+ dtype=tf.int32), dtype=tf.uint8)
+ images_list = tf.unstack(images, axis=0)
+ encoded_images_list = [tf.io.encode_jpeg(image) for image in images_list]
+ encoded_images = self.materialize_tensors(encoded_images_list)
+ timestamps = [100000, 110000, 120000]
+ is_annotated = [1, 0, 1]
+ bboxes = [
+ np.array([[0., 0., 0., 0.],
+ [0., 0., 1., 1.]], dtype=np.float32),
+ np.zeros([0, 4], dtype=np.float32),
+ np.array([], dtype=np.float32)
+ ]
+ label_strings = [
+ np.array(labels),
+ np.array([]),
+ np.array([])
+ ]
+
+ seq_example = seq_example_util.make_sequence_example(
+ dataset_name=dataset_name,
+ video_id=video_id,
+ encoded_images=encoded_images,
+ image_height=image_height,
+ image_width=image_width,
+ timestamps=timestamps,
+ is_annotated=is_annotated,
+ bboxes=bboxes,
+ label_strings=label_strings)
+
+ context_feature_dict = seq_example.context.feature
+ self.assertEqual(
+ dataset_name,
+ context_feature_dict['example/dataset_name'].bytes_list.value[0])
+ self.assertEqual(
+ timestamps[0],
+ context_feature_dict['clip/start/timestamp'].int64_list.value[0])
+ self.assertEqual(
+ timestamps[-1],
+ context_feature_dict['clip/end/timestamp'].int64_list.value[0])
+ self.assertEqual(
+ num_frames,
+ context_feature_dict['clip/frames'].int64_list.value[0])
+
+ seq_feature_dict = seq_example.feature_lists.feature_list
+ self.assertLen(
+ seq_feature_dict['image/encoded'].feature[:],
+ num_frames)
+ actual_timestamps = [
+ feature.int64_list.value[0] for feature
+ in seq_feature_dict['image/timestamp'].feature]
+ self.assertAllEqual(timestamps, actual_timestamps)
+ # Frame 0.
+ self.assertAllEqual(
+ is_annotated[0],
+ seq_feature_dict['region/is_annotated'].feature[0].int64_list.value[0])
+ self.assertAllClose(
+ [0., 0.],
+ seq_feature_dict['region/bbox/ymin'].feature[0].float_list.value[:])
+ self.assertAllClose(
+ [0., 0.],
+ seq_feature_dict['region/bbox/xmin'].feature[0].float_list.value[:])
+ self.assertAllClose(
+ [0., 1.],
+ seq_feature_dict['region/bbox/ymax'].feature[0].float_list.value[:])
+ self.assertAllClose(
+ [0., 1.],
+ seq_feature_dict['region/bbox/xmax'].feature[0].float_list.value[:])
+ self.assertAllEqual(
+ labels,
+ seq_feature_dict['region/label/string'].feature[0].bytes_list.value[:])
+
+ # Frame 1.
+ self.assertAllEqual(
+ is_annotated[1],
+ seq_feature_dict['region/is_annotated'].feature[1].int64_list.value[0])
+ self.assertAllClose(
+ [],
+ seq_feature_dict['region/bbox/ymin'].feature[1].float_list.value[:])
+ self.assertAllClose(
+ [],
+ seq_feature_dict['region/bbox/xmin'].feature[1].float_list.value[:])
+ self.assertAllClose(
+ [],
+ seq_feature_dict['region/bbox/ymax'].feature[1].float_list.value[:])
+ self.assertAllClose(
+ [],
+ seq_feature_dict['region/bbox/xmax'].feature[1].float_list.value[:])
+ self.assertAllEqual(
+ [],
+ seq_feature_dict['region/label/string'].feature[1].bytes_list.value[:])
+
+ def test_make_labeled_example_with_context_features(self):
num_frames = 2
image_height = 100
image_width = 200
@@ -128,6 +229,9 @@ class SeqExampleUtilTest(tf.test.TestCase):
np.array(labels),
np.array([])
]
+ context_features = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5]
+ context_feature_length = [3]
+ context_features_image_id_list = [b'im_1', b'im_2']
seq_example = seq_example_util.make_sequence_example(
dataset_name=dataset_name,
@@ -138,7 +242,10 @@ class SeqExampleUtilTest(tf.test.TestCase):
timestamps=timestamps,
is_annotated=is_annotated,
bboxes=bboxes,
- label_strings=label_strings)
+ label_strings=label_strings,
+ context_features=context_features,
+ context_feature_length=context_feature_length,
+ context_features_image_id_list=context_features_image_id_list)
context_feature_dict = seq_example.context.feature
self.assertEqual(
@@ -154,6 +261,18 @@ class SeqExampleUtilTest(tf.test.TestCase):
num_frames,
context_feature_dict['clip/frames'].int64_list.value[0])
+ self.assertAllClose(
+ context_features,
+ context_feature_dict['image/context_features'].float_list.value[:])
+ self.assertEqual(
+ context_feature_length[0],
+ context_feature_dict[
+ 'image/context_feature_length'].int64_list.value[0])
+ self.assertEqual(
+ context_features_image_id_list,
+ context_feature_dict[
+ 'image/context_features_image_id_list'].bytes_list.value[:])
+
seq_feature_dict = seq_example.feature_lists.feature_list
self.assertLen(
seq_feature_dict['image/encoded'].feature[:],
diff --git a/research/object_detection/eval_util.py b/research/object_detection/eval_util.py
index b6d57f738942d740277add29c15aabd0de4c9c18..f5e7fef600bfe79de22c92f12323a56e0923ac76 100644
--- a/research/object_detection/eval_util.py
+++ b/research/object_detection/eval_util.py
@@ -33,6 +33,7 @@ from object_detection.core import box_list_ops
from object_detection.core import keypoint_ops
from object_detection.core import standard_fields as fields
from object_detection.metrics import coco_evaluation
+from object_detection.metrics import lvis_evaluation
from object_detection.protos import eval_pb2
from object_detection.utils import label_map_util
from object_detection.utils import object_detection_evaluation
@@ -54,6 +55,8 @@ EVAL_METRICS_CLASS_DICT = {
coco_evaluation.CocoMaskEvaluator,
'coco_panoptic_metrics':
coco_evaluation.CocoPanopticSegmentationEvaluator,
+ 'lvis_mask_metrics':
+ lvis_evaluation.LVISMaskEvaluator,
'oid_challenge_detection_metrics':
object_detection_evaluation.OpenImagesDetectionChallengeEvaluator,
'oid_challenge_segmentation_metrics':
@@ -548,10 +551,38 @@ def _scale_box_to_absolute(args):
box_list.BoxList(boxes), image_shape[0], image_shape[1]).get()
-def _resize_detection_masks(args):
- detection_boxes, detection_masks, image_shape = args
+def _resize_detection_masks(arg_tuple):
+ """Resizes detection masks.
+
+ Args:
+ arg_tuple: A (detection_boxes, detection_masks, image_shape, pad_shape)
+ tuple where
+ detection_boxes is a tf.float32 tensor of size [num_masks, 4] containing
+ the box corners. Row i contains [ymin, xmin, ymax, xmax] of the box
+ corresponding to mask i. Note that the box corners are in
+ normalized coordinates.
+ detection_masks is a tensor of size
+ [num_masks, mask_height, mask_width].
+ image_shape is a tensor of shape [2]
+ pad_shape is a tensor of shape [2] --- this is assumed to be greater
+ than or equal to image_shape along both dimensions and represents a
+ shape to-be-padded-to.
+
+ Returns:
+ """
+
+ detection_boxes, detection_masks, image_shape, pad_shape = arg_tuple
+
detection_masks_reframed = ops.reframe_box_masks_to_image_masks(
detection_masks, detection_boxes, image_shape[0], image_shape[1])
+
+ pad_instance_dim = tf.zeros([3, 1], dtype=tf.int32)
+ pad_hw_dim = tf.concat([tf.zeros([1], dtype=tf.int32),
+ pad_shape - image_shape], axis=0)
+ pad_hw_dim = tf.expand_dims(pad_hw_dim, 1)
+ paddings = tf.concat([pad_instance_dim, pad_hw_dim], axis=1)
+ detection_masks_reframed = tf.pad(detection_masks_reframed, paddings)
+
# If the masks are currently float, binarize them. Otherwise keep them as
# integers, since they have already been thresholded.
if detection_masks_reframed.dtype == tf.float32:
@@ -559,9 +590,44 @@ def _resize_detection_masks(args):
return tf.cast(detection_masks_reframed, tf.uint8)
+def resize_detection_masks(detection_boxes, detection_masks,
+ original_image_spatial_shapes):
+ """Resizes per-box detection masks to be relative to the entire image.
+
+ Note that this function only works when the spatial size of all images in
+ the batch is the same. If not, this function should be used with batch_size=1.
+
+ Args:
+ detection_boxes: A [batch_size, num_instances, 4] float tensor containing
+ bounding boxes.
+ detection_masks: A [batch_size, num_instances, height, width] float tensor
+ containing binary instance masks per box.
+ original_image_spatial_shapes: a [batch_size, 3] shaped int tensor
+ holding the spatial dimensions of each image in the batch.
+ Returns:
+ masks: Masks resized to the spatial extents given by
+ (original_image_spatial_shapes[0, 0], original_image_spatial_shapes[0, 1])
+ """
+ # modify original image spatial shapes to be max along each dim
+ # in evaluator, should have access to original_image_spatial_shape field
+ # in add_Eval_Dict
+ max_spatial_shape = tf.reduce_max(
+ original_image_spatial_shapes, axis=0, keep_dims=True)
+ tiled_max_spatial_shape = tf.tile(
+ max_spatial_shape,
+ multiples=[tf.shape(original_image_spatial_shapes)[0], 1])
+ return shape_utils.static_or_dynamic_map_fn(
+ _resize_detection_masks,
+ elems=[detection_boxes,
+ detection_masks,
+ original_image_spatial_shapes,
+ tiled_max_spatial_shape],
+ dtype=tf.uint8)
+
+
def _resize_groundtruth_masks(args):
- """Resizes groundgtruth masks to the original image size."""
- mask, true_image_shape, original_image_shape = args
+ """Resizes groundtruth masks to the original image size."""
+ mask, true_image_shape, original_image_shape, pad_shape = args
true_height = true_image_shape[0]
true_width = true_image_shape[1]
mask = mask[:, :true_height, :true_width]
@@ -571,7 +637,15 @@ def _resize_groundtruth_masks(args):
original_image_shape,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
align_corners=True)
- return tf.cast(tf.squeeze(mask, 3), tf.uint8)
+
+ paddings = tf.concat(
+ [tf.zeros([3, 1], dtype=tf.int32),
+ tf.expand_dims(
+ tf.concat([tf.zeros([1], dtype=tf.int32),
+ pad_shape-original_image_shape], axis=0),
+ 1)], axis=1)
+ mask = tf.pad(tf.squeeze(mask, 3), paddings)
+ return tf.cast(mask, tf.uint8)
def _resize_surface_coordinate_masks(args):
@@ -698,7 +772,8 @@ def result_dict_for_batched_example(images,
scale_to_absolute=False,
original_image_spatial_shapes=None,
true_image_shapes=None,
- max_gt_boxes=None):
+ max_gt_boxes=None,
+ label_id_offset=1):
"""Merges all detection and groundtruth information for a single example.
Note that evaluation tools require classes that are 1-indexed, and so this
@@ -753,6 +828,7 @@ def result_dict_for_batched_example(images,
containing the size of the unpadded original_image.
max_gt_boxes: [batch_size] tensor representing the maximum number of
groundtruth boxes to pad.
+ label_id_offset: offset for class ids.
Returns:
A dictionary with:
@@ -807,8 +883,6 @@ def result_dict_for_batched_example(images,
ValueError: if true_image_shapes is not 2D int32 tensor of shape
[3].
"""
- label_id_offset = 1 # Applying label id offset (b/63711816)
-
input_data_fields = fields.InputDataFields
if original_image_spatial_shapes is None:
original_image_spatial_shapes = tf.tile(
@@ -869,12 +943,9 @@ def result_dict_for_batched_example(images,
if detection_fields.detection_masks in detections:
detection_masks = detections[detection_fields.detection_masks]
- output_dict[detection_fields.detection_masks] = (
- shape_utils.static_or_dynamic_map_fn(
- _resize_detection_masks,
- elems=[detection_boxes, detection_masks,
- original_image_spatial_shapes],
- dtype=tf.uint8))
+ output_dict[detection_fields.detection_masks] = resize_detection_masks(
+ detection_boxes, detection_masks, original_image_spatial_shapes)
+
if detection_fields.detection_surface_coords in detections:
detection_surface_coords = detections[
detection_fields.detection_surface_coords]
@@ -911,10 +982,17 @@ def result_dict_for_batched_example(images,
if input_data_fields.groundtruth_instance_masks in groundtruth:
masks = groundtruth[input_data_fields.groundtruth_instance_masks]
+ max_spatial_shape = tf.reduce_max(
+ original_image_spatial_shapes, axis=0, keep_dims=True)
+ tiled_max_spatial_shape = tf.tile(
+ max_spatial_shape,
+ multiples=[tf.shape(original_image_spatial_shapes)[0], 1])
groundtruth[input_data_fields.groundtruth_instance_masks] = (
shape_utils.static_or_dynamic_map_fn(
_resize_groundtruth_masks,
- elems=[masks, true_image_shapes, original_image_spatial_shapes],
+ elems=[masks, true_image_shapes,
+ original_image_spatial_shapes,
+ tiled_max_spatial_shape],
dtype=tf.uint8))
output_dict.update(groundtruth)
@@ -1095,11 +1173,39 @@ def evaluator_options_from_eval_config(eval_config):
eval_metric_fn_keys = eval_config.metrics_set
evaluator_options = {}
for eval_metric_fn_key in eval_metric_fn_keys:
- if eval_metric_fn_key in ('coco_detection_metrics', 'coco_mask_metrics'):
+ if eval_metric_fn_key in (
+ 'coco_detection_metrics', 'coco_mask_metrics', 'lvis_mask_metrics'):
evaluator_options[eval_metric_fn_key] = {
'include_metrics_per_category': (
eval_config.include_metrics_per_category)
}
+
+ if (hasattr(eval_config, 'all_metrics_per_category') and
+ eval_config.all_metrics_per_category):
+ evaluator_options[eval_metric_fn_key].update({
+ 'all_metrics_per_category': eval_config.all_metrics_per_category
+ })
+ # For coco detection eval, if the eval_config proto contains the
+ # "skip_predictions_for_unlabeled_class" field, include this field in
+ # evaluator_options.
+ if eval_metric_fn_key == 'coco_detection_metrics' and hasattr(
+ eval_config, 'skip_predictions_for_unlabeled_class'):
+ evaluator_options[eval_metric_fn_key].update({
+ 'skip_predictions_for_unlabeled_class':
+ (eval_config.skip_predictions_for_unlabeled_class)
+ })
+ for super_category in eval_config.super_categories:
+ if 'super_categories' not in evaluator_options[eval_metric_fn_key]:
+ evaluator_options[eval_metric_fn_key]['super_categories'] = {}
+ key = super_category
+ value = eval_config.super_categories[key].split(',')
+ evaluator_options[eval_metric_fn_key]['super_categories'][key] = value
+ if eval_metric_fn_key == 'lvis_mask_metrics' and hasattr(
+ eval_config, 'export_path'):
+ evaluator_options[eval_metric_fn_key].update({
+ 'export_path': eval_config.export_path
+ })
+
elif eval_metric_fn_key == 'precision_at_recall_detection_metrics':
evaluator_options[eval_metric_fn_key] = {
'recall_lower_bound': (eval_config.recall_lower_bound),
diff --git a/research/object_detection/eval_util_test.py b/research/object_detection/eval_util_test.py
index d0623f1fcda50482ee98eccb2e2e62ef10b88be3..a39a5ff16749fdfbb091448c444c02de5d524b36 100644
--- a/research/object_detection/eval_util_test.py
+++ b/research/object_detection/eval_util_test.py
@@ -25,6 +25,7 @@ import numpy as np
import six
from six.moves import range
import tensorflow.compat.v1 as tf
+from google.protobuf import text_format
from object_detection import eval_util
from object_detection.core import standard_fields as fields
@@ -84,6 +85,8 @@ class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
groundtruth_boxes = tf.constant([[0., 0., 1., 1.]])
groundtruth_classes = tf.constant([1])
groundtruth_instance_masks = tf.ones(shape=[1, 20, 20], dtype=tf.uint8)
+ original_image_spatial_shapes = tf.constant([[20, 20]], dtype=tf.int32)
+
groundtruth_keypoints = tf.constant([[0.0, 0.0], [0.5, 0.5], [1.0, 1.0]])
if resized_groundtruth_masks:
groundtruth_instance_masks = tf.ones(shape=[1, 10, 10], dtype=tf.uint8)
@@ -99,6 +102,8 @@ class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
groundtruth_keypoints = tf.tile(
tf.expand_dims(groundtruth_keypoints, 0),
multiples=[batch_size, 1, 1])
+ original_image_spatial_shapes = tf.tile(original_image_spatial_shapes,
+ multiples=[batch_size, 1])
detections = {
detection_fields.detection_boxes: detection_boxes,
@@ -111,7 +116,10 @@ class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
input_data_fields.groundtruth_boxes: groundtruth_boxes,
input_data_fields.groundtruth_classes: groundtruth_classes,
input_data_fields.groundtruth_keypoints: groundtruth_keypoints,
- input_data_fields.groundtruth_instance_masks: groundtruth_instance_masks
+ input_data_fields.groundtruth_instance_masks:
+ groundtruth_instance_masks,
+ input_data_fields.original_image_spatial_shape:
+ original_image_spatial_shapes
}
if batch_size > 1:
return eval_util.result_dict_for_batched_example(
@@ -239,6 +247,8 @@ class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
eval_config)
self.assertTrue(evaluator_options['coco_detection_metrics']
['include_metrics_per_category'])
+ self.assertFalse(evaluator_options['coco_detection_metrics']
+ ['skip_predictions_for_unlabeled_class'])
self.assertTrue(
evaluator_options['coco_mask_metrics']['include_metrics_per_category'])
self.assertAlmostEqual(
@@ -253,6 +263,7 @@ class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
eval_config.metrics_set.extend(
['coco_detection_metrics', 'precision_at_recall_detection_metrics'])
eval_config.include_metrics_per_category = True
+ eval_config.skip_predictions_for_unlabeled_class = True
eval_config.recall_lower_bound = 0.2
eval_config.recall_upper_bound = 0.6
categories = self._get_categories_list()
@@ -263,6 +274,7 @@ class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
evaluator_options)
self.assertTrue(evaluator[0]._include_metrics_per_category)
+ self.assertTrue(evaluator[0]._skip_predictions_for_unlabeled_class)
self.assertAlmostEqual(evaluator[1]._recall_lower_bound,
eval_config.recall_lower_bound)
self.assertAlmostEqual(evaluator[1]._recall_upper_bound,
@@ -402,6 +414,48 @@ class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
[[[0., 0.], [75., 150.], [150., 300.]]]],
detection_keypoints)
+ def test_evaluator_options_from_eval_config_no_super_categories(self):
+ eval_config_text_proto = """
+ metrics_set: "coco_detection_metrics"
+ metrics_set: "coco_mask_metrics"
+ include_metrics_per_category: true
+ use_moving_averages: false
+ batch_size: 1;
+ """
+ eval_config = eval_pb2.EvalConfig()
+ text_format.Merge(eval_config_text_proto, eval_config)
+ evaluator_options = eval_util.evaluator_options_from_eval_config(
+ eval_config)
+ self.assertNotIn('super_categories', evaluator_options['coco_mask_metrics'])
+
+ def test_evaluator_options_from_eval_config_with_super_categories(self):
+ eval_config_text_proto = """
+ metrics_set: "coco_detection_metrics"
+ metrics_set: "coco_mask_metrics"
+ include_metrics_per_category: true
+ use_moving_averages: false
+ batch_size: 1;
+ super_categories {
+ key: "supercat1"
+ value: "a,b,c"
+ }
+ super_categories {
+ key: "supercat2"
+ value: "d,e,f"
+ }
+ """
+ eval_config = eval_pb2.EvalConfig()
+ text_format.Merge(eval_config_text_proto, eval_config)
+ evaluator_options = eval_util.evaluator_options_from_eval_config(
+ eval_config)
+ self.assertIn('super_categories', evaluator_options['coco_mask_metrics'])
+ super_categories = evaluator_options[
+ 'coco_mask_metrics']['super_categories']
+ self.assertIn('supercat1', super_categories)
+ self.assertIn('supercat2', super_categories)
+ self.assertAllEqual(super_categories['supercat1'], ['a', 'b', 'c'])
+ self.assertAllEqual(super_categories['supercat2'], ['d', 'e', 'f'])
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/export_tflite_graph_lib_tf2.py b/research/object_detection/export_tflite_graph_lib_tf2.py
new file mode 100644
index 0000000000000000000000000000000000000000..66bc8ecf03c1156ca96e44418c906963ac159ac5
--- /dev/null
+++ b/research/object_detection/export_tflite_graph_lib_tf2.py
@@ -0,0 +1,375 @@
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library to export TFLite-compatible SavedModel from TF2 detection models."""
+import os
+import numpy as np
+import tensorflow.compat.v1 as tf1
+import tensorflow.compat.v2 as tf
+
+from object_detection.builders import model_builder
+from object_detection.builders import post_processing_builder
+from object_detection.core import box_list
+from object_detection.core import standard_fields as fields
+
+_DEFAULT_NUM_CHANNELS = 3
+_DEFAULT_NUM_COORD_BOX = 4
+_MAX_CLASSES_PER_DETECTION = 1
+_DETECTION_POSTPROCESS_FUNC = 'TFLite_Detection_PostProcess'
+
+
+def get_const_center_size_encoded_anchors(anchors):
+ """Exports center-size encoded anchors as a constant tensor.
+
+ Args:
+ anchors: a float32 tensor of shape [num_anchors, 4] containing the anchor
+ boxes
+
+ Returns:
+ encoded_anchors: a float32 constant tensor of shape [num_anchors, 4]
+ containing the anchor boxes.
+ """
+ anchor_boxlist = box_list.BoxList(anchors)
+ y, x, h, w = anchor_boxlist.get_center_coordinates_and_sizes()
+ num_anchors = y.get_shape().as_list()
+
+ with tf1.Session() as sess:
+ y_out, x_out, h_out, w_out = sess.run([y, x, h, w])
+ encoded_anchors = tf1.constant(
+ np.transpose(np.stack((y_out, x_out, h_out, w_out))),
+ dtype=tf1.float32,
+ shape=[num_anchors[0], _DEFAULT_NUM_COORD_BOX],
+ name='anchors')
+ return num_anchors[0], encoded_anchors
+
+
+class SSDModule(tf.Module):
+ """Inference Module for TFLite-friendly SSD models."""
+
+ def __init__(self, pipeline_config, detection_model, max_detections,
+ use_regular_nms):
+ """Initialization.
+
+ Args:
+ pipeline_config: The original pipeline_pb2.TrainEvalPipelineConfig
+ detection_model: The detection model to use for inference.
+ max_detections: Max detections desired from the TFLite model.
+ use_regular_nms: If True, TFLite model uses the (slower) multi-class NMS.
+ """
+ self._process_config(pipeline_config)
+ self._pipeline_config = pipeline_config
+ self._model = detection_model
+ self._max_detections = max_detections
+ self._use_regular_nms = use_regular_nms
+
+ def _process_config(self, pipeline_config):
+ self._num_classes = pipeline_config.model.ssd.num_classes
+ self._nms_score_threshold = pipeline_config.model.ssd.post_processing.batch_non_max_suppression.score_threshold
+ self._nms_iou_threshold = pipeline_config.model.ssd.post_processing.batch_non_max_suppression.iou_threshold
+ self._scale_values = {}
+ self._scale_values[
+ 'y_scale'] = pipeline_config.model.ssd.box_coder.faster_rcnn_box_coder.y_scale
+ self._scale_values[
+ 'x_scale'] = pipeline_config.model.ssd.box_coder.faster_rcnn_box_coder.x_scale
+ self._scale_values[
+ 'h_scale'] = pipeline_config.model.ssd.box_coder.faster_rcnn_box_coder.height_scale
+ self._scale_values[
+ 'w_scale'] = pipeline_config.model.ssd.box_coder.faster_rcnn_box_coder.width_scale
+
+ image_resizer_config = pipeline_config.model.ssd.image_resizer
+ image_resizer = image_resizer_config.WhichOneof('image_resizer_oneof')
+ self._num_channels = _DEFAULT_NUM_CHANNELS
+
+ if image_resizer == 'fixed_shape_resizer':
+ self._height = image_resizer_config.fixed_shape_resizer.height
+ self._width = image_resizer_config.fixed_shape_resizer.width
+ if image_resizer_config.fixed_shape_resizer.convert_to_grayscale:
+ self._num_channels = 1
+ else:
+ raise ValueError(
+ 'Only fixed_shape_resizer'
+ 'is supported with tflite. Found {}'.format(
+ image_resizer_config.WhichOneof('image_resizer_oneof')))
+
+ def input_shape(self):
+ """Returns shape of TFLite model input."""
+ return [1, self._height, self._width, self._num_channels]
+
+ def postprocess_implements_signature(self):
+ """Returns tf.implements signature for MLIR legalization of TFLite NMS."""
+ implements_signature = [
+ 'name: "%s"' % _DETECTION_POSTPROCESS_FUNC,
+ 'attr { key: "max_detections" value { i: %d } }' % self._max_detections,
+ 'attr { key: "max_classes_per_detection" value { i: %d } }' %
+ _MAX_CLASSES_PER_DETECTION,
+ 'attr { key: "use_regular_nms" value { b: %s } }' %
+ str(self._use_regular_nms).lower(),
+ 'attr { key: "nms_score_threshold" value { f: %f } }' %
+ self._nms_score_threshold,
+ 'attr { key: "nms_iou_threshold" value { f: %f } }' %
+ self._nms_iou_threshold,
+ 'attr { key: "y_scale" value { f: %f } }' %
+ self._scale_values['y_scale'],
+ 'attr { key: "x_scale" value { f: %f } }' %
+ self._scale_values['x_scale'],
+ 'attr { key: "h_scale" value { f: %f } }' %
+ self._scale_values['h_scale'],
+ 'attr { key: "w_scale" value { f: %f } }' %
+ self._scale_values['w_scale'],
+ 'attr { key: "num_classes" value { i: %d } }' % self._num_classes
+ ]
+ implements_signature = ' '.join(implements_signature)
+ return implements_signature
+
+ def _get_postprocess_fn(self, num_anchors, num_classes):
+ # There is no TF equivalent for TFLite's custom post-processing op.
+ # So we add an 'empty' composite function here, that is legalized to the
+ # custom op with MLIR.
+ @tf.function(
+ experimental_implements=self.postprocess_implements_signature())
+ # pylint: disable=g-unused-argument,unused-argument
+ def dummy_post_processing(box_encodings, class_predictions, anchors):
+ boxes = tf.constant(0.0, dtype=tf.float32, name='boxes')
+ scores = tf.constant(0.0, dtype=tf.float32, name='scores')
+ classes = tf.constant(0.0, dtype=tf.float32, name='classes')
+ num_detections = tf.constant(0.0, dtype=tf.float32, name='num_detections')
+ return boxes, classes, scores, num_detections
+
+ return dummy_post_processing
+
+ @tf.function
+ def inference_fn(self, image):
+ """Encapsulates SSD inference for TFLite conversion.
+
+ NOTE: The Args & Returns sections below indicate the TFLite model signature,
+ and not what the TF graph does (since the latter does not include the custom
+ NMS op used by TFLite)
+
+ Args:
+ image: a float32 tensor of shape [num_anchors, 4] containing the anchor
+ boxes
+
+ Returns:
+ num_detections: a float32 scalar denoting number of total detections.
+ classes: a float32 tensor denoting class ID for each detection.
+ scores: a float32 tensor denoting score for each detection.
+ boxes: a float32 tensor denoting coordinates of each detected box.
+ """
+ predicted_tensors = self._model.predict(image, true_image_shapes=None)
+ # The score conversion occurs before the post-processing custom op
+ _, score_conversion_fn = post_processing_builder.build(
+ self._pipeline_config.model.ssd.post_processing)
+ class_predictions = score_conversion_fn(
+ predicted_tensors['class_predictions_with_background'])
+
+ with tf.name_scope('raw_outputs'):
+ # 'raw_outputs/box_encodings': a float32 tensor of shape
+ # [1, num_anchors, 4] containing the encoded box predictions. Note that
+ # these are raw predictions and no Non-Max suppression is applied on
+ # them and no decode center size boxes is applied to them.
+ box_encodings = tf.identity(
+ predicted_tensors['box_encodings'], name='box_encodings')
+ # 'raw_outputs/class_predictions': a float32 tensor of shape
+ # [1, num_anchors, num_classes] containing the class scores for each
+ # anchor after applying score conversion.
+ class_predictions = tf.identity(
+ class_predictions, name='class_predictions')
+ # 'anchors': a float32 tensor of shape
+ # [4, num_anchors] containing the anchors as a constant node.
+ num_anchors, anchors = get_const_center_size_encoded_anchors(
+ predicted_tensors['anchors'])
+ anchors = tf.identity(anchors, name='anchors')
+
+ # tf.function@ seems to reverse order of inputs, so reverse them here.
+ return self._get_postprocess_fn(num_anchors,
+ self._num_classes)(box_encodings,
+ class_predictions,
+ anchors)[::-1]
+
+
+class CenterNetModule(tf.Module):
+ """Inference Module for TFLite-friendly CenterNet models.
+
+ The exported CenterNet model includes the preprocessing and postprocessing
+ logics so the caller should pass in the raw image pixel values. It supports
+ both object detection and keypoint estimation task.
+ """
+
+ def __init__(self, pipeline_config, max_detections, include_keypoints,
+ label_map_path=''):
+ """Initialization.
+
+ Args:
+ pipeline_config: The original pipeline_pb2.TrainEvalPipelineConfig
+ max_detections: Max detections desired from the TFLite model.
+ include_keypoints: If set true, the output dictionary will include the
+ keypoint coordinates and keypoint confidence scores.
+ label_map_path: Path to the label map which is used by CenterNet keypoint
+ estimation task. If provided, the label_map_path in the configuration
+ will be replaced by this one.
+ """
+ self._max_detections = max_detections
+ self._include_keypoints = include_keypoints
+ self._process_config(pipeline_config)
+ if include_keypoints and label_map_path:
+ pipeline_config.model.center_net.keypoint_label_map_path = label_map_path
+ self._pipeline_config = pipeline_config
+ self._model = model_builder.build(
+ self._pipeline_config.model, is_training=False)
+
+ def get_model(self):
+ return self._model
+
+ def _process_config(self, pipeline_config):
+ self._num_classes = pipeline_config.model.center_net.num_classes
+
+ center_net_config = pipeline_config.model.center_net
+ image_resizer_config = center_net_config.image_resizer
+ image_resizer = image_resizer_config.WhichOneof('image_resizer_oneof')
+ self._num_channels = _DEFAULT_NUM_CHANNELS
+
+ if image_resizer == 'fixed_shape_resizer':
+ self._height = image_resizer_config.fixed_shape_resizer.height
+ self._width = image_resizer_config.fixed_shape_resizer.width
+ if image_resizer_config.fixed_shape_resizer.convert_to_grayscale:
+ self._num_channels = 1
+ else:
+ raise ValueError(
+ 'Only fixed_shape_resizer'
+ 'is supported with tflite. Found {}'.format(image_resizer))
+
+ center_net_config.object_center_params.max_box_predictions = (
+ self._max_detections)
+
+ if not self._include_keypoints:
+ del center_net_config.keypoint_estimation_task[:]
+
+ def input_shape(self):
+ """Returns shape of TFLite model input."""
+ return [1, self._height, self._width, self._num_channels]
+
+ @tf.function
+ def inference_fn(self, image):
+ """Encapsulates CenterNet inference for TFLite conversion.
+
+ Args:
+ image: a float32 tensor of shape [1, image_height, image_width, channel]
+ denoting the image pixel values.
+
+ Returns:
+ A dictionary of predicted tensors:
+ classes: a float32 tensor with shape [1, max_detections] denoting class
+ ID for each detection.
+ scores: a float32 tensor with shape [1, max_detections] denoting score
+ for each detection.
+ boxes: a float32 tensor with shape [1, max_detections, 4] denoting
+ coordinates of each detected box.
+ keypoints: a float32 with shape [1, max_detections, num_keypoints, 2]
+ denoting the predicted keypoint coordinates (normalized in between
+ 0-1). Note that [:, :, :, 0] represents the y coordinates and
+ [:, :, :, 1] represents the x coordinates.
+ keypoint_scores: a float32 with shape [1, max_detections, num_keypoints]
+ denoting keypoint confidence scores.
+ """
+ image = tf.cast(image, tf.float32)
+ image, shapes = self._model.preprocess(image)
+ prediction_dict = self._model.predict(image, None)
+ detections = self._model.postprocess(
+ prediction_dict, true_image_shapes=shapes)
+
+ field_names = fields.DetectionResultFields
+ classes_field = field_names.detection_classes
+ classes = tf.cast(detections[classes_field], tf.float32)
+ num_detections = tf.cast(detections[field_names.num_detections], tf.float32)
+
+ if self._include_keypoints:
+ model_outputs = (detections[field_names.detection_boxes], classes,
+ detections[field_names.detection_scores], num_detections,
+ detections[field_names.detection_keypoints],
+ detections[field_names.detection_keypoint_scores])
+ else:
+ model_outputs = (detections[field_names.detection_boxes], classes,
+ detections[field_names.detection_scores], num_detections)
+
+ # tf.function@ seems to reverse order of inputs, so reverse them here.
+ return model_outputs[::-1]
+
+
+def export_tflite_model(pipeline_config, trained_checkpoint_dir,
+ output_directory, max_detections, use_regular_nms,
+ include_keypoints=False, label_map_path=''):
+ """Exports inference SavedModel for TFLite conversion.
+
+ NOTE: Only supports SSD meta-architectures for now, and the output model will
+ have static-shaped, single-batch input.
+
+ This function creates `output_directory` if it does not already exist,
+ which will hold the intermediate SavedModel that can be used with the TFLite
+ converter.
+
+ Args:
+ pipeline_config: pipeline_pb2.TrainAndEvalPipelineConfig proto.
+ trained_checkpoint_dir: Path to the trained checkpoint file.
+ output_directory: Path to write outputs.
+ max_detections: Max detections desired from the TFLite model.
+ use_regular_nms: If True, TFLite model uses the (slower) multi-class NMS.
+ Note that this argument is only used by the SSD model.
+ include_keypoints: Decides whether to also output the keypoint predictions.
+ Note that this argument is only used by the CenterNet model.
+ label_map_path: Path to the label map which is used by CenterNet keypoint
+ estimation task. If provided, the label_map_path in the configuration will
+ be replaced by this one.
+
+ Raises:
+ ValueError: if pipeline is invalid.
+ """
+ output_saved_model_directory = os.path.join(output_directory, 'saved_model')
+
+ # Build the underlying model using pipeline config.
+ # TODO(b/162842801): Add support for other architectures.
+ if pipeline_config.model.WhichOneof('model') == 'ssd':
+ detection_model = model_builder.build(
+ pipeline_config.model, is_training=False)
+ ckpt = tf.train.Checkpoint(model=detection_model)
+ # The module helps build a TF SavedModel appropriate for TFLite conversion.
+ detection_module = SSDModule(pipeline_config, detection_model,
+ max_detections, use_regular_nms)
+ elif pipeline_config.model.WhichOneof('model') == 'center_net':
+ detection_module = CenterNetModule(
+ pipeline_config, max_detections, include_keypoints,
+ label_map_path=label_map_path)
+ ckpt = tf.train.Checkpoint(model=detection_module.get_model())
+ else:
+ raise ValueError('Only ssd or center_net models are supported in tflite. '
+ 'Found {} in config'.format(
+ pipeline_config.model.WhichOneof('model')))
+
+ manager = tf.train.CheckpointManager(
+ ckpt, trained_checkpoint_dir, max_to_keep=1)
+ status = ckpt.restore(manager.latest_checkpoint).expect_partial()
+
+ # Getting the concrete function traces the graph and forces variables to
+ # be constructed; only after this can we save the saved model.
+ status.assert_existing_objects_matched()
+ concrete_function = detection_module.inference_fn.get_concrete_function(
+ tf.TensorSpec(
+ shape=detection_module.input_shape(), dtype=tf.float32, name='input'))
+ status.assert_existing_objects_matched()
+
+ # Export SavedModel.
+ tf.saved_model.save(
+ detection_module,
+ output_saved_model_directory,
+ signatures=concrete_function)
diff --git a/research/object_detection/export_tflite_graph_lib_tf2_test.py b/research/object_detection/export_tflite_graph_lib_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..6bb151fbda49dc6a67960d5bd998cd3aa91640cf
--- /dev/null
+++ b/research/object_detection/export_tflite_graph_lib_tf2_test.py
@@ -0,0 +1,342 @@
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test for export_tflite_graph_lib_tf2.py."""
+
+from __future__ import division
+import os
+import unittest
+import six
+
+import tensorflow.compat.v2 as tf
+
+from object_detection import export_tflite_graph_lib_tf2
+from object_detection.builders import model_builder
+from object_detection.core import model
+from object_detection.protos import pipeline_pb2
+from object_detection.utils import tf_version
+from google.protobuf import text_format
+
+if six.PY2:
+ import mock # pylint: disable=g-importing-member,g-import-not-at-top
+else:
+ from unittest import mock # pylint: disable=g-importing-member,g-import-not-at-top
+
+
+class FakeModel(model.DetectionModel):
+
+ def __init__(self):
+ super(FakeModel, self).__init__(num_classes=2)
+ self._conv = tf.keras.layers.Conv2D(
+ filters=1,
+ kernel_size=1,
+ strides=(1, 1),
+ padding='valid',
+ kernel_initializer=tf.keras.initializers.Constant(value=1.0))
+
+ def preprocess(self, inputs):
+ true_image_shapes = [] # Doesn't matter for the fake model.
+ return tf.identity(inputs), true_image_shapes
+
+ def predict(self, preprocessed_inputs, true_image_shapes):
+ prediction_tensors = {'image': self._conv(preprocessed_inputs)}
+ with tf.control_dependencies([prediction_tensors['image']]):
+ prediction_tensors['box_encodings'] = tf.constant(
+ [[[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 0.8, 0.8]]], tf.float32)
+ prediction_tensors['class_predictions_with_background'] = tf.constant(
+ [[[0.7, 0.6], [0.9, 0.0]]], tf.float32)
+ with tf.control_dependencies([
+ tf.convert_to_tensor(
+ prediction_tensors['image'].get_shape().as_list()[1:3])
+ ]):
+ prediction_tensors['anchors'] = tf.constant(
+ [[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 1.0, 1.0]], tf.float32)
+ return prediction_tensors
+
+ def postprocess(self, prediction_dict, true_image_shapes):
+ predict_tensor_sum = tf.reduce_sum(prediction_dict['image'])
+ with tf.control_dependencies(list(prediction_dict.values())):
+ postprocessed_tensors = {
+ 'detection_boxes':
+ tf.constant([[[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 0.8, 0.8]],
+ [[0.5, 0.5, 1.0, 1.0], [0.0, 0.0, 0.0, 0.0]]],
+ tf.float32),
+ 'detection_scores':
+ predict_tensor_sum +
+ tf.constant([[0.7, 0.6], [0.9, 0.0]], tf.float32),
+ 'detection_classes':
+ tf.constant([[0, 1], [1, 0]], tf.float32),
+ 'num_detections':
+ tf.constant([2, 1], tf.float32),
+ 'detection_keypoints':
+ tf.zeros([2, 17, 2], tf.float32),
+ 'detection_keypoint_scores':
+ tf.zeros([2, 17], tf.float32),
+ }
+ return postprocessed_tensors
+
+ def restore_map(self, checkpoint_path, from_detection_checkpoint):
+ pass
+
+ def restore_from_objects(self, fine_tune_checkpoint_type):
+ pass
+
+ def loss(self, prediction_dict, true_image_shapes):
+ pass
+
+ def regularization_losses(self):
+ pass
+
+ def updates(self):
+ pass
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class ExportTfLiteGraphTest(tf.test.TestCase):
+
+ def _save_checkpoint_from_mock_model(self, checkpoint_dir):
+ mock_model = FakeModel()
+ fake_image = tf.zeros(shape=[1, 10, 10, 3], dtype=tf.float32)
+ preprocessed_inputs, true_image_shapes = mock_model.preprocess(fake_image)
+ predictions = mock_model.predict(preprocessed_inputs, true_image_shapes)
+ mock_model.postprocess(predictions, true_image_shapes)
+
+ ckpt = tf.train.Checkpoint(model=mock_model)
+ exported_checkpoint_manager = tf.train.CheckpointManager(
+ ckpt, checkpoint_dir, max_to_keep=1)
+ exported_checkpoint_manager.save(checkpoint_number=0)
+
+ def _get_ssd_config(self):
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ pipeline_config.model.ssd.image_resizer.fixed_shape_resizer.height = 10
+ pipeline_config.model.ssd.image_resizer.fixed_shape_resizer.width = 10
+ pipeline_config.model.ssd.num_classes = 2
+ pipeline_config.model.ssd.box_coder.faster_rcnn_box_coder.y_scale = 10.0
+ pipeline_config.model.ssd.box_coder.faster_rcnn_box_coder.x_scale = 10.0
+ pipeline_config.model.ssd.box_coder.faster_rcnn_box_coder.height_scale = 5.0
+ pipeline_config.model.ssd.box_coder.faster_rcnn_box_coder.width_scale = 5.0
+ pipeline_config.model.ssd.post_processing.batch_non_max_suppression.iou_threshold = 0.5
+ return pipeline_config
+
+ def _get_center_net_config(self):
+ pipeline_config_text = """
+model {
+ center_net {
+ num_classes: 1
+ feature_extractor {
+ type: "mobilenet_v2_fpn"
+ }
+ image_resizer {
+ fixed_shape_resizer {
+ height: 10
+ width: 10
+ }
+ }
+ object_detection_task {
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ }
+ object_center_params {
+ classification_loss {
+ }
+ max_box_predictions: 20
+ }
+ keypoint_estimation_task {
+ loss {
+ localization_loss {
+ l1_localization_loss {
+ }
+ }
+ classification_loss {
+ penalty_reduced_logistic_focal_loss {
+ }
+ }
+ }
+ }
+ }
+}
+ """
+ return text_format.Parse(
+ pipeline_config_text, pipeline_pb2.TrainEvalPipelineConfig())
+
+ # The tf.implements signature is important since it ensures MLIR legalization,
+ # so we test it here.
+ def test_postprocess_implements_signature(self):
+ tmp_dir = self.get_temp_dir()
+ self._save_checkpoint_from_mock_model(tmp_dir)
+ pipeline_config = self._get_ssd_config()
+
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+
+ detection_model = model_builder.build(
+ pipeline_config.model, is_training=False)
+
+ ckpt = tf.train.Checkpoint(model=detection_model)
+ manager = tf.train.CheckpointManager(ckpt, tmp_dir, max_to_keep=1)
+ ckpt.restore(manager.latest_checkpoint).expect_partial()
+
+ # The module helps build a TF graph appropriate for TFLite conversion.
+ detection_module = export_tflite_graph_lib_tf2.SSDModule(
+ pipeline_config=pipeline_config,
+ detection_model=detection_model,
+ max_detections=20,
+ use_regular_nms=True)
+
+ expected_signature = ('name: "TFLite_Detection_PostProcess" attr { key: '
+ '"max_detections" value { i: 20 } } attr { key: '
+ '"max_classes_per_detection" value { i: 1 } } attr '
+ '{ key: "use_regular_nms" value { b: true } } attr '
+ '{ key: "nms_score_threshold" value { f: 0.000000 }'
+ ' } attr { key: "nms_iou_threshold" value { f: '
+ '0.500000 } } attr { key: "y_scale" value { f: '
+ '10.000000 } } attr { key: "x_scale" value { f: '
+ '10.000000 } } attr { key: "h_scale" value { f: '
+ '5.000000 } } attr { key: "w_scale" value { f: '
+ '5.000000 } } attr { key: "num_classes" value { i: '
+ '2 } }')
+
+ self.assertEqual(expected_signature,
+ detection_module.postprocess_implements_signature())
+
+ def test_unsupported_architecture(self):
+ tmp_dir = self.get_temp_dir()
+ self._save_checkpoint_from_mock_model(tmp_dir)
+
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ pipeline_config.model.faster_rcnn.num_classes = 10
+
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ output_directory = os.path.join(tmp_dir, 'output')
+ expected_message = 'Only ssd or center_net models are supported in tflite'
+ try:
+ export_tflite_graph_lib_tf2.export_tflite_model(
+ pipeline_config=pipeline_config,
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory,
+ max_detections=10,
+ use_regular_nms=False)
+ except ValueError as e:
+ if expected_message not in str(e):
+ raise
+ else:
+ raise AssertionError('Exception not raised: %s' % expected_message)
+
+ def test_export_yields_saved_model(self):
+ tmp_dir = self.get_temp_dir()
+ self._save_checkpoint_from_mock_model(tmp_dir)
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ output_directory = os.path.join(tmp_dir, 'output')
+ export_tflite_graph_lib_tf2.export_tflite_model(
+ pipeline_config=self._get_ssd_config(),
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory,
+ max_detections=10,
+ use_regular_nms=False)
+ self.assertTrue(
+ os.path.exists(
+ os.path.join(output_directory, 'saved_model', 'saved_model.pb')))
+ self.assertTrue(
+ os.path.exists(
+ os.path.join(output_directory, 'saved_model', 'variables',
+ 'variables.index')))
+ self.assertTrue(
+ os.path.exists(
+ os.path.join(output_directory, 'saved_model', 'variables',
+ 'variables.data-00000-of-00001')))
+
+ def test_exported_model_inference(self):
+ tmp_dir = self.get_temp_dir()
+ output_directory = os.path.join(tmp_dir, 'output')
+ self._save_checkpoint_from_mock_model(tmp_dir)
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ export_tflite_graph_lib_tf2.export_tflite_model(
+ pipeline_config=self._get_ssd_config(),
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory,
+ max_detections=10,
+ use_regular_nms=False)
+
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ detect_fn = tf.saved_model.load(saved_model_path)
+ detect_fn_sig = detect_fn.signatures['serving_default']
+ image = tf.zeros(shape=[1, 10, 10, 3], dtype=tf.float32)
+ detections = detect_fn_sig(image)
+
+ # The exported graph doesn't have numerically correct outputs, but there
+ # should be 4.
+ self.assertEqual(4, len(detections))
+
+ def test_center_net_inference_object_detection(self):
+ tmp_dir = self.get_temp_dir()
+ output_directory = os.path.join(tmp_dir, 'output')
+ self._save_checkpoint_from_mock_model(tmp_dir)
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ export_tflite_graph_lib_tf2.export_tflite_model(
+ pipeline_config=self._get_center_net_config(),
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory,
+ max_detections=10,
+ use_regular_nms=False)
+
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ detect_fn = tf.saved_model.load(saved_model_path)
+ detect_fn_sig = detect_fn.signatures['serving_default']
+ image = tf.zeros(shape=[1, 10, 10, 3], dtype=tf.float32)
+ detections = detect_fn_sig(image)
+
+ # The exported graph doesn't have numerically correct outputs, but there
+ # should be 4.
+ self.assertEqual(4, len(detections))
+
+ def test_center_net_inference_keypoint(self):
+ tmp_dir = self.get_temp_dir()
+ output_directory = os.path.join(tmp_dir, 'output')
+ self._save_checkpoint_from_mock_model(tmp_dir)
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ export_tflite_graph_lib_tf2.export_tflite_model(
+ pipeline_config=self._get_center_net_config(),
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory,
+ max_detections=10,
+ use_regular_nms=False,
+ include_keypoints=True)
+
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ detect_fn = tf.saved_model.load(saved_model_path)
+ detect_fn_sig = detect_fn.signatures['serving_default']
+ image = tf.zeros(shape=[1, 10, 10, 3], dtype=tf.float32)
+ detections = detect_fn_sig(image)
+
+ # The exported graph doesn't have numerically correct outputs, but there
+ # should be 6 (4 for boxes, 2 for keypoints).
+ self.assertEqual(6, len(detections))
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/export_tflite_graph_tf2.py b/research/object_detection/export_tflite_graph_tf2.py
new file mode 100644
index 0000000000000000000000000000000000000000..6bb40b15d8d69e72b802f931396072e1469bb445
--- /dev/null
+++ b/research/object_detection/export_tflite_graph_tf2.py
@@ -0,0 +1,161 @@
+# Lint as: python2, python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Exports TF2 detection SavedModel for conversion to TensorFlow Lite.
+
+Link to the TF2 Detection Zoo:
+https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md
+The output folder will contain an intermediate SavedModel that can be used with
+the TfLite converter.
+
+NOTE: This only supports SSD meta-architectures for now.
+
+One input:
+ image: a float32 tensor of shape[1, height, width, 3] containing the
+ *normalized* input image.
+ NOTE: See the `preprocess` function defined in the feature extractor class
+ in the object_detection/models directory.
+
+Four Outputs:
+ detection_boxes: a float32 tensor of shape [1, num_boxes, 4] with box
+ locations
+ detection_classes: a float32 tensor of shape [1, num_boxes]
+ with class indices
+ detection_scores: a float32 tensor of shape [1, num_boxes]
+ with class scores
+ num_boxes: a float32 tensor of size 1 containing the number of detected boxes
+
+Example Usage:
+--------------
+python object_detection/export_tflite_graph_tf2.py \
+ --pipeline_config_path path/to/ssd_model/pipeline.config \
+ --trained_checkpoint_dir path/to/ssd_model/checkpoint \
+ --output_directory path/to/exported_model_directory
+
+The expected output SavedModel would be in the directory
+path/to/exported_model_directory (which is created if it does not exist).
+
+Config overrides (see the `config_override` flag) are text protobufs
+(also of type pipeline_pb2.TrainEvalPipelineConfig) which are used to override
+certain fields in the provided pipeline_config_path. These are useful for
+making small changes to the inference graph that differ from the training or
+eval config.
+
+Example Usage 1 (in which we change the NMS iou_threshold to be 0.5 and
+NMS score_threshold to be 0.0):
+python object_detection/export_tflite_model_tf2.py \
+ --pipeline_config_path path/to/ssd_model/pipeline.config \
+ --trained_checkpoint_dir path/to/ssd_model/checkpoint \
+ --output_directory path/to/exported_model_directory
+ --config_override " \
+ model{ \
+ ssd{ \
+ post_processing { \
+ batch_non_max_suppression { \
+ score_threshold: 0.0 \
+ iou_threshold: 0.5 \
+ } \
+ } \
+ } \
+ } \
+ "
+
+Example Usage 2 (export CenterNet model for keypoint estimation task with fixed
+shape resizer and customized input resolution):
+python object_detection/export_tflite_model_tf2.py \
+ --pipeline_config_path path/to/ssd_model/pipeline.config \
+ --trained_checkpoint_dir path/to/ssd_model/checkpoint \
+ --output_directory path/to/exported_model_directory \
+ --keypoint_label_map_path path/to/label_map.txt \
+ --max_detections 10 \
+ --centernet_include_keypoints true \
+ --config_override " \
+ model{ \
+ center_net { \
+ image_resizer { \
+ fixed_shape_resizer { \
+ height: 320 \
+ width: 320 \
+ } \
+ } \
+ } \
+ }" \
+"""
+from absl import app
+from absl import flags
+
+import tensorflow.compat.v2 as tf
+from google.protobuf import text_format
+from object_detection import export_tflite_graph_lib_tf2
+from object_detection.protos import pipeline_pb2
+
+tf.enable_v2_behavior()
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string(
+ 'pipeline_config_path', None,
+ 'Path to a pipeline_pb2.TrainEvalPipelineConfig config '
+ 'file.')
+flags.DEFINE_string('trained_checkpoint_dir', None,
+ 'Path to trained checkpoint directory')
+flags.DEFINE_string('output_directory', None, 'Path to write outputs.')
+flags.DEFINE_string(
+ 'config_override', '', 'pipeline_pb2.TrainEvalPipelineConfig '
+ 'text proto to override pipeline_config_path.')
+flags.DEFINE_integer('max_detections', 10,
+ 'Maximum number of detections (boxes) to return.')
+# SSD-specific flags
+flags.DEFINE_bool(
+ 'ssd_use_regular_nms', False,
+ 'Flag to set postprocessing op to use Regular NMS instead of Fast NMS '
+ '(Default false).')
+# CenterNet-specific flags
+flags.DEFINE_bool(
+ 'centernet_include_keypoints', False,
+ 'Whether to export the predicted keypoint tensors. Only CenterNet model'
+ ' supports this flag.'
+)
+flags.DEFINE_string(
+ 'keypoint_label_map_path', None,
+ 'Path of the label map used by CenterNet keypoint estimation task. If'
+ ' provided, the label map path in the pipeline config will be replaced by'
+ ' this one. Note that it is only used when exporting CenterNet model for'
+ ' keypoint estimation task.'
+)
+
+
+def main(argv):
+ del argv # Unused.
+ flags.mark_flag_as_required('pipeline_config_path')
+ flags.mark_flag_as_required('trained_checkpoint_dir')
+ flags.mark_flag_as_required('output_directory')
+
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+
+ with tf.io.gfile.GFile(FLAGS.pipeline_config_path, 'r') as f:
+ text_format.Parse(f.read(), pipeline_config)
+ override_config = pipeline_pb2.TrainEvalPipelineConfig()
+ text_format.Parse(FLAGS.config_override, override_config)
+ pipeline_config.MergeFrom(override_config)
+
+ export_tflite_graph_lib_tf2.export_tflite_model(
+ pipeline_config, FLAGS.trained_checkpoint_dir, FLAGS.output_directory,
+ FLAGS.max_detections, FLAGS.ssd_use_regular_nms,
+ FLAGS.centernet_include_keypoints, FLAGS.keypoint_label_map_path)
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/research/object_detection/exporter.py b/research/object_detection/exporter.py
index 61c5f7f22db46c88c8bc5c1803b281da4c020967..a4848e35b52bc87d1e752927b0faeaf41dc9cca3 100644
--- a/research/object_detection/exporter.py
+++ b/research/object_detection/exporter.py
@@ -170,7 +170,7 @@ def replace_variable_values_with_moving_averages(graph,
with graph.as_default():
variable_averages = tf.train.ExponentialMovingAverage(0.0)
ema_variables_to_restore = variable_averages.variables_to_restore()
- ema_variables_to_restore = config_util.remove_unecessary_ema(
+ ema_variables_to_restore = config_util.remove_unnecessary_ema(
ema_variables_to_restore, no_ema_collection)
with tf.Session() as sess:
read_saver = tf.train.Saver(ema_variables_to_restore)
diff --git a/research/object_detection/exporter_lib_tf2_test.py b/research/object_detection/exporter_lib_tf2_test.py
index 99cbf263bece871d1a7d3b5a9f92e22c3f356412..cf8973fdf9d63183a5470a16ad64836ef504b24c 100644
--- a/research/object_detection/exporter_lib_tf2_test.py
+++ b/research/object_detection/exporter_lib_tf2_test.py
@@ -51,11 +51,13 @@ class FakeModel(model.DetectionModel):
value=conv_weight_scalar))
def preprocess(self, inputs):
- true_image_shapes = [] # Doesn't matter for the fake model.
- return tf.identity(inputs), true_image_shapes
+ return tf.identity(inputs), exporter_lib_v2.get_true_shapes(inputs)
- def predict(self, preprocessed_inputs, true_image_shapes):
- return {'image': self._conv(preprocessed_inputs)}
+ def predict(self, preprocessed_inputs, true_image_shapes, **side_inputs):
+ return_dict = {'image': self._conv(preprocessed_inputs)}
+ if 'side_inp_1' in side_inputs:
+ return_dict['image'] += side_inputs['side_inp_1']
+ return return_dict
def postprocess(self, prediction_dict, true_image_shapes):
predict_tensor_sum = tf.reduce_sum(prediction_dict['image'])
@@ -73,6 +75,13 @@ class FakeModel(model.DetectionModel):
}
return postprocessed_tensors
+ def predict_masks_from_boxes(self, prediction_dict, true_image_shapes, boxes):
+ output_dict = self.postprocess(prediction_dict, true_image_shapes)
+ output_dict.update({
+ 'detection_masks': tf.ones(shape=(1, 2, 16), dtype=tf.float32),
+ })
+ return output_dict
+
def restore_map(self, checkpoint_path, fine_tune_checkpoint_type):
pass
@@ -117,6 +126,7 @@ class ExportInferenceGraphTest(tf.test.TestCase, parameterized.TestCase):
with mock.patch.object(
model_builder, 'build', autospec=True) as mock_builder:
mock_builder.return_value = FakeModel()
+ exporter_lib_v2.INPUT_BUILDER_UTIL_MAP['model_build'] = mock_builder
output_directory = os.path.join(tmp_dir, 'output')
pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
exporter_lib_v2.export_inference_graph(
@@ -142,9 +152,9 @@ class ExportInferenceGraphTest(tf.test.TestCase, parameterized.TestCase):
"""Get dummy input for the given input type."""
if input_type == 'image_tensor':
- return np.zeros(shape=(1, 20, 20, 3), dtype=np.uint8)
+ return np.zeros((1, 20, 20, 3), dtype=np.uint8)
if input_type == 'float_image_tensor':
- return np.zeros(shape=(1, 20, 20, 3), dtype=np.float32)
+ return np.zeros((1, 20, 20, 3), dtype=np.float32)
elif input_type == 'encoded_image_string_tensor':
image = Image.new('RGB', (20, 20))
byte_io = io.BytesIO()
@@ -178,6 +188,7 @@ class ExportInferenceGraphTest(tf.test.TestCase, parameterized.TestCase):
with mock.patch.object(
model_builder, 'build', autospec=True) as mock_builder:
mock_builder.return_value = FakeModel()
+ exporter_lib_v2.INPUT_BUILDER_UTIL_MAP['model_build'] = mock_builder
output_directory = os.path.join(tmp_dir, 'output')
pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
exporter_lib_v2.export_inference_graph(
@@ -189,7 +200,7 @@ class ExportInferenceGraphTest(tf.test.TestCase, parameterized.TestCase):
saved_model_path = os.path.join(output_directory, 'saved_model')
detect_fn = tf.saved_model.load(saved_model_path)
image = self.get_dummy_input(input_type)
- detections = detect_fn(image)
+ detections = detect_fn(tf.constant(image))
detection_fields = fields.DetectionResultFields
self.assertAllClose(detections[detection_fields.detection_boxes],
@@ -203,12 +214,64 @@ class ExportInferenceGraphTest(tf.test.TestCase, parameterized.TestCase):
[[1, 2], [2, 1]])
self.assertAllClose(detections[detection_fields.num_detections], [2, 1])
+ @parameterized.parameters(
+ {'use_default_serving': True},
+ {'use_default_serving': False}
+ )
+ def test_export_saved_model_and_run_inference_with_side_inputs(
+ self, input_type='image_tensor', use_default_serving=True):
+ tmp_dir = self.get_temp_dir()
+ self._save_checkpoint_from_mock_model(tmp_dir)
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ exporter_lib_v2.INPUT_BUILDER_UTIL_MAP['model_build'] = mock_builder
+ output_directory = os.path.join(tmp_dir, 'output')
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ exporter_lib_v2.export_inference_graph(
+ input_type=input_type,
+ pipeline_config=pipeline_config,
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory,
+ use_side_inputs=True,
+ side_input_shapes='1/2,2',
+ side_input_names='side_inp_1,side_inp_2',
+ side_input_types='tf.float32,tf.uint8')
+
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ detect_fn = tf.saved_model.load(saved_model_path)
+ detect_fn_sig = detect_fn.signatures['serving_default']
+ image = tf.constant(self.get_dummy_input(input_type))
+ side_input_1 = np.ones((1,), dtype=np.float32)
+ side_input_2 = np.ones((2, 2), dtype=np.uint8)
+ if use_default_serving:
+ detections = detect_fn_sig(input_tensor=image,
+ side_inp_1=tf.constant(side_input_1),
+ side_inp_2=tf.constant(side_input_2))
+ else:
+ detections = detect_fn(image,
+ tf.constant(side_input_1),
+ tf.constant(side_input_2))
+
+ detection_fields = fields.DetectionResultFields
+ self.assertAllClose(detections[detection_fields.detection_boxes],
+ [[[0.0, 0.0, 0.5, 0.5],
+ [0.5, 0.5, 0.8, 0.8]],
+ [[0.5, 0.5, 1.0, 1.0],
+ [0.0, 0.0, 0.0, 0.0]]])
+ self.assertAllClose(detections[detection_fields.detection_scores],
+ [[400.7, 400.6], [400.9, 400.0]])
+ self.assertAllClose(detections[detection_fields.detection_classes],
+ [[1, 2], [2, 1]])
+ self.assertAllClose(detections[detection_fields.num_detections], [2, 1])
+
def test_export_checkpoint_and_run_inference_with_image(self):
tmp_dir = self.get_temp_dir()
self._save_checkpoint_from_mock_model(tmp_dir, conv_weight_scalar=2.0)
with mock.patch.object(
model_builder, 'build', autospec=True) as mock_builder:
mock_builder.return_value = FakeModel()
+ exporter_lib_v2.INPUT_BUILDER_UTIL_MAP['model_build'] = mock_builder
output_directory = os.path.join(tmp_dir, 'output')
pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
exporter_lib_v2.export_inference_graph(
@@ -235,6 +298,83 @@ class ExportInferenceGraphTest(tf.test.TestCase, parameterized.TestCase):
[[150 + 0.7, 150 + 0.6], [150 + 0.9, 150 + 0.0]])
+class DetectionFromImageAndBoxModuleTest(tf.test.TestCase):
+
+ def get_dummy_input(self, input_type):
+ """Get dummy input for the given input type."""
+
+ if input_type == 'image_tensor' or input_type == 'image_and_boxes_tensor':
+ return np.zeros((1, 20, 20, 3), dtype=np.uint8)
+ if input_type == 'float_image_tensor':
+ return np.zeros((1, 20, 20, 3), dtype=np.float32)
+ elif input_type == 'encoded_image_string_tensor':
+ image = Image.new('RGB', (20, 20))
+ byte_io = io.BytesIO()
+ image.save(byte_io, 'PNG')
+ return [byte_io.getvalue()]
+ elif input_type == 'tf_example':
+ image_tensor = tf.zeros((20, 20, 3), dtype=tf.uint8)
+ encoded_jpeg = tf.image.encode_jpeg(tf.constant(image_tensor)).numpy()
+ example = tf.train.Example(
+ features=tf.train.Features(
+ feature={
+ 'image/encoded':
+ dataset_util.bytes_feature(encoded_jpeg),
+ 'image/format':
+ dataset_util.bytes_feature(six.b('jpeg')),
+ 'image/source_id':
+ dataset_util.bytes_feature(six.b('image_id')),
+ })).SerializeToString()
+ return [example]
+
+ def _save_checkpoint_from_mock_model(self,
+ checkpoint_dir,
+ conv_weight_scalar=6.0):
+ mock_model = FakeModel(conv_weight_scalar)
+ fake_image = tf.zeros(shape=[1, 10, 10, 3], dtype=tf.float32)
+ preprocessed_inputs, true_image_shapes = mock_model.preprocess(fake_image)
+ predictions = mock_model.predict(preprocessed_inputs, true_image_shapes)
+ mock_model.postprocess(predictions, true_image_shapes)
+
+ ckpt = tf.train.Checkpoint(model=mock_model)
+ exported_checkpoint_manager = tf.train.CheckpointManager(
+ ckpt, checkpoint_dir, max_to_keep=1)
+ exported_checkpoint_manager.save(checkpoint_number=0)
+
+ def test_export_saved_model_and_run_inference_for_segmentation(
+ self, input_type='image_and_boxes_tensor'):
+ tmp_dir = self.get_temp_dir()
+ self._save_checkpoint_from_mock_model(tmp_dir)
+
+ with mock.patch.object(
+ model_builder, 'build', autospec=True) as mock_builder:
+ mock_builder.return_value = FakeModel()
+ exporter_lib_v2.INPUT_BUILDER_UTIL_MAP['model_build'] = mock_builder
+ output_directory = os.path.join(tmp_dir, 'output')
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ exporter_lib_v2.export_inference_graph(
+ input_type=input_type,
+ pipeline_config=pipeline_config,
+ trained_checkpoint_dir=tmp_dir,
+ output_directory=output_directory)
+
+ saved_model_path = os.path.join(output_directory, 'saved_model')
+ detect_fn = tf.saved_model.load(saved_model_path)
+ image = self.get_dummy_input(input_type)
+ boxes = tf.constant([
+ [
+ [0.0, 0.0, 0.5, 0.5],
+ [0.5, 0.5, 0.8, 0.8],
+ ],
+ ])
+ detections = detect_fn(tf.constant(image), boxes)
+
+ detection_fields = fields.DetectionResultFields
+ self.assertIn(detection_fields.detection_masks, detections)
+ self.assertListEqual(
+ list(detections[detection_fields.detection_masks].shape), [1, 2, 16])
+
+
if __name__ == '__main__':
tf.enable_v2_behavior()
tf.test.main()
diff --git a/research/object_detection/exporter_lib_v2.py b/research/object_detection/exporter_lib_v2.py
index a7ecb45adb14f1b20c2291a3cf67376ad07194eb..fd10187cf4e61543c552628bf29279b48f4323c8 100644
--- a/research/object_detection/exporter_lib_v2.py
+++ b/research/object_detection/exporter_lib_v2.py
@@ -15,7 +15,9 @@
# ==============================================================================
"""Functions to export object detection inference graph."""
+import ast
import os
+
import tensorflow.compat.v2 as tf
from object_detection.builders import model_builder
from object_detection.core import standard_fields as fields
@@ -23,6 +25,11 @@ from object_detection.data_decoders import tf_example_decoder
from object_detection.utils import config_util
+INPUT_BUILDER_UTIL_MAP = {
+ 'model_build': model_builder.build,
+}
+
+
def _decode_image(encoded_image_string_tensor):
image_tensor = tf.image.decode_image(encoded_image_string_tensor,
channels=3)
@@ -37,31 +44,87 @@ def _decode_tf_example(tf_example_string_tensor):
return image_tensor
+def _combine_side_inputs(side_input_shapes='',
+ side_input_types='',
+ side_input_names=''):
+ """Zips the side inputs together.
+
+ Args:
+ side_input_shapes: forward-slash-separated list of comma-separated lists
+ describing input shapes.
+ side_input_types: comma-separated list of the types of the inputs.
+ side_input_names: comma-separated list of the names of the inputs.
+
+ Returns:
+ a zipped list of side input tuples.
+ """
+ side_input_shapes = [
+ ast.literal_eval('[' + x + ']') for x in side_input_shapes.split('/')
+ ]
+ side_input_types = eval('[' + side_input_types + ']') # pylint: disable=eval-used
+ side_input_names = side_input_names.split(',')
+ return zip(side_input_shapes, side_input_types, side_input_names)
+
+
class DetectionInferenceModule(tf.Module):
"""Detection Inference Module."""
- def __init__(self, detection_model):
+ def __init__(self, detection_model,
+ use_side_inputs=False,
+ zipped_side_inputs=None):
"""Initializes a module for detection.
Args:
- detection_model: The detection model to use for inference.
+ detection_model: the detection model to use for inference.
+ use_side_inputs: whether to use side inputs.
+ zipped_side_inputs: the zipped side inputs.
"""
self._model = detection_model
- def _run_inference_on_images(self, image):
+ def _get_side_input_signature(self, zipped_side_inputs):
+ sig = []
+ side_input_names = []
+ for info in zipped_side_inputs:
+ sig.append(tf.TensorSpec(shape=info[0],
+ dtype=info[1],
+ name=info[2]))
+ side_input_names.append(info[2])
+ return sig
+
+ def _get_side_names_from_zip(self, zipped_side_inputs):
+ return [side[2] for side in zipped_side_inputs]
+
+ def _preprocess_input(self, batch_input, decode_fn):
+ # Input preprocessing happends on the CPU. We don't need to use the device
+ # placement as it is automatically handled by TF.
+ def _decode_and_preprocess(single_input):
+ image = decode_fn(single_input)
+ image = tf.cast(image, tf.float32)
+ image, true_shape = self._model.preprocess(image[tf.newaxis, :, :, :])
+ return image[0], true_shape[0]
+
+ images, true_shapes = tf.map_fn(
+ _decode_and_preprocess,
+ elems=batch_input,
+ parallel_iterations=32,
+ back_prop=False,
+ fn_output_signature=(tf.float32, tf.int32))
+ return images, true_shapes
+
+ def _run_inference_on_images(self, images, true_shapes, **kwargs):
"""Cast image to float and run inference.
Args:
- image: uint8 Tensor of shape [1, None, None, 3]
+ images: float32 Tensor of shape [None, None, None, 3].
+ true_shapes: int32 Tensor of form [batch, 3]
+ **kwargs: additional keyword arguments.
+
Returns:
Tensor dictionary holding detections.
"""
label_id_offset = 1
-
- image = tf.cast(image, tf.float32)
- image, shapes = self._model.preprocess(image)
- prediction_dict = self._model.predict(image, shapes)
- detections = self._model.postprocess(prediction_dict, shapes)
+ prediction_dict = self._model.predict(images, true_shapes, **kwargs)
+ detections = self._model.postprocess(prediction_dict, true_shapes)
classes_field = fields.DetectionResultFields.detection_classes
detections[classes_field] = (
tf.cast(detections[classes_field], tf.float32) + label_id_offset)
@@ -75,11 +138,44 @@ class DetectionInferenceModule(tf.Module):
class DetectionFromImageModule(DetectionInferenceModule):
"""Detection Inference Module for image inputs."""
- @tf.function(
- input_signature=[
- tf.TensorSpec(shape=[1, None, None, 3], dtype=tf.uint8)])
- def __call__(self, input_tensor):
- return self._run_inference_on_images(input_tensor)
+ def __init__(self, detection_model,
+ use_side_inputs=False,
+ zipped_side_inputs=None):
+ """Initializes a module for detection.
+
+ Args:
+ detection_model: the detection model to use for inference.
+ use_side_inputs: whether to use side inputs.
+ zipped_side_inputs: the zipped side inputs.
+ """
+ if zipped_side_inputs is None:
+ zipped_side_inputs = []
+ sig = [tf.TensorSpec(shape=[1, None, None, 3],
+ dtype=tf.uint8,
+ name='input_tensor')]
+ if use_side_inputs:
+ sig.extend(self._get_side_input_signature(zipped_side_inputs))
+ self._side_input_names = self._get_side_names_from_zip(zipped_side_inputs)
+
+ def call_func(input_tensor, *side_inputs):
+ kwargs = dict(zip(self._side_input_names, side_inputs))
+ images, true_shapes = self._preprocess_input(input_tensor, lambda x: x)
+ return self._run_inference_on_images(images, true_shapes, **kwargs)
+
+ self.__call__ = tf.function(call_func, input_signature=sig)
+
+ # TODO(kaushikshiv): Check if omitting the signature also works.
+ super(DetectionFromImageModule, self).__init__(detection_model,
+ use_side_inputs,
+ zipped_side_inputs)
+
+
+def get_true_shapes(input_tensor):
+ input_shape = tf.shape(input_tensor)
+ batch = input_shape[0]
+ image_shape = input_shape[1:]
+ true_shapes = tf.tile(image_shape[tf.newaxis, :], [batch, 1])
+ return true_shapes
class DetectionFromFloatImageModule(DetectionInferenceModule):
@@ -87,53 +183,40 @@ class DetectionFromFloatImageModule(DetectionInferenceModule):
@tf.function(
input_signature=[
- tf.TensorSpec(shape=[1, None, None, 3], dtype=tf.float32)])
+ tf.TensorSpec(shape=[None, None, None, 3], dtype=tf.float32)])
def __call__(self, input_tensor):
- return self._run_inference_on_images(input_tensor)
+ images, true_shapes = self._preprocess_input(input_tensor, lambda x: x)
+ return self._run_inference_on_images(images,
+ true_shapes)
class DetectionFromEncodedImageModule(DetectionInferenceModule):
"""Detection Inference Module for encoded image string inputs."""
- @tf.function(input_signature=[tf.TensorSpec(shape=[1], dtype=tf.string)])
+ @tf.function(input_signature=[tf.TensorSpec(shape=[None], dtype=tf.string)])
def __call__(self, input_tensor):
- with tf.device('cpu:0'):
- image = tf.map_fn(
- _decode_image,
- elems=input_tensor,
- dtype=tf.uint8,
- parallel_iterations=32,
- back_prop=False)
- return self._run_inference_on_images(image)
+ images, true_shapes = self._preprocess_input(input_tensor, _decode_image)
+ return self._run_inference_on_images(images, true_shapes)
class DetectionFromTFExampleModule(DetectionInferenceModule):
"""Detection Inference Module for TF.Example inputs."""
- @tf.function(input_signature=[tf.TensorSpec(shape=[1], dtype=tf.string)])
+ @tf.function(input_signature=[tf.TensorSpec(shape=[None], dtype=tf.string)])
def __call__(self, input_tensor):
- with tf.device('cpu:0'):
- image = tf.map_fn(
- _decode_tf_example,
- elems=input_tensor,
- dtype=tf.uint8,
- parallel_iterations=32,
- back_prop=False)
- return self._run_inference_on_images(image)
-
-DETECTION_MODULE_MAP = {
- 'image_tensor': DetectionFromImageModule,
- 'encoded_image_string_tensor':
- DetectionFromEncodedImageModule,
- 'tf_example': DetectionFromTFExampleModule,
- 'float_image_tensor': DetectionFromFloatImageModule
-}
+ images, true_shapes = self._preprocess_input(input_tensor,
+ _decode_tf_example)
+ return self._run_inference_on_images(images, true_shapes)
def export_inference_graph(input_type,
pipeline_config,
trained_checkpoint_dir,
- output_directory):
+ output_directory,
+ use_side_inputs=False,
+ side_input_shapes='',
+ side_input_types='',
+ side_input_names=''):
"""Exports inference graph for the model specified in the pipeline config.
This function creates `output_directory` if it does not already exist,
@@ -147,14 +230,20 @@ def export_inference_graph(input_type,
pipeline_config: pipeline_pb2.TrainAndEvalPipelineConfig proto.
trained_checkpoint_dir: Path to the trained checkpoint file.
output_directory: Path to write outputs.
+ use_side_inputs: boolean that determines whether side inputs should be
+ included in the input signature.
+ side_input_shapes: forward-slash-separated list of comma-separated lists
+ describing input shapes.
+ side_input_types: comma-separated list of the types of the inputs.
+ side_input_names: comma-separated list of the names of the inputs.
Raises:
ValueError: if input_type is invalid.
"""
output_checkpoint_directory = os.path.join(output_directory, 'checkpoint')
output_saved_model_directory = os.path.join(output_directory, 'saved_model')
- detection_model = model_builder.build(pipeline_config.model,
- is_training=False)
+ detection_model = INPUT_BUILDER_UTIL_MAP['model_build'](
+ pipeline_config.model, is_training=False)
ckpt = tf.train.Checkpoint(
model=detection_model)
@@ -164,7 +253,18 @@ def export_inference_graph(input_type,
if input_type not in DETECTION_MODULE_MAP:
raise ValueError('Unrecognized `input_type`')
- detection_module = DETECTION_MODULE_MAP[input_type](detection_model)
+ if use_side_inputs and input_type != 'image_tensor':
+ raise ValueError('Side inputs supported for image_tensor input type only.')
+
+ zipped_side_inputs = []
+ if use_side_inputs:
+ zipped_side_inputs = _combine_side_inputs(side_input_shapes,
+ side_input_types,
+ side_input_names)
+
+ detection_module = DETECTION_MODULE_MAP[input_type](detection_model,
+ use_side_inputs,
+ list(zipped_side_inputs))
# Getting the concrete function traces the graph and forces variables to
# be constructed --- only after this can we save the checkpoint and
# saved model.
@@ -180,3 +280,78 @@ def export_inference_graph(input_type,
signatures=concrete_function)
config_util.save_pipeline_config(pipeline_config, output_directory)
+
+
+class DetectionFromImageAndBoxModule(DetectionInferenceModule):
+ """Detection Inference Module for image with bounding box inputs.
+
+ The saved model will require two inputs (image and normalized boxes) and run
+ per-box mask prediction. To be compatible with this exporter, the detection
+ model has to implement a called predict_masks_from_boxes(
+ prediction_dict, true_image_shapes, provided_boxes, **params), where
+ - prediciton_dict is a dict returned by the predict method.
+ - true_image_shapes is a tensor of size [batch_size, 3], containing the
+ true shape of each image in case it is padded.
+ - provided_boxes is a [batch_size, num_boxes, 4] size tensor containing
+ boxes specified in normalized coordinates.
+ """
+
+ def __init__(self,
+ detection_model,
+ use_side_inputs=False,
+ zipped_side_inputs=None):
+ """Initializes a module for detection.
+
+ Args:
+ detection_model: the detection model to use for inference.
+ use_side_inputs: whether to use side inputs.
+ zipped_side_inputs: the zipped side inputs.
+ """
+ assert hasattr(detection_model, 'predict_masks_from_boxes')
+ super(DetectionFromImageAndBoxModule,
+ self).__init__(detection_model, use_side_inputs, zipped_side_inputs)
+
+ def _run_segmentation_on_images(self, image, boxes, **kwargs):
+ """Run segmentation on images with provided boxes.
+
+ Args:
+ image: uint8 Tensor of shape [1, None, None, 3].
+ boxes: float32 tensor of shape [1, None, 4] containing normalized box
+ coordinates.
+ **kwargs: additional keyword arguments.
+
+ Returns:
+ Tensor dictionary holding detections (including masks).
+ """
+ label_id_offset = 1
+
+ image = tf.cast(image, tf.float32)
+ image, shapes = self._model.preprocess(image)
+ prediction_dict = self._model.predict(image, shapes, **kwargs)
+ detections = self._model.predict_masks_from_boxes(prediction_dict, shapes,
+ boxes)
+ classes_field = fields.DetectionResultFields.detection_classes
+ detections[classes_field] = (
+ tf.cast(detections[classes_field], tf.float32) + label_id_offset)
+
+ for key, val in detections.items():
+ detections[key] = tf.cast(val, tf.float32)
+
+ return detections
+
+ @tf.function(input_signature=[
+ tf.TensorSpec(shape=[1, None, None, 3], dtype=tf.uint8),
+ tf.TensorSpec(shape=[1, None, 4], dtype=tf.float32)
+ ])
+ def __call__(self, input_tensor, boxes):
+ return self._run_segmentation_on_images(input_tensor, boxes)
+
+
+DETECTION_MODULE_MAP = {
+ 'image_tensor': DetectionFromImageModule,
+ 'encoded_image_string_tensor':
+ DetectionFromEncodedImageModule,
+ 'tf_example': DetectionFromTFExampleModule,
+ 'float_image_tensor': DetectionFromFloatImageModule,
+ 'image_and_boxes_tensor': DetectionFromImageAndBoxModule,
+}
diff --git a/research/object_detection/exporter_main_v2.py b/research/object_detection/exporter_main_v2.py
index a2ba8456039d4584e5998d619f36747d58018418..39630b65d2467ff759c3af4b4171fe8fd1733397 100644
--- a/research/object_detection/exporter_main_v2.py
+++ b/research/object_detection/exporter_main_v2.py
@@ -31,6 +31,11 @@ specified option.
* `tf_example`: Accepts a 1-D string tensor of shape [None] containing
serialized TFExample protos. Image resolutions are expected to be the same
if more than 1 image is provided.
+ * `image_and_boxes_tensor`: Accepts a 4-D image tensor of size
+ [1, None, None, 3] and a boxes tensor of size [1, None, 4] of normalized
+ bounding boxes. To be able to support this option, the model needs
+ to implement a predict_masks_from_boxes method. See the documentation
+ for DetectionFromImageAndBoxModule for details.
and the following output nodes returned by the model.postprocess(..):
* `num_detections`: Outputs float32 tensors of the form [batch]
@@ -50,6 +55,10 @@ python exporter_main_v2.py \
--pipeline_config_path path/to/ssd_inception_v2.config \
--trained_checkpoint_dir path/to/checkpoint \
--output_directory path/to/exported_model_directory
+ --use_side_inputs True/False \
+ --side_input_shapes dim_0,dim_1,...dim_a/.../dim_0,dim_1,...,dim_z \
+ --side_input_names name_a,name_b,...,name_c \
+ --side_input_types type_1,type_2
The expected output would be in the directory
path/to/exported_model_directory (which is created if it does not exist)
@@ -80,6 +89,13 @@ python exporter_main_v2.py \
} \
} \
}"
+
+If side inputs are desired, the following arguments could be appended
+(the example below is for Context R-CNN).
+ --use_side_inputs True \
+ --side_input_shapes 1,2000,2057/1 \
+ --side_input_names context_features,valid_context_size \
+ --side_input_types tf.float32,tf.int32
"""
from absl import app
from absl import flags
@@ -96,7 +112,8 @@ FLAGS = flags.FLAGS
flags.DEFINE_string('input_type', 'image_tensor', 'Type of input node. Can be '
'one of [`image_tensor`, `encoded_image_string_tensor`, '
- '`tf_example`, `float_image_tensor`]')
+ '`tf_example`, `float_image_tensor`, '
+ '`image_and_boxes_tensor`]')
flags.DEFINE_string('pipeline_config_path', None,
'Path to a pipeline_pb2.TrainEvalPipelineConfig config '
'file.')
@@ -106,6 +123,27 @@ flags.DEFINE_string('output_directory', None, 'Path to write outputs.')
flags.DEFINE_string('config_override', '',
'pipeline_pb2.TrainEvalPipelineConfig '
'text proto to override pipeline_config_path.')
+flags.DEFINE_boolean('use_side_inputs', False,
+ 'If True, uses side inputs as well as image inputs.')
+flags.DEFINE_string('side_input_shapes', '',
+ 'If use_side_inputs is True, this explicitly sets '
+ 'the shape of the side input tensors to a fixed size. The '
+ 'dimensions are to be provided as a comma-separated list '
+ 'of integers. A value of -1 can be used for unknown '
+ 'dimensions. A `/` denotes a break, starting the shape of '
+ 'the next side input tensor. This flag is required if '
+ 'using side inputs.')
+flags.DEFINE_string('side_input_types', '',
+ 'If use_side_inputs is True, this explicitly sets '
+ 'the type of the side input tensors. The '
+ 'dimensions are to be provided as a comma-separated list '
+ 'of types, each of `string`, `integer`, or `float`. '
+ 'This flag is required if using side inputs.')
+flags.DEFINE_string('side_input_names', '',
+ 'If use_side_inputs is True, this explicitly sets '
+ 'the names of the side input tensors required by the model '
+ 'assuming the names will be a comma-separated list of '
+ 'strings. This flag is required if using side inputs.')
flags.mark_flag_as_required('pipeline_config_path')
flags.mark_flag_as_required('trained_checkpoint_dir')
@@ -119,7 +157,8 @@ def main(_):
text_format.Merge(FLAGS.config_override, pipeline_config)
exporter_lib_v2.export_inference_graph(
FLAGS.input_type, pipeline_config, FLAGS.trained_checkpoint_dir,
- FLAGS.output_directory)
+ FLAGS.output_directory, FLAGS.use_side_inputs, FLAGS.side_input_shapes,
+ FLAGS.side_input_types, FLAGS.side_input_names)
if __name__ == '__main__':
diff --git a/research/object_detection/g3doc/deepmac.md b/research/object_detection/g3doc/deepmac.md
new file mode 100644
index 0000000000000000000000000000000000000000..effffbbba6d06bbfdc1ab4b23a10a5d533c733d7
--- /dev/null
+++ b/research/object_detection/g3doc/deepmac.md
@@ -0,0 +1,97 @@
+# DeepMAC model
+
+
+
+**DeepMAC** (Deep Mask heads Above CenterNet) is a neural network architecture
+that is designed for the partially supervised instance segmentation task. For
+details see the
+[The surprising impact of mask-head architecture on novel class segmentation](https://arxiv.org/abs/2104.00613)
+paper. The figure below shows improved mask predictions for unseen classes as we
+use better mask-head architectures.
+
+
+
+
+
+Just by using better mask-head architectures (no extra losses or modules) we
+achieve state-of-the-art performance in the partially supervised instance
+segmentation task.
+
+## Code structure
+
+* `deepmac_meta_arch.py` implements our main architecture, DeepMAC, on top of
+ the CenterNet detection architecture.
+* The proto message `DeepMACMaskEstimation` in `center_net.proto` controls the
+ configuration of the mask head used.
+* The field `allowed_masked_classes_ids` controls which classes recieve mask
+ supervision during training.
+* Mask R-CNN based ablations in the paper are implemented in the
+ [TF model garden](../../../official/vision/beta/projects/deepmac_maskrcnn)
+ code base.
+
+## Prerequisites
+
+1. Follow [TF2 install instructions](tf2.md) to install Object Detection API.
+2. Generate COCO dataset by using
+ [create_coco_tf_record.py](../../../official/vision/beta/data/create_coco_tf_record.py)
+
+## Configurations
+
+We provide pre-defined configs which can be run as a
+[TF2 training pipeline](tf2_training_and_evaluation.md). Each of these
+configurations needs to be passed as the `pipeline_config_path` argument to the
+`object_detection/model_main_tf2.py` binary. Note that the `512x512` resolution
+models require a TPU `v3-32` and the `1024x1024` resolution models require a TPU
+`v3-128` to train. The configs can be found in the [configs/tf2](../configs/tf2)
+directory. In the table below `X->Y` indicates that we train with masks from `X`
+and evaluate with masks from `Y`. Performance is measured on the `coco-val2017`
+set.
+
+### Partially supervised models
+
+Resolution | Mask head | Train->Eval | Config name | Mask mAP
+:--------- | :------------ | :------------- | :------------------------------------------------- | -------:
+512x512 | Hourglass-52 | VOC -> Non-VOC | `center_net_deepmac_512x512_voc_only.config` | 32.5
+1024x1024 | Hourglass-100 | VOC -> Non-VOC | `center_net_deepmac_1024x1024_voc_only.config` | 35.5
+1024x1024 | Hourglass-100 | Non-VOC -> VOC | `center_net_deepmac_1024x1024_non_voc_only.config` | 39.1
+
+### Fully supervised models
+
+Here we report the Mask mAP averaged over all COCO classes on the `test-dev2017`
+set .
+
+Resolution | Mask head | Config name | Mask mAP
+:--------- | :------------ | :----------------------------------------- | -------:
+1024x1024 | Hourglass-100 | `center_net_deepmac_1024x1024_coco.config` | 39.4
+
+## Demos
+
+* [DeepMAC Colab](../colab_tutorials/deepmac_colab.ipynb) lets you run a
+ pre-trained DeepMAC model on user-specified boxes. Note that you are not
+ restricted to COCO classes!
+* [iWildCam Notebook](https://www.kaggle.com/vighneshbgoogle/iwildcam-visualize-instance-masks)
+ to visualize instance masks generated by DeepMAC on the iWildCam dataset.
+
+## Pre-trained models
+
+* [COCO Checkpoint](http://download.tensorflow.org/models/object_detection/tf2/20210329/deepmac_1024x1024_coco17.tar.gz) -
+ Takes as input Image + Boxes and produces per-box instance masks as output.
+
+## See also
+
+* [Mask RCNN code](https://github.com/tensorflow/models/tree/master/official/vision/beta/projects/deepmac_maskrcnn)
+ in TF Model garden code base.
+* Project website - [git.io/deepmac](https://git.io/deepmac)
+
+## Citation
+
+```
+@misc{birodkar2021surprising,
+ title={The surprising impact of mask-head architecture on novel class segmentation},
+ author={Vighnesh Birodkar and Zhichao Lu and Siyang Li and Vivek Rathod and Jonathan Huang},
+ year={2021},
+ eprint={2104.00613},
+ archivePrefix={arXiv},
+ primaryClass={cs.CV}
+}
+```
diff --git a/research/object_detection/g3doc/img/mask_improvement.png b/research/object_detection/g3doc/img/mask_improvement.png
new file mode 100644
index 0000000000000000000000000000000000000000..3e35501b8b59083ba60a87ab917f1919c06cef27
Binary files /dev/null and b/research/object_detection/g3doc/img/mask_improvement.png differ
diff --git a/research/object_detection/g3doc/release_notes.md b/research/object_detection/g3doc/release_notes.md
index f69727d5d8547e908b6de76b43a678943c5bee09..21512397c9991e74bbe7a009e5cae40e11de63c6 100644
--- a/research/object_detection/g3doc/release_notes.md
+++ b/research/object_detection/g3doc/release_notes.md
@@ -1,5 +1,12 @@
# Release Notes
+### September 3rd, 2020
+
+TF2 OD API models can now be converted to TensorFlow Lite! Only SSD models
+currently supported. See documentation.
+
+**Thanks to contributors**: Sachin Joglekar
+
### July 10th, 2020
We are happy to announce that the TF OD API officially supports TF2! Our release
@@ -40,6 +47,18 @@ Sergi Caelles Prat, Shan Yang, Sudheendra Vijayanarasimhan, Tina Tian, Tomer
Kaftan, Vighnesh Birodkar, Vishnu Banna, Vivek Rathod, Yanhui Liang, Yiming Shi,
Yixin Shi, Yu-hui Chen, Zhichao Lu.
+### June 26th, 2020
+
+We have released SSDLite with MobileDet GPU backbone, which achieves 17% mAP
+higher than the MobileNetV2 SSDLite (27.5 mAP vs 23.5 mAP) on a NVIDIA Jetson
+Xavier at comparable latency (3.2ms vs 3.3ms).
+
+Along with the model definition, we are also releasing model checkpoints trained
+on the COCO dataset.
+
+Thanks to contributors: Yongzhe Wang, Bo Chen, Hanxiao Liu, Le An
+(NVIDIA), Yu-Te Cheng (NVIDIA), Oliver Knieps (NVIDIA), and Josh Park (NVIDIA).
+
### June 17th, 2020
We have released [Context R-CNN](https://arxiv.org/abs/1912.03538), a model that
diff --git a/research/object_detection/g3doc/running_on_mobile_tensorflowlite.md b/research/object_detection/g3doc/running_on_mobile_tensorflowlite.md
index 379652e34cb2241d6294679548c988e8916510bc..0acbf5dd34a5de1311b4dfb7fd0966c34c895ef1 100644
--- a/research/object_detection/g3doc/running_on_mobile_tensorflowlite.md
+++ b/research/object_detection/g3doc/running_on_mobile_tensorflowlite.md
@@ -51,22 +51,23 @@ will output the frozen graph that we can input to TensorFlow Lite directly and
is the one we’ll be using.
Next we’ll use TensorFlow Lite to get the optimized model by using
-[TOCO](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/toco),
+[TfLite Converter](https://www.tensorflow.org/lite/convert),
the TensorFlow Lite Optimizing Converter. This will convert the resulting frozen
graph (tflite_graph.pb) to the TensorFlow Lite flatbuffer format (detect.tflite)
via the following command. For a quantized model, run this from the tensorflow/
directory:
```shell
-bazel run -c opt tensorflow/lite/toco:toco -- \
---input_file=$OUTPUT_DIR/tflite_graph.pb \
+bazel run -c opt tensorflow/lite/python:tflite_convert -- \
+--enable_v1_converter \
+--graph_def_file=$OUTPUT_DIR/tflite_graph.pb \
--output_file=$OUTPUT_DIR/detect.tflite \
--input_shapes=1,300,300,3 \
--input_arrays=normalized_input_image_tensor \
--output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3' \
--inference_type=QUANTIZED_UINT8 \
--mean_values=128 \
---std_values=128 \
+--std_dev_values=128 \
--change_concat_input_ranges=false \
--allow_custom_ops
```
@@ -84,8 +85,9 @@ parameters and can be run via the TensorFlow Lite interpreter on the Android
device. For a floating point model, run this from the tensorflow/ directory:
```shell
-bazel run -c opt tensorflow/lite/toco:toco -- \
---input_file=$OUTPUT_DIR/tflite_graph.pb \
+bazel run -c opt tensorflow/lite/python:tflite_convert -- \
+--enable_v1_converter \
+--graph_def_file=$OUTPUT_DIR/tflite_graph.pb \
--output_file=$OUTPUT_DIR/detect.tflite \
--input_shapes=1,300,300,3 \
--input_arrays=normalized_input_image_tensor \
diff --git a/research/object_detection/g3doc/running_on_mobile_tf2.md b/research/object_detection/g3doc/running_on_mobile_tf2.md
new file mode 100644
index 0000000000000000000000000000000000000000..efa335c17b83590e76ada787b3ea76e3e97e66d9
--- /dev/null
+++ b/research/object_detection/g3doc/running_on_mobile_tf2.md
@@ -0,0 +1,145 @@
+# Running TF2 Detection API Models on mobile
+
+[](https://github.com/tensorflow/tensorflow/releases/tag/v2.4.0)
+[](https://www.python.org/downloads/release/python-360/)
+
+**NOTE:** This document talks about the *SSD* models in the detection zoo. For
+details on our (experimental) CenterNet support, see
+[this notebook](../colab_tutorials/centernet_on_device.ipynb).
+
+[TensorFlow Lite](https://www.tensorflow.org/mobile/tflite/)(TFLite) is
+TensorFlow’s lightweight solution for mobile and embedded devices. It enables
+on-device machine learning inference with low latency and a small binary size.
+TensorFlow Lite uses many techniques for this such as quantized kernels that
+allow smaller and faster (fixed-point math) models.
+
+This document shows how elgible models from the
+[TF2 Detection zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md)
+can be converted for inference with TFLite.
+
+For an end-to-end Python guide on how to fine-tune an SSD model for mobile
+inference, look at
+[this Colab](../colab_tutorials/eager_few_shot_od_training_tflite.ipynb).
+
+**NOTE:** TFLite currently only supports **SSD Architectures** (excluding
+EfficientDet) for boxes-based detection. Support for EfficientDet is coming
+soon.
+
+The output model has the following inputs & outputs:
+
+```
+One input:
+ image: a float32 tensor of shape[1, height, width, 3] containing the
+ *normalized* input image.
+ NOTE: See the `preprocess` function defined in the feature extractor class
+ in the object_detection/models directory.
+
+Four Outputs:
+ detection_boxes: a float32 tensor of shape [1, num_boxes, 4] with box
+ locations
+ detection_classes: a float32 tensor of shape [1, num_boxes]
+ with class indices
+ detection_scores: a float32 tensor of shape [1, num_boxes]
+ with class scores
+ num_boxes: a float32 tensor of size 1 containing the number of detected boxes
+```
+
+There are two steps to TFLite conversion:
+
+### Step 1: Export TFLite inference graph
+
+This step generates an intermediate SavedModel that can be used with the
+[TFLite Converter](https://www.tensorflow.org/lite/convert) via commandline or
+Python API.
+
+To use the script:
+
+```bash
+# From the tensorflow/models/research/ directory
+python object_detection/export_tflite_graph_tf2.py \
+ --pipeline_config_path path/to/ssd_model/pipeline.config \
+ --trained_checkpoint_dir path/to/ssd_model/checkpoint \
+ --output_directory path/to/exported_model_directory
+```
+
+Use `--help` with the above script to get the full list of supported parameters.
+These can fine-tune accuracy and speed for your model.
+
+### Step 2: Convert to TFLite
+
+Use the [TensorFlow Lite Converter](https://www.tensorflow.org/lite/convert) to
+convert the `SavedModel` to TFLite. Note that you need to use `from_saved_model`
+for TFLite conversion with the Python API.
+
+You can also leverage
+[Post-training Quantization](https://www.tensorflow.org/lite/performance/post_training_quantization)
+to
+[optimize performance](https://www.tensorflow.org/lite/performance/model_optimization)
+and obtain a smaller model. Note that this is only possible from the *Python
+API*. Be sure to use a
+[representative dataset](https://www.tensorflow.org/lite/performance/post_training_quantization#full_integer_quantization)
+and set the following options on the converter:
+
+```python
+converter.optimizations = [tf.lite.Optimize.DEFAULT]
+converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8,
+ tf.lite.OpsSet.TFLITE_BUILTINS]
+converter.representative_dataset = <...>
+```
+
+## Running our model on Android
+
+To run our TensorFlow Lite model on device, we will use Android Studio to build
+and run the TensorFlow Lite detection example with the new model. The example is
+found in the
+[TensorFlow examples repository](https://github.com/tensorflow/examples) under
+`/lite/examples/object_detection`. The example can be built with
+[Android Studio](https://developer.android.com/studio/index.html), and requires
+the
+[Android SDK with build tools](https://developer.android.com/tools/revisions/build-tools.html)
+that support API >= 21. Additional details are available on the
+[TensorFlow Lite example page](https://github.com/tensorflow/examples/tree/master/lite/examples/object_detection/android).
+
+Next we need to point the app to our new detect.tflite file and give it the
+names of our new labels. Specifically, we will copy our TensorFlow Lite
+flatbuffer to the app assets directory with the following command:
+
+```shell
+mkdir $TF_EXAMPLES/lite/examples/object_detection/android/app/src/main/assets
+cp /tmp/tflite/detect.tflite \
+ $TF_EXAMPLES/lite/examples/object_detection/android/app/src/main/assets
+```
+
+You will also need to copy your new labelmap labelmap.txt to the assets
+directory.
+
+We will now edit the gradle build file to use these assets. First, open the
+`build.gradle` file
+`$TF_EXAMPLES/lite/examples/object_detection/android/app/build.gradle`. Comment
+out the model download script to avoid your assets being overwritten:
+
+```shell
+// apply from:'download_model.gradle'
+```
+
+If your model is named `detect.tflite`, and your labels file `labelmap.txt`, the
+example will use them automatically as long as they've been properly copied into
+the base assets directory. If you need to use a custom path or filename, open up
+the
+$TF_EXAMPLES/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/demo/DetectorActivity.java
+file in a text editor and find the definition of TF_OD_API_LABELS_FILE. Update
+this path to point to your new label map file: "labels_list.txt". Note that if
+your model is quantized, the flag TF_OD_API_IS_QUANTIZED is set to true, and if
+your model is floating point, the flag TF_OD_API_IS_QUANTIZED is set to false.
+This new section of DetectorActivity.java should now look as follows for a
+quantized model:
+
+```java
+ private static final boolean TF_OD_API_IS_QUANTIZED = true;
+ private static final String TF_OD_API_MODEL_FILE = "detect.tflite";
+ private static final String TF_OD_API_LABELS_FILE = "labels_list.txt";
+```
+
+Once you’ve copied the TensorFlow Lite model and edited the gradle build script
+to not use the downloaded assets, you can build and deploy the app using the
+usual Android Studio build process.
diff --git a/research/object_detection/g3doc/tf1.md b/research/object_detection/g3doc/tf1.md
index f973ef38c3a70dbd2bc996b90141ae50eed84713..f1577600963e1af99b6fdd192028a12622240cc2 100644
--- a/research/object_detection/g3doc/tf1.md
+++ b/research/object_detection/g3doc/tf1.md
@@ -35,7 +35,7 @@ cd models/research
protoc object_detection/protos/*.proto --python_out=.
# Install TensorFlow Object Detection API.
cp object_detection/packages/tf1/setup.py .
-python -m pip install .
+python -m pip install --use-feature=2020-resolver .
```
```bash
@@ -73,6 +73,8 @@ the [Model Zoo](tf1_detection_zoo.md).
Supported object detection evaluation protocols
*
TPU compatible detection pipelines
+*
+ Training and evaluation guide (CPU, GPU, or TPU)
## Extras:
diff --git a/research/object_detection/g3doc/tf1_detection_zoo.md b/research/object_detection/g3doc/tf1_detection_zoo.md
index 15416bb7aec947933097af1ef26205f3b8d60e20..6f002cd09bb8208e02af3d4960212470cd291555 100644
--- a/research/object_detection/g3doc/tf1_detection_zoo.md
+++ b/research/object_detection/g3doc/tf1_detection_zoo.md
@@ -67,8 +67,7 @@ Some remarks on frozen inference graphs:
metrics.
* Our frozen inference graphs are generated using the
[v1.12.0](https://github.com/tensorflow/tensorflow/tree/v1.12.0) release
- version of TensorFlow and we do not guarantee that these will work with
- other versions; this being said, each frozen inference graph can be
+ version of TensorFlow; this being said, each frozen inference graph can be
regenerated using your current version of TensorFlow by re-running the
[exporter](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/exporting_models.md),
pointing it at the model directory as well as the corresponding config file
diff --git a/research/object_detection/g3doc/tf2.md b/research/object_detection/g3doc/tf2.md
index 3860c7ea2f302816fd1c38068862b5a8f837ff53..d45d157f3b94cff36a3a76fd18a9fe7b4f7a2d9c 100644
--- a/research/object_detection/g3doc/tf2.md
+++ b/research/object_detection/g3doc/tf2.md
@@ -35,7 +35,7 @@ cd models/research
protoc object_detection/protos/*.proto --python_out=.
# Install TensorFlow Object Detection API.
cp object_detection/packages/tf2/setup.py .
-python -m pip install .
+python -m pip install --use-feature=2020-resolver .
```
```bash
@@ -55,6 +55,9 @@ python object_detection/builders/model_builder_tf2_test.py
* Inference -
[Run inference with models from the zoo](../colab_tutorials/inference_tf2_colab.ipynb)
+* Few Shot Learning for Mobile Inference -
+ [Fine-tune a pre-trained detector for use with TensorFlow Lite](../colab_tutorials/eager_few_shot_od_training_tflite.ipynb)
+
## Training and Evaluation
@@ -80,3 +83,5 @@ We provide a large collection of models that are trained on COCO 2017 in the
Supported object detection evaluation protocols
*
TPU compatible detection pipelines
+*
+ Training and evaluation guide (CPU, GPU, or TPU)
diff --git a/research/object_detection/g3doc/tf2_detection_zoo.md b/research/object_detection/g3doc/tf2_detection_zoo.md
index b129b80003a9093f9bfa565f8b86625a3de6b747..249e5fa1bf8d0f304043050c5c65299ca705b390 100644
--- a/research/object_detection/g3doc/tf2_detection_zoo.md
+++ b/research/object_detection/g3doc/tf2_detection_zoo.md
@@ -15,6 +15,8 @@ They are also useful for initializing your models when training on novel
datasets. You can try this out on our few-shot training
[colab](../colab_tutorials/eager_few_shot_od_training_tf2_colab.ipynb).
+Please look at [this guide](running_on_mobile_tf2.md) for mobile inference.
+
Finally, if you would like to train these models from scratch, you can find the
@@ -23,15 +25,17 @@ model configs in this [directory](../configs/tf2) (also in the linked
Model name | Speed (ms) | COCO mAP | Outputs
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------: | :----------: | :-----:
-[CenterNet HourGlass104 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_hg104_512x512_coco17_tpu-8.tar.gz) | 70 | 41.6 | Boxes
+[CenterNet HourGlass104 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200713/centernet_hg104_512x512_coco17_tpu-8.tar.gz) | 70 | 41.9 | Boxes
[CenterNet HourGlass104 Keypoints 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_hg104_512x512_kpts_coco17_tpu-32.tar.gz) | 76 | 40.0/61.4 | Boxes/Keypoints
-[CenterNet HourGlass104 1024x1024](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_hg104_1024x1024_coco17_tpu-32.tar.gz) | 197 | 43.5 | Boxes
+[CenterNet HourGlass104 1024x1024](http://download.tensorflow.org/models/object_detection/tf2/20200713/centernet_hg104_1024x1024_coco17_tpu-32.tar.gz) | 197 | 44.5 | Boxes
[CenterNet HourGlass104 Keypoints 1024x1024](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_hg104_1024x1024_kpts_coco17_tpu-32.tar.gz) | 211 | 42.8/64.5 | Boxes/Keypoints
[CenterNet Resnet50 V1 FPN 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_resnet50_v1_fpn_512x512_coco17_tpu-8.tar.gz) | 27 | 31.2 | Boxes
[CenterNet Resnet50 V1 FPN Keypoints 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_resnet50_v1_fpn_512x512_kpts_coco17_tpu-8.tar.gz) | 30 | 29.3/50.7 | Boxes/Keypoints
[CenterNet Resnet101 V1 FPN 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_resnet101_v1_fpn_512x512_coco17_tpu-8.tar.gz) | 34 | 34.2 | Boxes
[CenterNet Resnet50 V2 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_resnet50_v2_512x512_coco17_tpu-8.tar.gz) | 27 | 29.5 | Boxes
[CenterNet Resnet50 V2 Keypoints 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200711/centernet_resnet50_v2_512x512_kpts_coco17_tpu-8.tar.gz) | 30 | 27.6/48.2 | Boxes/Keypoints
+[CenterNet MobileNetV2 FPN 512x512](http://download.tensorflow.org/models/object_detection/tf2/20210210/centernet_mobilenetv2fpn_512x512_coco17_od.tar.gz) | 6 | 23.4 | Boxes
+[CenterNet MobileNetV2 FPN Keypoints 512x512](http://download.tensorflow.org/models/object_detection/tf2/20210210/centernet_mobilenetv2fpn_512x512_coco17_kpts.tar.gz) | 6 | 41.7 | Keypoints
[EfficientDet D0 512x512](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d0_coco17_tpu-32.tar.gz) | 39 | 33.6 | Boxes
[EfficientDet D1 640x640](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d1_coco17_tpu-32.tar.gz) | 54 | 38.4 | Boxes
[EfficientDet D2 768x768](http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d2_coco17_tpu-32.tar.gz) | 67 | 41.8 | Boxes
diff --git a/research/object_detection/inference/infer_detections.py b/research/object_detection/inference/infer_detections.py
index 3579142fc9add1c439164f0d938d4efe9089c692..7bc662f4297436024dd2f9632fdd92133116d482 100644
--- a/research/object_detection/inference/infer_detections.py
+++ b/research/object_detection/inference/infer_detections.py
@@ -17,7 +17,7 @@ r"""Infers detections on a TFRecord of TFExamples given an inference graph.
Example usage:
./infer_detections \
--input_tfrecord_paths=/path/to/input/tfrecord1,/path/to/input/tfrecord2 \
- --output_tfrecord_path_prefix=/path/to/output/detections.tfrecord \
+ --output_tfrecord_path=/path/to/output/detections.tfrecord \
--inference_graph=/path/to/frozen_weights_inference_graph.pb
The output is a TFRecord of TFExamples. Each TFExample from the input is first
diff --git a/research/object_detection/inputs.py b/research/object_detection/inputs.py
index 59ed22d0531476c7550e7cabb3961480fe7e4f4d..bdb219b08cc1e2adaa1aa58eec02588733cd3c4d 100644
--- a/research/object_detection/inputs.py
+++ b/research/object_detection/inputs.py
@@ -68,8 +68,25 @@ def _multiclass_scores_or_one_hot_labels(multiclass_scores,
return tf.cond(tf.size(multiclass_scores) > 0, true_fn, false_fn)
-def _convert_labeled_classes_to_k_hot(groundtruth_labeled_classes, num_classes):
- """Returns k-hot encoding of the labeled classes."""
+def convert_labeled_classes_to_k_hot(groundtruth_labeled_classes,
+ num_classes,
+ map_empty_to_ones=False):
+ """Returns k-hot encoding of the labeled classes.
+
+ If map_empty_to_ones is enabled and the input labeled_classes is empty,
+ this function assumes all classes are exhaustively labeled, thus returning
+ an all-one encoding.
+
+ Args:
+ groundtruth_labeled_classes: a Tensor holding a sparse representation of
+ labeled classes.
+ num_classes: an integer representing the number of classes
+ map_empty_to_ones: boolean (default: False). Set this to be True to default
+ to an all-ones result if given an empty `groundtruth_labeled_classes`.
+ Returns:
+ A k-hot (and 0-indexed) tensor representation of
+ `groundtruth_labeled_classes`.
+ """
# If the input labeled_classes is empty, it assumes all classes are
# exhaustively labeled, thus returning an all-one encoding.
@@ -82,13 +99,16 @@ def _convert_labeled_classes_to_k_hot(groundtruth_labeled_classes, num_classes):
def false_fn():
return tf.ones(num_classes, dtype=tf.float32)
- return tf.cond(tf.size(groundtruth_labeled_classes) > 0, true_fn, false_fn)
+ if map_empty_to_ones:
+ return tf.cond(tf.size(groundtruth_labeled_classes) > 0, true_fn, false_fn)
+ return true_fn()
def _remove_unrecognized_classes(class_ids, unrecognized_label):
"""Returns class ids with unrecognized classes filtered out."""
- recognized_indices = tf.where(tf.greater(class_ids, unrecognized_label))
+ recognized_indices = tf.squeeze(
+ tf.where(tf.greater(class_ids, unrecognized_label)), -1)
return tf.gather(class_ids, recognized_indices)
@@ -197,51 +217,54 @@ def transform_input_data(tensor_dict,
"""
out_tensor_dict = tensor_dict.copy()
- labeled_classes_field = fields.InputDataFields.groundtruth_labeled_classes
- image_classes_field = fields.InputDataFields.groundtruth_image_classes
+ input_fields = fields.InputDataFields
+ labeled_classes_field = input_fields.groundtruth_labeled_classes
+ image_classes_field = input_fields.groundtruth_image_classes
+ verified_neg_classes_field = input_fields.groundtruth_verified_neg_classes
+ not_exhaustive_field = input_fields.groundtruth_not_exhaustive_classes
+
if (labeled_classes_field in out_tensor_dict and
image_classes_field in out_tensor_dict):
raise KeyError('groundtruth_labeled_classes and groundtruth_image_classes'
'are provided by the decoder, but only one should be set.')
- if labeled_classes_field in out_tensor_dict:
- # tf_example_decoder casts unrecognized labels to -1. Remove these
- # unrecognized labels before converting labeled_classes to k-hot vector.
- out_tensor_dict[labeled_classes_field] = _remove_unrecognized_classes(
- out_tensor_dict[labeled_classes_field], unrecognized_label=-1)
- out_tensor_dict[labeled_classes_field] = _convert_labeled_classes_to_k_hot(
- out_tensor_dict[labeled_classes_field], num_classes)
-
- if image_classes_field in out_tensor_dict:
- out_tensor_dict[labeled_classes_field] = _convert_labeled_classes_to_k_hot(
- out_tensor_dict[image_classes_field], num_classes)
-
- if fields.InputDataFields.multiclass_scores in out_tensor_dict:
+ for field, map_empty_to_ones in [
+ (labeled_classes_field, True),
+ (image_classes_field, True),
+ (verified_neg_classes_field, False),
+ (not_exhaustive_field, False)]:
+ if field in out_tensor_dict:
+ out_tensor_dict[field] = _remove_unrecognized_classes(
+ out_tensor_dict[field], unrecognized_label=-1)
+ out_tensor_dict[field] = convert_labeled_classes_to_k_hot(
+ out_tensor_dict[field], num_classes, map_empty_to_ones)
+
+ if input_fields.multiclass_scores in out_tensor_dict:
out_tensor_dict[
- fields.InputDataFields
+ input_fields
.multiclass_scores] = _multiclass_scores_or_one_hot_labels(
- out_tensor_dict[fields.InputDataFields.multiclass_scores],
- out_tensor_dict[fields.InputDataFields.groundtruth_boxes],
- out_tensor_dict[fields.InputDataFields.groundtruth_classes],
+ out_tensor_dict[input_fields.multiclass_scores],
+ out_tensor_dict[input_fields.groundtruth_boxes],
+ out_tensor_dict[input_fields.groundtruth_classes],
num_classes)
- if fields.InputDataFields.groundtruth_boxes in out_tensor_dict:
+ if input_fields.groundtruth_boxes in out_tensor_dict:
out_tensor_dict = util_ops.filter_groundtruth_with_nan_box_coordinates(
out_tensor_dict)
out_tensor_dict = util_ops.filter_unrecognized_classes(out_tensor_dict)
if retain_original_image:
- out_tensor_dict[fields.InputDataFields.original_image] = tf.cast(
- image_resizer_fn(out_tensor_dict[fields.InputDataFields.image],
+ out_tensor_dict[input_fields.original_image] = tf.cast(
+ image_resizer_fn(out_tensor_dict[input_fields.image],
None)[0], tf.uint8)
- if fields.InputDataFields.image_additional_channels in out_tensor_dict:
- channels = out_tensor_dict[fields.InputDataFields.image_additional_channels]
- out_tensor_dict[fields.InputDataFields.image] = tf.concat(
- [out_tensor_dict[fields.InputDataFields.image], channels], axis=2)
+ if input_fields.image_additional_channels in out_tensor_dict:
+ channels = out_tensor_dict[input_fields.image_additional_channels]
+ out_tensor_dict[input_fields.image] = tf.concat(
+ [out_tensor_dict[input_fields.image], channels], axis=2)
if retain_original_image_additional_channels:
out_tensor_dict[
- fields.InputDataFields.image_additional_channels] = tf.cast(
+ input_fields.image_additional_channels] = tf.cast(
image_resizer_fn(channels, None)[0], tf.uint8)
# Apply data augmentation ops.
@@ -249,7 +272,7 @@ def transform_input_data(tensor_dict,
out_tensor_dict = data_augmentation_fn(out_tensor_dict)
# Apply model preprocessing ops and resize instance masks.
- image = out_tensor_dict[fields.InputDataFields.image]
+ image = out_tensor_dict[input_fields.image]
preprocessed_resized_image, true_image_shape = model_preprocess_fn(
tf.expand_dims(tf.cast(image, dtype=tf.float32), axis=0))
@@ -262,35 +285,43 @@ def transform_input_data(tensor_dict,
tf.to_float(new_width) / tf.to_float(true_image_shape[0, 1])
])
- if fields.InputDataFields.groundtruth_boxes in tensor_dict:
- bboxes = out_tensor_dict[fields.InputDataFields.groundtruth_boxes]
+ if input_fields.groundtruth_boxes in tensor_dict:
+ bboxes = out_tensor_dict[input_fields.groundtruth_boxes]
boxlist = box_list.BoxList(bboxes)
realigned_bboxes = box_list_ops.change_coordinate_frame(boxlist, im_box)
realigned_boxes_tensor = realigned_bboxes.get()
valid_boxes_tensor = assert_or_prune_invalid_boxes(realigned_boxes_tensor)
out_tensor_dict[
- fields.InputDataFields.groundtruth_boxes] = valid_boxes_tensor
+ input_fields.groundtruth_boxes] = valid_boxes_tensor
- if fields.InputDataFields.groundtruth_keypoints in tensor_dict:
- keypoints = out_tensor_dict[fields.InputDataFields.groundtruth_keypoints]
+ if input_fields.groundtruth_keypoints in tensor_dict:
+ keypoints = out_tensor_dict[input_fields.groundtruth_keypoints]
realigned_keypoints = keypoint_ops.change_coordinate_frame(keypoints,
im_box)
out_tensor_dict[
- fields.InputDataFields.groundtruth_keypoints] = realigned_keypoints
- flds_gt_kpt = fields.InputDataFields.groundtruth_keypoints
- flds_gt_kpt_vis = fields.InputDataFields.groundtruth_keypoint_visibilities
- flds_gt_kpt_weights = fields.InputDataFields.groundtruth_keypoint_weights
+ input_fields.groundtruth_keypoints] = realigned_keypoints
+ flds_gt_kpt = input_fields.groundtruth_keypoints
+ flds_gt_kpt_vis = input_fields.groundtruth_keypoint_visibilities
+ flds_gt_kpt_weights = input_fields.groundtruth_keypoint_weights
if flds_gt_kpt_vis not in out_tensor_dict:
out_tensor_dict[flds_gt_kpt_vis] = tf.ones_like(
out_tensor_dict[flds_gt_kpt][:, :, 0],
dtype=tf.bool)
+ flds_gt_kpt_depth = fields.InputDataFields.groundtruth_keypoint_depths
+ flds_gt_kpt_depth_weight = (
+ fields.InputDataFields.groundtruth_keypoint_depth_weights)
+ if flds_gt_kpt_depth in out_tensor_dict:
+ out_tensor_dict[flds_gt_kpt_depth] = out_tensor_dict[flds_gt_kpt_depth]
+ out_tensor_dict[flds_gt_kpt_depth_weight] = out_tensor_dict[
+ flds_gt_kpt_depth_weight]
+
out_tensor_dict[flds_gt_kpt_weights] = (
keypoint_ops.keypoint_weights_from_visibilities(
out_tensor_dict[flds_gt_kpt_vis],
keypoint_type_weight))
- dp_surface_coords_fld = fields.InputDataFields.groundtruth_dp_surface_coords
+ dp_surface_coords_fld = input_fields.groundtruth_dp_surface_coords
if dp_surface_coords_fld in tensor_dict:
dp_surface_coords = out_tensor_dict[dp_surface_coords_fld]
realigned_dp_surface_coords = densepose_ops.change_coordinate_frame(
@@ -300,60 +331,60 @@ def transform_input_data(tensor_dict,
if use_bfloat16:
preprocessed_resized_image = tf.cast(
preprocessed_resized_image, tf.bfloat16)
- if fields.InputDataFields.context_features in out_tensor_dict:
- out_tensor_dict[fields.InputDataFields.context_features] = tf.cast(
- out_tensor_dict[fields.InputDataFields.context_features], tf.bfloat16)
- out_tensor_dict[fields.InputDataFields.image] = tf.squeeze(
+ if input_fields.context_features in out_tensor_dict:
+ out_tensor_dict[input_fields.context_features] = tf.cast(
+ out_tensor_dict[input_fields.context_features], tf.bfloat16)
+ out_tensor_dict[input_fields.image] = tf.squeeze(
preprocessed_resized_image, axis=0)
- out_tensor_dict[fields.InputDataFields.true_image_shape] = tf.squeeze(
+ out_tensor_dict[input_fields.true_image_shape] = tf.squeeze(
true_image_shape, axis=0)
- if fields.InputDataFields.groundtruth_instance_masks in out_tensor_dict:
- masks = out_tensor_dict[fields.InputDataFields.groundtruth_instance_masks]
+ if input_fields.groundtruth_instance_masks in out_tensor_dict:
+ masks = out_tensor_dict[input_fields.groundtruth_instance_masks]
_, resized_masks, _ = image_resizer_fn(image, masks)
if use_bfloat16:
resized_masks = tf.cast(resized_masks, tf.bfloat16)
out_tensor_dict[
- fields.InputDataFields.groundtruth_instance_masks] = resized_masks
+ input_fields.groundtruth_instance_masks] = resized_masks
zero_indexed_groundtruth_classes = out_tensor_dict[
- fields.InputDataFields.groundtruth_classes] - _LABEL_OFFSET
+ input_fields.groundtruth_classes] - _LABEL_OFFSET
if use_multiclass_scores:
out_tensor_dict[
- fields.InputDataFields.groundtruth_classes] = out_tensor_dict[
- fields.InputDataFields.multiclass_scores]
+ input_fields.groundtruth_classes] = out_tensor_dict[
+ input_fields.multiclass_scores]
else:
- out_tensor_dict[fields.InputDataFields.groundtruth_classes] = tf.one_hot(
+ out_tensor_dict[input_fields.groundtruth_classes] = tf.one_hot(
zero_indexed_groundtruth_classes, num_classes)
- out_tensor_dict.pop(fields.InputDataFields.multiclass_scores, None)
+ out_tensor_dict.pop(input_fields.multiclass_scores, None)
- if fields.InputDataFields.groundtruth_confidences in out_tensor_dict:
+ if input_fields.groundtruth_confidences in out_tensor_dict:
groundtruth_confidences = out_tensor_dict[
- fields.InputDataFields.groundtruth_confidences]
+ input_fields.groundtruth_confidences]
# Map the confidences to the one-hot encoding of classes
- out_tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
+ out_tensor_dict[input_fields.groundtruth_confidences] = (
tf.reshape(groundtruth_confidences, [-1, 1]) *
- out_tensor_dict[fields.InputDataFields.groundtruth_classes])
+ out_tensor_dict[input_fields.groundtruth_classes])
else:
groundtruth_confidences = tf.ones_like(
zero_indexed_groundtruth_classes, dtype=tf.float32)
- out_tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
- out_tensor_dict[fields.InputDataFields.groundtruth_classes])
+ out_tensor_dict[input_fields.groundtruth_confidences] = (
+ out_tensor_dict[input_fields.groundtruth_classes])
if merge_multiple_boxes:
merged_boxes, merged_classes, merged_confidences, _ = (
util_ops.merge_boxes_with_multiple_labels(
- out_tensor_dict[fields.InputDataFields.groundtruth_boxes],
+ out_tensor_dict[input_fields.groundtruth_boxes],
zero_indexed_groundtruth_classes,
groundtruth_confidences,
num_classes))
merged_classes = tf.cast(merged_classes, tf.float32)
- out_tensor_dict[fields.InputDataFields.groundtruth_boxes] = merged_boxes
- out_tensor_dict[fields.InputDataFields.groundtruth_classes] = merged_classes
- out_tensor_dict[fields.InputDataFields.groundtruth_confidences] = (
+ out_tensor_dict[input_fields.groundtruth_boxes] = merged_boxes
+ out_tensor_dict[input_fields.groundtruth_classes] = merged_classes
+ out_tensor_dict[input_fields.groundtruth_confidences] = (
merged_confidences)
- if fields.InputDataFields.groundtruth_boxes in out_tensor_dict:
- out_tensor_dict[fields.InputDataFields.num_groundtruth_boxes] = tf.shape(
- out_tensor_dict[fields.InputDataFields.groundtruth_boxes])[0]
+ if input_fields.groundtruth_boxes in out_tensor_dict:
+ out_tensor_dict[input_fields.num_groundtruth_boxes] = tf.shape(
+ out_tensor_dict[input_fields.groundtruth_boxes])[0]
return out_tensor_dict
@@ -397,113 +428,132 @@ def pad_input_data_to_static_shapes(tensor_dict,
max_num_context_features is not specified and context_features is in the
tensor dict.
"""
-
if not spatial_image_shape or spatial_image_shape == [-1, -1]:
height, width = None, None
else:
height, width = spatial_image_shape # pylint: disable=unpacking-non-sequence
+ input_fields = fields.InputDataFields
num_additional_channels = 0
- if fields.InputDataFields.image_additional_channels in tensor_dict:
+ if input_fields.image_additional_channels in tensor_dict:
num_additional_channels = shape_utils.get_dim_as_int(tensor_dict[
- fields.InputDataFields.image_additional_channels].shape[2])
+ input_fields.image_additional_channels].shape[2])
# We assume that if num_additional_channels > 0, then it has already been
# concatenated to the base image (but not the ground truth).
num_channels = 3
- if fields.InputDataFields.image in tensor_dict:
+ if input_fields.image in tensor_dict:
num_channels = shape_utils.get_dim_as_int(
- tensor_dict[fields.InputDataFields.image].shape[2])
+ tensor_dict[input_fields.image].shape[2])
if num_additional_channels:
if num_additional_channels >= num_channels:
raise ValueError(
'Image must be already concatenated with additional channels.')
- if (fields.InputDataFields.original_image in tensor_dict and
+ if (input_fields.original_image in tensor_dict and
shape_utils.get_dim_as_int(
- tensor_dict[fields.InputDataFields.original_image].shape[2]) ==
+ tensor_dict[input_fields.original_image].shape[2]) ==
num_channels):
raise ValueError(
'Image must be already concatenated with additional channels.')
- if fields.InputDataFields.context_features in tensor_dict and (
+ if input_fields.context_features in tensor_dict and (
max_num_context_features is None):
raise ValueError('max_num_context_features must be specified in the model '
'config if include_context is specified in the input '
'config')
padding_shapes = {
- fields.InputDataFields.image: [height, width, num_channels],
- fields.InputDataFields.original_image_spatial_shape: [2],
- fields.InputDataFields.image_additional_channels: [
+ input_fields.image: [height, width, num_channels],
+ input_fields.original_image_spatial_shape: [2],
+ input_fields.image_additional_channels: [
height, width, num_additional_channels
],
- fields.InputDataFields.source_id: [],
- fields.InputDataFields.filename: [],
- fields.InputDataFields.key: [],
- fields.InputDataFields.groundtruth_difficult: [max_num_boxes],
- fields.InputDataFields.groundtruth_boxes: [max_num_boxes, 4],
- fields.InputDataFields.groundtruth_classes: [max_num_boxes, num_classes],
- fields.InputDataFields.groundtruth_instance_masks: [
+ input_fields.source_id: [],
+ input_fields.filename: [],
+ input_fields.key: [],
+ input_fields.groundtruth_difficult: [max_num_boxes],
+ input_fields.groundtruth_boxes: [max_num_boxes, 4],
+ input_fields.groundtruth_classes: [max_num_boxes, num_classes],
+ input_fields.groundtruth_instance_masks: [
max_num_boxes, height, width
],
- fields.InputDataFields.groundtruth_is_crowd: [max_num_boxes],
- fields.InputDataFields.groundtruth_group_of: [max_num_boxes],
- fields.InputDataFields.groundtruth_area: [max_num_boxes],
- fields.InputDataFields.groundtruth_weights: [max_num_boxes],
- fields.InputDataFields.groundtruth_confidences: [
+ input_fields.groundtruth_is_crowd: [max_num_boxes],
+ input_fields.groundtruth_group_of: [max_num_boxes],
+ input_fields.groundtruth_area: [max_num_boxes],
+ input_fields.groundtruth_weights: [max_num_boxes],
+ input_fields.groundtruth_confidences: [
max_num_boxes, num_classes
],
- fields.InputDataFields.num_groundtruth_boxes: [],
- fields.InputDataFields.groundtruth_label_types: [max_num_boxes],
- fields.InputDataFields.groundtruth_label_weights: [max_num_boxes],
- fields.InputDataFields.true_image_shape: [3],
- fields.InputDataFields.groundtruth_image_classes: [num_classes],
- fields.InputDataFields.groundtruth_image_confidences: [num_classes],
- fields.InputDataFields.groundtruth_labeled_classes: [num_classes],
+ input_fields.num_groundtruth_boxes: [],
+ input_fields.groundtruth_label_types: [max_num_boxes],
+ input_fields.groundtruth_label_weights: [max_num_boxes],
+ input_fields.true_image_shape: [3],
+ input_fields.groundtruth_image_classes: [num_classes],
+ input_fields.groundtruth_image_confidences: [num_classes],
+ input_fields.groundtruth_labeled_classes: [num_classes],
}
- if fields.InputDataFields.original_image in tensor_dict:
- padding_shapes[fields.InputDataFields.original_image] = [
+ if input_fields.original_image in tensor_dict:
+ padding_shapes[input_fields.original_image] = [
height, width,
- shape_utils.get_dim_as_int(tensor_dict[fields.InputDataFields.
+ shape_utils.get_dim_as_int(tensor_dict[input_fields.
original_image].shape[2])
]
- if fields.InputDataFields.groundtruth_keypoints in tensor_dict:
+ if input_fields.groundtruth_keypoints in tensor_dict:
tensor_shape = (
- tensor_dict[fields.InputDataFields.groundtruth_keypoints].shape)
+ tensor_dict[input_fields.groundtruth_keypoints].shape)
padding_shape = [max_num_boxes,
shape_utils.get_dim_as_int(tensor_shape[1]),
shape_utils.get_dim_as_int(tensor_shape[2])]
- padding_shapes[fields.InputDataFields.groundtruth_keypoints] = padding_shape
- if fields.InputDataFields.groundtruth_keypoint_visibilities in tensor_dict:
- tensor_shape = tensor_dict[fields.InputDataFields.
+ padding_shapes[input_fields.groundtruth_keypoints] = padding_shape
+ if input_fields.groundtruth_keypoint_visibilities in tensor_dict:
+ tensor_shape = tensor_dict[input_fields.
groundtruth_keypoint_visibilities].shape
padding_shape = [max_num_boxes, shape_utils.get_dim_as_int(tensor_shape[1])]
- padding_shapes[fields.InputDataFields.
+ padding_shapes[input_fields.
groundtruth_keypoint_visibilities] = padding_shape
- if fields.InputDataFields.groundtruth_keypoint_weights in tensor_dict:
- tensor_shape = (
- tensor_dict[fields.InputDataFields.groundtruth_keypoint_weights].shape)
+ if fields.InputDataFields.groundtruth_keypoint_depths in tensor_dict:
+ tensor_shape = tensor_dict[fields.InputDataFields.
+ groundtruth_keypoint_depths].shape
padding_shape = [max_num_boxes, shape_utils.get_dim_as_int(tensor_shape[1])]
padding_shapes[fields.InputDataFields.
+ groundtruth_keypoint_depths] = padding_shape
+ padding_shapes[fields.InputDataFields.
+ groundtruth_keypoint_depth_weights] = padding_shape
+
+ if input_fields.groundtruth_keypoint_weights in tensor_dict:
+ tensor_shape = (
+ tensor_dict[input_fields.groundtruth_keypoint_weights].shape)
+ padding_shape = [max_num_boxes, shape_utils.get_dim_as_int(tensor_shape[1])]
+ padding_shapes[input_fields.
groundtruth_keypoint_weights] = padding_shape
- if fields.InputDataFields.groundtruth_dp_num_points in tensor_dict:
+ if input_fields.groundtruth_dp_num_points in tensor_dict:
padding_shapes[
- fields.InputDataFields.groundtruth_dp_num_points] = [max_num_boxes]
+ input_fields.groundtruth_dp_num_points] = [max_num_boxes]
padding_shapes[
- fields.InputDataFields.groundtruth_dp_part_ids] = [
+ input_fields.groundtruth_dp_part_ids] = [
max_num_boxes, max_dp_points]
padding_shapes[
- fields.InputDataFields.groundtruth_dp_surface_coords] = [
+ input_fields.groundtruth_dp_surface_coords] = [
max_num_boxes, max_dp_points, 4]
+ if input_fields.groundtruth_track_ids in tensor_dict:
+ padding_shapes[
+ input_fields.groundtruth_track_ids] = [max_num_boxes]
+
+ if input_fields.groundtruth_verified_neg_classes in tensor_dict:
+ padding_shapes[
+ input_fields.groundtruth_verified_neg_classes] = [num_classes]
+ if input_fields.groundtruth_not_exhaustive_classes in tensor_dict:
+ padding_shapes[
+ input_fields.groundtruth_not_exhaustive_classes] = [num_classes]
# Prepare for ContextRCNN related fields.
- if fields.InputDataFields.context_features in tensor_dict:
+ if input_fields.context_features in tensor_dict:
padding_shape = [max_num_context_features, context_feature_length]
- padding_shapes[fields.InputDataFields.context_features] = padding_shape
+ padding_shapes[input_fields.context_features] = padding_shape
tensor_shape = tf.shape(
tensor_dict[fields.InputDataFields.context_features])
@@ -511,9 +561,12 @@ def pad_input_data_to_static_shapes(tensor_dict,
padding_shapes[fields.InputDataFields.valid_context_size] = []
if fields.InputDataFields.context_feature_length in tensor_dict:
padding_shapes[fields.InputDataFields.context_feature_length] = []
+ if fields.InputDataFields.context_features_image_id_list in tensor_dict:
+ padding_shapes[fields.InputDataFields.context_features_image_id_list] = [
+ max_num_context_features]
- if fields.InputDataFields.is_annotated in tensor_dict:
- padding_shapes[fields.InputDataFields.is_annotated] = []
+ if input_fields.is_annotated in tensor_dict:
+ padding_shapes[input_fields.is_annotated] = []
padded_tensor_dict = {}
for tensor_name in tensor_dict:
@@ -522,10 +575,10 @@ def pad_input_data_to_static_shapes(tensor_dict,
# Make sure that the number of groundtruth boxes now reflects the
# padded/clipped tensors.
- if fields.InputDataFields.num_groundtruth_boxes in padded_tensor_dict:
- padded_tensor_dict[fields.InputDataFields.num_groundtruth_boxes] = (
+ if input_fields.num_groundtruth_boxes in padded_tensor_dict:
+ padded_tensor_dict[input_fields.num_groundtruth_boxes] = (
tf.minimum(
- padded_tensor_dict[fields.InputDataFields.num_groundtruth_boxes],
+ padded_tensor_dict[input_fields.num_groundtruth_boxes],
max_num_boxes))
return padded_tensor_dict
@@ -552,6 +605,8 @@ def augment_input_data(tensor_dict, data_augmentation_options):
in tensor_dict)
include_keypoint_visibilities = (
fields.InputDataFields.groundtruth_keypoint_visibilities in tensor_dict)
+ include_keypoint_depths = (
+ fields.InputDataFields.groundtruth_keypoint_depths in tensor_dict)
include_label_weights = (fields.InputDataFields.groundtruth_weights
in tensor_dict)
include_label_confidences = (fields.InputDataFields.groundtruth_confidences
@@ -571,7 +626,8 @@ def augment_input_data(tensor_dict, data_augmentation_options):
include_instance_masks=include_instance_masks,
include_keypoints=include_keypoints,
include_keypoint_visibilities=include_keypoint_visibilities,
- include_dense_pose=include_dense_pose))
+ include_dense_pose=include_dense_pose,
+ include_keypoint_depths=include_keypoint_depths))
tensor_dict[fields.InputDataFields.image] = tf.squeeze(
tensor_dict[fields.InputDataFields.image], axis=0)
return tensor_dict
@@ -593,6 +649,8 @@ def _get_labels_dict(input_dict):
fields.InputDataFields.groundtruth_confidences,
fields.InputDataFields.groundtruth_labeled_classes,
fields.InputDataFields.groundtruth_keypoints,
+ fields.InputDataFields.groundtruth_keypoint_depths,
+ fields.InputDataFields.groundtruth_keypoint_depth_weights,
fields.InputDataFields.groundtruth_instance_masks,
fields.InputDataFields.groundtruth_area,
fields.InputDataFields.groundtruth_is_crowd,
@@ -602,7 +660,10 @@ def _get_labels_dict(input_dict):
fields.InputDataFields.groundtruth_keypoint_weights,
fields.InputDataFields.groundtruth_dp_num_points,
fields.InputDataFields.groundtruth_dp_part_ids,
- fields.InputDataFields.groundtruth_dp_surface_coords
+ fields.InputDataFields.groundtruth_dp_surface_coords,
+ fields.InputDataFields.groundtruth_track_ids,
+ fields.InputDataFields.groundtruth_verified_neg_classes,
+ fields.InputDataFields.groundtruth_not_exhaustive_classes
]
for key in optional_label_keys:
@@ -673,6 +734,9 @@ def _get_features_dict(input_dict, include_source_id=False):
if fields.InputDataFields.valid_context_size in input_dict:
features[fields.InputDataFields.valid_context_size] = input_dict[
fields.InputDataFields.valid_context_size]
+ if fields.InputDataFields.context_features_image_id_list in input_dict:
+ features[fields.InputDataFields.context_features_image_id_list] = (
+ input_dict[fields.InputDataFields.context_features_image_id_list])
return features
@@ -762,6 +826,8 @@ def train_input(train_config, train_input_config,
DensePose surface coordinates. The format is (y, x, v, u), where (y, x)
are normalized image coordinates and (v, u) are normalized surface part
coordinates.
+ labels[fields.InputDataFields.groundtruth_track_ids] is a
+ [batch_size, num_boxes] int32 tensor with the track ID for each object.
Raises:
TypeError: if the `train_config`, `train_input_config` or `model_config`
@@ -853,7 +919,7 @@ def create_eval_input_fn(eval_config, eval_input_config, model_config):
def eval_input(eval_config, eval_input_config, model_config,
- model=None, params=None):
+ model=None, params=None, input_context=None):
"""Returns `features` and `labels` tensor dictionaries for evaluation.
Args:
@@ -863,6 +929,9 @@ def eval_input(eval_config, eval_input_config, model_config,
model: A pre-constructed Detection Model.
If None, one will be created from the config.
params: Parameter dictionary passed from the estimator.
+ input_context: optional, A tf.distribute.InputContext object used to
+ shard filenames and compute per-replica batch_size when this function
+ is being called per-replica.
Returns:
A tf.data.Dataset that holds (features, labels) tuple.
@@ -914,6 +983,8 @@ def eval_input(eval_config, eval_input_config, model_config,
DensePose surface coordinates. The format is (y, x, v, u), where (y, x)
are normalized image coordinates and (v, u) are normalized surface part
coordinates.
+ labels[fields.InputDataFields.groundtruth_track_ids] is a
+ [batch_size, num_boxes] int32 tensor with the track ID for each object.
Raises:
TypeError: if the `eval_config`, `eval_input_config` or `model_config`
@@ -981,6 +1052,7 @@ def eval_input(eval_config, eval_input_config, model_config,
eval_input_config,
batch_size=params['batch_size'] if params else eval_config.batch_size,
transform_input_data_fn=transform_and_pad_input_data_fn,
+ input_context=input_context,
reduce_to_frame_fn=reduce_to_frame_fn)
return dataset
@@ -1094,8 +1166,12 @@ def get_reduce_to_frame_fn(input_reader_config, is_training):
num_frames = tf.cast(
tf.shape(tensor_dict[fields.InputDataFields.source_id])[0],
dtype=tf.int32)
- frame_index = tf.random.uniform((), minval=0, maxval=num_frames,
- dtype=tf.int32)
+ if input_reader_config.frame_index == -1:
+ frame_index = tf.random.uniform((), minval=0, maxval=num_frames,
+ dtype=tf.int32)
+ else:
+ frame_index = tf.constant(input_reader_config.frame_index,
+ dtype=tf.int32)
out_tensor_dict = {}
for key in tensor_dict:
if key in fields.SEQUENCE_FIELDS:
diff --git a/research/object_detection/inputs_test.py b/research/object_detection/inputs_test.py
index ec44d04d81b4ff5097bdfd0b2670949e92f00a99..4716882e9a34dc1124d1ea881087ff8da70f28f5 100644
--- a/research/object_detection/inputs_test.py
+++ b/research/object_detection/inputs_test.py
@@ -61,7 +61,7 @@ def _get_configs_for_model(model_name):
configs, kwargs_dict=override_dict)
-def _get_configs_for_model_sequence_example(model_name):
+def _get_configs_for_model_sequence_example(model_name, frame_index=-1):
"""Returns configurations for model."""
fname = os.path.join(tf.resource_loader.get_data_files_path(),
'test_data/' + model_name + '.config')
@@ -74,7 +74,8 @@ def _get_configs_for_model_sequence_example(model_name):
override_dict = {
'train_input_path': data_path,
'eval_input_path': data_path,
- 'label_map_path': label_map_path
+ 'label_map_path': label_map_path,
+ 'frame_index': frame_index
}
return config_util.merge_external_params_with_configs(
configs, kwargs_dict=override_dict)
@@ -312,6 +313,87 @@ class InputFnTest(test_case.TestCase, parameterized.TestCase):
tf.float32,
labels[fields.InputDataFields.groundtruth_weights].dtype)
+ def test_context_rcnn_resnet50_eval_input_with_sequence_example_image_id_list(
+ self, eval_batch_size=8):
+ """Tests the eval input function for FasterRcnnResnet50."""
+ configs = _get_configs_for_model_sequence_example(
+ 'context_rcnn_camera_trap')
+ model_config = configs['model']
+ eval_config = configs['eval_config']
+ eval_config.batch_size = eval_batch_size
+ eval_input_config = configs['eval_input_configs'][0]
+ eval_input_config.load_context_image_ids = True
+ eval_input_fn = inputs.create_eval_input_fn(
+ eval_config, eval_input_config, model_config)
+ features, labels = _make_initializable_iterator(eval_input_fn()).get_next()
+ self.assertAllEqual([eval_batch_size, 640, 640, 3],
+ features[fields.InputDataFields.image].shape.as_list())
+ self.assertEqual(tf.float32, features[fields.InputDataFields.image].dtype)
+ self.assertAllEqual(
+ [eval_batch_size, 640, 640, 3],
+ features[fields.InputDataFields.original_image].shape.as_list())
+ self.assertEqual(tf.uint8,
+ features[fields.InputDataFields.original_image].dtype)
+ self.assertAllEqual([eval_batch_size],
+ features[inputs.HASH_KEY].shape.as_list())
+ self.assertEqual(tf.int32, features[inputs.HASH_KEY].dtype)
+ self.assertAllEqual(
+ [eval_batch_size, 100, 4],
+ labels[fields.InputDataFields.groundtruth_boxes].shape.as_list())
+ self.assertEqual(tf.float32,
+ labels[fields.InputDataFields.groundtruth_boxes].dtype)
+ self.assertAllEqual(
+ [eval_batch_size, 100, model_config.faster_rcnn.num_classes],
+ labels[fields.InputDataFields.groundtruth_classes].shape.as_list())
+ self.assertEqual(tf.float32,
+ labels[fields.InputDataFields.groundtruth_classes].dtype)
+ self.assertAllEqual(
+ [eval_batch_size, 100],
+ labels[fields.InputDataFields.groundtruth_weights].shape.as_list())
+ self.assertEqual(
+ tf.float32,
+ labels[fields.InputDataFields.groundtruth_weights].dtype)
+
+ def test_context_rcnn_resnet50_train_input_with_sequence_example_frame_index(
+ self, train_batch_size=8):
+ """Tests the training input function for FasterRcnnResnet50."""
+ configs = _get_configs_for_model_sequence_example(
+ 'context_rcnn_camera_trap', frame_index=2)
+ model_config = configs['model']
+ train_config = configs['train_config']
+ train_config.batch_size = train_batch_size
+ train_input_fn = inputs.create_train_input_fn(
+ train_config, configs['train_input_config'], model_config)
+ features, labels = _make_initializable_iterator(train_input_fn()).get_next()
+
+ self.assertAllEqual([train_batch_size, 640, 640, 3],
+ features[fields.InputDataFields.image].shape.as_list())
+ self.assertEqual(tf.float32, features[fields.InputDataFields.image].dtype)
+ self.assertAllEqual([train_batch_size],
+ features[inputs.HASH_KEY].shape.as_list())
+ self.assertEqual(tf.int32, features[inputs.HASH_KEY].dtype)
+ self.assertAllEqual(
+ [train_batch_size, 100, 4],
+ labels[fields.InputDataFields.groundtruth_boxes].shape.as_list())
+ self.assertEqual(tf.float32,
+ labels[fields.InputDataFields.groundtruth_boxes].dtype)
+ self.assertAllEqual(
+ [train_batch_size, 100, model_config.faster_rcnn.num_classes],
+ labels[fields.InputDataFields.groundtruth_classes].shape.as_list())
+ self.assertEqual(tf.float32,
+ labels[fields.InputDataFields.groundtruth_classes].dtype)
+ self.assertAllEqual(
+ [train_batch_size, 100],
+ labels[fields.InputDataFields.groundtruth_weights].shape.as_list())
+ self.assertEqual(tf.float32,
+ labels[fields.InputDataFields.groundtruth_weights].dtype)
+ self.assertAllEqual(
+ [train_batch_size, 100, model_config.faster_rcnn.num_classes],
+ labels[fields.InputDataFields.groundtruth_confidences].shape.as_list())
+ self.assertEqual(
+ tf.float32,
+ labels[fields.InputDataFields.groundtruth_confidences].dtype)
+
def test_ssd_inceptionV2_train_input(self):
"""Tests the training input function for SSDInceptionV2."""
configs = _get_configs_for_model('ssd_inception_v2_pets')
@@ -1338,6 +1420,49 @@ class DataTransformationFnTest(test_case.TestCase, parameterized.TestCase):
[[[0., 0., 0., 0.,], [0., 0., 0., 0.,]],
[[0.1, 0.1, 0.3, 0.4,], [0.6, 0.4, 0.6, 0.7,]]])
+ def test_groundtruth_keypoint_depths(self):
+ def graph_fn():
+ tensor_dict = {
+ fields.InputDataFields.image:
+ tf.constant(np.random.rand(100, 50, 3).astype(np.float32)),
+ fields.InputDataFields.groundtruth_boxes:
+ tf.constant(np.array([[.5, .5, 1, 1], [.0, .0, .5, .5]],
+ np.float32)),
+ fields.InputDataFields.groundtruth_classes:
+ tf.constant(np.array([1, 2], np.int32)),
+ fields.InputDataFields.groundtruth_keypoints:
+ tf.constant([[[0.1, 0.2], [0.3, 0.4]],
+ [[0.5, 0.6], [0.7, 0.8]]]),
+ fields.InputDataFields.groundtruth_keypoint_visibilities:
+ tf.constant([[True, False], [True, True]]),
+ fields.InputDataFields.groundtruth_keypoint_depths:
+ tf.constant([[1.0, 0.9], [0.8, 0.7]]),
+ fields.InputDataFields.groundtruth_keypoint_depth_weights:
+ tf.constant([[0.7, 0.8], [0.9, 1.0]]),
+ }
+
+ num_classes = 3
+ keypoint_type_weight = [1.0, 2.0]
+ input_transformation_fn = functools.partial(
+ inputs.transform_input_data,
+ model_preprocess_fn=_fake_resize50_preprocess_fn,
+ image_resizer_fn=_fake_image_resizer_fn,
+ num_classes=num_classes,
+ keypoint_type_weight=keypoint_type_weight)
+ transformed_inputs = input_transformation_fn(tensor_dict=tensor_dict)
+ return (transformed_inputs[
+ fields.InputDataFields.groundtruth_keypoint_depths],
+ transformed_inputs[
+ fields.InputDataFields.groundtruth_keypoint_depth_weights])
+
+ keypoint_depths, keypoint_depth_weights = self.execute_cpu(graph_fn, [])
+ self.assertAllClose(
+ keypoint_depths,
+ [[1.0, 0.9], [0.8, 0.7]])
+ self.assertAllClose(
+ keypoint_depth_weights,
+ [[0.7, 0.8], [0.9, 1.0]])
+
class PadInputDataToStaticShapesFnTest(test_case.TestCase):
@@ -1528,6 +1653,22 @@ class PadInputDataToStaticShapesFnTest(test_case.TestCase):
padded_tensor_dict[fields.InputDataFields.groundtruth_dp_surface_coords]
.shape.as_list(), [3, 200, 4])
+ def test_pad_input_data_to_static_shapes_for_trackid(self):
+ input_tensor_dict = {
+ fields.InputDataFields.groundtruth_track_ids:
+ tf.constant([0, 1], dtype=tf.int32),
+ }
+
+ padded_tensor_dict = inputs.pad_input_data_to_static_shapes(
+ tensor_dict=input_tensor_dict,
+ max_num_boxes=3,
+ num_classes=1,
+ spatial_image_shape=[128, 128])
+
+ self.assertAllEqual(
+ padded_tensor_dict[fields.InputDataFields.groundtruth_track_ids]
+ .shape.as_list(), [3])
+
def test_context_features(self):
context_memory_size = 8
context_feature_length = 10
diff --git a/research/object_detection/matchers/hungarian_matcher.py b/research/object_detection/matchers/hungarian_matcher.py
new file mode 100644
index 0000000000000000000000000000000000000000..63ee5d9f228a94406b1b3c1707eb493572749a91
--- /dev/null
+++ b/research/object_detection/matchers/hungarian_matcher.py
@@ -0,0 +1,58 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Hungarian bipartite matcher implementation."""
+
+import numpy as np
+from scipy.optimize import linear_sum_assignment
+
+import tensorflow.compat.v1 as tf
+from object_detection.core import matcher
+
+
+class HungarianBipartiteMatcher(matcher.Matcher):
+ """Wraps a Hungarian bipartite matcher into TensorFlow."""
+
+ def _match(self, similarity_matrix, valid_rows):
+ """Optimally bipartite matches a collection rows and columns.
+
+ Args:
+ similarity_matrix: Float tensor of shape [N, M] with pairwise similarity
+ where higher values mean more similar.
+ valid_rows: A boolean tensor of shape [N] indicating the rows that are
+ valid.
+
+ Returns:
+ match_results: int32 tensor of shape [M] with match_results[i]=-1
+ meaning that column i is not matched and otherwise that it is matched to
+ row match_results[i].
+ """
+ valid_row_sim_matrix = tf.gather(similarity_matrix,
+ tf.squeeze(tf.where(valid_rows), axis=-1))
+ distance_matrix = -1 * valid_row_sim_matrix
+
+ def numpy_wrapper(inputs):
+ def numpy_matching(input_matrix):
+ row_indices, col_indices = linear_sum_assignment(input_matrix)
+ match_results = np.full(input_matrix.shape[1], -1)
+ match_results[col_indices] = row_indices
+ return match_results.astype(np.int32)
+
+ return tf.numpy_function(numpy_matching, inputs, Tout=[tf.int32])
+
+ matching_result = tf.autograph.experimental.do_not_convert(
+ numpy_wrapper)([distance_matrix])
+
+ return tf.reshape(matching_result, [-1])
diff --git a/research/object_detection/matchers/hungarian_matcher_tf2_test.py b/research/object_detection/matchers/hungarian_matcher_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..bbac858a42db5ca53ab89b40d2fd95010d2d18fd
--- /dev/null
+++ b/research/object_detection/matchers/hungarian_matcher_tf2_test.py
@@ -0,0 +1,105 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for object_detection.core.bipartite_matcher."""
+import unittest
+import numpy as np
+import tensorflow.compat.v1 as tf
+
+from object_detection.utils import test_case
+from object_detection.utils import tf_version
+
+if tf_version.is_tf2():
+ from object_detection.matchers import hungarian_matcher # pylint: disable=g-import-not-at-top
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class HungarianBipartiteMatcherTest(test_case.TestCase):
+
+ def test_get_expected_matches_when_all_rows_are_valid(self):
+ similarity_matrix = np.array([[0.50, 0.1, 0.8], [0.15, 0.2, 0.3]],
+ dtype=np.float32)
+ valid_rows = np.ones([2], dtype=np.bool)
+ expected_match_results = [-1, 1, 0]
+
+ matcher = hungarian_matcher.HungarianBipartiteMatcher()
+ match_results_out = matcher.match(similarity_matrix, valid_rows=valid_rows)
+
+ self.assertAllEqual(match_results_out._match_results.numpy(),
+ expected_match_results)
+
+ def test_get_expected_matches_with_all_rows_be_default(self):
+ similarity_matrix = np.array([[0.50, 0.1, 0.8], [0.15, 0.2, 0.3]],
+ dtype=np.float32)
+ expected_match_results = [-1, 1, 0]
+
+ matcher = hungarian_matcher.HungarianBipartiteMatcher()
+ match_results_out = matcher.match(similarity_matrix)
+
+ self.assertAllEqual(match_results_out._match_results.numpy(),
+ expected_match_results)
+
+ def test_get_no_matches_with_zero_valid_rows(self):
+ similarity_matrix = np.array([[0.50, 0.1, 0.8], [0.15, 0.2, 0.3]],
+ dtype=np.float32)
+ valid_rows = np.zeros([2], dtype=np.bool)
+ expected_match_results = [-1, -1, -1]
+
+ matcher = hungarian_matcher.HungarianBipartiteMatcher()
+ match_results_out = matcher.match(similarity_matrix, valid_rows=valid_rows)
+
+ self.assertAllEqual(match_results_out._match_results.numpy(),
+ expected_match_results)
+
+ def test_get_expected_matches_with_only_one_valid_row(self):
+ similarity_matrix = np.array([[0.50, 0.1, 0.8], [0.15, 0.2, 0.3]],
+ dtype=np.float32)
+ valid_rows = np.array([True, False], dtype=np.bool)
+ expected_match_results = [-1, -1, 0]
+
+ matcher = hungarian_matcher.HungarianBipartiteMatcher()
+ match_results_out = matcher.match(similarity_matrix, valid_rows=valid_rows)
+
+ self.assertAllEqual(match_results_out._match_results.numpy(),
+ expected_match_results)
+
+ def test_get_expected_matches_with_only_one_valid_row_at_bottom(self):
+ similarity_matrix = np.array([[0.15, 0.2, 0.3], [0.50, 0.1, 0.8]],
+ dtype=np.float32)
+ valid_rows = np.array([False, True], dtype=np.bool)
+ expected_match_results = [-1, -1, 0]
+
+ matcher = hungarian_matcher.HungarianBipartiteMatcher()
+ match_results_out = matcher.match(similarity_matrix, valid_rows=valid_rows)
+
+ self.assertAllEqual(match_results_out._match_results.numpy(),
+ expected_match_results)
+
+ def test_get_expected_matches_with_two_valid_rows(self):
+ similarity_matrix = np.array([[0.15, 0.2, 0.3], [0.50, 0.1, 0.8],
+ [0.84, 0.32, 0.2]],
+ dtype=np.float32)
+ valid_rows = np.array([True, False, True], dtype=np.bool)
+ expected_match_results = [1, -1, 0]
+
+ matcher = hungarian_matcher.HungarianBipartiteMatcher()
+ match_results_out = matcher.match(similarity_matrix, valid_rows=valid_rows)
+
+ self.assertAllEqual(match_results_out._match_results.numpy(),
+ expected_match_results)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/meta_architectures/center_net_meta_arch.py b/research/object_detection/meta_architectures/center_net_meta_arch.py
index fc242c99020dc8db4819fb2ba4076e6c3913bd70..2131c530642ca41e4db3001492e3bc6be2123562 100644
--- a/research/object_detection/meta_architectures/center_net_meta_arch.py
+++ b/research/object_detection/meta_architectures/center_net_meta_arch.py
@@ -21,7 +21,6 @@
import abc
import collections
import functools
-import numpy as np
import tensorflow.compat.v1 as tf
import tensorflow.compat.v2 as tf2
@@ -32,6 +31,9 @@ from object_detection.core import model
from object_detection.core import standard_fields as fields
from object_detection.core import target_assigner as cn_assigner
from object_detection.utils import shape_utils
+from object_detection.utils import target_assigner_utils as ta_utils
+from object_detection.utils import tf_version
+
# Number of channels needed to predict size and offsets.
NUM_OFFSET_CHANNELS = 2
@@ -40,10 +42,6 @@ NUM_SIZE_CHANNELS = 2
# Error range for detecting peaks.
PEAK_EPSILON = 1e-6
-# Constants shared between all keypoint tasks.
-UNMATCHED_KEYPOINT_SCORE = 0.1
-KEYPOINT_CANDIDATE_SEARCH_SCALE = 0.3
-
class CenterNetFeatureExtractor(tf.keras.Model):
"""Base class for feature extractors for the CenterNet meta architecture.
@@ -118,9 +116,29 @@ class CenterNetFeatureExtractor(tf.keras.Model):
"""Ther number of feature outputs returned by the feature extractor."""
pass
+ @property
+ @abc.abstractmethod
+ def supported_sub_model_types(self):
+ """Valid sub model types supported by the get_sub_model function."""
+ pass
+
+ @abc.abstractmethod
+ def get_sub_model(self, sub_model_type):
+ """Returns the underlying keras model for the given sub_model_type.
+
+ This function is useful when we only want to get a subset of weights to
+ be restored from a checkpoint.
+
+ Args:
+ sub_model_type: string, the type of sub model. Currently, CenterNet
+ feature extractors support 'detection' and 'classification'.
+ """
+ pass
+
-def make_prediction_net(num_out_channels, kernel_size=3, num_filters=256,
- bias_fill=None):
+def make_prediction_net(num_out_channels, kernel_sizes=(3), num_filters=(256),
+ bias_fill=None, use_depthwise=False, name=None,
+ unit_height_conv=True):
"""Creates a network to predict the given number of output channels.
This function is intended to make the prediction heads for the CenterNet
@@ -128,29 +146,57 @@ def make_prediction_net(num_out_channels, kernel_size=3, num_filters=256,
Args:
num_out_channels: Number of output channels.
- kernel_size: The size of the conv kernel in the intermediate layer
- num_filters: The number of filters in the intermediate conv layer.
+ kernel_sizes: A list representing the sizes of the conv kernel in the
+ intermediate layer. Note that the length of the list indicates the number
+ of intermediate conv layers and it must be the same as the length of the
+ num_filters.
+ num_filters: A list representing the number of filters in the intermediate
+ conv layer. Note that the length of the list indicates the number of
+ intermediate conv layers.
bias_fill: If not None, is used to initialize the bias in the final conv
layer.
+ use_depthwise: If true, use SeparableConv2D to construct the Sequential
+ layers instead of Conv2D.
+ name: Optional name for the prediction net.
+ unit_height_conv: If True, Conv2Ds have asymmetric kernels with height=1.
Returns:
net: A keras module which when called on an input tensor of size
[batch_size, height, width, num_in_channels] returns an output
of size [batch_size, height, width, num_out_channels]
"""
+ if isinstance(kernel_sizes, int) and isinstance(num_filters, int):
+ kernel_sizes = [kernel_sizes]
+ num_filters = [num_filters]
+ assert len(kernel_sizes) == len(num_filters)
+ if use_depthwise:
+ conv_fn = tf.keras.layers.SeparableConv2D
+ else:
+ conv_fn = tf.keras.layers.Conv2D
- out_conv = tf.keras.layers.Conv2D(num_out_channels, kernel_size=1)
+ # We name the convolution operations explicitly because Keras, by default,
+ # uses different names during training and evaluation. By setting the names
+ # here, we avoid unexpected pipeline breakage in TF1.
+ out_conv = tf.keras.layers.Conv2D(
+ num_out_channels,
+ kernel_size=1,
+ name='conv1' if tf_version.is_tf1() else None)
if bias_fill is not None:
out_conv.bias_initializer = tf.keras.initializers.constant(bias_fill)
- net = tf.keras.Sequential(
- [tf.keras.layers.Conv2D(num_filters, kernel_size=kernel_size,
- padding='same'),
- tf.keras.layers.ReLU(),
- out_conv]
- )
-
+ layers = []
+ for idx, (kernel_size,
+ num_filter) in enumerate(zip(kernel_sizes, num_filters)):
+ layers.append(
+ conv_fn(
+ num_filter,
+ kernel_size=[1, kernel_size] if unit_height_conv else kernel_size,
+ padding='same',
+ name='conv2_%d' % idx if tf_version.is_tf1() else None))
+ layers.append(tf.keras.layers.ReLU())
+ layers.append(out_conv)
+ net = tf.keras.Sequential(layers, name=name)
return net
@@ -159,7 +205,7 @@ def _to_float32(x):
def _get_shape(tensor, num_dims):
- tf.Assert(tensor.get_shape().ndims == num_dims, [tensor])
+ assert len(tensor.shape.as_list()) == num_dims
return shape_utils.combined_static_and_dynamic_shape(tensor)
@@ -169,12 +215,44 @@ def _flatten_spatial_dimensions(batch_images):
channels])
+def _multi_range(limit,
+ value_repetitions=1,
+ range_repetitions=1,
+ dtype=tf.int32):
+ """Creates a sequence with optional value duplication and range repetition.
+
+ As an example (see the Args section for more details),
+ _multi_range(limit=2, value_repetitions=3, range_repetitions=4) returns:
+
+ [0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1]
+
+ Args:
+ limit: A 0-D Tensor (scalar). Upper limit of sequence, exclusive.
+ value_repetitions: Integer. The number of times a value in the sequence is
+ repeated. With value_repetitions=3, the result is [0, 0, 0, 1, 1, 1, ..].
+ range_repetitions: Integer. The number of times the range is repeated. With
+ range_repetitions=3, the result is [0, 1, 2, .., 0, 1, 2, ..].
+ dtype: The type of the elements of the resulting tensor.
+
+ Returns:
+ A 1-D tensor of type `dtype` and size
+ [`limit` * `value_repetitions` * `range_repetitions`] that contains the
+ specified range with given repetitions.
+ """
+ return tf.reshape(
+ tf.tile(
+ tf.expand_dims(tf.range(limit, dtype=dtype), axis=-1),
+ multiples=[range_repetitions, value_repetitions]), [-1])
+
+
def top_k_feature_map_locations(feature_map, max_pool_kernel_size=3, k=100,
per_channel=False):
"""Returns the top k scores and their locations in a feature map.
Given a feature map, the top k values (based on activation) are returned. If
- `per_channel` is True, the top k values **per channel** are returned.
+ `per_channel` is True, the top k values **per channel** are returned. Note
+ that when k equals to 1, ths function uses reduce_max and argmax instead of
+ top_k to make the logics more efficient.
The `max_pool_kernel_size` argument allows for selecting local peaks in a
region. This filtering is done per channel, so nothing prevents two values at
@@ -224,12 +302,21 @@ def top_k_feature_map_locations(feature_map, max_pool_kernel_size=3, k=100,
batch_size, _, width, num_channels = _get_shape(feature_map, 4)
if per_channel:
- # Perform top k over batch and channels.
- feature_map_peaks_transposed = tf.transpose(feature_map_peaks,
- perm=[0, 3, 1, 2])
- feature_map_peaks_transposed = tf.reshape(
- feature_map_peaks_transposed, [batch_size, num_channels, -1])
- scores, peak_flat_indices = tf.math.top_k(feature_map_peaks_transposed, k=k)
+ if k == 1:
+ feature_map_flattened = tf.reshape(
+ feature_map_peaks, [batch_size, -1, num_channels])
+ scores = tf.math.reduce_max(feature_map_flattened, axis=1)
+ peak_flat_indices = tf.math.argmax(
+ feature_map_flattened, axis=1, output_type=tf.dtypes.int32)
+ peak_flat_indices = tf.expand_dims(peak_flat_indices, axis=-1)
+ else:
+ # Perform top k over batch and channels.
+ feature_map_peaks_transposed = tf.transpose(feature_map_peaks,
+ perm=[0, 3, 1, 2])
+ feature_map_peaks_transposed = tf.reshape(
+ feature_map_peaks_transposed, [batch_size, num_channels, -1])
+ scores, peak_flat_indices = tf.math.top_k(
+ feature_map_peaks_transposed, k=k)
# Convert the indices such that they represent the location in the full
# (flattened) feature map of size [batch, height * width * channels].
channel_idx = tf.range(num_channels)[tf.newaxis, :, tf.newaxis]
@@ -237,8 +324,14 @@ def top_k_feature_map_locations(feature_map, max_pool_kernel_size=3, k=100,
scores = tf.reshape(scores, [batch_size, -1])
peak_flat_indices = tf.reshape(peak_flat_indices, [batch_size, -1])
else:
- feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1])
- scores, peak_flat_indices = tf.math.top_k(feature_map_peaks_flat, k=k)
+ if k == 1:
+ feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1])
+ scores = tf.math.reduce_max(feature_map_peaks_flat, axis=1, keepdims=True)
+ peak_flat_indices = tf.expand_dims(tf.math.argmax(
+ feature_map_peaks_flat, axis=1, output_type=tf.dtypes.int32), axis=-1)
+ else:
+ feature_map_peaks_flat = tf.reshape(feature_map_peaks, [batch_size, -1])
+ scores, peak_flat_indices = tf.math.top_k(feature_map_peaks_flat, k=k)
# Get x, y and channel indices corresponding to the top indices in the flat
# array.
@@ -248,20 +341,15 @@ def top_k_feature_map_locations(feature_map, max_pool_kernel_size=3, k=100,
return scores, y_indices, x_indices, channel_indices
-def prediction_tensors_to_boxes(detection_scores, y_indices, x_indices,
- channel_indices, height_width_predictions,
+def prediction_tensors_to_boxes(y_indices, x_indices, height_width_predictions,
offset_predictions):
"""Converts CenterNet class-center, offset and size predictions to boxes.
Args:
- detection_scores: A [batch, num_boxes] float32 tensor with detection
- scores in range [0, 1].
y_indices: A [batch, num_boxes] int32 tensor with y indices corresponding to
object center locations (expressed in output coordinate frame).
x_indices: A [batch, num_boxes] int32 tensor with x indices corresponding to
object center locations (expressed in output coordinate frame).
- channel_indices: A [batch, num_boxes] int32 tensor with channel indices
- corresponding to object classes.
height_width_predictions: A float tensor of shape [batch_size, height,
width, 2] representing the height and width of a box centered at each
pixel.
@@ -272,49 +360,87 @@ def prediction_tensors_to_boxes(detection_scores, y_indices, x_indices,
Returns:
detection_boxes: A tensor of shape [batch_size, num_boxes, 4] holding the
the raw bounding box coordinates of boxes.
- detection_classes: An integer tensor of shape [batch_size, num_boxes]
- indicating the predicted class for each box.
- detection_scores: A float tensor of shape [batch_size, num_boxes] indicating
- the score for each box.
- num_detections: An integer tensor of shape [batch_size,] indicating the
- number of boxes detected for each sample in the batch.
-
"""
- _, _, width, _ = _get_shape(height_width_predictions, 4)
+ batch_size, num_boxes = _get_shape(y_indices, 2)
+ _, height, width, _ = _get_shape(height_width_predictions, 4)
+ height, width = tf.cast(height, tf.float32), tf.cast(width, tf.float32)
+
+ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use
+ # tf_gather_nd instead and here we prepare the indices for that.
+ combined_indices = tf.stack([
+ _multi_range(batch_size, value_repetitions=num_boxes),
+ tf.reshape(y_indices, [-1]),
+ tf.reshape(x_indices, [-1])
+ ], axis=1)
+ new_height_width = tf.gather_nd(height_width_predictions, combined_indices)
+ new_height_width = tf.reshape(new_height_width, [batch_size, num_boxes, 2])
+
+ new_offsets = tf.gather_nd(offset_predictions, combined_indices)
+ offsets = tf.reshape(new_offsets, [batch_size, num_boxes, 2])
- peak_spatial_indices = flattened_indices_from_row_col_indices(
- y_indices, x_indices, width)
y_indices = _to_float32(y_indices)
x_indices = _to_float32(x_indices)
- height_width_flat = _flatten_spatial_dimensions(height_width_predictions)
- offsets_flat = _flatten_spatial_dimensions(offset_predictions)
-
- height_width = tf.gather(height_width_flat, peak_spatial_indices,
- batch_dims=1)
- offsets = tf.gather(offsets_flat, peak_spatial_indices, batch_dims=1)
-
+ height_width = tf.maximum(new_height_width, 0)
heights, widths = tf.unstack(height_width, axis=2)
y_offsets, x_offsets = tf.unstack(offsets, axis=2)
- detection_classes = channel_indices
+ ymin = y_indices + y_offsets - heights / 2.0
+ xmin = x_indices + x_offsets - widths / 2.0
+ ymax = y_indices + y_offsets + heights / 2.0
+ xmax = x_indices + x_offsets + widths / 2.0
- num_detections = tf.reduce_sum(tf.to_int32(detection_scores > 0), axis=1)
+ ymin = tf.clip_by_value(ymin, 0., height)
+ xmin = tf.clip_by_value(xmin, 0., width)
+ ymax = tf.clip_by_value(ymax, 0., height)
+ xmax = tf.clip_by_value(xmax, 0., width)
+ boxes = tf.stack([ymin, xmin, ymax, xmax], axis=2)
- boxes = tf.stack([y_indices + y_offsets - heights / 2.0,
- x_indices + x_offsets - widths / 2.0,
- y_indices + y_offsets + heights / 2.0,
- x_indices + x_offsets + widths / 2.0], axis=2)
+ return boxes
- return boxes, detection_classes, detection_scores, num_detections
+def prediction_tensors_to_temporal_offsets(
+ y_indices, x_indices, offset_predictions):
+ """Converts CenterNet temporal offset map predictions to batched format.
-def prediction_tensors_to_keypoint_candidates(
- keypoint_heatmap_predictions,
- keypoint_heatmap_offsets,
- keypoint_score_threshold=0.1,
- max_pool_kernel_size=1,
- max_candidates=20):
+ This function is similar to the box offset conversion function, as both
+ temporal offsets and box offsets are size-2 vectors.
+
+ Args:
+ y_indices: A [batch, num_boxes] int32 tensor with y indices corresponding to
+ object center locations (expressed in output coordinate frame).
+ x_indices: A [batch, num_boxes] int32 tensor with x indices corresponding to
+ object center locations (expressed in output coordinate frame).
+ offset_predictions: A float tensor of shape [batch_size, height, width, 2]
+ representing the y and x offsets of a box's center across adjacent frames.
+
+ Returns:
+ offsets: A tensor of shape [batch_size, num_boxes, 2] holding the
+ the object temporal offsets of (y, x) dimensions.
+
+ """
+ batch_size, num_boxes = _get_shape(y_indices, 2)
+
+ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use
+ # tf_gather_nd instead and here we prepare the indices for that.
+ combined_indices = tf.stack([
+ _multi_range(batch_size, value_repetitions=num_boxes),
+ tf.reshape(y_indices, [-1]),
+ tf.reshape(x_indices, [-1])
+ ], axis=1)
+
+ new_offsets = tf.gather_nd(offset_predictions, combined_indices)
+ offsets = tf.reshape(new_offsets, [batch_size, num_boxes, -1])
+
+ return offsets
+
+
+def prediction_tensors_to_keypoint_candidates(keypoint_heatmap_predictions,
+ keypoint_heatmap_offsets,
+ keypoint_score_threshold=0.1,
+ max_pool_kernel_size=1,
+ max_candidates=20,
+ keypoint_depths=None):
"""Convert keypoint heatmap predictions and offsets to keypoint candidates.
Args:
@@ -323,14 +449,17 @@ def prediction_tensors_to_keypoint_candidates(
keypoint_heatmap_offsets: A float tensor of shape [batch_size, height,
width, 2] (or [batch_size, height, width, 2 * num_keypoints] if
'per_keypoint_offset' is set True) representing the per-keypoint offsets.
- keypoint_score_threshold: float, the threshold for considering a keypoint
- a candidate.
+ keypoint_score_threshold: float, the threshold for considering a keypoint a
+ candidate.
max_pool_kernel_size: integer, the max pool kernel size to use to pull off
peak score locations in a neighborhood. For example, to make sure no two
neighboring values for the same keypoint are returned, set
max_pool_kernel_size=3. If None or 1, will not apply any local filtering.
- max_candidates: integer, maximum number of keypoint candidates per
- keypoint type.
+ max_candidates: integer, maximum number of keypoint candidates per keypoint
+ type.
+ keypoint_depths: (optional) A float tensor of shape [batch_size, height,
+ width, 1] (or [batch_size, height, width, num_keypoints] if
+ 'per_keypoint_depth' is set True) representing the per-keypoint depths.
Returns:
keypoint_candidates: A tensor of shape
@@ -344,9 +473,11 @@ def prediction_tensors_to_keypoint_candidates(
[batch_size, num_keypoints] with the number of candidates for each
keypoint type, as it's possible to filter some candidates due to the score
threshold.
+ depth_candidates: A tensor of shape [batch_size, max_candidates,
+ num_keypoints] representing the estimated depth of each keypoint
+ candidate. Return None if the input keypoint_depths is None.
"""
- batch_size, _, width, num_keypoints = _get_shape(
- keypoint_heatmap_predictions, 4)
+ batch_size, _, _, num_keypoints = _get_shape(keypoint_heatmap_predictions, 4)
# Get x, y and channel indices corresponding to the top indices in the
# keypoint heatmap predictions.
# Note that the top k candidates are produced for **each keypoint type**.
@@ -358,19 +489,42 @@ def prediction_tensors_to_keypoint_candidates(
k=max_candidates,
per_channel=True))
- peak_spatial_indices = flattened_indices_from_row_col_indices(
- y_indices, x_indices, width)
+ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use
+ # tf_gather_nd instead and here we prepare the indices for that.
+ _, num_indices = _get_shape(y_indices, 2)
+ combined_indices = tf.stack([
+ _multi_range(batch_size, value_repetitions=num_indices),
+ tf.reshape(y_indices, [-1]),
+ tf.reshape(x_indices, [-1])
+ ], axis=1)
+
+ selected_offsets_flat = tf.gather_nd(keypoint_heatmap_offsets,
+ combined_indices)
+ selected_offsets = tf.reshape(selected_offsets_flat,
+ [batch_size, num_indices, -1])
+
y_indices = _to_float32(y_indices)
x_indices = _to_float32(x_indices)
- offsets_flat = _flatten_spatial_dimensions(keypoint_heatmap_offsets)
-
- selected_offsets = tf.gather(offsets_flat, peak_spatial_indices, batch_dims=1)
- _, num_indices, num_channels = _get_shape(selected_offsets, 3)
+ _, _, num_channels = _get_shape(selected_offsets, 3)
if num_channels > 2:
+ # Offsets are per keypoint and the last dimension of selected_offsets
+ # contains all those offsets, so reshape the offsets to make sure that the
+ # last dimension contains (y_offset, x_offset) for a single keypoint.
reshaped_offsets = tf.reshape(selected_offsets,
[batch_size, num_indices, -1, 2])
- offsets = tf.gather(reshaped_offsets, channel_indices, batch_dims=2)
+
+ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use
+ # tf_gather_nd instead and here we prepare the indices for that. In this
+ # case, channel_indices indicates which keypoint to use the offset from.
+ channel_combined_indices = tf.stack([
+ _multi_range(batch_size, value_repetitions=num_indices),
+ _multi_range(num_indices, range_repetitions=batch_size),
+ tf.reshape(channel_indices, [-1])
+ ], axis=1)
+
+ offsets = tf.gather_nd(reshaped_offsets, channel_combined_indices)
+ offsets = tf.reshape(offsets, [batch_size, num_indices, -1])
else:
offsets = selected_offsets
y_offsets, x_offsets = tf.unstack(offsets, axis=2)
@@ -388,7 +542,250 @@ def prediction_tensors_to_keypoint_candidates(
num_candidates = tf.reduce_sum(
tf.to_int32(keypoint_scores >= keypoint_score_threshold), axis=1)
- return keypoint_candidates, keypoint_scores, num_candidates
+ depth_candidates = None
+ if keypoint_depths is not None:
+ selected_depth_flat = tf.gather_nd(keypoint_depths, combined_indices)
+ selected_depth = tf.reshape(selected_depth_flat,
+ [batch_size, num_indices, -1])
+ _, _, num_depth_channels = _get_shape(selected_depth, 3)
+ if num_depth_channels > 1:
+ combined_indices = tf.stack([
+ _multi_range(batch_size, value_repetitions=num_indices),
+ _multi_range(num_indices, range_repetitions=batch_size),
+ tf.reshape(channel_indices, [-1])
+ ], axis=1)
+ depth = tf.gather_nd(selected_depth, combined_indices)
+ depth = tf.reshape(depth, [batch_size, num_indices, -1])
+ else:
+ depth = selected_depth
+ depth_candidates = tf.reshape(depth,
+ [batch_size, num_keypoints, max_candidates])
+ depth_candidates = tf.transpose(depth_candidates, [0, 2, 1])
+
+ return keypoint_candidates, keypoint_scores, num_candidates, depth_candidates
+
+
+def argmax_feature_map_locations(feature_map):
+ """Returns the peak locations in the feature map."""
+ batch_size, _, width, num_channels = _get_shape(feature_map, 4)
+
+ feature_map_flattened = tf.reshape(
+ feature_map, [batch_size, -1, num_channels])
+ peak_flat_indices = tf.math.argmax(
+ feature_map_flattened, axis=1, output_type=tf.dtypes.int32)
+ # Get x and y indices corresponding to the top indices in the flat array.
+ y_indices, x_indices = (
+ row_col_indices_from_flattened_indices(peak_flat_indices, width))
+ channel_indices = tf.tile(
+ tf.range(num_channels)[tf.newaxis, :], [batch_size, 1])
+ return y_indices, x_indices, channel_indices
+
+
+def prediction_tensors_to_single_instance_kpts(
+ keypoint_heatmap_predictions,
+ keypoint_heatmap_offsets,
+ keypoint_score_heatmap=None):
+ """Convert keypoint heatmap predictions and offsets to keypoint candidates.
+
+ Args:
+ keypoint_heatmap_predictions: A float tensor of shape [batch_size, height,
+ width, num_keypoints] representing the per-keypoint heatmaps which is
+ used for finding the best keypoint candidate locations.
+ keypoint_heatmap_offsets: A float tensor of shape [batch_size, height,
+ width, 2] (or [batch_size, height, width, 2 * num_keypoints] if
+ 'per_keypoint_offset' is set True) representing the per-keypoint offsets.
+ keypoint_score_heatmap: (optional) A float tensor of shape [batch_size,
+ height, width, num_keypoints] representing the heatmap which is used for
+ reporting the confidence scores. If not provided, then the values in the
+ keypoint_heatmap_predictions will be used.
+
+ Returns:
+ keypoint_candidates: A tensor of shape
+ [batch_size, max_candidates, num_keypoints, 2] holding the
+ location of keypoint candidates in [y, x] format (expressed in absolute
+ coordinates in the output coordinate frame).
+ keypoint_scores: A float tensor of shape
+ [batch_size, max_candidates, num_keypoints] with the scores for each
+ keypoint candidate. The scores come directly from the heatmap predictions.
+ num_keypoint_candidates: An integer tensor of shape
+ [batch_size, num_keypoints] with the number of candidates for each
+ keypoint type, as it's possible to filter some candidates due to the score
+ threshold.
+ """
+ batch_size, height, width, num_keypoints = _get_shape(
+ keypoint_heatmap_predictions, 4)
+ # Get x, y and channel indices corresponding to the top indices in the
+ # keypoint heatmap predictions.
+ y_indices, x_indices, channel_indices = argmax_feature_map_locations(
+ keypoint_heatmap_predictions)
+
+ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use
+ # tf_gather_nd instead and here we prepare the indices for that.
+ _, num_keypoints = _get_shape(y_indices, 2)
+ combined_indices = tf.stack([
+ _multi_range(batch_size, value_repetitions=num_keypoints),
+ tf.reshape(y_indices, [-1]),
+ tf.reshape(x_indices, [-1]),
+ tf.reshape(channel_indices, [-1])
+ ], axis=1)
+
+ # Reshape the offsets predictions to shape:
+ # [batch_size, height, width, num_keypoints, 2]
+ keypoint_heatmap_offsets = tf.reshape(
+ keypoint_heatmap_offsets, [batch_size, height, width, num_keypoints, -1])
+
+ # shape: [num_keypoints, 2]
+ selected_offsets_flat = tf.gather_nd(keypoint_heatmap_offsets,
+ combined_indices)
+ y_offsets, x_offsets = tf.unstack(selected_offsets_flat, axis=1)
+
+ keypoint_candidates = tf.stack([
+ tf.cast(y_indices, dtype=tf.float32) + tf.expand_dims(y_offsets, axis=0),
+ tf.cast(x_indices, dtype=tf.float32) + tf.expand_dims(x_offsets, axis=0)
+ ], axis=2)
+ keypoint_candidates = tf.expand_dims(keypoint_candidates, axis=0)
+ if keypoint_score_heatmap is None:
+ keypoint_scores = tf.gather_nd(
+ keypoint_heatmap_predictions, combined_indices)
+ else:
+ keypoint_scores = tf.gather_nd(keypoint_score_heatmap, combined_indices)
+ keypoint_scores = tf.expand_dims(
+ tf.expand_dims(keypoint_scores, axis=0), axis=0)
+ return keypoint_candidates, keypoint_scores
+
+
+def _score_to_distance_map(y_grid, x_grid, heatmap, points_y, points_x,
+ score_distance_offset):
+ """Rescores heatmap using the distance information.
+
+ Rescore the heatmap scores using the formula:
+ score / (d + score_distance_offset), where the d is the distance from each
+ pixel location to the target point location.
+
+ Args:
+ y_grid: A float tensor with shape [height, width] representing the
+ y-coordinate of each pixel grid.
+ x_grid: A float tensor with shape [height, width] representing the
+ x-coordinate of each pixel grid.
+ heatmap: A float tensor with shape [1, height, width, channel]
+ representing the heatmap to be rescored.
+ points_y: A float tensor with shape [channel] representing the y
+ coordinates of the target points for each channel.
+ points_x: A float tensor with shape [channel] representing the x
+ coordinates of the target points for each channel.
+ score_distance_offset: A constant used in the above formula.
+
+ Returns:
+ A float tensor with shape [1, height, width, channel] representing the
+ rescored heatmap.
+ """
+ y_diff = y_grid[:, :, tf.newaxis] - points_y
+ x_diff = x_grid[:, :, tf.newaxis] - points_x
+ distance = tf.math.sqrt(y_diff**2 + x_diff**2)
+ return tf.math.divide(heatmap, distance + score_distance_offset)
+
+
+def prediction_to_single_instance_keypoints(
+ object_heatmap,
+ keypoint_heatmap,
+ keypoint_offset,
+ keypoint_regression,
+ kp_params,
+ keypoint_depths=None):
+ """Postprocess function to predict single instance keypoints.
+
+ This is a simplified postprocessing function based on the assumption that
+ there is only one instance in the image. If there are multiple instances in
+ the image, the model prefers to predict the one that is closest to the image
+ center. Here is a high-level description of what this function does:
+ 1) Object heatmap re-weighted by the distance between each pixel to the
+ image center is used to determine the instance center.
+ 2) Regressed keypoint locations are retrieved from the instance center. The
+ Gaussian kernel is applied to the regressed keypoint locations to
+ re-weight the keypoint heatmap. This is to select the keypoints that are
+ associated with the center instance without using top_k op.
+ 3) The keypoint locations are computed by the re-weighted keypoint heatmap
+ and the keypoint offset.
+
+ Args:
+ object_heatmap: A float tensor of shape [1, height, width, 1] representing
+ the heapmap of the class.
+ keypoint_heatmap: A float tensor of shape [1, height, width, num_keypoints]
+ representing the per-keypoint heatmaps.
+ keypoint_offset: A float tensor of shape [1, height, width, 2] (or [1,
+ height, width, 2 * num_keypoints] if 'per_keypoint_offset' is set True)
+ representing the per-keypoint offsets.
+ keypoint_regression: A float tensor of shape [1, height, width, 2 *
+ num_keypoints] representing the joint regression prediction.
+ kp_params: A `KeypointEstimationParams` object with parameters for a single
+ keypoint class.
+ keypoint_depths: (optional) A float tensor of shape [batch_size, height,
+ width, 1] (or [batch_size, height, width, num_keypoints] if
+ 'per_keypoint_depth' is set True) representing the per-keypoint depths.
+
+ Returns:
+ A tuple of two tensors:
+ keypoint_candidates: A float tensor with shape [1, 1, num_keypoints, 2]
+ representing the yx-coordinates of the keypoints in the output feature
+ map space.
+ keypoint_scores: A float tensor with shape [1, 1, num_keypoints]
+ representing the keypoint prediction scores.
+
+ Raises:
+ ValueError: if the input keypoint_std_dev doesn't have valid number of
+ elements (1 or num_keypoints).
+ """
+ # TODO(yuhuic): add the keypoint depth prediction logics in the browser
+ # postprocessing back.
+ del keypoint_depths
+
+ num_keypoints = len(kp_params.keypoint_std_dev)
+ batch_size, height, width, _ = _get_shape(keypoint_heatmap, 4)
+
+ # Create the image center location.
+ image_center_y = tf.convert_to_tensor([0.5 * height], dtype=tf.float32)
+ image_center_x = tf.convert_to_tensor([0.5 * width], dtype=tf.float32)
+ (y_grid, x_grid) = ta_utils.image_shape_to_grids(height, width)
+ # Rescore the object heatmap by the distnace to the image center.
+ object_heatmap = _score_to_distance_map(
+ y_grid, x_grid, object_heatmap, image_center_y,
+ image_center_x, kp_params.score_distance_offset)
+
+ # Pick the highest score and location of the weighted object heatmap.
+ y_indices, x_indices, _ = argmax_feature_map_locations(object_heatmap)
+ _, num_indices = _get_shape(y_indices, 2)
+ combined_indices = tf.stack([
+ _multi_range(batch_size, value_repetitions=num_indices),
+ tf.reshape(y_indices, [-1]),
+ tf.reshape(x_indices, [-1])
+ ], axis=1)
+
+ # Select the regression vectors from the object center.
+ selected_regression_flat = tf.gather_nd(keypoint_regression, combined_indices)
+ # shape: [num_keypoints, 2]
+ regression_offsets = tf.reshape(selected_regression_flat, [num_keypoints, -1])
+ (y_reg, x_reg) = tf.unstack(regression_offsets, axis=1)
+ y_regressed = tf.cast(y_indices, dtype=tf.float32) + y_reg
+ x_regressed = tf.cast(x_indices, dtype=tf.float32) + x_reg
+
+ if kp_params.candidate_ranking_mode == 'score_distance_ratio':
+ reweighted_keypoint_heatmap = _score_to_distance_map(
+ y_grid, x_grid, keypoint_heatmap, y_regressed, x_regressed,
+ kp_params.score_distance_offset)
+ else:
+ raise ValueError('Unsupported candidate_ranking_mode: %s' %
+ kp_params.candidate_ranking_mode)
+
+ # Get the keypoint locations/scores:
+ # keypoint_candidates: [1, 1, num_keypoints, 2]
+ # keypoint_scores: [1, 1, num_keypoints]
+ # depth_candidates: [1, 1, num_keypoints]
+ (keypoint_candidates, keypoint_scores
+ ) = prediction_tensors_to_single_instance_kpts(
+ reweighted_keypoint_heatmap,
+ keypoint_offset,
+ keypoint_score_heatmap=keypoint_heatmap)
+ return keypoint_candidates, keypoint_scores, None
def regressed_keypoints_at_object_centers(regressed_keypoint_predictions,
@@ -415,16 +812,18 @@ def regressed_keypoints_at_object_centers(regressed_keypoint_predictions,
regressed keypoints are gathered at the provided locations, and converted
to absolute coordinates in the output coordinate frame.
"""
- batch_size, _, width, _ = _get_shape(regressed_keypoint_predictions, 4)
- flattened_indices = flattened_indices_from_row_col_indices(
- y_indices, x_indices, width)
- _, num_instances = _get_shape(flattened_indices, 2)
-
- regressed_keypoints_flat = _flatten_spatial_dimensions(
- regressed_keypoint_predictions)
-
- relative_regressed_keypoints = tf.gather(
- regressed_keypoints_flat, flattened_indices, batch_dims=1)
+ batch_size, num_instances = _get_shape(y_indices, 2)
+
+ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use
+ # tf_gather_nd instead and here we prepare the indices for that.
+ combined_indices = tf.stack([
+ _multi_range(batch_size, value_repetitions=num_instances),
+ tf.reshape(y_indices, [-1]),
+ tf.reshape(x_indices, [-1])
+ ], axis=1)
+
+ relative_regressed_keypoints = tf.gather_nd(regressed_keypoint_predictions,
+ combined_indices)
relative_regressed_keypoints = tf.reshape(
relative_regressed_keypoints,
[batch_size, num_instances, -1, 2])
@@ -440,11 +839,18 @@ def regressed_keypoints_at_object_centers(regressed_keypoint_predictions,
[batch_size, num_instances, -1])
-def refine_keypoints(regressed_keypoints, keypoint_candidates, keypoint_scores,
- num_keypoint_candidates, bboxes=None,
- unmatched_keypoint_score=0.1, box_scale=1.2,
+def refine_keypoints(regressed_keypoints,
+ keypoint_candidates,
+ keypoint_scores,
+ num_keypoint_candidates,
+ bboxes=None,
+ unmatched_keypoint_score=0.1,
+ box_scale=1.2,
candidate_search_scale=0.3,
- candidate_ranking_mode='min_distance'):
+ candidate_ranking_mode='min_distance',
+ score_distance_offset=1e-6,
+ keypoint_depth_candidates=None,
+ keypoint_score_threshold=0.1):
"""Refines regressed keypoints by snapping to the nearest candidate keypoints.
The initial regressed keypoints represent a full set of keypoints regressed
@@ -500,6 +906,16 @@ def refine_keypoints(regressed_keypoints, keypoint_candidates, keypoint_scores,
candidate_ranking_mode: A string as one of ['min_distance',
'score_distance_ratio'] indicating how to select the candidate. If invalid
value is provided, an ValueError will be raised.
+ score_distance_offset: The distance offset to apply in the denominator when
+ candidate_ranking_mode is 'score_distance_ratio'. The metric to maximize
+ in this scenario is score / (distance + score_distance_offset). Larger
+ values of score_distance_offset make the keypoint score gain more relative
+ importance.
+ keypoint_depth_candidates: (optional) A float tensor of shape
+ [batch_size, max_candidates, num_keypoints] indicating the depths for
+ keypoint candidates.
+ keypoint_score_threshold: float, The heatmap score threshold for
+ a keypoint to become a valid candidate.
Returns:
A tuple with:
@@ -526,26 +942,39 @@ def refine_keypoints(regressed_keypoints, keypoint_candidates, keypoint_scores,
num_candidates_tiled = tf.tile(tf.expand_dims(num_keypoint_candidates, 1),
[1, max_candidates, 1])
invalid_candidates = range_tiled >= num_candidates_tiled
- nan_mask = tf.where(
- invalid_candidates,
- np.nan * tf.ones_like(invalid_candidates, dtype=tf.float32),
- tf.ones_like(invalid_candidates, dtype=tf.float32))
- keypoint_candidates_with_nans = tf.math.multiply(
- keypoint_candidates, tf.expand_dims(nan_mask, -1))
# Pairwise squared distances between regressed keypoints and candidate
# keypoints (for a single keypoint type).
- # Shape [batch_size, num_instances, max_candidates, num_keypoints].
+ # Shape [batch_size, num_instances, 1, num_keypoints, 2].
regressed_keypoint_expanded = tf.expand_dims(regressed_keypoints,
axis=2)
+ # Shape [batch_size, 1, max_candidates, num_keypoints, 2].
keypoint_candidates_expanded = tf.expand_dims(
- keypoint_candidates_with_nans, axis=1)
- sqrd_distances = tf.math.reduce_sum(
- tf.math.squared_difference(regressed_keypoint_expanded,
- keypoint_candidates_expanded),
- axis=-1)
+ keypoint_candidates, axis=1)
+ # Use explicit tensor shape broadcasting (since the tensor dimensions are
+ # expanded to 5D) to make it tf.lite compatible.
+ regressed_keypoint_expanded = tf.tile(
+ regressed_keypoint_expanded, multiples=[1, 1, max_candidates, 1, 1])
+ keypoint_candidates_expanded = tf.tile(
+ keypoint_candidates_expanded, multiples=[1, num_instances, 1, 1, 1])
+ # Replace tf.math.squared_difference by "-" operator and tf.multiply ops since
+ # tf.lite convert doesn't support squared_difference with undetermined
+ # dimension.
+ diff = regressed_keypoint_expanded - keypoint_candidates_expanded
+ sqrd_distances = tf.math.reduce_sum(tf.multiply(diff, diff), axis=-1)
distances = tf.math.sqrt(sqrd_distances)
+ # Replace the invalid candidated with large constant (10^5) to make sure the
+ # following reduce_min/argmin behaves properly.
+ max_dist = 1e5
+ distances = tf.where(
+ tf.tile(
+ tf.expand_dims(invalid_candidates, axis=1),
+ multiples=[1, num_instances, 1, 1]),
+ tf.ones_like(distances) * max_dist,
+ distances
+ )
+
# Determine the candidates that have the minimum distance to the regressed
# keypoints. Shape [batch_size, num_instances, num_keypoints].
min_distances = tf.math.reduce_min(distances, axis=2)
@@ -557,7 +986,7 @@ def refine_keypoints(regressed_keypoints, keypoint_candidates, keypoint_scores,
tiled_keypoint_scores = tf.tile(
tf.expand_dims(keypoint_scores, axis=1),
multiples=[1, num_instances, 1, 1])
- ranking_scores = tiled_keypoint_scores / (distances + 1e-6)
+ ranking_scores = tiled_keypoint_scores / (distances + score_distance_offset)
nearby_candidate_inds = tf.math.argmax(ranking_scores, axis=2)
else:
raise ValueError('Not recognized candidate_ranking_mode: %s' %
@@ -566,47 +995,47 @@ def refine_keypoints(regressed_keypoints, keypoint_candidates, keypoint_scores,
# Gather the coordinates and scores corresponding to the closest candidates.
# Shape of tensors are [batch_size, num_instances, num_keypoints, 2] and
# [batch_size, num_instances, num_keypoints], respectively.
- nearby_candidate_coords, nearby_candidate_scores = (
- _gather_candidates_at_indices(keypoint_candidates, keypoint_scores,
- nearby_candidate_inds))
+ (nearby_candidate_coords, nearby_candidate_scores,
+ nearby_candidate_depths) = (
+ _gather_candidates_at_indices(keypoint_candidates, keypoint_scores,
+ nearby_candidate_inds,
+ keypoint_depth_candidates))
if bboxes is None:
- # Create bboxes from regressed keypoints.
- # Shape [batch_size * num_instances, 4].
- regressed_keypoints_flattened = tf.reshape(
- regressed_keypoints, [-1, num_keypoints, 2])
- bboxes_flattened = keypoint_ops.keypoints_to_enclosing_bounding_boxes(
- regressed_keypoints_flattened)
+ # Filter out the chosen candidate with score lower than unmatched
+ # keypoint score.
+ mask = tf.cast(nearby_candidate_scores <
+ keypoint_score_threshold, tf.int32)
else:
bboxes_flattened = tf.reshape(bboxes, [-1, 4])
- # Scale the bounding boxes.
- # Shape [batch_size, num_instances, 4].
- boxlist = box_list.BoxList(bboxes_flattened)
- boxlist_scaled = box_list_ops.scale_height_width(
- boxlist, box_scale, box_scale)
- bboxes_scaled = boxlist_scaled.get()
- bboxes = tf.reshape(bboxes_scaled, [batch_size, num_instances, 4])
-
- # Get ymin, xmin, ymax, xmax bounding box coordinates, tiled per keypoint.
- # Shape [batch_size, num_instances, num_keypoints].
- bboxes_tiled = tf.tile(tf.expand_dims(bboxes, 2), [1, 1, num_keypoints, 1])
- ymin, xmin, ymax, xmax = tf.unstack(bboxes_tiled, axis=3)
-
- # Produce a mask that indicates whether the original regressed keypoint
- # should be used instead of a candidate keypoint.
- # Shape [batch_size, num_instances, num_keypoints].
- search_radius = (
- tf.math.maximum(ymax - ymin, xmax - xmin) * candidate_search_scale)
- mask = (tf.cast(nearby_candidate_coords[:, :, :, 0] < ymin, tf.int32) +
- tf.cast(nearby_candidate_coords[:, :, :, 0] > ymax, tf.int32) +
- tf.cast(nearby_candidate_coords[:, :, :, 1] < xmin, tf.int32) +
- tf.cast(nearby_candidate_coords[:, :, :, 1] > xmax, tf.int32) +
- # Filter out the chosen candidate with score lower than unmatched
- # keypoint score.
- tf.cast(nearby_candidate_scores <
- unmatched_keypoint_score, tf.int32) +
- tf.cast(min_distances > search_radius, tf.int32))
+ # Scale the bounding boxes.
+ # Shape [batch_size, num_instances, 4].
+ boxlist = box_list.BoxList(bboxes_flattened)
+ boxlist_scaled = box_list_ops.scale_height_width(
+ boxlist, box_scale, box_scale)
+ bboxes_scaled = boxlist_scaled.get()
+ bboxes = tf.reshape(bboxes_scaled, [batch_size, num_instances, 4])
+
+ # Get ymin, xmin, ymax, xmax bounding box coordinates, tiled per keypoint.
+ # Shape [batch_size, num_instances, num_keypoints].
+ bboxes_tiled = tf.tile(tf.expand_dims(bboxes, 2), [1, 1, num_keypoints, 1])
+ ymin, xmin, ymax, xmax = tf.unstack(bboxes_tiled, axis=3)
+
+ # Produce a mask that indicates whether the original regressed keypoint
+ # should be used instead of a candidate keypoint.
+ # Shape [batch_size, num_instances, num_keypoints].
+ search_radius = (
+ tf.math.maximum(ymax - ymin, xmax - xmin) * candidate_search_scale)
+ mask = (tf.cast(nearby_candidate_coords[:, :, :, 0] < ymin, tf.int32) +
+ tf.cast(nearby_candidate_coords[:, :, :, 0] > ymax, tf.int32) +
+ tf.cast(nearby_candidate_coords[:, :, :, 1] < xmin, tf.int32) +
+ tf.cast(nearby_candidate_coords[:, :, :, 1] > xmax, tf.int32) +
+ # Filter out the chosen candidate with score lower than unmatched
+ # keypoint score.
+ tf.cast(nearby_candidate_scores <
+ keypoint_score_threshold, tf.int32) +
+ tf.cast(min_distances > search_radius, tf.int32))
mask = mask > 0
# Create refined keypoints where candidate keypoints replace original
@@ -625,7 +1054,12 @@ def refine_keypoints(regressed_keypoints, keypoint_candidates, keypoint_scores,
unmatched_keypoint_score * tf.ones_like(nearby_candidate_scores),
nearby_candidate_scores)
- return refined_keypoints, refined_scores
+ refined_depths = None
+ if nearby_candidate_depths is not None:
+ refined_depths = tf.where(mask, tf.zeros_like(nearby_candidate_depths),
+ nearby_candidate_depths)
+
+ return refined_keypoints, refined_scores, refined_depths
def _pad_to_full_keypoint_dim(keypoint_coords, keypoint_scores, keypoint_inds,
@@ -706,8 +1140,10 @@ def _pad_to_full_instance_dim(keypoint_coords, keypoint_scores, instance_inds,
return keypoint_coords_padded, keypoint_scores_padded
-def _gather_candidates_at_indices(keypoint_candidates, keypoint_scores,
- indices):
+def _gather_candidates_at_indices(keypoint_candidates,
+ keypoint_scores,
+ indices,
+ keypoint_depth_candidates=None):
"""Gathers keypoint candidate coordinates and scores at indices.
Args:
@@ -717,31 +1153,72 @@ def _gather_candidates_at_indices(keypoint_candidates, keypoint_scores,
num_keypoints] with keypoint scores.
indices: an integer tensor of shape [batch_size, num_indices, num_keypoints]
with indices.
+ keypoint_depth_candidates: (optional) a float tensor of shape [batch_size,
+ max_candidates, num_keypoints] with keypoint depths.
Returns:
A tuple with
gathered_keypoint_candidates: a float tensor of shape [batch_size,
num_indices, num_keypoints, 2] with gathered coordinates.
gathered_keypoint_scores: a float tensor of shape [batch_size,
- num_indices, num_keypoints, 2].
+ num_indices, num_keypoints].
+ gathered_keypoint_depths: a float tensor of shape [batch_size,
+ num_indices, num_keypoints]. Return None if the input
+ keypoint_depth_candidates is None.
"""
+ batch_size, num_indices, num_keypoints = _get_shape(indices, 3)
+
# Transpose tensors so that all batch dimensions are up front.
keypoint_candidates_transposed = tf.transpose(keypoint_candidates,
[0, 2, 1, 3])
keypoint_scores_transposed = tf.transpose(keypoint_scores, [0, 2, 1])
- nearby_candidate_inds_transposed = tf.transpose(indices,
- [0, 2, 1])
- nearby_candidate_coords_tranposed = tf.gather(
- keypoint_candidates_transposed, nearby_candidate_inds_transposed,
- batch_dims=2)
- nearby_candidate_scores_transposed = tf.gather(
- keypoint_scores_transposed, nearby_candidate_inds_transposed,
- batch_dims=2)
- gathered_keypoint_candidates = tf.transpose(nearby_candidate_coords_tranposed,
- [0, 2, 1, 3])
+ nearby_candidate_inds_transposed = tf.transpose(indices, [0, 2, 1])
+
+ # TF Lite does not support tf.gather with batch_dims > 0, so we need to use
+ # tf_gather_nd instead and here we prepare the indices for that.
+ combined_indices = tf.stack([
+ _multi_range(
+ batch_size,
+ value_repetitions=num_keypoints * num_indices,
+ dtype=tf.int64),
+ _multi_range(
+ num_keypoints,
+ value_repetitions=num_indices,
+ range_repetitions=batch_size,
+ dtype=tf.int64),
+ tf.reshape(nearby_candidate_inds_transposed, [-1])
+ ], axis=1)
+
+ nearby_candidate_coords_transposed = tf.gather_nd(
+ keypoint_candidates_transposed, combined_indices)
+ nearby_candidate_coords_transposed = tf.reshape(
+ nearby_candidate_coords_transposed,
+ [batch_size, num_keypoints, num_indices, -1])
+
+ nearby_candidate_scores_transposed = tf.gather_nd(keypoint_scores_transposed,
+ combined_indices)
+ nearby_candidate_scores_transposed = tf.reshape(
+ nearby_candidate_scores_transposed,
+ [batch_size, num_keypoints, num_indices])
+
+ gathered_keypoint_candidates = tf.transpose(
+ nearby_candidate_coords_transposed, [0, 2, 1, 3])
gathered_keypoint_scores = tf.transpose(nearby_candidate_scores_transposed,
[0, 2, 1])
- return gathered_keypoint_candidates, gathered_keypoint_scores
+
+ gathered_keypoint_depths = None
+ if keypoint_depth_candidates is not None:
+ keypoint_depths_transposed = tf.transpose(keypoint_depth_candidates,
+ [0, 2, 1])
+ nearby_candidate_depths_transposed = tf.gather_nd(
+ keypoint_depths_transposed, combined_indices)
+ nearby_candidate_depths_transposed = tf.reshape(
+ nearby_candidate_depths_transposed,
+ [batch_size, num_keypoints, num_indices])
+ gathered_keypoint_depths = tf.transpose(nearby_candidate_depths_transposed,
+ [0, 2, 1])
+ return (gathered_keypoint_candidates, gathered_keypoint_scores,
+ gathered_keypoint_depths)
def flattened_indices_from_row_col_indices(row_indices, col_indices, num_cols):
@@ -768,13 +1245,45 @@ def row_col_channel_indices_from_flattened_indices(indices, num_cols,
indices.
"""
+ # Be careful with this function when running a model in float16 precision
+ # (e.g. TF.js with WebGL) because the array indices may not be represented
+ # accurately if they are too large, resulting in incorrect channel indices.
+ # See:
+ # https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Precision_limitations_on_integer_values
+ #
+ # Avoid using mod operator to make the ops more easy to be compatible with
+ # different environments, e.g. WASM.
row_indices = (indices // num_channels) // num_cols
- col_indices = (indices // num_channels) % num_cols
- channel_indices = indices % num_channels
+ col_indices = (indices // num_channels) - row_indices * num_cols
+ channel_indices_temp = indices // num_channels
+ channel_indices = indices - channel_indices_temp * num_channels
return row_indices, col_indices, channel_indices
+def row_col_indices_from_flattened_indices(indices, num_cols):
+ """Computes row and column indices from flattened indices.
+
+ Args:
+ indices: An integer tensor of any shape holding the indices in the flattened
+ space.
+ num_cols: Number of columns in the image (width).
+
+ Returns:
+ row_indices: The row indices corresponding to each of the input indices.
+ Same shape as indices.
+ col_indices: The column indices corresponding to each of the input indices.
+ Same shape as indices.
+
+ """
+ # Avoid using mod operator to make the ops more easy to be compatible with
+ # different environments, e.g. WASM.
+ row_indices = indices // num_cols
+ col_indices = indices - row_indices * num_cols
+
+ return row_indices, col_indices
+
+
def get_valid_anchor_weights_in_flattened_image(true_image_shapes, height,
width):
"""Computes valid anchor weights for an image assuming pixels will be flattened.
@@ -833,27 +1342,12 @@ def convert_strided_predictions_to_normalized_boxes(boxes, stride,
boxes: A tensor of shape [batch_size, num_boxes, 4] representing the
coordinates of the normalized boxes.
"""
-
- def _normalize_boxlist(args):
-
- boxes, height, width = args
- boxes = box_list_ops.scale(boxes, stride, stride)
- boxes = box_list_ops.to_normalized_coordinates(boxes, height, width)
- boxes = box_list_ops.clip_to_window(boxes, [0., 0., 1., 1.],
- filter_nonoverlapping=False)
- return boxes
-
- box_lists = [box_list.BoxList(boxes) for boxes in tf.unstack(boxes, axis=0)]
- true_heights, true_widths, _ = tf.unstack(true_image_shapes, axis=1)
-
- true_heights_list = tf.unstack(true_heights, axis=0)
- true_widths_list = tf.unstack(true_widths, axis=0)
-
- box_lists = list(map(_normalize_boxlist,
- zip(box_lists, true_heights_list, true_widths_list)))
- boxes = tf.stack([box_list_instance.get() for
- box_list_instance in box_lists], axis=0)
-
+ # Note: We use tf ops instead of functions in box_list_ops to make this
+ # function compatible with dynamic batch size.
+ boxes = boxes * stride
+ true_image_shapes = tf.tile(true_image_shapes[:, tf.newaxis, :2], [1, 1, 2])
+ boxes = boxes / tf.cast(true_image_shapes, tf.float32)
+ boxes = tf.clip_by_value(boxes, 0.0, 1.0)
return boxes
@@ -915,9 +1409,16 @@ def convert_strided_predictions_to_normalized_keypoints(
def clip_to_window(inputs):
keypoints, window = inputs
return keypoint_ops.clip_to_window(keypoints, window)
+
+ # Specify the TensorSpec explicitly in the tf.map_fn to make it tf.lite
+ # compatible.
+ kpts_dims = _get_shape(keypoint_coords_normalized, 4)
+ output_spec = tf.TensorSpec(
+ shape=[kpts_dims[1], kpts_dims[2], kpts_dims[3]], dtype=tf.float32)
keypoint_coords_normalized = tf.map_fn(
clip_to_window, (keypoint_coords_normalized, batch_window),
- dtype=tf.float32, back_prop=False)
+ dtype=tf.float32, back_prop=False,
+ fn_output_signature=output_spec)
keypoint_scores = tf.where(valid_indices, keypoint_scores,
tf.zeros_like(keypoint_scores))
return keypoint_coords_normalized, keypoint_scores
@@ -1151,6 +1652,33 @@ def gather_surface_coords_for_parts(surface_coords_cropped,
return tf.reshape(vu_coords_flattened, [max_detections, height, width, 2])
+def predicted_embeddings_at_object_centers(embedding_predictions,
+ y_indices, x_indices):
+ """Returns the predicted embeddings at specified object centers.
+
+ Args:
+ embedding_predictions: A float tensor of shape [batch_size, height, width,
+ reid_embed_size] holding predicted embeddings.
+ y_indices: A [batch, num_instances] int tensor holding y indices for object
+ centers. These indices correspond to locations in the output feature map.
+ x_indices: A [batch, num_instances] int tensor holding x indices for object
+ centers. These indices correspond to locations in the output feature map.
+
+ Returns:
+ A float tensor of shape [batch_size, num_objects, reid_embed_size] where
+ predicted embeddings are gathered at the provided locations.
+ """
+ batch_size, _, width, _ = _get_shape(embedding_predictions, 4)
+ flattened_indices = flattened_indices_from_row_col_indices(
+ y_indices, x_indices, width)
+ _, num_instances = _get_shape(flattened_indices, 2)
+ embeddings_flat = _flatten_spatial_dimensions(embedding_predictions)
+ embeddings = tf.gather(embeddings_flat, flattened_indices, batch_dims=1)
+ embeddings = tf.reshape(embeddings, [batch_size, num_instances, -1])
+
+ return embeddings
+
+
class ObjectDetectionParams(
collections.namedtuple('ObjectDetectionParams', [
'localization_loss', 'scale_loss_weight', 'offset_loss_weight',
@@ -1201,7 +1729,13 @@ class KeypointEstimationParams(
'heatmap_bias_init', 'num_candidates_per_keypoint', 'task_loss_weight',
'peak_max_pool_kernel_size', 'unmatched_keypoint_score', 'box_scale',
'candidate_search_scale', 'candidate_ranking_mode',
- 'offset_peak_radius', 'per_keypoint_offset'
+ 'offset_peak_radius', 'per_keypoint_offset', 'predict_depth',
+ 'per_keypoint_depth', 'keypoint_depth_loss_weight',
+ 'score_distance_offset', 'clip_out_of_frame_keypoints',
+ 'rescore_instances', 'heatmap_head_num_filters',
+ 'heatmap_head_kernel_sizes', 'offset_head_num_filters',
+ 'offset_head_kernel_sizes', 'regress_head_num_filters',
+ 'regress_head_kernel_sizes'
])):
"""Namedtuple to host object detection related parameters.
@@ -1234,7 +1768,19 @@ class KeypointEstimationParams(
candidate_search_scale=0.3,
candidate_ranking_mode='min_distance',
offset_peak_radius=0,
- per_keypoint_offset=False):
+ per_keypoint_offset=False,
+ predict_depth=False,
+ per_keypoint_depth=False,
+ keypoint_depth_loss_weight=1.0,
+ score_distance_offset=1e-6,
+ clip_out_of_frame_keypoints=False,
+ rescore_instances=False,
+ heatmap_head_num_filters=(256),
+ heatmap_head_kernel_sizes=(3),
+ offset_head_num_filters=(256),
+ offset_head_kernel_sizes=(3),
+ regress_head_num_filters=(256),
+ regress_head_kernel_sizes=(3)):
"""Constructor with default values for KeypointEstimationParams.
Args:
@@ -1298,6 +1844,34 @@ class KeypointEstimationParams(
original paper). If set True, the output offset target has the shape
[batch_size, out_height, out_width, 2 * num_keypoints] (recommended when
the offset_peak_radius is not zero).
+ predict_depth: A bool indicates whether to predict the depth of each
+ keypoints.
+ per_keypoint_depth: A bool indicates whether the model predicts the depth
+ of each keypoints in independent channels. Similar to
+ per_keypoint_offset but for the keypoint depth.
+ keypoint_depth_loss_weight: The weight of the keypoint depth loss.
+ score_distance_offset: The distance offset to apply in the denominator
+ when candidate_ranking_mode is 'score_distance_ratio'. The metric to
+ maximize in this scenario is score / (distance + score_distance_offset).
+ Larger values of score_distance_offset make the keypoint score gain more
+ relative importance.
+ clip_out_of_frame_keypoints: Whether keypoints outside the image frame
+ should be clipped back to the image boundary. If True, the keypoints
+ that are clipped have scores set to 0.0.
+ rescore_instances: Whether to rescore instances based on a combination of
+ detection score and keypoint scores.
+ heatmap_head_num_filters: filter numbers of the convolutional layers used
+ by the keypoint heatmap prediction head.
+ heatmap_head_kernel_sizes: kernel size of the convolutional layers used
+ by the keypoint heatmap prediction head.
+ offset_head_num_filters: filter numbers of the convolutional layers used
+ by the keypoint offset prediction head.
+ offset_head_kernel_sizes: kernel size of the convolutional layers used
+ by the keypoint offset prediction head.
+ regress_head_num_filters: filter numbers of the convolutional layers used
+ by the keypoint regression prediction head.
+ regress_head_kernel_sizes: kernel size of the convolutional layers used
+ by the keypoint regression prediction head.
Returns:
An initialized KeypointEstimationParams namedtuple.
@@ -1310,13 +1884,20 @@ class KeypointEstimationParams(
heatmap_bias_init, num_candidates_per_keypoint, task_loss_weight,
peak_max_pool_kernel_size, unmatched_keypoint_score, box_scale,
candidate_search_scale, candidate_ranking_mode, offset_peak_radius,
- per_keypoint_offset)
+ per_keypoint_offset, predict_depth, per_keypoint_depth,
+ keypoint_depth_loss_weight, score_distance_offset,
+ clip_out_of_frame_keypoints, rescore_instances,
+ heatmap_head_num_filters, heatmap_head_kernel_sizes,
+ offset_head_num_filters, offset_head_kernel_sizes,
+ regress_head_num_filters, regress_head_kernel_sizes)
class ObjectCenterParams(
collections.namedtuple('ObjectCenterParams', [
'classification_loss', 'object_center_loss_weight', 'heatmap_bias_init',
- 'min_box_overlap_iou', 'max_box_predictions', 'use_only_known_classes'
+ 'min_box_overlap_iou', 'max_box_predictions', 'use_labeled_classes',
+ 'keypoint_weights_for_center', 'center_head_num_filters',
+ 'center_head_kernel_sizes'
])):
"""Namedtuple to store object center prediction related parameters."""
@@ -1328,7 +1909,10 @@ class ObjectCenterParams(
heatmap_bias_init=-2.19,
min_box_overlap_iou=0.7,
max_box_predictions=100,
- use_labeled_classes=False):
+ use_labeled_classes=False,
+ keypoint_weights_for_center=None,
+ center_head_num_filters=(256),
+ center_head_kernel_sizes=(3)):
"""Constructor with default values for ObjectCenterParams.
Args:
@@ -1343,7 +1927,16 @@ class ObjectCenterParams(
computing the class specific center heatmaps.
max_box_predictions: int, the maximum number of boxes to predict.
use_labeled_classes: boolean, compute the loss only labeled classes.
-
+ keypoint_weights_for_center: (optional) The keypoint weights used for
+ calculating the location of object center. If provided, the number of
+ weights need to be the same as the number of keypoints. The object
+ center is calculated by the weighted mean of the keypoint locations. If
+ not provided, the object center is determined by the center of the
+ bounding box (default behavior).
+ center_head_num_filters: filter numbers of the convolutional layers used
+ by the object center prediction head.
+ center_head_kernel_sizes: kernel size of the convolutional layers used
+ by the object center prediction head.
Returns:
An initialized ObjectCenterParams namedtuple.
"""
@@ -1351,7 +1944,8 @@ class ObjectCenterParams(
cls).__new__(cls, classification_loss,
object_center_loss_weight, heatmap_bias_init,
min_box_overlap_iou, max_box_predictions,
- use_labeled_classes)
+ use_labeled_classes, keypoint_weights_for_center,
+ center_head_num_filters, center_head_kernel_sizes)
class MaskParams(
@@ -1451,6 +2045,68 @@ class DensePoseParams(
task_loss_weight, upsample_to_input_res,
upsample_method, heatmap_bias_init)
+
+class TrackParams(
+ collections.namedtuple('TrackParams', [
+ 'num_track_ids', 'reid_embed_size', 'num_fc_layers',
+ 'classification_loss', 'task_loss_weight'
+ ])):
+ """Namedtuple to store tracking prediction related parameters."""
+
+ __slots__ = ()
+
+ def __new__(cls,
+ num_track_ids,
+ reid_embed_size,
+ num_fc_layers,
+ classification_loss,
+ task_loss_weight=1.0):
+ """Constructor with default values for TrackParams.
+
+ Args:
+ num_track_ids: int. The maximum track ID in the dataset. Used for ReID
+ embedding classification task.
+ reid_embed_size: int. The embedding size for ReID task.
+ num_fc_layers: int. The number of (fully-connected, batch-norm, relu)
+ layers for track ID classification head.
+ classification_loss: an object_detection.core.losses.Loss object to
+ compute the loss for the ReID embedding in CenterNet.
+ task_loss_weight: float, the loss weight for the tracking task.
+
+ Returns:
+ An initialized TrackParams namedtuple.
+ """
+ return super(TrackParams,
+ cls).__new__(cls, num_track_ids, reid_embed_size,
+ num_fc_layers, classification_loss,
+ task_loss_weight)
+
+
+class TemporalOffsetParams(
+ collections.namedtuple('TemporalOffsetParams', [
+ 'localization_loss', 'task_loss_weight'
+ ])):
+ """Namedtuple to store temporal offset related parameters."""
+
+ __slots__ = ()
+
+ def __new__(cls,
+ localization_loss,
+ task_loss_weight=1.0):
+ """Constructor with default values for TrackParams.
+
+ Args:
+ localization_loss: an object_detection.core.losses.Loss object to
+ compute the loss for the temporal offset in CenterNet.
+ task_loss_weight: float, the loss weight for the temporal offset
+ task.
+
+ Returns:
+ An initialized TemporalOffsetParams namedtuple.
+ """
+ return super(TemporalOffsetParams,
+ cls).__new__(cls, localization_loss, task_loss_weight)
+
# The following constants are used to generate the keys of the
# (prediction, loss, target assigner,...) dictionaries used in CenterNetMetaArch
# class.
@@ -1461,12 +2117,17 @@ BOX_OFFSET = 'box/offset'
KEYPOINT_REGRESSION = 'keypoint/regression'
KEYPOINT_HEATMAP = 'keypoint/heatmap'
KEYPOINT_OFFSET = 'keypoint/offset'
+KEYPOINT_DEPTH = 'keypoint/depth'
SEGMENTATION_TASK = 'segmentation_task'
SEGMENTATION_HEATMAP = 'segmentation/heatmap'
DENSEPOSE_TASK = 'densepose_task'
DENSEPOSE_HEATMAP = 'densepose/heatmap'
DENSEPOSE_REGRESSION = 'densepose/regression'
LOSS_KEY_PREFIX = 'Loss'
+TRACK_TASK = 'track_task'
+TRACK_REID = 'track/reid'
+TEMPORALOFFSET_TASK = 'temporal_offset_task'
+TEMPORAL_OFFSET = 'track/offset'
def get_keypoint_name(task_name, head_name):
@@ -1510,7 +2171,13 @@ class CenterNetMetaArch(model.DetectionModel):
object_detection_params=None,
keypoint_params_dict=None,
mask_params=None,
- densepose_params=None):
+ densepose_params=None,
+ track_params=None,
+ temporal_offset_params=None,
+ use_depthwise=False,
+ compute_heatmap_sparse=False,
+ non_max_suppression_fn=None,
+ unit_height_conv=False):
"""Initializes a CenterNet model.
Args:
@@ -1542,6 +2209,20 @@ class CenterNetMetaArch(model.DetectionModel):
hyper-parameters for DensePose prediction. Please see the class
definition for more details. Note that if this is provided, it is
expected that `mask_params` is also provided.
+ track_params: A TrackParams namedtuple. This object
+ holds the hyper-parameters for tracking. Please see the class
+ definition for more details.
+ temporal_offset_params: A TemporalOffsetParams namedtuple. This object
+ holds the hyper-parameters for offset prediction based tracking.
+ use_depthwise: If true, all task heads will be constructed using
+ separable_conv. Otherwise, standard convoltuions will be used.
+ compute_heatmap_sparse: bool, whether or not to use the sparse version of
+ the Op that computes the center heatmaps. The sparse version scales
+ better with number of channels in the heatmap, but in some cases is
+ known to cause an OOM error. See b/170989061.
+ non_max_suppression_fn: Optional Non Max Suppression function to apply.
+ unit_height_conv: If True, Conv2Ds in prediction heads have asymmetric
+ kernels with height=1.
"""
assert object_detection_params or keypoint_params_dict
# Shorten the name for convenience and better formatting.
@@ -1549,6 +2230,7 @@ class CenterNetMetaArch(model.DetectionModel):
# The Objects as Points paper attaches loss functions to multiple
# (`num_feature_outputs`) feature maps in the the backbone. E.g.
# for the hourglass backbone, `num_feature_outputs` is 2.
+ self._num_classes = num_classes
self._feature_extractor = feature_extractor
self._num_feature_outputs = feature_extractor.num_feature_outputs
self._stride = self._feature_extractor.out_stride
@@ -1561,12 +2243,21 @@ class CenterNetMetaArch(model.DetectionModel):
raise ValueError('To run DensePose prediction, `mask_params` must also '
'be supplied.')
self._densepose_params = densepose_params
+ self._track_params = track_params
+ self._temporal_offset_params = temporal_offset_params
+
+ self._use_depthwise = use_depthwise
+ self._compute_heatmap_sparse = compute_heatmap_sparse
+ # subclasses may not implement the unit_height_conv arg, so only provide it
+ # as a kwarg if it is True.
+ kwargs = {'unit_height_conv': unit_height_conv} if unit_height_conv else {}
# Construct the prediction head nets.
self._prediction_head_dict = self._construct_prediction_heads(
num_classes,
self._num_feature_outputs,
- class_prediction_bias_init=self._center_params.heatmap_bias_init)
+ class_prediction_bias_init=self._center_params.heatmap_bias_init,
+ **kwargs)
# Initialize the target assigners.
self._target_assigner_dict = self._initialize_target_assigners(
stride=self._stride,
@@ -1574,6 +2265,7 @@ class CenterNetMetaArch(model.DetectionModel):
# Will be used in VOD single_frame_meta_arch for tensor reshape.
self._batched_prediction_tensor_names = []
+ self._non_max_suppression_fn = non_max_suppression_fn
super(CenterNetMetaArch, self).__init__(num_classes)
@@ -1584,8 +2276,26 @@ class CenterNetMetaArch(model.DetectionModel):
'tensor names.')
return self._batched_prediction_tensor_names
+ def _make_prediction_net_list(self, num_feature_outputs, num_out_channels,
+ kernel_sizes=(3), num_filters=(256),
+ bias_fill=None, name=None,
+ unit_height_conv=False):
+ prediction_net_list = []
+ for i in range(num_feature_outputs):
+ prediction_net_list.append(
+ make_prediction_net(
+ num_out_channels,
+ kernel_sizes=kernel_sizes,
+ num_filters=num_filters,
+ bias_fill=bias_fill,
+ use_depthwise=self._use_depthwise,
+ name='{}_{}'.format(name, i) if name else name,
+ unit_height_conv=unit_height_conv))
+ return prediction_net_list
+
def _construct_prediction_heads(self, num_classes, num_feature_outputs,
- class_prediction_bias_init):
+ class_prediction_bias_init,
+ unit_height_conv=False):
"""Constructs the prediction heads based on the specific parameters.
Args:
@@ -1597,62 +2307,126 @@ class CenterNetMetaArch(model.DetectionModel):
class_prediction_bias_init: float, the initial value of bias in the
convolutional kernel of the class prediction head. If set to None, the
bias is initialized with zeros.
+ unit_height_conv: If True, Conv2Ds have asymmetric kernels with height=1.
Returns:
A dictionary of keras modules generated by calling make_prediction_net
- function.
+ function. It will also create and set a private member of the class when
+ learning the tracking task.
"""
prediction_heads = {}
- prediction_heads[OBJECT_CENTER] = [
- make_prediction_net(num_classes, bias_fill=class_prediction_bias_init)
- for _ in range(num_feature_outputs)
- ]
+ prediction_heads[OBJECT_CENTER] = self._make_prediction_net_list(
+ num_feature_outputs,
+ num_classes,
+ kernel_sizes=self._center_params.center_head_kernel_sizes,
+ num_filters=self._center_params.center_head_num_filters,
+ bias_fill=class_prediction_bias_init,
+ name='center',
+ unit_height_conv=unit_height_conv)
+
if self._od_params is not None:
- prediction_heads[BOX_SCALE] = [
- make_prediction_net(NUM_SIZE_CHANNELS)
- for _ in range(num_feature_outputs)
- ]
- prediction_heads[BOX_OFFSET] = [
- make_prediction_net(NUM_OFFSET_CHANNELS)
- for _ in range(num_feature_outputs)
- ]
+ prediction_heads[BOX_SCALE] = self._make_prediction_net_list(
+ num_feature_outputs, NUM_SIZE_CHANNELS, name='box_scale',
+ unit_height_conv=unit_height_conv)
+ prediction_heads[BOX_OFFSET] = self._make_prediction_net_list(
+ num_feature_outputs, NUM_OFFSET_CHANNELS, name='box_offset',
+ unit_height_conv=unit_height_conv)
+
if self._kp_params_dict is not None:
for task_name, kp_params in self._kp_params_dict.items():
num_keypoints = len(kp_params.keypoint_indices)
- prediction_heads[get_keypoint_name(task_name, KEYPOINT_HEATMAP)] = [
- make_prediction_net(
- num_keypoints, bias_fill=kp_params.heatmap_bias_init)
- for _ in range(num_feature_outputs)
- ]
- prediction_heads[get_keypoint_name(task_name, KEYPOINT_REGRESSION)] = [
- make_prediction_net(NUM_OFFSET_CHANNELS * num_keypoints)
- for _ in range(num_feature_outputs)
- ]
+ prediction_heads[get_keypoint_name(
+ task_name, KEYPOINT_HEATMAP)] = self._make_prediction_net_list(
+ num_feature_outputs,
+ num_keypoints,
+ kernel_sizes=kp_params.heatmap_head_kernel_sizes,
+ num_filters=kp_params.heatmap_head_num_filters,
+ bias_fill=kp_params.heatmap_bias_init,
+ name='kpt_heatmap',
+ unit_height_conv=unit_height_conv)
+ prediction_heads[get_keypoint_name(
+ task_name, KEYPOINT_REGRESSION)] = self._make_prediction_net_list(
+ num_feature_outputs,
+ NUM_OFFSET_CHANNELS * num_keypoints,
+ kernel_sizes=kp_params.regress_head_kernel_sizes,
+ num_filters=kp_params.regress_head_num_filters,
+ name='kpt_regress',
+ unit_height_conv=unit_height_conv)
+
if kp_params.per_keypoint_offset:
- prediction_heads[get_keypoint_name(task_name, KEYPOINT_OFFSET)] = [
- make_prediction_net(NUM_OFFSET_CHANNELS * num_keypoints)
- for _ in range(num_feature_outputs)
- ]
+ prediction_heads[get_keypoint_name(
+ task_name, KEYPOINT_OFFSET)] = self._make_prediction_net_list(
+ num_feature_outputs,
+ NUM_OFFSET_CHANNELS * num_keypoints,
+ kernel_sizes=kp_params.offset_head_kernel_sizes,
+ num_filters=kp_params.offset_head_num_filters,
+ name='kpt_offset',
+ unit_height_conv=unit_height_conv)
else:
- prediction_heads[get_keypoint_name(task_name, KEYPOINT_OFFSET)] = [
- make_prediction_net(NUM_OFFSET_CHANNELS)
- for _ in range(num_feature_outputs)
- ]
+ prediction_heads[get_keypoint_name(
+ task_name, KEYPOINT_OFFSET)] = self._make_prediction_net_list(
+ num_feature_outputs,
+ NUM_OFFSET_CHANNELS,
+ kernel_sizes=kp_params.offset_head_kernel_sizes,
+ num_filters=kp_params.offset_head_num_filters,
+ name='kpt_offset',
+ unit_height_conv=unit_height_conv)
+
+ if kp_params.predict_depth:
+ num_depth_channel = (
+ num_keypoints if kp_params.per_keypoint_depth else 1)
+ prediction_heads[get_keypoint_name(
+ task_name, KEYPOINT_DEPTH)] = self._make_prediction_net_list(
+ num_feature_outputs, num_depth_channel, name='kpt_depth',
+ unit_height_conv=unit_height_conv)
+
if self._mask_params is not None:
- prediction_heads[SEGMENTATION_HEATMAP] = [
- make_prediction_net(num_classes,
- bias_fill=self._mask_params.heatmap_bias_init)
- for _ in range(num_feature_outputs)]
+ prediction_heads[SEGMENTATION_HEATMAP] = self._make_prediction_net_list(
+ num_feature_outputs,
+ num_classes,
+ bias_fill=self._mask_params.heatmap_bias_init,
+ name='seg_heatmap',
+ unit_height_conv=unit_height_conv)
+
if self._densepose_params is not None:
- prediction_heads[DENSEPOSE_HEATMAP] = [
- make_prediction_net( # pylint: disable=g-complex-comprehension
- self._densepose_params.num_parts,
- bias_fill=self._densepose_params.heatmap_bias_init)
- for _ in range(num_feature_outputs)]
- prediction_heads[DENSEPOSE_REGRESSION] = [
- make_prediction_net(2 * self._densepose_params.num_parts)
- for _ in range(num_feature_outputs)
- ]
+ prediction_heads[DENSEPOSE_HEATMAP] = self._make_prediction_net_list(
+ num_feature_outputs,
+ self._densepose_params.num_parts,
+ bias_fill=self._densepose_params.heatmap_bias_init,
+ name='dense_pose_heatmap',
+ unit_height_conv=unit_height_conv)
+ prediction_heads[DENSEPOSE_REGRESSION] = self._make_prediction_net_list(
+ num_feature_outputs,
+ 2 * self._densepose_params.num_parts,
+ name='dense_pose_regress',
+ unit_height_conv=unit_height_conv)
+
+ if self._track_params is not None:
+ prediction_heads[TRACK_REID] = self._make_prediction_net_list(
+ num_feature_outputs,
+ self._track_params.reid_embed_size,
+ name='track_reid',
+ unit_height_conv=unit_height_conv)
+
+ # Creates a classification network to train object embeddings by learning
+ # a projection from embedding space to object track ID space.
+ self.track_reid_classification_net = tf.keras.Sequential()
+ for _ in range(self._track_params.num_fc_layers - 1):
+ self.track_reid_classification_net.add(
+ tf.keras.layers.Dense(self._track_params.reid_embed_size,
+ input_shape=(
+ self._track_params.reid_embed_size,)))
+ self.track_reid_classification_net.add(
+ tf.keras.layers.BatchNormalization())
+ self.track_reid_classification_net.add(tf.keras.layers.ReLU())
+ self.track_reid_classification_net.add(
+ tf.keras.layers.Dense(self._track_params.num_track_ids,
+ input_shape=(
+ self._track_params.reid_embed_size,)))
+ if self._temporal_offset_params is not None:
+ prediction_heads[TEMPORAL_OFFSET] = self._make_prediction_net_list(
+ num_feature_outputs, NUM_OFFSET_CHANNELS, name='temporal_offset',
+ unit_height_conv=unit_height_conv)
return prediction_heads
def _initialize_target_assigners(self, stride, min_box_overlap_iou):
@@ -1668,9 +2442,31 @@ class CenterNetMetaArch(model.DetectionModel):
A dictionary of initialized target assigners for each task.
"""
target_assigners = {}
- target_assigners[OBJECT_CENTER] = (
- cn_assigner.CenterNetCenterHeatmapTargetAssigner(
- stride, min_box_overlap_iou))
+ keypoint_weights_for_center = (
+ self._center_params.keypoint_weights_for_center)
+ if not keypoint_weights_for_center:
+ target_assigners[OBJECT_CENTER] = (
+ cn_assigner.CenterNetCenterHeatmapTargetAssigner(
+ stride, min_box_overlap_iou, self._compute_heatmap_sparse))
+ self._center_from_keypoints = False
+ else:
+ # Determining the object center location by keypoint location is only
+ # supported when there is exactly one keypoint prediction task and no
+ # object detection task is specified.
+ assert len(self._kp_params_dict) == 1 and self._od_params is None
+ kp_params = next(iter(self._kp_params_dict.values()))
+ # The number of keypoint_weights_for_center needs to be the same as the
+ # number of keypoints.
+ assert len(keypoint_weights_for_center) == len(kp_params.keypoint_indices)
+ target_assigners[OBJECT_CENTER] = (
+ cn_assigner.CenterNetCenterHeatmapTargetAssigner(
+ stride,
+ min_box_overlap_iou,
+ self._compute_heatmap_sparse,
+ keypoint_class_id=kp_params.class_id,
+ keypoint_indices=kp_params.keypoint_indices,
+ keypoint_weights_for_center=keypoint_weights_for_center))
+ self._center_from_keypoints = True
if self._od_params is not None:
target_assigners[DETECTION_TASK] = (
cn_assigner.CenterNetBoxTargetAssigner(stride))
@@ -1683,7 +2479,9 @@ class CenterNetMetaArch(model.DetectionModel):
keypoint_indices=kp_params.keypoint_indices,
keypoint_std_dev=kp_params.keypoint_std_dev,
peak_radius=kp_params.offset_peak_radius,
- per_keypoint_offset=kp_params.per_keypoint_offset))
+ per_keypoint_offset=kp_params.per_keypoint_offset,
+ compute_heatmap_sparse=self._compute_heatmap_sparse,
+ per_keypoint_depth=kp_params.per_keypoint_depth))
if self._mask_params is not None:
target_assigners[SEGMENTATION_TASK] = (
cn_assigner.CenterNetMaskTargetAssigner(stride))
@@ -1691,6 +2489,13 @@ class CenterNetMetaArch(model.DetectionModel):
dp_stride = 1 if self._densepose_params.upsample_to_input_res else stride
target_assigners[DENSEPOSE_TASK] = (
cn_assigner.CenterNetDensePoseTargetAssigner(dp_stride))
+ if self._track_params is not None:
+ target_assigners[TRACK_TASK] = (
+ cn_assigner.CenterNetTrackTargetAssigner(
+ stride, self._track_params.num_track_ids))
+ if self._temporal_offset_params is not None:
+ target_assigners[TEMPORALOFFSET_TASK] = (
+ cn_assigner.CenterNetTemporalOffsetTargetAssigner(stride))
return target_assigners
@@ -1711,11 +2516,10 @@ class CenterNetMetaArch(model.DetectionModel):
Returns:
A float scalar tensor representing the object center loss per instance.
"""
- gt_boxes_list = self.groundtruth_lists(fields.BoxListFields.boxes)
gt_classes_list = self.groundtruth_lists(fields.BoxListFields.classes)
gt_weights_list = self.groundtruth_lists(fields.BoxListFields.weights)
- if self._center_params.use_only_known_classes:
+ if self._center_params.use_labeled_classes:
gt_labeled_classes_list = self.groundtruth_lists(
fields.InputDataFields.groundtruth_labeled_classes)
batch_labeled_classes = tf.stack(gt_labeled_classes_list, axis=0)
@@ -1727,12 +2531,22 @@ class CenterNetMetaArch(model.DetectionModel):
# Convert the groundtruth to targets.
assigner = self._target_assigner_dict[OBJECT_CENTER]
- heatmap_targets = assigner.assign_center_targets_from_boxes(
- height=input_height,
- width=input_width,
- gt_boxes_list=gt_boxes_list,
- gt_classes_list=gt_classes_list,
- gt_weights_list=gt_weights_list)
+ if self._center_from_keypoints:
+ gt_keypoints_list = self.groundtruth_lists(fields.BoxListFields.keypoints)
+ heatmap_targets = assigner.assign_center_targets_from_keypoints(
+ height=input_height,
+ width=input_width,
+ gt_classes_list=gt_classes_list,
+ gt_keypoints_list=gt_keypoints_list,
+ gt_weights_list=gt_weights_list)
+ else:
+ gt_boxes_list = self.groundtruth_lists(fields.BoxListFields.boxes)
+ heatmap_targets = assigner.assign_center_targets_from_boxes(
+ height=input_height,
+ width=input_width,
+ gt_boxes_list=gt_boxes_list,
+ gt_classes_list=gt_classes_list,
+ gt_weights_list=gt_weights_list)
flattened_heatmap_targets = _flatten_spatial_dimensions(heatmap_targets)
num_boxes = _to_float32(get_num_instances_from_weights(gt_weights_list))
@@ -1847,6 +2661,7 @@ class CenterNetMetaArch(model.DetectionModel):
heatmap_key = get_keypoint_name(task_name, KEYPOINT_HEATMAP)
offset_key = get_keypoint_name(task_name, KEYPOINT_OFFSET)
regression_key = get_keypoint_name(task_name, KEYPOINT_REGRESSION)
+ depth_key = get_keypoint_name(task_name, KEYPOINT_DEPTH)
heatmap_loss = self._compute_kp_heatmap_loss(
input_height=input_height,
input_width=input_width,
@@ -1874,6 +2689,14 @@ class CenterNetMetaArch(model.DetectionModel):
kp_params.keypoint_offset_loss_weight * offset_loss)
loss_dict[regression_key] = (
kp_params.keypoint_regression_loss_weight * reg_loss)
+ if kp_params.predict_depth:
+ depth_loss = self._compute_kp_depth_loss(
+ input_height=input_height,
+ input_width=input_width,
+ task_name=task_name,
+ depth_predictions=prediction_dict[depth_key],
+ localization_loss_fn=kp_params.localization_loss)
+ loss_dict[depth_key] = kp_params.keypoint_depth_loss_weight * depth_loss
return loss_dict
def _compute_kp_heatmap_loss(self, input_height, input_width, task_name,
@@ -2043,6 +2866,65 @@ class CenterNetMetaArch(model.DetectionModel):
tf.maximum(tf.reduce_sum(batch_weights), 1.0))
return loss
+ def _compute_kp_depth_loss(self, input_height, input_width, task_name,
+ depth_predictions, localization_loss_fn):
+ """Computes the loss of the keypoint depth estimation.
+
+ Args:
+ input_height: An integer scalar tensor representing input image height.
+ input_width: An integer scalar tensor representing input image width.
+ task_name: A string representing the name of the keypoint task.
+ depth_predictions: A list of float tensors of shape [batch_size,
+ out_height, out_width, 1 (or num_keypoints)] representing the prediction
+ heads of the model for keypoint depth.
+ localization_loss_fn: An object_detection.core.losses.Loss object to
+ compute the loss for the keypoint offset predictions in CenterNet.
+
+ Returns:
+ loss: A float scalar tensor representing the keypoint depth loss
+ normalized by number of total keypoints.
+ """
+ kp_params = self._kp_params_dict[task_name]
+ gt_keypoints_list = self.groundtruth_lists(fields.BoxListFields.keypoints)
+ gt_classes_list = self.groundtruth_lists(fields.BoxListFields.classes)
+ gt_weights_list = self.groundtruth_lists(fields.BoxListFields.weights)
+ gt_keypoint_depths_list = self.groundtruth_lists(
+ fields.BoxListFields.keypoint_depths)
+ gt_keypoint_depth_weights_list = self.groundtruth_lists(
+ fields.BoxListFields.keypoint_depth_weights)
+
+ assigner = self._target_assigner_dict[task_name]
+ (batch_indices, batch_depths,
+ batch_weights) = assigner.assign_keypoints_depth_targets(
+ height=input_height,
+ width=input_width,
+ gt_keypoints_list=gt_keypoints_list,
+ gt_weights_list=gt_weights_list,
+ gt_classes_list=gt_classes_list,
+ gt_keypoint_depths_list=gt_keypoint_depths_list,
+ gt_keypoint_depth_weights_list=gt_keypoint_depth_weights_list)
+
+ # Keypoint offset loss.
+ loss = 0.0
+ for prediction in depth_predictions:
+ if kp_params.per_keypoint_depth:
+ prediction = tf.expand_dims(prediction, axis=-1)
+ selected_depths = cn_assigner.get_batch_predictions_from_indices(
+ prediction, batch_indices)
+ # The dimensions passed are not as per the doc string but the loss
+ # still computes the correct value.
+ unweighted_loss = localization_loss_fn(
+ selected_depths,
+ batch_depths,
+ weights=tf.expand_dims(tf.ones_like(batch_weights), -1))
+ # Apply the weights after the loss function to have full control over it.
+ loss += batch_weights * tf.squeeze(unweighted_loss, axis=1)
+
+ loss = tf.reduce_sum(loss) / (
+ float(len(depth_predictions)) *
+ tf.maximum(tf.reduce_sum(batch_weights), 1.0))
+ return loss
+
def _compute_segmentation_losses(self, prediction_dict, per_pixel_weights):
"""Computes all the losses associated with segmentation.
@@ -2209,6 +3091,193 @@ class CenterNetMetaArch(model.DetectionModel):
num_predictions * num_valid_points)
return part_prediction_loss, surface_coord_loss
+ def _compute_track_losses(self, input_height, input_width, prediction_dict):
+ """Computes all the losses associated with tracking.
+
+ Args:
+ input_height: An integer scalar tensor representing input image height.
+ input_width: An integer scalar tensor representing input image width.
+ prediction_dict: The dictionary returned from the predict() method.
+
+ Returns:
+ A dictionary with tracking losses.
+ """
+ object_reid_predictions = prediction_dict[TRACK_REID]
+ embedding_loss = self._compute_track_embedding_loss(
+ input_height=input_height,
+ input_width=input_width,
+ object_reid_predictions=object_reid_predictions)
+ losses = {
+ TRACK_REID: embedding_loss
+ }
+ return losses
+
+ def _compute_track_embedding_loss(self, input_height, input_width,
+ object_reid_predictions):
+ """Computes the object ReID loss.
+
+ The embedding is trained as a classification task where the target is the
+ ID of each track among all tracks in the whole dataset.
+
+ Args:
+ input_height: An integer scalar tensor representing input image height.
+ input_width: An integer scalar tensor representing input image width.
+ object_reid_predictions: A list of float tensors of shape [batch_size,
+ out_height, out_width, reid_embed_size] representing the object
+ embedding feature maps.
+
+ Returns:
+ A float scalar tensor representing the object ReID loss per instance.
+ """
+ gt_track_ids_list = self.groundtruth_lists(fields.BoxListFields.track_ids)
+ gt_boxes_list = self.groundtruth_lists(fields.BoxListFields.boxes)
+ gt_weights_list = self.groundtruth_lists(fields.BoxListFields.weights)
+ num_boxes = _to_float32(get_num_instances_from_weights(gt_weights_list))
+
+ # Convert the groundtruth to targets.
+ assigner = self._target_assigner_dict[TRACK_TASK]
+ batch_indices, batch_weights, track_targets = assigner.assign_track_targets(
+ height=input_height,
+ width=input_width,
+ gt_track_ids_list=gt_track_ids_list,
+ gt_boxes_list=gt_boxes_list,
+ gt_weights_list=gt_weights_list)
+ batch_weights = tf.expand_dims(batch_weights, -1)
+
+ loss = 0.0
+ object_reid_loss = self._track_params.classification_loss
+ # Loop through each feature output head.
+ for pred in object_reid_predictions:
+ embedding_pred = cn_assigner.get_batch_predictions_from_indices(
+ pred, batch_indices)
+
+ reid_classification = self.track_reid_classification_net(embedding_pred)
+
+ loss += object_reid_loss(
+ reid_classification, track_targets, weights=batch_weights)
+
+ loss_per_instance = tf.reduce_sum(loss) / (
+ float(len(object_reid_predictions)) * num_boxes)
+
+ return loss_per_instance
+
+ def _compute_temporal_offset_loss(self, input_height,
+ input_width, prediction_dict):
+ """Computes the temporal offset loss for tracking.
+
+ Args:
+ input_height: An integer scalar tensor representing input image height.
+ input_width: An integer scalar tensor representing input image width.
+ prediction_dict: The dictionary returned from the predict() method.
+
+ Returns:
+ A dictionary with track/temporal_offset losses.
+ """
+ gt_boxes_list = self.groundtruth_lists(fields.BoxListFields.boxes)
+ gt_offsets_list = self.groundtruth_lists(
+ fields.BoxListFields.temporal_offsets)
+ gt_match_list = self.groundtruth_lists(
+ fields.BoxListFields.track_match_flags)
+ gt_weights_list = self.groundtruth_lists(fields.BoxListFields.weights)
+ num_boxes = tf.cast(
+ get_num_instances_from_weights(gt_weights_list), tf.float32)
+
+ offset_predictions = prediction_dict[TEMPORAL_OFFSET]
+ num_predictions = float(len(offset_predictions))
+
+ assigner = self._target_assigner_dict[TEMPORALOFFSET_TASK]
+ (batch_indices, batch_offset_targets,
+ batch_weights) = assigner.assign_temporal_offset_targets(
+ height=input_height,
+ width=input_width,
+ gt_boxes_list=gt_boxes_list,
+ gt_offsets_list=gt_offsets_list,
+ gt_match_list=gt_match_list,
+ gt_weights_list=gt_weights_list)
+ batch_weights = tf.expand_dims(batch_weights, -1)
+
+ offset_loss_fn = self._temporal_offset_params.localization_loss
+ loss_dict = {}
+ offset_loss = 0
+ for offset_pred in offset_predictions:
+ offset_pred = cn_assigner.get_batch_predictions_from_indices(
+ offset_pred, batch_indices)
+ offset_loss += offset_loss_fn(offset_pred[:, None],
+ batch_offset_targets[:, None],
+ weights=batch_weights)
+ offset_loss = tf.reduce_sum(offset_loss) / (num_predictions * num_boxes)
+ loss_dict[TEMPORAL_OFFSET] = offset_loss
+ return loss_dict
+
+ def _should_clip_keypoints(self):
+ """Returns a boolean indicating whether keypoint clipping should occur.
+
+ If there is only one keypoint task, clipping is controlled by the field
+ `clip_out_of_frame_keypoints`. If there are multiple keypoint tasks,
+ clipping logic is defined based on unanimous agreement of keypoint
+ parameters. If there is any ambiguity, clip_out_of_frame_keypoints is set
+ to False (default).
+ """
+ kp_params_iterator = iter(self._kp_params_dict.values())
+ if len(self._kp_params_dict) == 1:
+ kp_params = next(kp_params_iterator)
+ return kp_params.clip_out_of_frame_keypoints
+
+ # Multi-task setting.
+ kp_params = next(kp_params_iterator)
+ should_clip = kp_params.clip_out_of_frame_keypoints
+ for kp_params in kp_params_iterator:
+ if kp_params.clip_out_of_frame_keypoints != should_clip:
+ return False
+ return should_clip
+
+ def _rescore_instances(self, classes, scores, keypoint_scores):
+ """Rescores instances based on detection and keypoint scores.
+
+ Args:
+ classes: A [batch, max_detections] int32 tensor with detection classes.
+ scores: A [batch, max_detections] float32 tensor with detection scores.
+ keypoint_scores: A [batch, max_detections, total_num_keypoints] float32
+ tensor with keypoint scores.
+
+ Returns:
+ A [batch, max_detections] float32 tensor with possibly altered detection
+ scores.
+ """
+ batch, max_detections, total_num_keypoints = (
+ shape_utils.combined_static_and_dynamic_shape(keypoint_scores))
+ classes_tiled = tf.tile(classes[:, :, tf.newaxis],
+ multiples=[1, 1, total_num_keypoints])
+ # TODO(yuhuic): Investigate whether this function will create subgraphs in
+ # tflite that will cause the model to run slower at inference.
+ for kp_params in self._kp_params_dict.values():
+ if not kp_params.rescore_instances:
+ continue
+ class_id = kp_params.class_id
+ keypoint_indices = kp_params.keypoint_indices
+ kpt_mask = tf.reduce_sum(
+ tf.one_hot(keypoint_indices, depth=total_num_keypoints), axis=0)
+ kpt_mask_tiled = tf.tile(kpt_mask[tf.newaxis, tf.newaxis, :],
+ multiples=[batch, max_detections, 1])
+ class_and_keypoint_mask = tf.math.logical_and(
+ classes_tiled == class_id,
+ kpt_mask_tiled == 1.0)
+ class_and_keypoint_mask_float = tf.cast(class_and_keypoint_mask,
+ dtype=tf.float32)
+ visible_keypoints = tf.math.greater(keypoint_scores, 0.0)
+ num_visible_keypoints = tf.reduce_sum(
+ class_and_keypoint_mask_float *
+ tf.cast(visible_keypoints, tf.float32), axis=-1)
+ num_visible_keypoints = tf.math.maximum(num_visible_keypoints, 1.0)
+ scores_for_class = (1./num_visible_keypoints) * (
+ tf.reduce_sum(class_and_keypoint_mask_float *
+ scores[:, :, tf.newaxis] *
+ keypoint_scores, axis=-1))
+ scores = tf.where(classes == class_id,
+ scores_for_class,
+ scores)
+ return scores
+
def preprocess(self, inputs):
outputs = shape_utils.resize_images_and_return_shapes(
inputs, self._image_resizer_fn)
@@ -2303,7 +3372,9 @@ class CenterNetMetaArch(model.DetectionModel):
'Loss/$TASK_NAME/keypoint/regression', (optional)
'Loss/segmentation/heatmap', (optional)
'Loss/densepose/heatmap', (optional)
- 'Loss/densepose/regression]' (optional)
+ 'Loss/densepose/regression', (optional)
+ 'Loss/track/reid'] (optional)
+ 'Loss/track/offset'] (optional)
scalar tensors corresponding to the losses for different tasks. Note the
$TASK_NAME is provided by the KeypointEstimation namedtuple used to
differentiate between different keypoint tasks.
@@ -2312,8 +3383,8 @@ class CenterNetMetaArch(model.DetectionModel):
_, input_height, input_width, _ = _get_shape(
prediction_dict['preprocessed_inputs'], 4)
- output_height, output_width = (input_height // self._stride,
- input_width // self._stride)
+ output_height, output_width = (tf.maximum(input_height // self._stride, 1),
+ tf.maximum(input_width // self._stride, 1))
# TODO(vighneshb) Explore whether using floor here is safe.
output_true_image_shapes = tf.ceil(
@@ -2371,6 +3442,26 @@ class CenterNetMetaArch(model.DetectionModel):
densepose_losses[key] * self._densepose_params.task_loss_weight)
losses.update(densepose_losses)
+ if self._track_params is not None:
+ track_losses = self._compute_track_losses(
+ input_height=input_height,
+ input_width=input_width,
+ prediction_dict=prediction_dict)
+ for key in track_losses:
+ track_losses[key] = (
+ track_losses[key] * self._track_params.task_loss_weight)
+ losses.update(track_losses)
+
+ if self._temporal_offset_params is not None:
+ offset_losses = self._compute_temporal_offset_loss(
+ input_height=input_height,
+ input_width=input_width,
+ prediction_dict=prediction_dict)
+ for key in offset_losses:
+ offset_losses[key] = (
+ offset_losses[key] * self._temporal_offset_params.task_loss_weight)
+ losses.update(offset_losses)
+
# Prepend the LOSS_KEY_PREFIX to the keys in the dictionary such that the
# losses will be grouped together in Tensorboard.
return dict([('%s/%s' % (LOSS_KEY_PREFIX, key), val)
@@ -2394,8 +3485,13 @@ class CenterNetMetaArch(model.DetectionModel):
detections: a dictionary containing the following fields
detection_boxes - A tensor of shape [batch, max_detections, 4]
holding the predicted boxes.
+ detection_boxes_strided: A tensor of shape [batch_size, num_detections,
+ 4] holding the predicted boxes in absolute coordinates of the
+ feature extractor's final layer output.
detection_scores: A tensor of shape [batch, max_detections] holding
the predicted score for each box.
+ detection_multiclass_scores: A tensor of shape [batch, max_detection,
+ num_classes] holding multiclass score for each box.
detection_classes: An integer tensor of shape [batch, max_detections]
containing the detected class for each box.
num_detections: An integer tensor of shape [batch] containing the
@@ -2413,6 +3509,8 @@ class CenterNetMetaArch(model.DetectionModel):
detection_surface_coords: (Optional) A float32 tensor of shape [batch,
max_detection, mask_height, mask_width, 2] with DensePose surface
coordinates, in (v, u) format.
+ detection_embeddings: (Optional) A float tensor of shape [batch,
+ max_detections, reid_embed_size] containing object embeddings.
"""
object_center_prob = tf.nn.sigmoid(prediction_dict[OBJECT_CENTER][-1])
# Get x, y and channel indices corresponding to the top indices in the class
@@ -2421,35 +3519,84 @@ class CenterNetMetaArch(model.DetectionModel):
top_k_feature_map_locations(
object_center_prob, max_pool_kernel_size=3,
k=self._center_params.max_box_predictions))
+ multiclass_scores = tf.gather_nd(
+ object_center_prob, tf.stack([y_indices, x_indices], -1), batch_dims=1)
- boxes_strided, classes, scores, num_detections = (
- prediction_tensors_to_boxes(
- detection_scores, y_indices, x_indices, channel_indices,
- prediction_dict[BOX_SCALE][-1], prediction_dict[BOX_OFFSET][-1]))
-
- boxes = convert_strided_predictions_to_normalized_boxes(
- boxes_strided, self._stride, true_image_shapes)
-
+ num_detections = tf.reduce_sum(tf.to_int32(detection_scores > 0), axis=1)
postprocess_dict = {
- fields.DetectionResultFields.detection_boxes: boxes,
- fields.DetectionResultFields.detection_scores: scores,
- fields.DetectionResultFields.detection_classes: classes,
+ fields.DetectionResultFields.detection_scores: detection_scores,
+ fields.DetectionResultFields.detection_multiclass_scores:
+ multiclass_scores,
+ fields.DetectionResultFields.detection_classes: channel_indices,
fields.DetectionResultFields.num_detections: num_detections,
}
+ boxes_strided = None
+ if self._od_params:
+ boxes_strided = (
+ prediction_tensors_to_boxes(y_indices, x_indices,
+ prediction_dict[BOX_SCALE][-1],
+ prediction_dict[BOX_OFFSET][-1]))
+
+ boxes = convert_strided_predictions_to_normalized_boxes(
+ boxes_strided, self._stride, true_image_shapes)
+
+ postprocess_dict.update({
+ fields.DetectionResultFields.detection_boxes: boxes,
+ 'detection_boxes_strided': boxes_strided
+ })
+
if self._kp_params_dict:
- keypoints, keypoint_scores = self._postprocess_keypoints(
- prediction_dict, classes, y_indices, x_indices,
- boxes_strided, num_detections)
- keypoints, keypoint_scores = (
- convert_strided_predictions_to_normalized_keypoints(
- keypoints, keypoint_scores, self._stride, true_image_shapes,
- clip_out_of_frame_keypoints=True))
+ # If the model is trained to predict only one class of object and its
+ # keypoint, we fall back to a simpler postprocessing function which uses
+ # the ops that are supported by tf.lite on GPU.
+ clip_keypoints = self._should_clip_keypoints()
+ if len(self._kp_params_dict) == 1 and self._num_classes == 1:
+ (keypoints, keypoint_scores,
+ keypoint_depths) = self._postprocess_keypoints_single_class(
+ prediction_dict, channel_indices, y_indices, x_indices,
+ boxes_strided, num_detections)
+ keypoints, keypoint_scores = (
+ convert_strided_predictions_to_normalized_keypoints(
+ keypoints, keypoint_scores, self._stride, true_image_shapes,
+ clip_out_of_frame_keypoints=clip_keypoints))
+ if keypoint_depths is not None:
+ postprocess_dict.update({
+ fields.DetectionResultFields.detection_keypoint_depths:
+ keypoint_depths
+ })
+ else:
+ # Multi-class keypoint estimation task does not support depth
+ # estimation.
+ assert all([
+ not kp_dict.predict_depth
+ for kp_dict in self._kp_params_dict.values()
+ ])
+ keypoints, keypoint_scores = self._postprocess_keypoints_multi_class(
+ prediction_dict, channel_indices, y_indices, x_indices,
+ None, num_detections)
+ keypoints, keypoint_scores = (
+ convert_strided_predictions_to_normalized_keypoints(
+ keypoints, keypoint_scores, self._stride, true_image_shapes,
+ clip_out_of_frame_keypoints=clip_keypoints))
+
+ # Update instance scores based on keypoints.
+ scores = self._rescore_instances(
+ channel_indices, detection_scores, keypoint_scores)
postprocess_dict.update({
+ fields.DetectionResultFields.detection_scores: scores,
fields.DetectionResultFields.detection_keypoints: keypoints,
fields.DetectionResultFields.detection_keypoint_scores:
keypoint_scores
})
+ if self._od_params is None:
+ # Still output the box prediction by enclosing the keypoints for
+ # evaluation purpose.
+ boxes = keypoint_ops.keypoints_to_enclosing_bounding_boxes(
+ keypoints, keypoints_axis=2)
+ postprocess_dict.update({
+ fields.DetectionResultFields.detection_boxes: boxes,
+ })
if self._mask_params:
masks = tf.nn.sigmoid(prediction_dict[SEGMENTATION_HEATMAP][-1])
@@ -2461,7 +3608,7 @@ class CenterNetMetaArch(model.DetectionModel):
densepose_class_index = self._densepose_params.class_id
instance_masks, surface_coords = (
convert_strided_predictions_to_instance_masks(
- boxes, classes, masks, true_image_shapes,
+ boxes, channel_indices, masks, true_image_shapes,
densepose_part_heatmap, densepose_surface_coords,
stride=self._stride, mask_height=self._mask_params.mask_height,
mask_width=self._mask_params.mask_width,
@@ -2474,12 +3621,196 @@ class CenterNetMetaArch(model.DetectionModel):
fields.DetectionResultFields.detection_surface_coords] = (
surface_coords)
+ if self._track_params:
+ embeddings = self._postprocess_embeddings(prediction_dict,
+ y_indices, x_indices)
+ postprocess_dict.update({
+ fields.DetectionResultFields.detection_embeddings: embeddings
+ })
+
+ if self._temporal_offset_params:
+ offsets = prediction_tensors_to_temporal_offsets(
+ y_indices, x_indices,
+ prediction_dict[TEMPORAL_OFFSET][-1])
+ postprocess_dict[fields.DetectionResultFields.detection_offsets] = offsets
+
+ if self._non_max_suppression_fn:
+ boxes = tf.expand_dims(
+ postprocess_dict.pop(fields.DetectionResultFields.detection_boxes),
+ axis=-2)
+ multiclass_scores = postprocess_dict[
+ fields.DetectionResultFields.detection_multiclass_scores]
+ num_valid_boxes = postprocess_dict.pop(
+ fields.DetectionResultFields.num_detections)
+ # Remove scores and classes as NMS will compute these form multiclass
+ # scores.
+ postprocess_dict.pop(fields.DetectionResultFields.detection_scores)
+ postprocess_dict.pop(fields.DetectionResultFields.detection_classes)
+ (nmsed_boxes, nmsed_scores, nmsed_classes, _, nmsed_additional_fields,
+ num_detections) = self._non_max_suppression_fn(
+ boxes,
+ multiclass_scores,
+ additional_fields=postprocess_dict,
+ num_valid_boxes=num_valid_boxes)
+ postprocess_dict = nmsed_additional_fields
+ postprocess_dict[
+ fields.DetectionResultFields.detection_boxes] = nmsed_boxes
+ postprocess_dict[
+ fields.DetectionResultFields.detection_scores] = nmsed_scores
+ postprocess_dict[
+ fields.DetectionResultFields.detection_classes] = nmsed_classes
+ postprocess_dict[
+ fields.DetectionResultFields.num_detections] = num_detections
+ postprocess_dict.update(nmsed_additional_fields)
+ return postprocess_dict
+
+ def postprocess_single_instance_keypoints(
+ self,
+ prediction_dict,
+ true_image_shapes):
+ """Postprocess for predicting single instance keypoints.
+
+ This postprocess function is a special case of predicting the keypoint of
+ a single instance in the image (original CenterNet postprocess supports
+ multi-instance prediction). Due to the simplification assumption, this
+ postprocessing function achieves much faster inference time.
+ Here is a short list of the modifications made in this function:
+
+ 1) Assume the model predicts only single class keypoint.
+ 2) Assume there is only one instance in the image. If multiple instances
+ appear in the image, the model tends to predict the one that is closer
+ to the image center (the other ones are considered as background and
+ are rejected by the model).
+ 3) Avoid using top_k ops in the postprocessing logics since it is slower
+ than using argmax.
+ 4) The predictions other than the keypoints are ignored, e.g. boxes.
+ 5) The input batch size is assumed to be 1.
+
+ Args:
+ prediction_dict: a dictionary holding predicted tensors from "predict"
+ function.
+ true_image_shapes: int32 tensor of shape [batch, 3] where each row is of
+ the form [height, width, channels] indicating the shapes of true images
+ in the resized images, as resized images can be padded with zeros.
+
+ Returns:
+ detections: a dictionary containing the following fields
+ detection_keypoints: A float tensor of shape
+ [1, 1, num_keypoints, 2] with normalized keypoints. Any invalid
+ keypoints have their coordinates and scores set to 0.0.
+ detection_keypoint_scores: A float tensor of shape
+ [1, 1, num_keypoints] with scores for each keypoint.
+ """
+ # The number of keypoint task is expected to be 1.
+ assert len(self._kp_params_dict) == 1
+ task_name, kp_params = next(iter(self._kp_params_dict.items()))
+ keypoint_heatmap = tf.nn.sigmoid(prediction_dict[get_keypoint_name(
+ task_name, KEYPOINT_HEATMAP)][-1])
+ keypoint_offset = prediction_dict[get_keypoint_name(task_name,
+ KEYPOINT_OFFSET)][-1]
+ keypoint_regression = prediction_dict[get_keypoint_name(
+ task_name, KEYPOINT_REGRESSION)][-1]
+ object_heatmap = tf.nn.sigmoid(prediction_dict[OBJECT_CENTER][-1])
+
+ keypoint_depths = None
+ if kp_params.predict_depth:
+ keypoint_depths = prediction_dict[get_keypoint_name(
+ task_name, KEYPOINT_DEPTH)][-1]
+ keypoints, keypoint_scores, keypoint_depths = (
+ prediction_to_single_instance_keypoints(
+ object_heatmap=object_heatmap,
+ keypoint_heatmap=keypoint_heatmap,
+ keypoint_offset=keypoint_offset,
+ keypoint_regression=keypoint_regression,
+ kp_params=kp_params,
+ keypoint_depths=keypoint_depths))
+
+ keypoints, keypoint_scores = (
+ convert_strided_predictions_to_normalized_keypoints(
+ keypoints,
+ keypoint_scores,
+ self._stride,
+ true_image_shapes,
+ clip_out_of_frame_keypoints=False))
+ postprocess_dict = {
+ fields.DetectionResultFields.detection_keypoints: keypoints,
+ fields.DetectionResultFields.detection_keypoint_scores: keypoint_scores
+ }
+
+ if kp_params.predict_depth:
+ postprocess_dict.update({
+ fields.DetectionResultFields.detection_keypoint_depths:
+ keypoint_depths
+ })
return postprocess_dict
- def _postprocess_keypoints(self, prediction_dict, classes, y_indices,
- x_indices, boxes, num_detections):
+ def _postprocess_embeddings(self, prediction_dict, y_indices, x_indices):
+ """Performs postprocessing on embedding predictions.
+
+ Args:
+ prediction_dict: a dictionary holding predicted tensors, returned from the
+ predict() method. This dictionary should contain embedding prediction
+ feature maps for tracking task.
+ y_indices: A [batch_size, max_detections] int tensor with y indices for
+ all object centers.
+ x_indices: A [batch_size, max_detections] int tensor with x indices for
+ all object centers.
+
+ Returns:
+ embeddings: A [batch_size, max_detection, reid_embed_size] float32
+ tensor with L2 normalized embeddings extracted from detection box
+ centers.
+ """
+ embedding_predictions = prediction_dict[TRACK_REID][-1]
+ embeddings = predicted_embeddings_at_object_centers(
+ embedding_predictions, y_indices, x_indices)
+ embeddings, _ = tf.linalg.normalize(embeddings, axis=-1)
+
+ return embeddings
+
+ def _scatter_keypoints_to_batch(self, num_ind, kpt_coords_for_example,
+ kpt_scores_for_example,
+ instance_inds_for_example, max_detections,
+ total_num_keypoints):
+ """Helper function to convert scattered keypoints into batch."""
+ def left_fn(kpt_coords_for_example, kpt_scores_for_example,
+ instance_inds_for_example):
+ # Scatter into tensor where instances align with original detection
+ # instances. New shape of keypoint coordinates and scores are
+ # [1, max_detections, num_total_keypoints, 2] and
+ # [1, max_detections, num_total_keypoints], respectively.
+ return _pad_to_full_instance_dim(
+ kpt_coords_for_example, kpt_scores_for_example,
+ instance_inds_for_example,
+ self._center_params.max_box_predictions)
+
+ def right_fn():
+ kpt_coords_for_example_all_det = tf.zeros(
+ [1, max_detections, total_num_keypoints, 2], dtype=tf.float32)
+ kpt_scores_for_example_all_det = tf.zeros(
+ [1, max_detections, total_num_keypoints], dtype=tf.float32)
+ return (kpt_coords_for_example_all_det,
+ kpt_scores_for_example_all_det)
+
+ left_fn = functools.partial(left_fn, kpt_coords_for_example,
+ kpt_scores_for_example,
+ instance_inds_for_example)
+
+ # Use dimension values instead of tf.size for tf.lite compatibility.
+ return tf.cond(num_ind[0] > 0, left_fn, right_fn)
+
+ def _postprocess_keypoints_multi_class(self, prediction_dict, classes,
+ y_indices, x_indices, boxes,
+ num_detections):
"""Performs postprocessing on keypoint predictions.
+ This is the most general keypoint postprocessing function which supports
+ multiple keypoint tasks (e.g. human and dog keypoints) and multiple object
+ detection classes. Note that it is the most expensive postprocessing logics
+ and is currently not tf.lite/tf.js compatible. See
+ _postprocess_keypoints_single_class if you plan to export the model in more
+ portable format.
+
Args:
prediction_dict: a dictionary holding predicted tensors, returned from the
predict() method. This dictionary should contain keypoint prediction
@@ -2504,7 +3835,7 @@ class CenterNetMetaArch(model.DetectionModel):
"""
total_num_keypoints = sum(len(kp_dict.keypoint_indices) for kp_dict
in self._kp_params_dict.values())
- batch_size, max_detections, _ = _get_shape(boxes, 3)
+ batch_size, max_detections = _get_shape(classes, 2)
kpt_coords_for_example_list = []
kpt_scores_for_example_list = []
for ex_ind in range(batch_size):
@@ -2520,29 +3851,39 @@ class CenterNetMetaArch(model.DetectionModel):
get_keypoint_name(task_name, KEYPOINT_REGRESSION)][-1]
instance_inds = self._get_instance_indices(
classes, num_detections, ex_ind, kp_params.class_id)
+ num_ind = _get_shape(instance_inds, 1)
- def true_fn(
- keypoint_heatmap, keypoint_offsets, keypoint_regression,
- classes, y_indices, x_indices, boxes, instance_inds,
- ex_ind, kp_params):
+ def true_fn(keypoint_heatmap, keypoint_offsets, keypoint_regression,
+ classes, y_indices, x_indices, boxes, instance_inds, ex_ind,
+ kp_params):
"""Logics to execute when instance_inds is not an empty set."""
+ # Gather the feature map locations corresponding to the object class.
+ y_indices_for_kpt_class = tf.gather(y_indices, instance_inds, axis=1)
+ x_indices_for_kpt_class = tf.gather(x_indices, instance_inds, axis=1)
+ if boxes is None:
+ boxes_for_kpt_class = None
+ else:
+ boxes_for_kpt_class = tf.gather(boxes, instance_inds, axis=1)
+
# Postprocess keypoints and scores for class and single image. Shapes
# are [1, num_instances_i, num_keypoints_i, 2] and
# [1, num_instances_i, num_keypoints_i], respectively. Note that
# num_instances_i and num_keypoints_i refers to the number of
# instances and keypoints for class i, respectively.
- kpt_coords_for_class, kpt_scores_for_class = (
+ (kpt_coords_for_class, kpt_scores_for_class, _) = (
self._postprocess_keypoints_for_class_and_image(
keypoint_heatmap, keypoint_offsets, keypoint_regression,
- classes, y_indices, x_indices, boxes, instance_inds,
- ex_ind, kp_params))
+ classes, y_indices_for_kpt_class, x_indices_for_kpt_class,
+ boxes_for_kpt_class, ex_ind, kp_params))
+
# Expand keypoint dimension (with padding) so that coordinates and
# scores have shape [1, num_instances_i, num_total_keypoints, 2] and
# [1, num_instances_i, num_total_keypoints], respectively.
kpts_coords_for_class_padded, kpt_scores_for_class_padded = (
- _pad_to_full_keypoint_dim(
- kpt_coords_for_class, kpt_scores_for_class,
- kp_params.keypoint_indices, total_num_keypoints))
+ _pad_to_full_keypoint_dim(kpt_coords_for_class,
+ kpt_scores_for_class,
+ kp_params.keypoint_indices,
+ total_num_keypoints))
return kpts_coords_for_class_padded, kpt_scores_for_class_padded
def false_fn():
@@ -2554,7 +3895,8 @@ class CenterNetMetaArch(model.DetectionModel):
true_fn, keypoint_heatmap, keypoint_offsets, keypoint_regression,
classes, y_indices, x_indices, boxes, instance_inds, ex_ind,
kp_params)
- results = tf.cond(tf.size(instance_inds) > 0, true_fn, false_fn)
+ # Use dimension values instead of tf.size for tf.lite compatibility.
+ results = tf.cond(num_ind[0] > 0, true_fn, false_fn)
kpt_coords_for_class_list.append(results[0])
kpt_scores_for_class_list.append(results[1])
@@ -2566,24 +3908,13 @@ class CenterNetMetaArch(model.DetectionModel):
instance_inds_for_example = tf.concat(instance_inds_for_class_list,
axis=0)
- if tf.size(instance_inds_for_example) > 0:
- # Scatter into tensor where instances align with original detection
- # instances. New shape of keypoint coordinates and scores are
- # [1, max_detections, num_total_keypoints, 2] and
- # [1, max_detections, num_total_keypoints], respectively.
- kpt_coords_for_example_all_det, kpt_scores_for_example_all_det = (
- _pad_to_full_instance_dim(
- kpt_coords_for_example, kpt_scores_for_example,
- instance_inds_for_example,
- self._center_params.max_box_predictions))
- else:
- kpt_coords_for_example_all_det = tf.zeros(
- [1, max_detections, total_num_keypoints, 2], dtype=tf.float32)
- kpt_scores_for_example_all_det = tf.zeros(
- [1, max_detections, total_num_keypoints], dtype=tf.float32)
+ (kpt_coords_for_example_all_det,
+ kpt_scores_for_example_all_det) = self._scatter_keypoints_to_batch(
+ num_ind, kpt_coords_for_example, kpt_scores_for_example,
+ instance_inds_for_example, max_detections, total_num_keypoints)
- kpt_coords_for_example_list.append(kpt_coords_for_example_all_det)
- kpt_scores_for_example_list.append(kpt_scores_for_example_all_det)
+ kpt_coords_for_example_list.append(kpt_coords_for_example_all_det)
+ kpt_scores_for_example_list.append(kpt_scores_for_example_all_det)
# Concatenate all keypoints and scores from all examples in the batch.
# Shapes are [batch_size, max_detections, num_total_keypoints, 2] and
@@ -2593,6 +3924,91 @@ class CenterNetMetaArch(model.DetectionModel):
return keypoints, keypoint_scores
+ def _postprocess_keypoints_single_class(self, prediction_dict, classes,
+ y_indices, x_indices, boxes,
+ num_detections):
+ """Performs postprocessing on keypoint predictions (single class only).
+
+ This function handles the special case of keypoint task that the model
+ predicts only one class of the bounding box/keypoint (e.g. person). By the
+ assumption, the function uses only tf.lite supported ops and should run
+ faster.
+
+ Args:
+ prediction_dict: a dictionary holding predicted tensors, returned from the
+ predict() method. This dictionary should contain keypoint prediction
+ feature maps for each keypoint task.
+ classes: A [batch_size, max_detections] int tensor with class indices for
+ all detected objects.
+ y_indices: A [batch_size, max_detections] int tensor with y indices for
+ all object centers.
+ x_indices: A [batch_size, max_detections] int tensor with x indices for
+ all object centers.
+ boxes: A [batch_size, max_detections, 4] float32 tensor with bounding
+ boxes in (un-normalized) output space.
+ num_detections: A [batch_size] int tensor with the number of valid
+ detections for each image.
+
+ Returns:
+ A tuple of
+ keypoints: a [batch_size, max_detection, num_total_keypoints, 2] float32
+ tensor with keypoints in the output (strided) coordinate frame.
+ keypoint_scores: a [batch_size, max_detections, num_total_keypoints]
+ float32 tensor with keypoint scores.
+ """
+ # This function only works when there is only one keypoint task and the
+ # number of classes equal to one. For more general use cases, please use
+ # _postprocess_keypoints instead.
+ assert len(self._kp_params_dict) == 1 and self._num_classes == 1
+ task_name, kp_params = next(iter(self._kp_params_dict.items()))
+ keypoint_heatmap = prediction_dict[
+ get_keypoint_name(task_name, KEYPOINT_HEATMAP)][-1]
+ keypoint_offsets = prediction_dict[
+ get_keypoint_name(task_name, KEYPOINT_OFFSET)][-1]
+ keypoint_regression = prediction_dict[
+ get_keypoint_name(task_name, KEYPOINT_REGRESSION)][-1]
+ keypoint_depth_predictions = None
+ if kp_params.predict_depth:
+ keypoint_depth_predictions = prediction_dict[get_keypoint_name(
+ task_name, KEYPOINT_DEPTH)][-1]
+
+ batch_size, _ = _get_shape(classes, 2)
+ kpt_coords_for_example_list = []
+ kpt_scores_for_example_list = []
+ kpt_depths_for_example_list = []
+ for ex_ind in range(batch_size):
+ # Postprocess keypoints and scores for class and single image. Shapes
+ # are [1, max_detections, num_keypoints, 2] and
+ # [1, max_detections, num_keypoints], respectively.
+ (kpt_coords_for_class, kpt_scores_for_class, kpt_depths_for_class) = (
+ self._postprocess_keypoints_for_class_and_image(
+ keypoint_heatmap,
+ keypoint_offsets,
+ keypoint_regression,
+ classes,
+ y_indices,
+ x_indices,
+ boxes,
+ ex_ind,
+ kp_params,
+ keypoint_depth_predictions=keypoint_depth_predictions))
+
+ kpt_coords_for_example_list.append(kpt_coords_for_class)
+ kpt_scores_for_example_list.append(kpt_scores_for_class)
+ kpt_depths_for_example_list.append(kpt_depths_for_class)
+
+ # Concatenate all keypoints and scores from all examples in the batch.
+ # Shapes are [batch_size, max_detections, num_keypoints, 2] and
+ # [batch_size, max_detections, num_keypoints], respectively.
+ keypoints = tf.concat(kpt_coords_for_example_list, axis=0)
+ keypoint_scores = tf.concat(kpt_scores_for_example_list, axis=0)
+
+ keypoint_depths = None
+ if kp_params.predict_depth:
+ keypoint_depths = tf.concat(kpt_depths_for_example_list, axis=0)
+
+ return keypoints, keypoint_scores, keypoint_depths
+
def _get_instance_indices(self, classes, num_detections, batch_index,
class_id):
"""Gets the instance indices that match the target class ID.
@@ -2606,7 +4022,7 @@ class CenterNetMetaArch(model.DetectionModel):
class_id: Class id
Returns:
- instance_inds: A [num_instances] int tensor where each element indicates
+ instance_inds: A [num_instances] int32 tensor where each element indicates
the instance location within the `classes` tensor. This is useful to
associate the refined keypoints with the original detections (i.e.
boxes)
@@ -2615,26 +4031,29 @@ class CenterNetMetaArch(model.DetectionModel):
_, max_detections = shape_utils.combined_static_and_dynamic_shape(
classes)
# Get the detection indices corresponding to the target class.
+ # Call tf.math.equal with matched tensor shape to make it tf.lite
+ # compatible.
valid_detections_with_kpt_class = tf.math.logical_and(
tf.range(max_detections) < num_detections[batch_index],
- classes[0] == class_id)
+ tf.math.equal(classes[0], tf.fill(classes[0].shape, class_id)))
instance_inds = tf.where(valid_detections_with_kpt_class)[:, 0]
- return instance_inds
+ # Cast the indices tensor to int32 for tf.lite compatibility.
+ return tf.cast(instance_inds, tf.int32)
def _postprocess_keypoints_for_class_and_image(
- self, keypoint_heatmap, keypoint_offsets, keypoint_regression, classes,
- y_indices, x_indices, boxes, indices_with_kpt_class, batch_index,
- kp_params):
+ self,
+ keypoint_heatmap,
+ keypoint_offsets,
+ keypoint_regression,
+ classes,
+ y_indices,
+ x_indices,
+ boxes,
+ batch_index,
+ kp_params,
+ keypoint_depth_predictions=None):
"""Postprocess keypoints for a single image and class.
- This function performs the following postprocessing operations on a single
- image and single keypoint class:
- - Converts keypoints scores to range [0, 1] with sigmoid.
- - Determines the detections that correspond to the specified keypoint class.
- - Gathers the regressed keypoints at the detection (i.e. box) centers.
- - Gathers keypoint candidates from the keypoint heatmaps.
- - Snaps regressed keypoints to nearby keypoint candidates.
-
Args:
keypoint_heatmap: A [batch_size, height, width, num_keypoints] float32
tensor with keypoint heatmaps.
@@ -2650,13 +4069,11 @@ class CenterNetMetaArch(model.DetectionModel):
all object centers.
boxes: A [batch_size, max_detections, 4] float32 tensor with detected
boxes in the output (strided) frame.
- indices_with_kpt_class: A [num_instances] int tensor where each element
- indicates the instance location within the `classes` tensor. This is
- useful to associate the refined keypoints with the original detections
- (i.e. boxes)
batch_index: An integer specifying the index for an example in the batch.
kp_params: A `KeypointEstimationParams` object with parameters for a
single keypoint class.
+ keypoint_depth_predictions: (optional) A [batch_size, height, width, 1]
+ float32 tensor representing the keypoint depth prediction.
Returns:
A tuple of
@@ -2667,28 +4084,31 @@ class CenterNetMetaArch(model.DetectionModel):
for the specific class.
refined_scores: A [1, num_instances, num_keypoints] float32 tensor with
keypoint scores.
+ refined_depths: A [1, num_instances, num_keypoints] float32 tensor with
+ keypoint depths. Return None if the input keypoint_depth_predictions is
+ None.
"""
- keypoint_indices = kp_params.keypoint_indices
- num_keypoints = len(keypoint_indices)
+ num_keypoints = len(kp_params.keypoint_indices)
keypoint_heatmap = tf.nn.sigmoid(
keypoint_heatmap[batch_index:batch_index+1, ...])
keypoint_offsets = keypoint_offsets[batch_index:batch_index+1, ...]
keypoint_regression = keypoint_regression[batch_index:batch_index+1, ...]
+ keypoint_depths = None
+ if keypoint_depth_predictions is not None:
+ keypoint_depths = keypoint_depth_predictions[batch_index:batch_index + 1,
+ ...]
y_indices = y_indices[batch_index:batch_index+1, ...]
x_indices = x_indices[batch_index:batch_index+1, ...]
-
- # Gather the feature map locations corresponding to the object class.
- y_indices_for_kpt_class = tf.gather(y_indices, indices_with_kpt_class,
- axis=1)
- x_indices_for_kpt_class = tf.gather(x_indices, indices_with_kpt_class,
- axis=1)
- boxes_for_kpt_class = tf.gather(boxes, indices_with_kpt_class, axis=1)
+ if boxes is None:
+ boxes_slice = None
+ else:
+ boxes_slice = boxes[batch_index:batch_index+1, ...]
# Gather the regressed keypoints. Final tensor has shape
# [1, num_instances, num_keypoints, 2].
regressed_keypoints_for_objects = regressed_keypoints_at_object_centers(
- keypoint_regression, y_indices_for_kpt_class, x_indices_for_kpt_class)
+ keypoint_regression, y_indices, x_indices)
regressed_keypoints_for_objects = tf.reshape(
regressed_keypoints_for_objects, [1, -1, num_keypoints, 2])
@@ -2696,26 +4116,36 @@ class CenterNetMetaArch(model.DetectionModel):
# The shape of keypoint_candidates and keypoint_scores is:
# [1, num_candidates_per_keypoint, num_keypoints, 2] and
# [1, num_candidates_per_keypoint, num_keypoints], respectively.
- keypoint_candidates, keypoint_scores, num_keypoint_candidates = (
- prediction_tensors_to_keypoint_candidates(
- keypoint_heatmap, keypoint_offsets,
- keypoint_score_threshold=(
- kp_params.keypoint_candidate_score_threshold),
- max_pool_kernel_size=kp_params.peak_max_pool_kernel_size,
- max_candidates=kp_params.num_candidates_per_keypoint))
+ (keypoint_candidates, keypoint_scores, num_keypoint_candidates,
+ keypoint_depth_candidates) = (
+ prediction_tensors_to_keypoint_candidates(
+ keypoint_heatmap,
+ keypoint_offsets,
+ keypoint_score_threshold=(
+ kp_params.keypoint_candidate_score_threshold),
+ max_pool_kernel_size=kp_params.peak_max_pool_kernel_size,
+ max_candidates=kp_params.num_candidates_per_keypoint,
+ keypoint_depths=keypoint_depths))
# Get the refined keypoints and scores, of shape
# [1, num_instances, num_keypoints, 2] and
# [1, num_instances, num_keypoints], respectively.
- refined_keypoints, refined_scores = refine_keypoints(
- regressed_keypoints_for_objects, keypoint_candidates, keypoint_scores,
- num_keypoint_candidates, bboxes=boxes_for_kpt_class,
+ (refined_keypoints, refined_scores, refined_depths) = refine_keypoints(
+ regressed_keypoints_for_objects,
+ keypoint_candidates,
+ keypoint_scores,
+ num_keypoint_candidates,
+ bboxes=boxes_slice,
unmatched_keypoint_score=kp_params.unmatched_keypoint_score,
box_scale=kp_params.box_scale,
candidate_search_scale=kp_params.candidate_search_scale,
- candidate_ranking_mode=kp_params.candidate_ranking_mode)
+ candidate_ranking_mode=kp_params.candidate_ranking_mode,
+ score_distance_offset=kp_params.score_distance_offset,
+ keypoint_depth_candidates=keypoint_depth_candidates,
+ keypoint_score_threshold=(
+ kp_params.keypoint_candidate_score_threshold))
- return refined_keypoints, refined_scores
+ return refined_keypoints, refined_scores, refined_depths
def regularization_losses(self):
return []
@@ -2748,25 +4178,38 @@ class CenterNetMetaArch(model.DetectionModel):
fine_tune_checkpoint_type: whether to restore from a full detection
checkpoint (with compatible variable names) or to restore from a
classification checkpoint for initialization prior to training.
- Valid values: `detection`, `classification`. Default 'detection'.
- 'detection': used when loading in the Hourglass model pre-trained on
- other detection task.
- 'classification': used when loading in the ResNet model pre-trained on
- image classification task. Note that only the image feature encoding
- part is loaded but not those upsampling layers.
+ Valid values: `detection`, `classification`, `fine_tune`.
+ Default 'detection'.
+ 'detection': used when loading models pre-trained on other detection
+ tasks. With this checkpoint type the weights of the feature extractor
+ are expected under the attribute 'feature_extractor'.
+ 'classification': used when loading models pre-trained on an image
+ classification task. Note that only the encoder section of the network
+ is loaded and not the upsampling layers. With this checkpoint type,
+ the weights of only the encoder section are expected under the
+ attribute 'feature_extractor'.
'fine_tune': used when loading the entire CenterNet feature extractor
pre-trained on other tasks. The checkpoints saved during CenterNet
- model training can be directly loaded using this mode.
+ model training can be directly loaded using this type. With this
+ checkpoint type, the weights of the feature extractor are expected
+ under the attribute 'model._feature_extractor'.
+ For more details, see the tensorflow section on Loading mechanics.
+ https://www.tensorflow.org/guide/checkpoint#loading_mechanics
Returns:
A dict mapping keys to Trackable objects (tf.Module or Checkpoint).
"""
- if fine_tune_checkpoint_type == 'classification':
- return {'feature_extractor': self._feature_extractor.get_base_model()}
+ supported_types = self._feature_extractor.supported_sub_model_types
+ supported_types += ['fine_tune']
- elif fine_tune_checkpoint_type == 'detection':
- return {'feature_extractor': self._feature_extractor.get_model()}
+ if fine_tune_checkpoint_type not in supported_types:
+ message = ('Checkpoint type "{}" not supported for {}. '
+ 'Supported types are {}')
+ raise ValueError(
+ message.format(fine_tune_checkpoint_type,
+ self._feature_extractor.__class__.__name__,
+ supported_types))
elif fine_tune_checkpoint_type == 'fine_tune':
feature_extractor_model = tf.train.Checkpoint(
@@ -2774,9 +4217,17 @@ class CenterNetMetaArch(model.DetectionModel):
return {'model': feature_extractor_model}
else:
- raise ValueError('Not supported fine tune checkpoint type - {}'.format(
- fine_tune_checkpoint_type))
+ return {'feature_extractor': self._feature_extractor.get_sub_model(
+ fine_tune_checkpoint_type)}
def updates(self):
- raise RuntimeError('This model is intended to be used with model_lib_v2 '
- 'which does not support updates()')
+ if tf_version.is_tf2():
+ raise RuntimeError('This model is intended to be used with model_lib_v2 '
+ 'which does not support updates()')
+ else:
+ update_ops = []
+ slim_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
+ # Copy the slim ops to avoid modifying the collection
+ if slim_update_ops:
+ update_ops.extend(slim_update_ops)
+ return update_ops
diff --git a/research/object_detection/meta_architectures/center_net_meta_arch_tf2_test.py b/research/object_detection/meta_architectures/center_net_meta_arch_tf2_test.py
index c2474b53ceff59e824a2e9975afb436ed5de9c5c..d3784ef8419fba6ad1577becec5571be29b222ea 100644
--- a/research/object_detection/meta_architectures/center_net_meta_arch_tf2_test.py
+++ b/research/object_detection/meta_architectures/center_net_meta_arch_tf2_test.py
@@ -17,27 +17,35 @@
from __future__ import division
import functools
+import re
import unittest
+
from absl.testing import parameterized
import numpy as np
import tensorflow.compat.v1 as tf
+from object_detection.builders import post_processing_builder
+from object_detection.core import keypoint_ops
from object_detection.core import losses
from object_detection.core import preprocessor
from object_detection.core import standard_fields as fields
from object_detection.core import target_assigner as cn_assigner
from object_detection.meta_architectures import center_net_meta_arch as cnma
from object_detection.models import center_net_resnet_feature_extractor
+from object_detection.protos import post_processing_pb2
from object_detection.utils import test_case
from object_detection.utils import tf_version
@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
-class CenterNetMetaArchPredictionHeadTest(test_case.TestCase):
+class CenterNetMetaArchPredictionHeadTest(
+ test_case.TestCase, parameterized.TestCase):
"""Test CenterNet meta architecture prediction head."""
- def test_prediction_head(self):
- head = cnma.make_prediction_net(num_out_channels=7)
+ @parameterized.parameters([True, False])
+ def test_prediction_head(self, use_depthwise):
+ head = cnma.make_prediction_net(num_out_channels=7,
+ use_depthwise=use_depthwise)
output = head(np.zeros((4, 128, 128, 8)))
self.assertEqual((4, 128, 128, 7), output.shape)
@@ -47,7 +55,7 @@ class CenterNetMetaArchPredictionHeadTest(test_case.TestCase):
class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
"""Test for CenterNet meta architecture related functions."""
- def test_row_col_indices_from_flattened_indices(self):
+ def test_row_col_channel_indices_from_flattened_indices(self):
"""Tests that the computation of row, col, channel indices is correct."""
r_grid, c_grid, ch_grid = (np.zeros((5, 4, 3), dtype=np.int),
@@ -81,6 +89,21 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
np.testing.assert_array_equal(ci, c_grid.flatten())
np.testing.assert_array_equal(chi, ch_grid.flatten())
+ def test_row_col_indices_from_flattened_indices(self):
+ """Tests that the computation of row, col indices is correct."""
+
+ r_grid = np.array([[0, 0, 0, 0], [1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3],
+ [4, 4, 4, 4]])
+
+ c_grid = np.array([[0, 1, 2, 3], [0, 1, 2, 3], [0, 1, 2, 3], [0, 1, 2, 3],
+ [0, 1, 2, 3]])
+
+ indices = np.arange(20)
+ ri, ci, = cnma.row_col_indices_from_flattened_indices(indices, 4)
+
+ np.testing.assert_array_equal(ri, r_grid.flatten())
+ np.testing.assert_array_equal(ci, c_grid.flatten())
+
def test_flattened_indices_from_row_col_indices(self):
r = np.array(
@@ -479,6 +502,70 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
np.testing.assert_array_equal([1, 0, 0, 2], x_inds[1])
np.testing.assert_array_equal([0, 0, 1, 1], channel_inds[1])
+ def test_top_k_feature_map_locations_k1(self):
+ feature_map_np = np.zeros((2, 3, 3, 2), dtype=np.float32)
+ feature_map_np[0, 2, 0, 0] = 1.0 # Selected.
+ feature_map_np[0, 2, 1, 0] = 0.9
+ feature_map_np[0, 0, 1, 0] = 0.7
+ feature_map_np[0, 2, 2, 1] = 0.5
+ feature_map_np[0, 0, 0, 1] = 0.3
+ feature_map_np[1, 2, 1, 0] = 0.7
+ feature_map_np[1, 1, 0, 0] = 0.4
+ feature_map_np[1, 1, 2, 0] = 0.3
+ feature_map_np[1, 1, 0, 1] = 0.8 # Selected.
+ feature_map_np[1, 1, 2, 1] = 0.3
+
+ def graph_fn():
+ feature_map = tf.constant(feature_map_np)
+ scores, y_inds, x_inds, channel_inds = (
+ cnma.top_k_feature_map_locations(
+ feature_map, max_pool_kernel_size=3, k=1, per_channel=False))
+ return scores, y_inds, x_inds, channel_inds
+
+ scores, y_inds, x_inds, channel_inds = self.execute(graph_fn, [])
+
+ np.testing.assert_allclose([1.0], scores[0])
+ np.testing.assert_array_equal([2], y_inds[0])
+ np.testing.assert_array_equal([0], x_inds[0])
+ np.testing.assert_array_equal([0], channel_inds[0])
+
+ np.testing.assert_allclose([0.8], scores[1])
+ np.testing.assert_array_equal([1], y_inds[1])
+ np.testing.assert_array_equal([0], x_inds[1])
+ np.testing.assert_array_equal([1], channel_inds[1])
+
+ def test_top_k_feature_map_locations_k1_per_channel(self):
+ feature_map_np = np.zeros((2, 3, 3, 2), dtype=np.float32)
+ feature_map_np[0, 2, 0, 0] = 1.0 # Selected.
+ feature_map_np[0, 2, 1, 0] = 0.9
+ feature_map_np[0, 0, 1, 0] = 0.7
+ feature_map_np[0, 2, 2, 1] = 0.5 # Selected.
+ feature_map_np[0, 0, 0, 1] = 0.3
+ feature_map_np[1, 2, 1, 0] = 0.7 # Selected.
+ feature_map_np[1, 1, 0, 0] = 0.4
+ feature_map_np[1, 1, 2, 0] = 0.3
+ feature_map_np[1, 1, 0, 1] = 0.8 # Selected.
+ feature_map_np[1, 1, 2, 1] = 0.3
+
+ def graph_fn():
+ feature_map = tf.constant(feature_map_np)
+ scores, y_inds, x_inds, channel_inds = (
+ cnma.top_k_feature_map_locations(
+ feature_map, max_pool_kernel_size=3, k=1, per_channel=True))
+ return scores, y_inds, x_inds, channel_inds
+
+ scores, y_inds, x_inds, channel_inds = self.execute(graph_fn, [])
+
+ np.testing.assert_allclose([1.0, 0.5], scores[0])
+ np.testing.assert_array_equal([2, 2], y_inds[0])
+ np.testing.assert_array_equal([0, 2], x_inds[0])
+ np.testing.assert_array_equal([0, 1], channel_inds[0])
+
+ np.testing.assert_allclose([0.7, 0.8], scores[1])
+ np.testing.assert_array_equal([2, 1], y_inds[1])
+ np.testing.assert_array_equal([1, 0], x_inds[1])
+ np.testing.assert_array_equal([0, 1], channel_inds[1])
+
def test_box_prediction(self):
class_pred = np.zeros((3, 128, 128, 5), dtype=np.float32)
@@ -517,33 +604,69 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
hw_pred_tensor = tf.constant(hw_pred)
offset_pred_tensor = tf.constant(offset_pred)
- detection_scores, y_indices, x_indices, channel_indices = (
+ _, y_indices, x_indices, _ = (
cnma.top_k_feature_map_locations(
class_pred_tensor, max_pool_kernel_size=3, k=2))
- boxes, classes, scores, num_dets = cnma.prediction_tensors_to_boxes(
- detection_scores, y_indices, x_indices, channel_indices,
- hw_pred_tensor, offset_pred_tensor)
- return boxes, classes, scores, num_dets
+ boxes = cnma.prediction_tensors_to_boxes(
+ y_indices, x_indices, hw_pred_tensor, offset_pred_tensor)
+ return boxes
- boxes, classes, scores, num_dets = self.execute(graph_fn, [])
-
- np.testing.assert_array_equal(num_dets, [2, 2, 2])
+ boxes = self.execute(graph_fn, [])
np.testing.assert_allclose(
- [[-9, -8, 31, 52], [25, 35, 75, 85]], boxes[0])
+ [[0, 0, 31, 52], [25, 35, 75, 85]], boxes[0])
np.testing.assert_allclose(
[[96, 98, 106, 108], [96, 98, 106, 108]], boxes[1])
np.testing.assert_allclose(
[[69.5, 74.5, 90.5, 99.5], [40, 75, 80, 105]], boxes[2])
- np.testing.assert_array_equal(classes[0], [1, 0])
- np.testing.assert_array_equal(classes[1], [2, 1])
- np.testing.assert_array_equal(classes[2], [0, 4])
+ def test_offset_prediction(self):
+
+ class_pred = np.zeros((3, 128, 128, 5), dtype=np.float32)
+ offset_pred = np.zeros((3, 128, 128, 2), dtype=np.float32)
+
+ # Sample 1, 2 boxes
+ class_pred[0, 10, 20] = [0.3, .7, 0.0, 0.0, 0.0]
+ offset_pred[0, 10, 20] = [1, 2]
+
+ class_pred[0, 50, 60] = [0.55, 0.0, 0.0, 0.0, 0.45]
+ offset_pred[0, 50, 60] = [0, 0]
+
+ # Sample 2, 2 boxes (at same location)
+ class_pred[1, 100, 100] = [0.0, 0.1, 0.9, 0.0, 0.0]
+ offset_pred[1, 100, 100] = [1, 3]
+
+ # Sample 3, 3 boxes
+ class_pred[2, 60, 90] = [0.0, 0.0, 0.0, 0.2, 0.8]
+ offset_pred[2, 60, 90] = [0, 0]
+
+ class_pred[2, 65, 95] = [0.0, 0.7, 0.3, 0.0, 0.0]
+ offset_pred[2, 65, 95] = [1, 2]
+
+ class_pred[2, 75, 85] = [1.0, 0.0, 0.0, 0.0, 0.0]
+ offset_pred[2, 75, 85] = [5, 2]
+
+ def graph_fn():
+ class_pred_tensor = tf.constant(class_pred)
+ offset_pred_tensor = tf.constant(offset_pred)
+
+ _, y_indices, x_indices, _ = (
+ cnma.top_k_feature_map_locations(
+ class_pred_tensor, max_pool_kernel_size=3, k=2))
- np.testing.assert_allclose(scores[0], [.7, .55])
- np.testing.assert_allclose(scores[1][:1], [.9])
- np.testing.assert_allclose(scores[2], [1., .8])
+ offsets = cnma.prediction_tensors_to_temporal_offsets(
+ y_indices, x_indices, offset_pred_tensor)
+ return offsets
+
+ offsets = self.execute(graph_fn, [])
+
+ np.testing.assert_allclose(
+ [[1, 2], [0, 0]], offsets[0])
+ np.testing.assert_allclose(
+ [[1, 3], [1, 3]], offsets[1])
+ np.testing.assert_allclose(
+ [[5, 2], [0, 0]], offsets[2])
def test_keypoint_candidate_prediction(self):
keypoint_heatmap_np = np.zeros((2, 3, 3, 2), dtype=np.float32)
@@ -577,7 +700,7 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
keypoint_heatmap_offsets = tf.constant(
keypoint_heatmap_offsets_np, dtype=tf.float32)
- keypoint_cands, keypoint_scores, num_keypoint_candidates = (
+ (keypoint_cands, keypoint_scores, num_keypoint_candidates, _) = (
cnma.prediction_tensors_to_keypoint_candidates(
keypoint_heatmap,
keypoint_heatmap_offsets,
@@ -618,6 +741,73 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
np.testing.assert_array_equal(expected_num_keypoint_candidates,
num_keypoint_candidates)
+ def test_prediction_to_single_instance_keypoints(self):
+ image_size = (9, 9)
+ object_heatmap_np = np.zeros((1, image_size[0], image_size[1], 1),
+ dtype=np.float32)
+ # This should be picked.
+ object_heatmap_np[0, 4, 4, 0] = 0.9
+ # This shouldn't be picked since it's farther away from the center.
+ object_heatmap_np[0, 2, 2, 0] = 1.0
+
+ keypoint_heatmap_np = np.zeros((1, image_size[0], image_size[1], 4),
+ dtype=np.float32)
+ # Top-left corner should be picked.
+ keypoint_heatmap_np[0, 1, 1, 0] = 0.9
+ keypoint_heatmap_np[0, 4, 4, 0] = 1.0
+ # Top-right corner should be picked.
+ keypoint_heatmap_np[0, 1, 7, 1] = 0.9
+ keypoint_heatmap_np[0, 4, 4, 1] = 1.0
+ # Bottom-left corner should be picked.
+ keypoint_heatmap_np[0, 7, 1, 2] = 0.9
+ keypoint_heatmap_np[0, 4, 4, 2] = 1.0
+ # Bottom-right corner should be picked.
+ keypoint_heatmap_np[0, 7, 7, 3] = 0.9
+ keypoint_heatmap_np[0, 4, 4, 3] = 1.0
+
+ keypoint_offset_np = np.zeros((1, image_size[0], image_size[1], 8),
+ dtype=np.float32)
+ keypoint_offset_np[0, 1, 1] = [0.5, 0.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+ keypoint_offset_np[0, 1, 7] = [0.0, 0.0, 0.5, -0.5, 0.0, 0.0, 0.0, 0.0]
+ keypoint_offset_np[0, 7, 1] = [0.0, 0.0, 0.0, 0.0, -0.5, 0.5, 0.0, 0.0]
+ keypoint_offset_np[0, 7, 7] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.5, -0.5]
+
+ keypoint_regression_np = np.zeros((1, image_size[0], image_size[1], 8),
+ dtype=np.float32)
+ keypoint_regression_np[0, 4, 4] = [-3, -3, -3, 3, 3, -3, 3, 3]
+
+ kp_params = get_fake_kp_params(
+ candidate_ranking_mode='score_distance_ratio')
+
+ def graph_fn():
+ object_heatmap = tf.constant(object_heatmap_np, dtype=tf.float32)
+ keypoint_heatmap = tf.constant(keypoint_heatmap_np, dtype=tf.float32)
+ keypoint_offset = tf.constant(keypoint_offset_np, dtype=tf.float32)
+ keypoint_regression = tf.constant(
+ keypoint_regression_np, dtype=tf.float32)
+
+ (keypoint_cands, keypoint_scores, _) = (
+ cnma.prediction_to_single_instance_keypoints(
+ object_heatmap,
+ keypoint_heatmap,
+ keypoint_offset,
+ keypoint_regression,
+ kp_params=kp_params))
+
+ return keypoint_cands, keypoint_scores
+
+ (keypoint_cands, keypoint_scores) = self.execute(graph_fn, [])
+
+ expected_keypoint_candidates = [[[
+ [1.5, 1.5], # top-left
+ [1.5, 6.5], # top-right
+ [6.5, 1.5], # bottom-left
+ [6.5, 6.5], # bottom-right
+ ]]]
+ expected_keypoint_scores = [[[0.9, 0.9, 0.9, 0.9]]]
+ np.testing.assert_allclose(expected_keypoint_candidates, keypoint_cands)
+ np.testing.assert_allclose(expected_keypoint_scores, keypoint_scores)
+
def test_keypoint_candidate_prediction_per_keypoints(self):
keypoint_heatmap_np = np.zeros((2, 3, 3, 2), dtype=np.float32)
keypoint_heatmap_np[0, 0, 0, 0] = 1.0
@@ -652,7 +842,7 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
keypoint_heatmap_offsets = tf.constant(
keypoint_heatmap_offsets_np, dtype=tf.float32)
- keypoint_cands, keypoint_scores, num_keypoint_candidates = (
+ (keypoint_cands, keypoint_scores, num_keypoint_candidates, _) = (
cnma.prediction_tensors_to_keypoint_candidates(
keypoint_heatmap,
keypoint_heatmap_offsets,
@@ -693,6 +883,89 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
np.testing.assert_array_equal(expected_num_keypoint_candidates,
num_keypoint_candidates)
+ @parameterized.parameters({'per_keypoint_depth': True},
+ {'per_keypoint_depth': False})
+ def test_keypoint_candidate_prediction_depth(self, per_keypoint_depth):
+ keypoint_heatmap_np = np.zeros((2, 3, 3, 2), dtype=np.float32)
+ keypoint_heatmap_np[0, 0, 0, 0] = 1.0
+ keypoint_heatmap_np[0, 2, 1, 0] = 0.7
+ keypoint_heatmap_np[0, 1, 1, 0] = 0.6
+ keypoint_heatmap_np[0, 0, 2, 1] = 0.7
+ keypoint_heatmap_np[0, 1, 1, 1] = 0.3 # Filtered by low score.
+ keypoint_heatmap_np[0, 2, 2, 1] = 0.2
+ keypoint_heatmap_np[1, 1, 0, 0] = 0.6
+ keypoint_heatmap_np[1, 2, 1, 0] = 0.5
+ keypoint_heatmap_np[1, 0, 0, 0] = 0.4
+ keypoint_heatmap_np[1, 0, 0, 1] = 1.0
+ keypoint_heatmap_np[1, 0, 1, 1] = 0.9
+ keypoint_heatmap_np[1, 2, 0, 1] = 0.8
+
+ if per_keypoint_depth:
+ keypoint_depths_np = np.zeros((2, 3, 3, 2), dtype=np.float32)
+ keypoint_depths_np[0, 0, 0, 0] = -1.5
+ keypoint_depths_np[0, 2, 1, 0] = -1.0
+ keypoint_depths_np[0, 0, 2, 1] = 1.5
+ else:
+ keypoint_depths_np = np.zeros((2, 3, 3, 1), dtype=np.float32)
+ keypoint_depths_np[0, 0, 0, 0] = -1.5
+ keypoint_depths_np[0, 2, 1, 0] = -1.0
+ keypoint_depths_np[0, 0, 2, 0] = 1.5
+
+ keypoint_heatmap_offsets_np = np.zeros((2, 3, 3, 2), dtype=np.float32)
+ keypoint_heatmap_offsets_np[0, 0, 0] = [0.5, 0.25]
+ keypoint_heatmap_offsets_np[0, 2, 1] = [-0.25, 0.5]
+ keypoint_heatmap_offsets_np[0, 1, 1] = [0.0, 0.0]
+ keypoint_heatmap_offsets_np[0, 0, 2] = [1.0, 0.0]
+ keypoint_heatmap_offsets_np[0, 2, 2] = [1.0, 1.0]
+ keypoint_heatmap_offsets_np[1, 1, 0] = [0.25, 0.5]
+ keypoint_heatmap_offsets_np[1, 2, 1] = [0.5, 0.0]
+ keypoint_heatmap_offsets_np[1, 0, 0] = [0.0, -0.5]
+ keypoint_heatmap_offsets_np[1, 0, 1] = [0.5, -0.5]
+ keypoint_heatmap_offsets_np[1, 2, 0] = [-1.0, -0.5]
+
+ def graph_fn():
+ keypoint_heatmap = tf.constant(keypoint_heatmap_np, dtype=tf.float32)
+ keypoint_heatmap_offsets = tf.constant(
+ keypoint_heatmap_offsets_np, dtype=tf.float32)
+
+ keypoint_depths = tf.constant(keypoint_depths_np, dtype=tf.float32)
+ (keypoint_cands, keypoint_scores, num_keypoint_candidates,
+ keypoint_depths) = (
+ cnma.prediction_tensors_to_keypoint_candidates(
+ keypoint_heatmap,
+ keypoint_heatmap_offsets,
+ keypoint_score_threshold=0.5,
+ max_pool_kernel_size=1,
+ max_candidates=2,
+ keypoint_depths=keypoint_depths))
+ return (keypoint_cands, keypoint_scores, num_keypoint_candidates,
+ keypoint_depths)
+
+ (_, keypoint_scores, _, keypoint_depths) = self.execute(graph_fn, [])
+
+ expected_keypoint_scores = [
+ [ # Example 0.
+ [1.0, 0.7], # Keypoint 1.
+ [0.7, 0.3], # Keypoint 2.
+ ],
+ [ # Example 1.
+ [0.6, 1.0], # Keypoint 1.
+ [0.5, 0.9], # Keypoint 2.
+ ],
+ ]
+ expected_keypoint_depths = [
+ [
+ [-1.5, 1.5],
+ [-1.0, 0.0],
+ ],
+ [
+ [0., 0.],
+ [0., 0.],
+ ],
+ ]
+ np.testing.assert_allclose(expected_keypoint_scores, keypoint_scores)
+ np.testing.assert_allclose(expected_keypoint_depths, keypoint_depths)
+
def test_regressed_keypoints_at_object_centers(self):
batch_size = 2
num_keypoints = 5
@@ -798,11 +1071,22 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
keypoint_scores = tf.constant(keypoint_scores_np, dtype=tf.float32)
num_keypoint_candidates = tf.constant(num_keypoints_candidates_np,
dtype=tf.int32)
- refined_keypoints, refined_scores = cnma.refine_keypoints(
- regressed_keypoints, keypoint_candidates, keypoint_scores,
- num_keypoint_candidates, bboxes=None,
+ # The behavior of bboxes=None is different now. We provide the bboxes
+ # explicitly by using the regressed keypoints to create the same
+ # behavior.
+ regressed_keypoints_flattened = tf.reshape(
+ regressed_keypoints, [-1, 3, 2])
+ bboxes_flattened = keypoint_ops.keypoints_to_enclosing_bounding_boxes(
+ regressed_keypoints_flattened)
+ (refined_keypoints, refined_scores, _) = cnma.refine_keypoints(
+ regressed_keypoints,
+ keypoint_candidates,
+ keypoint_scores,
+ num_keypoint_candidates,
+ bboxes=bboxes_flattened,
unmatched_keypoint_score=unmatched_keypoint_score,
- box_scale=1.2, candidate_search_scale=0.3,
+ box_scale=1.2,
+ candidate_search_scale=0.3,
candidate_ranking_mode=candidate_ranking_mode)
return refined_keypoints, refined_scores
@@ -870,7 +1154,87 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
np.testing.assert_allclose(expected_refined_keypoints, refined_keypoints)
np.testing.assert_allclose(expected_refined_scores, refined_scores)
- def test_refine_keypoints_with_bboxes(self):
+ def test_refine_keypoints_without_bbox(self):
+ regressed_keypoints_np = np.array(
+ [
+ # Example 0.
+ [
+ [[2.0, 2.0], [6.0, 10.0], [14.0, 7.0]], # Instance 0.
+ [[0.0, 6.0], [3.0, 3.0], [5.0, 7.0]], # Instance 1.
+ ],
+ ], dtype=np.float32)
+ keypoint_candidates_np = np.array(
+ [
+ # Example 0.
+ [
+ [[2.0, 2.5], [6.0, 10.5], [4.0, 7.0]], # Candidate 0.
+ [[1.0, 8.0], [0.0, 0.0], [2.0, 2.0]], # Candidate 1.
+ [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], # Candidate 2.
+ ],
+ ], dtype=np.float32)
+ keypoint_scores_np = np.array(
+ [
+ # Example 0.
+ [
+ [0.8, 0.9, 1.0], # Candidate 0.
+ [0.6, 0.1, 0.9], # Candidate 1.
+ [0.0, 0.0, 0.0], # Candidate 1.
+ ],
+ ], dtype=np.float32)
+ num_keypoints_candidates_np = np.array(
+ [
+ # Example 0.
+ [2, 2, 2],
+ ], dtype=np.int32)
+ unmatched_keypoint_score = 0.1
+
+ def graph_fn():
+ regressed_keypoints = tf.constant(
+ regressed_keypoints_np, dtype=tf.float32)
+ keypoint_candidates = tf.constant(
+ keypoint_candidates_np, dtype=tf.float32)
+ keypoint_scores = tf.constant(keypoint_scores_np, dtype=tf.float32)
+ num_keypoint_candidates = tf.constant(num_keypoints_candidates_np,
+ dtype=tf.int32)
+ (refined_keypoints, refined_scores, _) = cnma.refine_keypoints(
+ regressed_keypoints,
+ keypoint_candidates,
+ keypoint_scores,
+ num_keypoint_candidates,
+ bboxes=None,
+ unmatched_keypoint_score=unmatched_keypoint_score,
+ box_scale=1.2,
+ candidate_search_scale=0.3,
+ candidate_ranking_mode='min_distance')
+ return refined_keypoints, refined_scores
+
+ refined_keypoints, refined_scores = self.execute(graph_fn, [])
+
+ # The expected refined keypoints pick the ones that are closest to the
+ # regressed keypoint locations without filtering out the candidates which
+ # are outside of the bounding box.
+ expected_refined_keypoints = np.array(
+ [
+ # Example 0.
+ [
+ [[2.0, 2.5], [6.0, 10.5], [4.0, 7.0]], # Instance 0.
+ [[1.0, 8.0], [0.0, 0.0], [4.0, 7.0]], # Instance 1.
+ ],
+ ], dtype=np.float32)
+ expected_refined_scores = np.array(
+ [
+ # Example 0.
+ [
+ [0.8, 0.9, 1.0], # Instance 0.
+ [0.6, 0.1, 1.0], # Instance 1.
+ ],
+ ], dtype=np.float32)
+
+ np.testing.assert_allclose(expected_refined_keypoints, refined_keypoints)
+ np.testing.assert_allclose(expected_refined_scores, refined_scores)
+
+ @parameterized.parameters({'predict_depth': True}, {'predict_depth': False})
+ def test_refine_keypoints_with_bboxes(self, predict_depth):
regressed_keypoints_np = np.array(
[
# Example 0.
@@ -909,7 +1273,22 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
[0.7, 0.4, 0.0], # Candidate 0.
[0.6, 0.1, 0.0], # Candidate 1.
]
- ], dtype=np.float32)
+ ],
+ dtype=np.float32)
+ keypoint_depths_np = np.array(
+ [
+ # Example 0.
+ [
+ [-0.8, -0.9, -1.0], # Candidate 0.
+ [-0.6, -0.1, -0.9], # Candidate 1.
+ ],
+ # Example 1.
+ [
+ [-0.7, -0.4, -0.0], # Candidate 0.
+ [-0.6, -0.1, -0.0], # Candidate 1.
+ ]
+ ],
+ dtype=np.float32)
num_keypoints_candidates_np = np.array(
[
# Example 0.
@@ -938,17 +1317,28 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
keypoint_candidates = tf.constant(
keypoint_candidates_np, dtype=tf.float32)
keypoint_scores = tf.constant(keypoint_scores_np, dtype=tf.float32)
+ if predict_depth:
+ keypoint_depths = tf.constant(keypoint_depths_np, dtype=tf.float32)
+ else:
+ keypoint_depths = None
num_keypoint_candidates = tf.constant(num_keypoints_candidates_np,
dtype=tf.int32)
bboxes = tf.constant(bboxes_np, dtype=tf.float32)
- refined_keypoints, refined_scores = cnma.refine_keypoints(
- regressed_keypoints, keypoint_candidates, keypoint_scores,
- num_keypoint_candidates, bboxes=bboxes,
- unmatched_keypoint_score=unmatched_keypoint_score,
- box_scale=1.0, candidate_search_scale=0.3)
- return refined_keypoints, refined_scores
-
- refined_keypoints, refined_scores = self.execute(graph_fn, [])
+ (refined_keypoints, refined_scores,
+ refined_depths) = cnma.refine_keypoints(
+ regressed_keypoints,
+ keypoint_candidates,
+ keypoint_scores,
+ num_keypoint_candidates,
+ bboxes=bboxes,
+ unmatched_keypoint_score=unmatched_keypoint_score,
+ box_scale=1.0,
+ candidate_search_scale=0.3,
+ keypoint_depth_candidates=keypoint_depths)
+ if predict_depth:
+ return refined_keypoints, refined_scores, refined_depths
+ else:
+ return refined_keypoints, refined_scores
expected_refined_keypoints = np.array(
[
@@ -979,8 +1369,17 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
],
], dtype=np.float32)
- np.testing.assert_allclose(expected_refined_keypoints, refined_keypoints)
- np.testing.assert_allclose(expected_refined_scores, refined_scores)
+ if predict_depth:
+ refined_keypoints, refined_scores, refined_depths = self.execute(
+ graph_fn, [])
+ expected_refined_depths = np.array([[[-0.8, 0.0, 0.0], [0.0, 0.0, -1.0]],
+ [[-0.7, -0.1, 0.0], [-0.7, -0.4,
+ 0.0]]])
+ np.testing.assert_allclose(expected_refined_depths, refined_depths)
+ else:
+ refined_keypoints, refined_scores = self.execute(graph_fn, [])
+ np.testing.assert_allclose(expected_refined_keypoints, refined_keypoints)
+ np.testing.assert_allclose(expected_refined_scores, refined_scores)
def test_pad_to_full_keypoint_dim(self):
batch_size = 4
@@ -1044,6 +1443,41 @@ class CenterNetMetaArchHelpersTest(test_case.TestCase, parameterized.TestCase):
np.testing.assert_allclose(kpt_scores_np[:, i, :],
kpt_scores_padded[:, inst_ind, :])
+ def test_predicted_embeddings_at_object_centers(self):
+ batch_size = 2
+ embedding_size = 5
+ num_instances = 6
+ predicted_embedding_feature_map_np = np.random.randn(
+ batch_size, 10, 10, embedding_size).astype(np.float32)
+ y_indices = np.random.choice(10, (batch_size, num_instances))
+ x_indices = np.random.choice(10, (batch_size, num_instances))
+
+ def graph_fn():
+ predicted_embedding_feature_map = tf.constant(
+ predicted_embedding_feature_map_np, dtype=tf.float32)
+
+ gathered_predicted_embeddings = (
+ cnma.predicted_embeddings_at_object_centers(
+ predicted_embedding_feature_map,
+ tf.constant(y_indices, dtype=tf.int32),
+ tf.constant(x_indices, dtype=tf.int32)))
+ return gathered_predicted_embeddings
+
+ gathered_predicted_embeddings = self.execute(graph_fn, [])
+
+ expected_gathered_embeddings_0 = predicted_embedding_feature_map_np[
+ 0, y_indices[0], x_indices[0], :]
+ expected_gathered_embeddings_1 = predicted_embedding_feature_map_np[
+ 1, y_indices[1], x_indices[1], :]
+ expected_gathered_embeddings = np.stack([
+ expected_gathered_embeddings_0,
+ expected_gathered_embeddings_1], axis=0)
+ expected_gathered_embeddings = np.reshape(
+ expected_gathered_embeddings,
+ [batch_size, num_instances, embedding_size])
+ np.testing.assert_allclose(expected_gathered_embeddings,
+ gathered_predicted_embeddings)
+
# Common parameters for setting up testing examples across tests.
_NUM_CLASSES = 10
@@ -1051,16 +1485,21 @@ _KEYPOINT_INDICES = [0, 1, 2, 3]
_NUM_KEYPOINTS = len(_KEYPOINT_INDICES)
_DENSEPOSE_NUM_PARTS = 24
_TASK_NAME = 'human_pose'
+_NUM_TRACK_IDS = 3
+_REID_EMBED_SIZE = 2
+_NUM_FC_LAYERS = 1
-def get_fake_center_params():
+def get_fake_center_params(max_box_predictions=5):
"""Returns the fake object center parameter namedtuple."""
return cnma.ObjectCenterParams(
classification_loss=losses.WeightedSigmoidClassificationLoss(),
object_center_loss_weight=1.0,
min_box_overlap_iou=1.0,
- max_box_predictions=5,
- use_labeled_classes=False)
+ max_box_predictions=max_box_predictions,
+ use_labeled_classes=False,
+ center_head_num_filters=[128],
+ center_head_kernel_sizes=[5])
def get_fake_od_params():
@@ -1071,7 +1510,12 @@ def get_fake_od_params():
scale_loss_weight=0.1)
-def get_fake_kp_params():
+def get_fake_kp_params(num_candidates_per_keypoint=100,
+ per_keypoint_offset=False,
+ predict_depth=False,
+ per_keypoint_depth=False,
+ peak_radius=0,
+ candidate_ranking_mode='min_distance'):
"""Returns the fake keypoint estimation parameter namedtuple."""
return cnma.KeypointEstimationParams(
task_name=_TASK_NAME,
@@ -1080,7 +1524,14 @@ def get_fake_kp_params():
keypoint_std_dev=[0.00001] * len(_KEYPOINT_INDICES),
classification_loss=losses.WeightedSigmoidClassificationLoss(),
localization_loss=losses.L1LocalizationLoss(),
- keypoint_candidate_score_threshold=0.1)
+ unmatched_keypoint_score=0.1,
+ keypoint_candidate_score_threshold=0.1,
+ num_candidates_per_keypoint=num_candidates_per_keypoint,
+ per_keypoint_offset=per_keypoint_offset,
+ predict_depth=predict_depth,
+ per_keypoint_depth=per_keypoint_depth,
+ offset_peak_radius=peak_radius,
+ candidate_ranking_mode=candidate_ranking_mode)
def get_fake_mask_params():
@@ -1106,7 +1557,34 @@ def get_fake_densepose_params():
upsample_method='nearest')
-def build_center_net_meta_arch(build_resnet=False):
+def get_fake_track_params():
+ """Returns the fake object tracking parameter namedtuple."""
+ return cnma.TrackParams(
+ num_track_ids=_NUM_TRACK_IDS,
+ reid_embed_size=_REID_EMBED_SIZE,
+ num_fc_layers=_NUM_FC_LAYERS,
+ classification_loss=losses.WeightedSoftmaxClassificationLoss(),
+ task_loss_weight=1.0)
+
+
+def get_fake_temporal_offset_params():
+ """Returns the fake temporal offset parameter namedtuple."""
+ return cnma.TemporalOffsetParams(
+ localization_loss=losses.WeightedSmoothL1LocalizationLoss(),
+ task_loss_weight=1.0)
+
+
+def build_center_net_meta_arch(build_resnet=False,
+ num_classes=_NUM_CLASSES,
+ max_box_predictions=5,
+ apply_non_max_suppression=False,
+ detection_only=False,
+ per_keypoint_offset=False,
+ predict_depth=False,
+ per_keypoint_depth=False,
+ peak_radius=0,
+ keypoint_only=False,
+ candidate_ranking_mode='min_distance'):
"""Builds the CenterNet meta architecture."""
if build_resnet:
feature_extractor = (
@@ -1124,17 +1602,82 @@ def build_center_net_meta_arch(build_resnet=False):
min_dimension=128,
max_dimension=128,
pad_to_max_dimesnion=True)
- return cnma.CenterNetMetaArch(
- is_training=True,
- add_summaries=False,
- num_classes=_NUM_CLASSES,
- feature_extractor=feature_extractor,
- image_resizer_fn=image_resizer_fn,
- object_center_params=get_fake_center_params(),
- object_detection_params=get_fake_od_params(),
- keypoint_params_dict={_TASK_NAME: get_fake_kp_params()},
- mask_params=get_fake_mask_params(),
- densepose_params=get_fake_densepose_params())
+
+ non_max_suppression_fn = None
+ if apply_non_max_suppression:
+ post_processing_proto = post_processing_pb2.PostProcessing()
+ post_processing_proto.batch_non_max_suppression.iou_threshold = 1.0
+ post_processing_proto.batch_non_max_suppression.score_threshold = 0.6
+ (post_processing_proto.batch_non_max_suppression.max_total_detections
+ ) = max_box_predictions
+ (post_processing_proto.batch_non_max_suppression.max_detections_per_class
+ ) = max_box_predictions
+ (post_processing_proto.batch_non_max_suppression.change_coordinate_frame
+ ) = False
+ non_max_suppression_fn, _ = post_processing_builder.build(
+ post_processing_proto)
+
+ if keypoint_only:
+ num_candidates_per_keypoint = 100 if max_box_predictions > 1 else 1
+ return cnma.CenterNetMetaArch(
+ is_training=True,
+ add_summaries=False,
+ num_classes=num_classes,
+ feature_extractor=feature_extractor,
+ image_resizer_fn=image_resizer_fn,
+ object_center_params=get_fake_center_params(max_box_predictions),
+ keypoint_params_dict={
+ _TASK_NAME:
+ get_fake_kp_params(num_candidates_per_keypoint,
+ per_keypoint_offset, predict_depth,
+ per_keypoint_depth, peak_radius,
+ candidate_ranking_mode)
+ },
+ non_max_suppression_fn=non_max_suppression_fn)
+ elif detection_only:
+ return cnma.CenterNetMetaArch(
+ is_training=True,
+ add_summaries=False,
+ num_classes=num_classes,
+ feature_extractor=feature_extractor,
+ image_resizer_fn=image_resizer_fn,
+ object_center_params=get_fake_center_params(max_box_predictions),
+ object_detection_params=get_fake_od_params(),
+ non_max_suppression_fn=non_max_suppression_fn)
+ elif num_classes == 1:
+ num_candidates_per_keypoint = 100 if max_box_predictions > 1 else 1
+ return cnma.CenterNetMetaArch(
+ is_training=True,
+ add_summaries=False,
+ num_classes=num_classes,
+ feature_extractor=feature_extractor,
+ image_resizer_fn=image_resizer_fn,
+ object_center_params=get_fake_center_params(max_box_predictions),
+ object_detection_params=get_fake_od_params(),
+ keypoint_params_dict={
+ _TASK_NAME:
+ get_fake_kp_params(num_candidates_per_keypoint,
+ per_keypoint_offset, predict_depth,
+ per_keypoint_depth, peak_radius,
+ candidate_ranking_mode)
+ },
+ non_max_suppression_fn=non_max_suppression_fn)
+ else:
+ return cnma.CenterNetMetaArch(
+ is_training=True,
+ add_summaries=False,
+ num_classes=num_classes,
+ feature_extractor=feature_extractor,
+ image_resizer_fn=image_resizer_fn,
+ object_center_params=get_fake_center_params(),
+ object_detection_params=get_fake_od_params(),
+ keypoint_params_dict={_TASK_NAME: get_fake_kp_params(
+ candidate_ranking_mode=candidate_ranking_mode)},
+ mask_params=get_fake_mask_params(),
+ densepose_params=get_fake_densepose_params(),
+ track_params=get_fake_track_params(),
+ temporal_offset_params=get_fake_temporal_offset_params(),
+ non_max_suppression_fn=non_max_suppression_fn)
def _logit(p):
@@ -1228,6 +1771,16 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
fake_feature_map)
self.assertEqual((4, 128, 128, 2 * _DENSEPOSE_NUM_PARTS), output.shape)
+ # "track embedding" head:
+ output = model._prediction_head_dict[cnma.TRACK_REID][-1](
+ fake_feature_map)
+ self.assertEqual((4, 128, 128, _REID_EMBED_SIZE), output.shape)
+
+ # "temporal offset" head:
+ output = model._prediction_head_dict[cnma.TEMPORAL_OFFSET][-1](
+ fake_feature_map)
+ self.assertEqual((4, 128, 128, 2), output.shape)
+
def test_initialize_target_assigners(self):
model = build_center_net_meta_arch()
assigner_dict = model._initialize_target_assigners(
@@ -1255,6 +1808,14 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
self.assertIsInstance(assigner_dict[cnma.DENSEPOSE_TASK],
cn_assigner.CenterNetDensePoseTargetAssigner)
+ # Track estimation target assigner:
+ self.assertIsInstance(assigner_dict[cnma.TRACK_TASK],
+ cn_assigner.CenterNetTrackTargetAssigner)
+
+ # Temporal Offset target assigner:
+ self.assertIsInstance(assigner_dict[cnma.TEMPORALOFFSET_TASK],
+ cn_assigner.CenterNetTemporalOffsetTargetAssigner)
+
def test_predict(self):
"""Test the predict function."""
@@ -1279,6 +1840,10 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
(2, 32, 32, _DENSEPOSE_NUM_PARTS))
self.assertEqual(prediction_dict[cnma.DENSEPOSE_REGRESSION][0].shape,
(2, 32, 32, 2 * _DENSEPOSE_NUM_PARTS))
+ self.assertEqual(prediction_dict[cnma.TRACK_REID][0].shape,
+ (2, 32, 32, _REID_EMBED_SIZE))
+ self.assertEqual(prediction_dict[cnma.TEMPORAL_OFFSET][0].shape,
+ (2, 32, 32, 2))
def test_loss(self):
"""Test the loss function."""
@@ -1297,7 +1862,20 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
groundtruth_dp_part_ids_list=groundtruth_dict[
fields.BoxListFields.densepose_part_ids],
groundtruth_dp_surface_coords_list=groundtruth_dict[
- fields.BoxListFields.densepose_surface_coords])
+ fields.BoxListFields.densepose_surface_coords],
+ groundtruth_track_ids_list=groundtruth_dict[
+ fields.BoxListFields.track_ids],
+ groundtruth_track_match_flags_list=groundtruth_dict[
+ fields.BoxListFields.track_match_flags],
+ groundtruth_temporal_offsets_list=groundtruth_dict[
+ fields.BoxListFields.temporal_offsets])
+
+ kernel_initializer = tf.constant_initializer(
+ [[1, 1, 0], [-1000000, -1000000, 1000000]])
+ model.track_reid_classification_net = tf.keras.layers.Dense(
+ _NUM_TRACK_IDS,
+ kernel_initializer=kernel_initializer,
+ input_shape=(_REID_EMBED_SIZE,))
prediction_dict = get_fake_prediction_dict(
input_height=16, input_width=32, stride=4)
@@ -1339,6 +1917,12 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
self.assertGreater(
0.01, loss_dict['%s/%s' % (cnma.LOSS_KEY_PREFIX,
cnma.DENSEPOSE_REGRESSION)])
+ self.assertGreater(
+ 0.01, loss_dict['%s/%s' % (cnma.LOSS_KEY_PREFIX,
+ cnma.TRACK_REID)])
+ self.assertGreater(
+ 0.01, loss_dict['%s/%s' % (cnma.LOSS_KEY_PREFIX,
+ cnma.TEMPORAL_OFFSET)])
@parameterized.parameters(
{'target_class_id': 1},
@@ -1349,15 +1933,18 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
model = build_center_net_meta_arch()
max_detection = model._center_params.max_box_predictions
num_keypoints = len(model._kp_params_dict[_TASK_NAME].keypoint_indices)
+ unmatched_keypoint_score = (
+ model._kp_params_dict[_TASK_NAME].unmatched_keypoint_score)
class_center = np.zeros((1, 32, 32, 10), dtype=np.float32)
height_width = np.zeros((1, 32, 32, 2), dtype=np.float32)
offset = np.zeros((1, 32, 32, 2), dtype=np.float32)
- keypoint_heatmaps = np.zeros((1, 32, 32, num_keypoints), dtype=np.float32)
+ keypoint_heatmaps = np.ones(
+ (1, 32, 32, num_keypoints), dtype=np.float32) * _logit(0.001)
keypoint_offsets = np.zeros((1, 32, 32, 2), dtype=np.float32)
keypoint_regression = np.random.randn(1, 32, 32, num_keypoints * 2)
- class_probs = np.zeros(10)
+ class_probs = np.ones(10) * _logit(0.25)
class_probs[target_class_id] = _logit(0.75)
class_center[0, 16, 16] = class_probs
height_width[0, 16, 16] = [5, 10]
@@ -1384,6 +1971,14 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
dp_surf_coords = np.random.randn(1, 32, 32, 2 * _DENSEPOSE_NUM_PARTS)
+ embedding_size = 100
+ track_reid_embedding = np.zeros((1, 32, 32, embedding_size),
+ dtype=np.float32)
+ track_reid_embedding[0, 16, 16, :] = np.ones(embedding_size)
+
+ temporal_offsets = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ temporal_offsets[..., 1] = 1
+
class_center = tf.constant(class_center)
height_width = tf.constant(height_width)
offset = tf.constant(offset)
@@ -1393,6 +1988,8 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
segmentation_heatmap = tf.constant(segmentation_heatmap, dtype=tf.float32)
dp_part_heatmap = tf.constant(dp_part_heatmap, dtype=tf.float32)
dp_surf_coords = tf.constant(dp_surf_coords, dtype=tf.float32)
+ track_reid_embedding = tf.constant(track_reid_embedding, dtype=tf.float32)
+ temporal_offsets = tf.constant(temporal_offsets, dtype=tf.float32)
prediction_dict = {
cnma.OBJECT_CENTER: [class_center],
@@ -1406,7 +2003,9 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
[keypoint_regression],
cnma.SEGMENTATION_HEATMAP: [segmentation_heatmap],
cnma.DENSEPOSE_HEATMAP: [dp_part_heatmap],
- cnma.DENSEPOSE_REGRESSION: [dp_surf_coords]
+ cnma.DENSEPOSE_REGRESSION: [dp_surf_coords],
+ cnma.TRACK_REID: [track_reid_embedding],
+ cnma.TEMPORAL_OFFSET: [temporal_offsets],
}
def graph_fn():
@@ -1415,11 +2014,23 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
return detections
detections = self.execute_cpu(graph_fn, [])
-
self.assertAllClose(detections['detection_boxes'][0, 0],
np.array([55, 46, 75, 86]) / 128.0)
self.assertAllClose(detections['detection_scores'][0],
[.75, .5, .5, .5, .5])
+ expected_multiclass_scores = [.25] * 10
+ expected_multiclass_scores[target_class_id] = .75
+ self.assertAllClose(expected_multiclass_scores,
+ detections['detection_multiclass_scores'][0][0])
+
+ # The output embedding extracted at the object center will be a 3-D array of
+ # shape [batch, num_boxes, embedding_size]. The valid predicted embedding
+ # will be the first embedding in the first batch. It is a 1-D array of
+ # shape [embedding_size] with values all ones. All the values of the
+ # embedding will then be divided by the square root of 'embedding_size'
+ # after the L2 normalization.
+ self.assertAllClose(detections['detection_embeddings'][0, 0],
+ np.ones(embedding_size) / embedding_size**0.5)
self.assertEqual(detections['detection_classes'][0, 0], target_class_id)
self.assertEqual(detections['num_detections'], [5])
self.assertAllEqual([1, max_detection, num_keypoints, 2],
@@ -1428,6 +2039,10 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
detections['detection_keypoint_scores'].shape)
self.assertAllEqual([1, max_detection, 4, 4],
detections['detection_masks'].shape)
+ self.assertAllEqual([1, max_detection, embedding_size],
+ detections['detection_embeddings'].shape)
+ self.assertAllEqual([1, max_detection, 2],
+ detections['detection_temporal_offsets'].shape)
# Masks should be empty for everything but the first detection.
self.assertAllEqual(
@@ -1441,7 +2056,7 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
expected_kpts_for_obj_0 = np.array(
[[14., 14.], [14., 18.], [18., 14.], [17., 17.]]) / 32.
expected_kpt_scores_for_obj_0 = np.array(
- [0.9, 0.9, 0.9, cnma.UNMATCHED_KEYPOINT_SCORE])
+ [0.9, 0.9, 0.9, unmatched_keypoint_score])
np.testing.assert_allclose(detections['detection_keypoints'][0][0],
expected_kpts_for_obj_0, rtol=1e-6)
np.testing.assert_allclose(detections['detection_keypoint_scores'][0][0],
@@ -1471,7 +2086,308 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
detections['detection_surface_coords'][0, 0, :, :],
np.zeros_like(detections['detection_surface_coords'][0, 0, :, :]))
- def test_get_instance_indices(self):
+ def test_postprocess_kpts_no_od(self):
+ """Test the postprocess function."""
+ target_class_id = 1
+ model = build_center_net_meta_arch(keypoint_only=True)
+ max_detection = model._center_params.max_box_predictions
+ num_keypoints = len(model._kp_params_dict[_TASK_NAME].keypoint_indices)
+
+ class_center = np.zeros((1, 32, 32, 10), dtype=np.float32)
+ keypoint_heatmaps = np.zeros((1, 32, 32, num_keypoints), dtype=np.float32)
+ keypoint_offsets = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ keypoint_regression = np.random.randn(1, 32, 32, num_keypoints * 2)
+
+ class_probs = np.ones(10) * _logit(0.25)
+ class_probs[target_class_id] = _logit(0.75)
+ class_center[0, 16, 16] = class_probs
+ keypoint_regression[0, 16, 16] = [
+ -1., -1.,
+ -1., 1.,
+ 1., -1.,
+ 1., 1.]
+ keypoint_heatmaps[0, 14, 14, 0] = _logit(0.9)
+ keypoint_heatmaps[0, 14, 18, 1] = _logit(0.9)
+ keypoint_heatmaps[0, 18, 14, 2] = _logit(0.9)
+ keypoint_heatmaps[0, 18, 18, 3] = _logit(0.05) # Note the low score.
+
+ class_center = tf.constant(class_center)
+ keypoint_heatmaps = tf.constant(keypoint_heatmaps, dtype=tf.float32)
+ keypoint_offsets = tf.constant(keypoint_offsets, dtype=tf.float32)
+ keypoint_regression = tf.constant(keypoint_regression, dtype=tf.float32)
+
+ prediction_dict = {
+ cnma.OBJECT_CENTER: [class_center],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_HEATMAP):
+ [keypoint_heatmaps],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_OFFSET):
+ [keypoint_offsets],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_REGRESSION):
+ [keypoint_regression],
+ }
+
+ # def graph_fn():
+ detections = model.postprocess(prediction_dict,
+ tf.constant([[128, 128, 3]]))
+ # return detections
+
+ # detections = self.execute_cpu(graph_fn, [])
+ self.assertAllClose(detections['detection_scores'][0],
+ [.75, .5, .5, .5, .5])
+ expected_multiclass_scores = [.25] * 10
+ expected_multiclass_scores[target_class_id] = .75
+ self.assertAllClose(expected_multiclass_scores,
+ detections['detection_multiclass_scores'][0][0])
+
+ self.assertEqual(detections['detection_classes'][0, 0], target_class_id)
+ self.assertEqual(detections['num_detections'], [5])
+ self.assertAllEqual([1, max_detection, num_keypoints, 2],
+ detections['detection_keypoints'].shape)
+ self.assertAllEqual([1, max_detection, num_keypoints],
+ detections['detection_keypoint_scores'].shape)
+
+ def test_non_max_suppression(self):
+ """Tests application of NMS on CenterNet detections."""
+ target_class_id = 1
+ model = build_center_net_meta_arch(apply_non_max_suppression=True,
+ detection_only=True)
+
+ class_center = np.zeros((1, 32, 32, 10), dtype=np.float32)
+ height_width = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ offset = np.zeros((1, 32, 32, 2), dtype=np.float32)
+
+ class_probs = np.ones(10) * _logit(0.25)
+ class_probs[target_class_id] = _logit(0.75)
+ class_center[0, 16, 16] = class_probs
+ height_width[0, 16, 16] = [5, 10]
+ offset[0, 16, 16] = [.25, .5]
+
+ class_center = tf.constant(class_center)
+ height_width = tf.constant(height_width)
+ offset = tf.constant(offset)
+
+ prediction_dict = {
+ cnma.OBJECT_CENTER: [class_center],
+ cnma.BOX_SCALE: [height_width],
+ cnma.BOX_OFFSET: [offset],
+ }
+
+ def graph_fn():
+ detections = model.postprocess(prediction_dict,
+ tf.constant([[128, 128, 3]]))
+ return detections
+
+ detections = self.execute_cpu(graph_fn, [])
+ num_detections = int(detections['num_detections'])
+ self.assertEqual(num_detections, 1)
+ self.assertAllClose(detections['detection_boxes'][0, 0],
+ np.array([55, 46, 75, 86]) / 128.0)
+ self.assertAllClose(detections['detection_scores'][0][:num_detections],
+ [.75])
+ expected_multiclass_scores = [.25] * 10
+ expected_multiclass_scores[target_class_id] = .75
+ self.assertAllClose(expected_multiclass_scores,
+ detections['detection_multiclass_scores'][0][0])
+
+ def test_postprocess_single_class(self):
+ """Test the postprocess function."""
+ model = build_center_net_meta_arch(num_classes=1)
+ max_detection = model._center_params.max_box_predictions
+ num_keypoints = len(model._kp_params_dict[_TASK_NAME].keypoint_indices)
+
+ class_center = np.zeros((1, 32, 32, 1), dtype=np.float32)
+ height_width = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ offset = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ keypoint_heatmaps = np.zeros((1, 32, 32, num_keypoints), dtype=np.float32)
+ keypoint_offsets = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ keypoint_regression = np.random.randn(1, 32, 32, num_keypoints * 2)
+
+ class_probs = np.zeros(1)
+ class_probs[0] = _logit(0.75)
+ class_center[0, 16, 16] = class_probs
+ height_width[0, 16, 16] = [5, 10]
+ offset[0, 16, 16] = [.25, .5]
+ keypoint_regression[0, 16, 16] = [
+ -1., -1.,
+ -1., 1.,
+ 1., -1.,
+ 1., 1.]
+ keypoint_heatmaps[0, 14, 14, 0] = _logit(0.9)
+ keypoint_heatmaps[0, 14, 18, 1] = _logit(0.9)
+ keypoint_heatmaps[0, 18, 14, 2] = _logit(0.9)
+ keypoint_heatmaps[0, 18, 18, 3] = _logit(0.05) # Note the low score.
+
+ class_center = tf.constant(class_center)
+ height_width = tf.constant(height_width)
+ offset = tf.constant(offset)
+ keypoint_heatmaps = tf.constant(keypoint_heatmaps, dtype=tf.float32)
+ keypoint_offsets = tf.constant(keypoint_offsets, dtype=tf.float32)
+ keypoint_regression = tf.constant(keypoint_regression, dtype=tf.float32)
+
+ prediction_dict = {
+ cnma.OBJECT_CENTER: [class_center],
+ cnma.BOX_SCALE: [height_width],
+ cnma.BOX_OFFSET: [offset],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_HEATMAP):
+ [keypoint_heatmaps],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_OFFSET):
+ [keypoint_offsets],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_REGRESSION):
+ [keypoint_regression],
+ }
+
+ def graph_fn():
+ detections = model.postprocess(prediction_dict,
+ tf.constant([[128, 128, 3]]))
+ return detections
+
+ detections = self.execute_cpu(graph_fn, [])
+
+ self.assertAllClose(detections['detection_boxes'][0, 0],
+ np.array([55, 46, 75, 86]) / 128.0)
+ self.assertAllClose(detections['detection_scores'][0],
+ [.75, .5, .5, .5, .5])
+
+ self.assertEqual(detections['detection_classes'][0, 0], 0)
+ self.assertEqual(detections['num_detections'], [5])
+ self.assertAllEqual([1, max_detection, num_keypoints, 2],
+ detections['detection_keypoints'].shape)
+ self.assertAllEqual([1, max_detection, num_keypoints],
+ detections['detection_keypoint_scores'].shape)
+
+ def test_postprocess_single_instance(self):
+ """Test the postprocess single instance function."""
+ model = build_center_net_meta_arch(
+ num_classes=1, candidate_ranking_mode='score_distance_ratio')
+ num_keypoints = len(model._kp_params_dict[_TASK_NAME].keypoint_indices)
+
+ class_center = np.zeros((1, 32, 32, 1), dtype=np.float32)
+ keypoint_heatmaps = np.zeros((1, 32, 32, num_keypoints), dtype=np.float32)
+ keypoint_offsets = np.zeros(
+ (1, 32, 32, num_keypoints * 2), dtype=np.float32)
+ keypoint_regression = np.random.randn(1, 32, 32, num_keypoints * 2)
+
+ class_probs = np.zeros(1)
+ class_probs[0] = _logit(0.75)
+ class_center[0, 16, 16] = class_probs
+ keypoint_regression[0, 16, 16] = [
+ -1., -1.,
+ -1., 1.,
+ 1., -1.,
+ 1., 1.]
+ keypoint_heatmaps[0, 14, 14, 0] = _logit(0.9)
+ keypoint_heatmaps[0, 14, 18, 1] = _logit(0.9)
+ keypoint_heatmaps[0, 18, 14, 2] = _logit(0.9)
+ keypoint_heatmaps[0, 18, 18, 3] = _logit(0.05) # Note the low score.
+
+ class_center = tf.constant(class_center)
+ keypoint_heatmaps = tf.constant(keypoint_heatmaps, dtype=tf.float32)
+ keypoint_offsets = tf.constant(keypoint_offsets, dtype=tf.float32)
+ keypoint_regression = tf.constant(keypoint_regression, dtype=tf.float32)
+
+ prediction_dict = {
+ cnma.OBJECT_CENTER: [class_center],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_HEATMAP):
+ [keypoint_heatmaps],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_OFFSET):
+ [keypoint_offsets],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_REGRESSION):
+ [keypoint_regression],
+ }
+
+ def graph_fn():
+ detections = model.postprocess_single_instance_keypoints(
+ prediction_dict,
+ tf.constant([[128, 128, 3]]))
+ return detections
+
+ detections = self.execute_cpu(graph_fn, [])
+
+ self.assertAllEqual([1, 1, num_keypoints, 2],
+ detections['detection_keypoints'].shape)
+ self.assertAllEqual([1, 1, num_keypoints],
+ detections['detection_keypoint_scores'].shape)
+
+ @parameterized.parameters(
+ {'per_keypoint_depth': False},
+ {'per_keypoint_depth': True},
+ )
+ def test_postprocess_single_class_depth(self, per_keypoint_depth):
+ """Test the postprocess function."""
+ model = build_center_net_meta_arch(
+ num_classes=1,
+ per_keypoint_offset=per_keypoint_depth,
+ predict_depth=True,
+ per_keypoint_depth=per_keypoint_depth)
+ num_keypoints = len(model._kp_params_dict[_TASK_NAME].keypoint_indices)
+
+ class_center = np.zeros((1, 32, 32, 1), dtype=np.float32)
+ height_width = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ offset = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ keypoint_heatmaps = np.ones(
+ (1, 32, 32, num_keypoints), dtype=np.float32) * _logit(0.001)
+ keypoint_offsets = np.zeros((1, 32, 32, 2), dtype=np.float32)
+ keypoint_regression = np.random.randn(1, 32, 32, num_keypoints * 2)
+
+ class_probs = np.zeros(1)
+ class_probs[0] = _logit(0.75)
+ class_center[0, 16, 16] = class_probs
+ height_width[0, 16, 16] = [5, 10]
+ offset[0, 16, 16] = [.25, .5]
+ keypoint_regression[0, 16, 16] = [-1., -1., -1., 1., 1., -1., 1., 1.]
+ keypoint_heatmaps[0, 14, 14, 0] = _logit(0.9)
+ keypoint_heatmaps[0, 14, 18, 1] = _logit(0.9)
+ keypoint_heatmaps[0, 18, 14, 2] = _logit(0.9)
+ keypoint_heatmaps[0, 18, 18, 3] = _logit(0.05) # Note the low score.
+
+ if per_keypoint_depth:
+ keypoint_depth = np.zeros((1, 32, 32, num_keypoints), dtype=np.float32)
+ keypoint_depth[0, 14, 14, 0] = -1.0
+ keypoint_depth[0, 14, 18, 1] = -1.1
+ keypoint_depth[0, 18, 14, 2] = -1.2
+ keypoint_depth[0, 18, 18, 3] = -1.3
+ else:
+ keypoint_depth = np.zeros((1, 32, 32, 1), dtype=np.float32)
+ keypoint_depth[0, 14, 14, 0] = -1.0
+ keypoint_depth[0, 14, 18, 0] = -1.1
+ keypoint_depth[0, 18, 14, 0] = -1.2
+ keypoint_depth[0, 18, 18, 0] = -1.3
+
+ class_center = tf.constant(class_center)
+ height_width = tf.constant(height_width)
+ offset = tf.constant(offset)
+ keypoint_heatmaps = tf.constant(keypoint_heatmaps, dtype=tf.float32)
+ keypoint_offsets = tf.constant(keypoint_offsets, dtype=tf.float32)
+ keypoint_regression = tf.constant(keypoint_regression, dtype=tf.float32)
+ keypoint_depth = tf.constant(keypoint_depth, dtype=tf.float32)
+
+ prediction_dict = {
+ cnma.OBJECT_CENTER: [class_center],
+ cnma.BOX_SCALE: [height_width],
+ cnma.BOX_OFFSET: [offset],
+ cnma.get_keypoint_name(_TASK_NAME,
+ cnma.KEYPOINT_HEATMAP): [keypoint_heatmaps],
+ cnma.get_keypoint_name(_TASK_NAME,
+ cnma.KEYPOINT_OFFSET): [keypoint_offsets],
+ cnma.get_keypoint_name(_TASK_NAME,
+ cnma.KEYPOINT_REGRESSION): [keypoint_regression],
+ cnma.get_keypoint_name(_TASK_NAME,
+ cnma.KEYPOINT_DEPTH): [keypoint_depth]
+ }
+
+ def graph_fn():
+ detections = model.postprocess(prediction_dict,
+ tf.constant([[128, 128, 3]]))
+ return detections
+
+ detections = self.execute_cpu(graph_fn, [])
+
+ self.assertAllClose(detections['detection_keypoint_depths'][0, 0],
+ np.array([-1.0, -1.1, -1.2, 0.0]))
+ self.assertAllClose(detections['detection_keypoint_scores'][0, 0],
+ np.array([0.9, 0.9, 0.9, 0.1]))
+
+ def test_get_instance_indices(self):
classes = tf.constant([[0, 1, 2, 0], [2, 1, 2, 2]], dtype=tf.int32)
num_detections = tf.constant([1, 3], dtype=tf.int32)
batch_index = 1
@@ -1481,8 +2397,72 @@ class CenterNetMetaArchTest(test_case.TestCase, parameterized.TestCase):
classes, num_detections, batch_index, class_id)
self.assertAllEqual(valid_indices.numpy(), [0, 2])
+ def test_rescore_instances(self):
+ feature_extractor = DummyFeatureExtractor(
+ channel_means=(1.0, 2.0, 3.0),
+ channel_stds=(10., 20., 30.),
+ bgr_ordering=False,
+ num_feature_outputs=2,
+ stride=4)
+ image_resizer_fn = functools.partial(
+ preprocessor.resize_to_range,
+ min_dimension=128,
+ max_dimension=128,
+ pad_to_max_dimesnion=True)
+
+ kp_params_1 = cnma.KeypointEstimationParams(
+ task_name='kpt_task_1',
+ class_id=0,
+ keypoint_indices=[0, 1, 2],
+ keypoint_std_dev=[0.00001] * 3,
+ classification_loss=losses.WeightedSigmoidClassificationLoss(),
+ localization_loss=losses.L1LocalizationLoss(),
+ keypoint_candidate_score_threshold=0.1,
+ rescore_instances=True) # Note rescoring for class_id = 0.
+ kp_params_2 = cnma.KeypointEstimationParams(
+ task_name='kpt_task_2',
+ class_id=1,
+ keypoint_indices=[3, 4],
+ keypoint_std_dev=[0.00001] * 2,
+ classification_loss=losses.WeightedSigmoidClassificationLoss(),
+ localization_loss=losses.L1LocalizationLoss(),
+ keypoint_candidate_score_threshold=0.1,
+ rescore_instances=False)
+ model = cnma.CenterNetMetaArch(
+ is_training=True,
+ add_summaries=False,
+ num_classes=2,
+ feature_extractor=feature_extractor,
+ image_resizer_fn=image_resizer_fn,
+ object_center_params=get_fake_center_params(),
+ object_detection_params=get_fake_od_params(),
+ keypoint_params_dict={
+ 'kpt_task_1': kp_params_1,
+ 'kpt_task_2': kp_params_2,
+ })
-def get_fake_prediction_dict(input_height, input_width, stride):
+ def graph_fn():
+ classes = tf.constant([[1, 0]], dtype=tf.int32)
+ scores = tf.constant([[0.5, 0.75]], dtype=tf.float32)
+ keypoint_scores = tf.constant(
+ [
+ [[0.1, 0.0, 0.3, 0.4, 0.5],
+ [0.1, 0.2, 0.3, 0.4, 0.5]],
+ ])
+ new_scores = model._rescore_instances(classes, scores, keypoint_scores)
+ return new_scores
+
+ new_scores = self.execute_cpu(graph_fn, [])
+ expected_scores = np.array(
+ [[0.5, 0.75 * (0.1 + 0.3)/2]]
+ )
+ self.assertAllClose(expected_scores, new_scores)
+
+
+def get_fake_prediction_dict(input_height,
+ input_width,
+ stride,
+ per_keypoint_depth=False):
"""Prepares the fake prediction dictionary."""
output_height = input_height // stride
output_width = input_width // stride
@@ -1517,6 +2497,11 @@ def get_fake_prediction_dict(input_height, input_width, stride):
dtype=np.float32)
keypoint_offset[0, 2, 4] = 0.2, 0.4
+ keypoint_depth = np.zeros((2, output_height, output_width,
+ _NUM_KEYPOINTS if per_keypoint_depth else 1),
+ dtype=np.float32)
+ keypoint_depth[0, 2, 4] = 3.0
+
keypoint_regression = np.zeros(
(2, output_height, output_width, 2 * _NUM_KEYPOINTS), dtype=np.float32)
keypoint_regression[0, 2, 4] = 0.0, 0.0, 0.2, 0.4, 0.0, 0.0, 0.2, 0.4
@@ -1537,6 +2522,14 @@ def get_fake_prediction_dict(input_height, input_width, stride):
# (5 * 2, 5 * 2 + 1), or (10, 11).
densepose_regression[0, 2, 4, 10:12] = 0.4, 0.7
+ track_reid_embedding = np.zeros((2, output_height, output_width,
+ _REID_EMBED_SIZE), dtype=np.float32)
+ track_reid_embedding[0, 2, 4, :] = np.arange(_REID_EMBED_SIZE)
+
+ temporal_offsets = np.zeros((2, output_height, output_width, 2),
+ dtype=np.float32)
+ temporal_offsets[0, 2, 4, :] = 5
+
prediction_dict = {
'preprocessed_inputs':
tf.zeros((2, input_height, input_width, 3)),
@@ -1544,14 +2537,10 @@ def get_fake_prediction_dict(input_height, input_width, stride):
tf.constant(object_center),
tf.constant(object_center)
],
- cnma.BOX_SCALE: [
- tf.constant(object_scale),
- tf.constant(object_scale)
- ],
- cnma.BOX_OFFSET: [
- tf.constant(object_offset),
- tf.constant(object_offset)
- ],
+ cnma.BOX_SCALE: [tf.constant(object_scale),
+ tf.constant(object_scale)],
+ cnma.BOX_OFFSET: [tf.constant(object_offset),
+ tf.constant(object_offset)],
cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_HEATMAP): [
tf.constant(keypoint_heatmap),
tf.constant(keypoint_heatmap)
@@ -1564,6 +2553,10 @@ def get_fake_prediction_dict(input_height, input_width, stride):
tf.constant(keypoint_regression),
tf.constant(keypoint_regression)
],
+ cnma.get_keypoint_name(_TASK_NAME, cnma.KEYPOINT_DEPTH): [
+ tf.constant(keypoint_depth),
+ tf.constant(keypoint_depth)
+ ],
cnma.SEGMENTATION_HEATMAP: [
tf.constant(mask_heatmap),
tf.constant(mask_heatmap)
@@ -1575,12 +2568,23 @@ def get_fake_prediction_dict(input_height, input_width, stride):
cnma.DENSEPOSE_REGRESSION: [
tf.constant(densepose_regression),
tf.constant(densepose_regression),
- ]
+ ],
+ cnma.TRACK_REID: [
+ tf.constant(track_reid_embedding),
+ tf.constant(track_reid_embedding),
+ ],
+ cnma.TEMPORAL_OFFSET: [
+ tf.constant(temporal_offsets),
+ tf.constant(temporal_offsets),
+ ],
}
return prediction_dict
-def get_fake_groundtruth_dict(input_height, input_width, stride):
+def get_fake_groundtruth_dict(input_height,
+ input_width,
+ stride,
+ has_depth=False):
"""Prepares the fake groundtruth dictionary."""
# A small box with center at (0.55, 0.55).
boxes = [
@@ -1609,6 +2613,26 @@ def get_fake_groundtruth_dict(input_height, input_width, stride):
axis=2),
multiples=[1, 1, 2]),
]
+ if has_depth:
+ keypoint_depths = [
+ tf.constant([[float('nan'), 3.0,
+ float('nan'), 3.0, 0.55, 0.0]]),
+ tf.constant([[float('nan'), 0.55,
+ float('nan'), 0.55, 0.55, 0.0]])
+ ]
+ keypoint_depth_weights = [
+ tf.constant([[1.0, 1.0, 1.0, 1.0, 0.0, 0.0]]),
+ tf.constant([[1.0, 1.0, 1.0, 1.0, 0.0, 0.0]])
+ ]
+ else:
+ keypoint_depths = [
+ tf.constant([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]),
+ tf.constant([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])
+ ]
+ keypoint_depth_weights = [
+ tf.constant([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]),
+ tf.constant([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])
+ ]
labeled_classes = [
tf.one_hot([1], depth=_NUM_CLASSES) + tf.one_hot([2], depth=_NUM_CLASSES),
tf.one_hot([0], depth=_NUM_CLASSES) + tf.one_hot([1], depth=_NUM_CLASSES),
@@ -1633,23 +2657,39 @@ def get_fake_groundtruth_dict(input_height, input_width, stride):
tf.constant(densepose_surface_coords_np),
tf.zeros_like(densepose_surface_coords_np)
]
+ track_ids = [
+ tf.constant([2], dtype=tf.int32),
+ tf.constant([1], dtype=tf.int32),
+ ]
+ temporal_offsets = [
+ tf.constant([[5.0, 5.0]], dtype=tf.float32),
+ tf.constant([[2.0, 3.0]], dtype=tf.float32),
+ ]
+ track_match_flags = [
+ tf.constant([1.0], dtype=tf.float32),
+ tf.constant([1.0], dtype=tf.float32),
+ ]
groundtruth_dict = {
fields.BoxListFields.boxes: boxes,
fields.BoxListFields.weights: weights,
fields.BoxListFields.classes: classes,
fields.BoxListFields.keypoints: keypoints,
+ fields.BoxListFields.keypoint_depths: keypoint_depths,
+ fields.BoxListFields.keypoint_depth_weights: keypoint_depth_weights,
fields.BoxListFields.masks: masks,
fields.BoxListFields.densepose_num_points: densepose_num_points,
fields.BoxListFields.densepose_part_ids: densepose_part_ids,
- fields.BoxListFields.densepose_surface_coords:
- densepose_surface_coords,
+ fields.BoxListFields.densepose_surface_coords: densepose_surface_coords,
+ fields.BoxListFields.track_ids: track_ids,
+ fields.BoxListFields.temporal_offsets: temporal_offsets,
+ fields.BoxListFields.track_match_flags: track_match_flags,
fields.InputDataFields.groundtruth_labeled_classes: labeled_classes,
}
return groundtruth_dict
@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
-class CenterNetMetaComputeLossTest(test_case.TestCase):
+class CenterNetMetaComputeLossTest(test_case.TestCase, parameterized.TestCase):
"""Test for CenterNet loss compuation related functions."""
def setUp(self):
@@ -1697,17 +2737,16 @@ class CenterNetMetaComputeLossTest(test_case.TestCase):
# The prediction and groundtruth are curated to produce very low loss.
self.assertGreater(0.01, loss)
- default_value = self.model._center_params.use_only_known_classes
+ default_value = self.model._center_params.use_labeled_classes
self.model._center_params = (
- self.model._center_params._replace(use_only_known_classes=True))
+ self.model._center_params._replace(use_labeled_classes=True))
loss = self.model._compute_object_center_loss(
object_center_predictions=self.prediction_dict[cnma.OBJECT_CENTER],
input_height=self.input_height,
input_width=self.input_width,
per_pixel_weights=self.per_pixel_weights)
self.model._center_params = (
- self.model._center_params._replace(
- use_only_known_classes=default_value))
+ self.model._center_params._replace(use_labeled_classes=default_value))
# The prediction and groundtruth are curated to produce very low loss.
self.assertGreater(0.01, loss)
@@ -1776,6 +2815,66 @@ class CenterNetMetaComputeLossTest(test_case.TestCase):
# The prediction and groundtruth are curated to produce very low loss.
self.assertGreater(0.01, loss)
+ @parameterized.parameters(
+ {'per_keypoint_depth': False},
+ {'per_keypoint_depth': True},
+ )
+ def test_compute_kp_depth_loss(self, per_keypoint_depth):
+ prediction_dict = get_fake_prediction_dict(
+ self.input_height,
+ self.input_width,
+ self.stride,
+ per_keypoint_depth=per_keypoint_depth)
+ model = build_center_net_meta_arch(
+ num_classes=1,
+ per_keypoint_offset=per_keypoint_depth,
+ predict_depth=True,
+ per_keypoint_depth=per_keypoint_depth,
+ peak_radius=1 if per_keypoint_depth else 0)
+ model._groundtruth_lists = get_fake_groundtruth_dict(
+ self.input_height, self.input_width, self.stride, has_depth=True)
+
+ def graph_fn():
+ loss = model._compute_kp_depth_loss(
+ input_height=self.input_height,
+ input_width=self.input_width,
+ task_name=_TASK_NAME,
+ depth_predictions=prediction_dict[cnma.get_keypoint_name(
+ _TASK_NAME, cnma.KEYPOINT_DEPTH)],
+ localization_loss_fn=self.localization_loss_fn)
+ return loss
+
+ loss = self.execute(graph_fn, [])
+
+ if per_keypoint_depth:
+ # The loss is computed on a disk with radius 1 but only the center pixel
+ # has the accurate prediction. The final loss is (4 * |3-0|) / 5 = 2.4
+ self.assertAlmostEqual(2.4, loss, delta=1e-4)
+ else:
+ # The prediction and groundtruth are curated to produce very low loss.
+ self.assertGreater(0.01, loss)
+
+ def test_compute_track_embedding_loss(self):
+ default_fc = self.model.track_reid_classification_net
+ # Initialize the kernel to extreme values so that the classification score
+ # is close to (0, 0, 1) after the softmax layer.
+ kernel_initializer = tf.constant_initializer(
+ [[1, 1, 0], [-1000000, -1000000, 1000000]])
+ self.model.track_reid_classification_net = tf.keras.layers.Dense(
+ _NUM_TRACK_IDS,
+ kernel_initializer=kernel_initializer,
+ input_shape=(_REID_EMBED_SIZE,))
+
+ loss = self.model._compute_track_embedding_loss(
+ input_height=self.input_height,
+ input_width=self.input_width,
+ object_reid_predictions=self.prediction_dict[cnma.TRACK_REID])
+
+ self.model.track_reid_classification_net = default_fc
+
+ # The prediction and groundtruth are curated to produce very low loss.
+ self.assertGreater(0.01, loss)
+
@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
class CenterNetMetaArchRestoreTest(test_case.TestCase):
@@ -1788,6 +2887,16 @@ class CenterNetMetaArchRestoreTest(test_case.TestCase):
self.assertIsInstance(restore_from_objects_map['feature_extractor'],
tf.keras.Model)
+ def test_retore_map_error(self):
+ """Test that restoring unsupported checkpoint type raises an error."""
+
+ model = build_center_net_meta_arch(build_resnet=True)
+ msg = ("Checkpoint type \"detection\" not supported for "
+ "CenterNetResnetFeatureExtractor. Supported types are "
+ "['classification', 'fine_tune']")
+ with self.assertRaisesRegex(ValueError, re.escape(msg)):
+ model.restore_from_objects('detection')
+
class DummyFeatureExtractor(cnma.CenterNetFeatureExtractor):
@@ -1886,6 +2995,162 @@ class CenterNetFeatureExtractorTest(test_case.TestCase):
self.assertAllClose(output[..., 2], 3 * np.ones((2, 32, 32)))
+class Dummy1dFeatureExtractor(cnma.CenterNetFeatureExtractor):
+ """Returns a static tensor."""
+
+ def __init__(self, tensor, out_stride=1, channel_means=(0., 0., 0.),
+ channel_stds=(1., 1., 1.), bgr_ordering=False):
+ """Intializes the feature extractor.
+
+ Args:
+ tensor: The tensor to return as the processed feature.
+ out_stride: The out_stride to return if asked.
+ channel_means: Ignored, but provided for API compatability.
+ channel_stds: Ignored, but provided for API compatability.
+ bgr_ordering: Ignored, but provided for API compatability.
+ """
+
+ super().__init__(
+ channel_means=channel_means, channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
+ self._tensor = tensor
+ self._out_stride = out_stride
+
+ def call(self, inputs):
+ return [self._tensor]
+
+ @property
+ def out_stride(self):
+ """The stride in the output image of the network."""
+ return self._out_stride
+
+ @property
+ def num_feature_outputs(self):
+ """Ther number of feature outputs returned by the feature extractor."""
+ return 1
+
+ @property
+ def supported_sub_model_types(self):
+ return ['detection']
+
+ def get_sub_model(self, sub_model_type):
+ if sub_model_type == 'detection':
+ return self._network
+ else:
+ ValueError('Sub model type "{}" not supported.'.format(sub_model_type))
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class CenterNetMetaArch1dTest(test_case.TestCase, parameterized.TestCase):
+
+ @parameterized.parameters([1, 2])
+ def test_outputs_with_correct_shape(self, stride):
+ # The 1D case reuses code from the 2D cases. These tests only check that
+ # the output shapes are correct, and relies on other tests for correctness.
+ batch_size = 2
+ height = 1
+ width = 32
+ channels = 16
+ unstrided_inputs = np.random.randn(
+ batch_size, height, width, channels)
+ fixed_output_features = np.random.randn(
+ batch_size, height, width // stride, channels)
+ max_boxes = 10
+ num_classes = 3
+ feature_extractor = Dummy1dFeatureExtractor(fixed_output_features, stride)
+ arch = cnma.CenterNetMetaArch(
+ is_training=True,
+ add_summaries=True,
+ num_classes=num_classes,
+ feature_extractor=feature_extractor,
+ image_resizer_fn=None,
+ object_center_params=cnma.ObjectCenterParams(
+ classification_loss=losses.PenaltyReducedLogisticFocalLoss(),
+ object_center_loss_weight=1.0,
+ max_box_predictions=max_boxes,
+ ),
+ object_detection_params=cnma.ObjectDetectionParams(
+ localization_loss=losses.L1LocalizationLoss(),
+ scale_loss_weight=1.0,
+ offset_loss_weight=1.0,
+ ),
+ keypoint_params_dict=None,
+ mask_params=None,
+ densepose_params=None,
+ track_params=None,
+ temporal_offset_params=None,
+ use_depthwise=False,
+ compute_heatmap_sparse=False,
+ non_max_suppression_fn=None,
+ unit_height_conv=True)
+ arch.provide_groundtruth(
+ groundtruth_boxes_list=[
+ tf.constant([[0, 0.5, 1.0, 0.75],
+ [0, 0.1, 1.0, 0.25]], tf.float32),
+ tf.constant([[0, 0, 1.0, 1.0],
+ [0, 0, 0.0, 0.0]], tf.float32)
+ ],
+ groundtruth_classes_list=[
+ tf.constant([[0, 0, 1],
+ [0, 1, 0]], tf.float32),
+ tf.constant([[1, 0, 0],
+ [0, 0, 0]], tf.float32)
+ ],
+ groundtruth_weights_list=[
+ tf.constant([1.0, 1.0]),
+ tf.constant([1.0, 0.0])]
+ )
+
+ predictions = arch.predict(None, None) # input is hardcoded above.
+ predictions['preprocessed_inputs'] = tf.constant(unstrided_inputs)
+ true_shapes = tf.constant([[1, 32, 16], [1, 24, 16]], tf.int32)
+ postprocess_output = arch.postprocess(predictions, true_shapes)
+ losses_output = arch.loss(predictions, true_shapes)
+
+ self.assertIn('%s/%s' % (cnma.LOSS_KEY_PREFIX, cnma.OBJECT_CENTER),
+ losses_output)
+ self.assertEqual((), losses_output['%s/%s' % (
+ cnma.LOSS_KEY_PREFIX, cnma.OBJECT_CENTER)].shape)
+ self.assertIn('%s/%s' % (cnma.LOSS_KEY_PREFIX, cnma.BOX_SCALE),
+ losses_output)
+ self.assertEqual((), losses_output['%s/%s' % (
+ cnma.LOSS_KEY_PREFIX, cnma.BOX_SCALE)].shape)
+ self.assertIn('%s/%s' % (cnma.LOSS_KEY_PREFIX, cnma.BOX_OFFSET),
+ losses_output)
+ self.assertEqual((), losses_output['%s/%s' % (
+ cnma.LOSS_KEY_PREFIX, cnma.BOX_OFFSET)].shape)
+
+ self.assertIn('detection_scores', postprocess_output)
+ self.assertEqual(postprocess_output['detection_scores'].shape,
+ (batch_size, max_boxes))
+ self.assertIn('detection_multiclass_scores', postprocess_output)
+ self.assertEqual(postprocess_output['detection_multiclass_scores'].shape,
+ (batch_size, max_boxes, num_classes))
+ self.assertIn('detection_classes', postprocess_output)
+ self.assertEqual(postprocess_output['detection_classes'].shape,
+ (batch_size, max_boxes))
+ self.assertIn('num_detections', postprocess_output)
+ self.assertEqual(postprocess_output['num_detections'].shape,
+ (batch_size,))
+ self.assertIn('detection_boxes', postprocess_output)
+ self.assertEqual(postprocess_output['detection_boxes'].shape,
+ (batch_size, max_boxes, 4))
+ self.assertIn('detection_boxes_strided', postprocess_output)
+ self.assertEqual(postprocess_output['detection_boxes_strided'].shape,
+ (batch_size, max_boxes, 4))
+
+ self.assertIn(cnma.OBJECT_CENTER, predictions)
+ self.assertEqual(predictions[cnma.OBJECT_CENTER][0].shape,
+ (batch_size, height, width // stride, num_classes))
+ self.assertIn(cnma.BOX_SCALE, predictions)
+ self.assertEqual(predictions[cnma.BOX_SCALE][0].shape,
+ (batch_size, height, width // stride, 2))
+ self.assertIn(cnma.BOX_OFFSET, predictions)
+ self.assertEqual(predictions[cnma.BOX_OFFSET][0].shape,
+ (batch_size, height, width // stride, 2))
+ self.assertIn('preprocessed_inputs', predictions)
+
+
if __name__ == '__main__':
tf.enable_v2_behavior()
tf.test.main()
diff --git a/research/object_detection/meta_architectures/context_rcnn_lib.py b/research/object_detection/meta_architectures/context_rcnn_lib.py
index 902a88c77669cd27eb36490d645740041600fcac..30e5f848e7b2d1b62fbfa1fc7fec7b8c5c58783f 100644
--- a/research/object_detection/meta_architectures/context_rcnn_lib.py
+++ b/research/object_detection/meta_architectures/context_rcnn_lib.py
@@ -67,10 +67,13 @@ def filter_weight_value(weights, values, valid_mask):
# Force the invalid weights to be very negative so it won't contribute to
# the softmax.
- weights += tf.transpose(
- tf.cast(tf.math.logical_not(valid_mask), weights.dtype) *
- _NEGATIVE_PADDING_VALUE,
- perm=[0, 2, 1])
+
+ very_negative_mask = tf.ones(
+ weights.shape, dtype=weights.dtype) * _NEGATIVE_PADDING_VALUE
+ valid_weight_mask = tf.tile(tf.transpose(valid_mask, perm=[0, 2, 1]),
+ [1, weights.shape[1], 1])
+ weights = tf.where(valid_weight_mask,
+ x=weights, y=very_negative_mask)
# Force the invalid values to be 0.
values *= tf.cast(valid_mask, values.dtype)
@@ -140,8 +143,9 @@ def project_features(features, projection_dimension, is_training, normalize):
def attention_block(input_features, context_features, bottleneck_dimension,
- output_dimension, attention_temperature, valid_mask,
- is_training):
+ output_dimension, attention_temperature,
+ keys_values_valid_mask, queries_valid_mask,
+ is_training, block_name="AttentionBlock"):
"""Generic attention block.
Args:
@@ -156,14 +160,18 @@ def attention_block(input_features, context_features, bottleneck_dimension,
attention_temperature: A float Tensor. It controls the temperature of the
softmax for weights calculation. The formula for calculation as follows:
weights = exp(weights / temperature) / sum(exp(weights / temperature))
- valid_mask: A boolean Tensor of shape [batch_size, context_size].
+ keys_values_valid_mask: A boolean Tensor of shape
+ [batch_size, context_size].
+ queries_valid_mask: A boolean Tensor of shape
+ [batch_size, max_num_proposals].
is_training: A boolean Tensor (affecting batch normalization).
+ block_name: A string to specify names for different attention blocks
Returns:
A float Tensor of shape [batch_size, input_size, output_dimension].
"""
- with tf.variable_scope("AttentionBlock"):
+ with tf.variable_scope(block_name):
queries = project_features(
input_features, bottleneck_dimension, is_training, normalize=True)
keys = project_features(
@@ -171,27 +179,42 @@ def attention_block(input_features, context_features, bottleneck_dimension,
values = project_features(
context_features, bottleneck_dimension, is_training, normalize=True)
- weights = tf.matmul(queries, keys, transpose_b=True)
+ # masking out any keys which are padding
+ keys *= tf.cast(keys_values_valid_mask[..., tf.newaxis], keys.dtype)
+ queries *= tf.cast(queries_valid_mask[..., tf.newaxis], queries.dtype)
+
+ weights = tf.matmul(queries, keys, transpose_b=True)
+
+ weights, values = filter_weight_value(weights, values,
+ keys_values_valid_mask)
- weights, values = filter_weight_value(weights, values, valid_mask)
+ weights = tf.identity(tf.nn.softmax(weights / attention_temperature),
+ name=block_name+"AttentionWeights")
- weights = tf.nn.softmax(weights / attention_temperature)
+ features = tf.matmul(weights, values)
- features = tf.matmul(weights, values)
output_features = project_features(
features, output_dimension, is_training, normalize=False)
return output_features
-def compute_box_context_attention(box_features, context_features,
- valid_context_size, bottleneck_dimension,
- attention_temperature, is_training):
+def _compute_box_context_attention(box_features, num_proposals,
+ context_features, valid_context_size,
+ bottleneck_dimension,
+ attention_temperature, is_training,
+ max_num_proposals,
+ use_self_attention=False,
+ use_long_term_attention=True,
+ self_attention_in_sequence=False,
+ num_attention_heads=1,
+ num_attention_layers=1):
"""Computes the attention feature from the context given a batch of box.
Args:
- box_features: A float Tensor of shape [batch_size, max_num_proposals,
+ box_features: A float Tensor of shape [batch_size * max_num_proposals,
height, width, channels]. It is pooled features from first stage
proposals.
+ num_proposals: The number of valid box proposals.
context_features: A float Tensor of shape [batch_size, context_size,
num_context_features].
valid_context_size: A int32 Tensor of shape [batch_size].
@@ -201,22 +224,78 @@ def compute_box_context_attention(box_features, context_features,
softmax for weights calculation. The formula for calculation as follows:
weights = exp(weights / temperature) / sum(exp(weights / temperature))
is_training: A boolean Tensor (affecting batch normalization).
+ max_num_proposals: The number of box proposals for each image.
+ use_self_attention: Whether to use an attention block across the
+ first stage predicted box features for the input image.
+ use_long_term_attention: Whether to use an attention block into the context
+ features.
+ self_attention_in_sequence: Whether self-attention and long term attention
+ should be in sequence or parallel.
+ num_attention_heads: Number of heads for multi-headed attention.
+ num_attention_layers: Number of heads for multi-layered attention.
Returns:
A float Tensor of shape [batch_size, max_num_proposals, 1, 1, channels].
"""
_, context_size, _ = context_features.shape
- valid_mask = compute_valid_mask(valid_context_size, context_size)
+ context_valid_mask = compute_valid_mask(valid_context_size, context_size)
+
+ total_proposals, height, width, channels = box_features.shape
+
+ batch_size = total_proposals // max_num_proposals
+ box_features = tf.reshape(
+ box_features,
+ [batch_size,
+ max_num_proposals,
+ height,
+ width,
+ channels])
- channels = box_features.shape[-1]
# Average pools over height and width dimension so that the shape of
# box_features becomes [batch_size, max_num_proposals, channels].
box_features = tf.reduce_mean(box_features, [2, 3])
-
- output_features = attention_block(box_features, context_features,
- bottleneck_dimension, channels.value,
- attention_temperature, valid_mask,
- is_training)
+ box_valid_mask = compute_valid_mask(
+ num_proposals,
+ box_features.shape[1])
+
+ if use_self_attention:
+ self_attention_box_features = attention_block(
+ box_features, box_features, bottleneck_dimension, channels.value,
+ attention_temperature, keys_values_valid_mask=box_valid_mask,
+ queries_valid_mask=box_valid_mask, is_training=is_training,
+ block_name="SelfAttentionBlock")
+
+ if use_long_term_attention:
+ if use_self_attention and self_attention_in_sequence:
+ input_features = tf.add(self_attention_box_features, box_features)
+ input_features = tf.divide(input_features, 2)
+ else:
+ input_features = box_features
+ original_input_features = input_features
+ for jdx in range(num_attention_layers):
+ layer_features = tf.zeros_like(input_features)
+ for idx in range(num_attention_heads):
+ block_name = "AttentionBlock" + str(idx) + "_AttentionLayer" +str(jdx)
+ attention_features = attention_block(
+ input_features,
+ context_features,
+ bottleneck_dimension,
+ channels.value,
+ attention_temperature,
+ keys_values_valid_mask=context_valid_mask,
+ queries_valid_mask=box_valid_mask,
+ is_training=is_training,
+ block_name=block_name)
+ layer_features = tf.add(layer_features, attention_features)
+ layer_features = tf.divide(layer_features, num_attention_heads)
+ input_features = tf.add(input_features, layer_features)
+ output_features = tf.add(input_features, original_input_features)
+ if not self_attention_in_sequence and use_self_attention:
+ output_features = tf.add(self_attention_box_features, output_features)
+ elif use_self_attention:
+ output_features = self_attention_box_features
+ else:
+ output_features = tf.zeros(self_attention_box_features.shape)
# Expands the dimension back to match with the original feature map.
output_features = output_features[:, :, tf.newaxis, tf.newaxis, :]
diff --git a/research/object_detection/meta_architectures/context_rcnn_lib_tf1_test.py b/research/object_detection/meta_architectures/context_rcnn_lib_tf1_test.py
index a0b3b848d835dcad37f6c75f05b869fbaec4facb..a4c9404f6ee9bbb98dccaf305692ddde185971be 100644
--- a/research/object_detection/meta_architectures/context_rcnn_lib_tf1_test.py
+++ b/research/object_detection/meta_architectures/context_rcnn_lib_tf1_test.py
@@ -50,9 +50,9 @@ class ContextRcnnLibTest(parameterized.TestCase, test_case.TestCase,
filtered_weights, filtered_values = context_rcnn_lib.filter_weight_value(
weights, values, valid_mask)
expected_weights = tf.constant([[[4, 4], [4, 4], [4, 4]],
- [[4, _NEGATIVE_PADDING_VALUE + 4],
- [4, _NEGATIVE_PADDING_VALUE + 4],
- [4, _NEGATIVE_PADDING_VALUE + 4]]])
+ [[4, _NEGATIVE_PADDING_VALUE],
+ [4, _NEGATIVE_PADDING_VALUE],
+ [4, _NEGATIVE_PADDING_VALUE]]])
expected_values = tf.constant([[[1, 1, 1, 1], [1, 1, 1, 1]],
[[1, 1, 1, 1], [0, 0, 0, 0]]])
@@ -66,9 +66,9 @@ class ContextRcnnLibTest(parameterized.TestCase, test_case.TestCase,
weights, values, valid_mask)
expected_weights = tf.constant(
[[[4, 4], [4, 4], [4, 4]],
- [[_NEGATIVE_PADDING_VALUE + 4, _NEGATIVE_PADDING_VALUE + 4],
- [_NEGATIVE_PADDING_VALUE + 4, _NEGATIVE_PADDING_VALUE + 4],
- [_NEGATIVE_PADDING_VALUE + 4, _NEGATIVE_PADDING_VALUE + 4]]])
+ [[_NEGATIVE_PADDING_VALUE, _NEGATIVE_PADDING_VALUE],
+ [_NEGATIVE_PADDING_VALUE, _NEGATIVE_PADDING_VALUE],
+ [_NEGATIVE_PADDING_VALUE, _NEGATIVE_PADDING_VALUE]]])
expected_values = tf.constant([[[1, 1, 1, 1], [1, 1, 1, 1]],
[[0, 0, 0, 0], [0, 0, 0, 0]]])
@@ -100,27 +100,67 @@ class ContextRcnnLibTest(parameterized.TestCase, test_case.TestCase,
input_features = tf.ones([2, 3, 4], tf.float32)
context_features = tf.ones([2, 2, 3], tf.float32)
valid_mask = tf.constant([[True, True], [False, False]], tf.bool)
+ box_valid_mask = tf.constant([[True, True, True], [False, False, False]],
+ tf.bool)
is_training = False
output_features = context_rcnn_lib.attention_block(
input_features, context_features, bottleneck_dimension,
- output_dimension, attention_temperature, valid_mask, is_training)
+ output_dimension, attention_temperature,
+ keys_values_valid_mask=valid_mask,
+ queries_valid_mask=box_valid_mask,
+ is_training=is_training)
# Makes sure the shape is correct.
self.assertAllEqual(output_features.shape, [2, 3, output_dimension])
@parameterized.parameters(True, False)
def test_compute_box_context_attention(self, is_training):
- box_features = tf.ones([2, 3, 4, 4, 4], tf.float32)
+ box_features = tf.ones([2 * 3, 4, 4, 4], tf.float32)
context_features = tf.ones([2, 5, 6], tf.float32)
valid_context_size = tf.constant((2, 3), tf.int32)
+ num_proposals = tf.constant((2, 3), tf.int32)
bottleneck_dimension = 10
attention_temperature = 1
- attention_features = context_rcnn_lib.compute_box_context_attention(
- box_features, context_features, valid_context_size,
- bottleneck_dimension, attention_temperature, is_training)
+ attention_features = context_rcnn_lib._compute_box_context_attention(
+ box_features, num_proposals, context_features, valid_context_size,
+ bottleneck_dimension, attention_temperature, is_training,
+ max_num_proposals=3)
# Makes sure the shape is correct.
self.assertAllEqual(attention_features.shape, [2, 3, 1, 1, 4])
+ @parameterized.parameters(True, False)
+ def test_compute_box_context_attention_with_self_attention(self, is_training):
+ box_features = tf.ones([2 * 3, 4, 4, 4], tf.float32)
+ context_features = tf.ones([2, 5, 6], tf.float32)
+ valid_context_size = tf.constant((2, 3), tf.int32)
+ num_proposals = tf.constant((2, 3), tf.int32)
+ bottleneck_dimension = 10
+ attention_temperature = 1
+ attention_features = context_rcnn_lib._compute_box_context_attention(
+ box_features, num_proposals, context_features, valid_context_size,
+ bottleneck_dimension, attention_temperature, is_training,
+ max_num_proposals=3,
+ use_self_attention=True)
+ # Makes sure the shape is correct.
+ self.assertAllEqual(attention_features.shape, [2, 3, 1, 1, 4])
+
+ @parameterized.parameters(True, False)
+ def test_compute_box_context_attention_with_layers_and_heads(
+ self, is_training):
+ box_features = tf.ones([2 * 3, 4, 4, 4], tf.float32)
+ context_features = tf.ones([2, 5, 6], tf.float32)
+ valid_context_size = tf.constant((2, 3), tf.int32)
+ num_proposals = tf.constant((2, 3), tf.int32)
+ bottleneck_dimension = 10
+ attention_temperature = 1
+ attention_features = context_rcnn_lib._compute_box_context_attention(
+ box_features, num_proposals, context_features, valid_context_size,
+ bottleneck_dimension, attention_temperature, is_training,
+ max_num_proposals=3,
+ num_attention_layers=3,
+ num_attention_heads=3)
+ # Makes sure the shape is correct.
+ self.assertAllEqual(attention_features.shape, [2, 3, 1, 1, 4])
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/meta_architectures/context_rcnn_lib_tf2.py b/research/object_detection/meta_architectures/context_rcnn_lib_tf2.py
new file mode 100644
index 0000000000000000000000000000000000000000..b795d60a91a0874a8d40dc1e140216a126901b0b
--- /dev/null
+++ b/research/object_detection/meta_architectures/context_rcnn_lib_tf2.py
@@ -0,0 +1,264 @@
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library functions for Context R-CNN."""
+import tensorflow as tf
+
+from object_detection.core import freezable_batch_norm
+
+# The negative value used in padding the invalid weights.
+_NEGATIVE_PADDING_VALUE = -100000
+
+
+class ContextProjection(tf.keras.layers.Layer):
+ """Custom layer to do batch normalization and projection."""
+
+ def __init__(self, projection_dimension, **kwargs):
+ self.batch_norm = freezable_batch_norm.FreezableBatchNorm(
+ epsilon=0.001,
+ center=True,
+ scale=True,
+ momentum=0.97,
+ trainable=True)
+ self.projection = tf.keras.layers.Dense(units=projection_dimension,
+ use_bias=True)
+ self.projection_dimension = projection_dimension
+ super(ContextProjection, self).__init__(**kwargs)
+
+ def build(self, input_shape):
+ self.projection.build(input_shape)
+ self.batch_norm.build(input_shape[:1] + [self.projection_dimension])
+
+ def call(self, input_features, is_training=False):
+ return tf.nn.relu6(self.batch_norm(self.projection(input_features),
+ is_training))
+
+
+class AttentionBlock(tf.keras.layers.Layer):
+ """Custom layer to perform all attention."""
+
+ def __init__(self, bottleneck_dimension, attention_temperature,
+ output_dimension=None, is_training=False,
+ name='AttentionBlock', max_num_proposals=100,
+ **kwargs):
+ """Constructs an attention block.
+
+ Args:
+ bottleneck_dimension: A int32 Tensor representing the bottleneck dimension
+ for intermediate projections.
+ attention_temperature: A float Tensor. It controls the temperature of the
+ softmax for weights calculation. The formula for calculation as follows:
+ weights = exp(weights / temperature) / sum(exp(weights / temperature))
+ output_dimension: A int32 Tensor representing the last dimension of the
+ output feature.
+ is_training: A boolean Tensor (affecting batch normalization).
+ name: A string describing what to name the variables in this block.
+ max_num_proposals: The number of box proposals for each image
+ **kwargs: Additional keyword arguments.
+ """
+
+ self._key_proj = ContextProjection(bottleneck_dimension)
+ self._val_proj = ContextProjection(bottleneck_dimension)
+ self._query_proj = ContextProjection(bottleneck_dimension)
+ self._feature_proj = None
+ self._attention_temperature = attention_temperature
+ self._bottleneck_dimension = bottleneck_dimension
+ self._is_training = is_training
+ self._output_dimension = output_dimension
+ self._max_num_proposals = max_num_proposals
+ if self._output_dimension:
+ self._feature_proj = ContextProjection(self._output_dimension)
+ super(AttentionBlock, self).__init__(name=name, **kwargs)
+
+ def build(self, input_shapes):
+ """Finishes building the attention block.
+
+ Args:
+ input_shapes: the shape of the primary input box features.
+ """
+ if not self._feature_proj:
+ self._output_dimension = input_shapes[-1]
+ self._feature_proj = ContextProjection(self._output_dimension)
+
+ def call(self, box_features, context_features, valid_context_size,
+ num_proposals):
+ """Handles a call by performing attention.
+
+ Args:
+ box_features: A float Tensor of shape [batch_size * input_size, height,
+ width, num_input_features].
+ context_features: A float Tensor of shape [batch_size, context_size,
+ num_context_features].
+ valid_context_size: A int32 Tensor of shape [batch_size].
+ num_proposals: A [batch_size] int32 Tensor specifying the number of valid
+ proposals per image in the batch.
+
+ Returns:
+ A float Tensor with shape [batch_size, input_size, num_input_features]
+ containing output features after attention with context features.
+ """
+
+ _, context_size, _ = context_features.shape
+ keys_values_valid_mask = compute_valid_mask(
+ valid_context_size, context_size)
+
+ total_proposals, height, width, channels = box_features.shape
+ batch_size = total_proposals // self._max_num_proposals
+ box_features = tf.reshape(
+ box_features,
+ [batch_size,
+ self._max_num_proposals,
+ height,
+ width,
+ channels])
+
+ # Average pools over height and width dimension so that the shape of
+ # box_features becomes [batch_size, max_num_proposals, channels].
+ box_features = tf.reduce_mean(box_features, [2, 3])
+
+ queries_valid_mask = compute_valid_mask(num_proposals,
+ box_features.shape[1])
+
+ queries = project_features(
+ box_features, self._bottleneck_dimension, self._is_training,
+ self._query_proj, normalize=True)
+ keys = project_features(
+ context_features, self._bottleneck_dimension, self._is_training,
+ self._key_proj, normalize=True)
+ values = project_features(
+ context_features, self._bottleneck_dimension, self._is_training,
+ self._val_proj, normalize=True)
+
+ # masking out any keys which are padding
+ keys *= tf.cast(keys_values_valid_mask[..., tf.newaxis], keys.dtype)
+ queries *= tf.cast(queries_valid_mask[..., tf.newaxis], queries.dtype)
+
+ weights = tf.matmul(queries, keys, transpose_b=True)
+ weights, values = filter_weight_value(weights, values,
+ keys_values_valid_mask)
+ weights = tf.nn.softmax(weights / self._attention_temperature)
+
+ features = tf.matmul(weights, values)
+ output_features = project_features(
+ features, self._output_dimension, self._is_training,
+ self._feature_proj, normalize=False)
+
+ output_features = output_features[:, :, tf.newaxis, tf.newaxis, :]
+
+ return output_features
+
+
+def filter_weight_value(weights, values, valid_mask):
+ """Filters weights and values based on valid_mask.
+
+ _NEGATIVE_PADDING_VALUE will be added to invalid elements in the weights to
+ avoid their contribution in softmax. 0 will be set for the invalid elements in
+ the values.
+
+ Args:
+ weights: A float Tensor of shape [batch_size, input_size, context_size].
+ values: A float Tensor of shape [batch_size, context_size,
+ projected_dimension].
+ valid_mask: A boolean Tensor of shape [batch_size, context_size]. True means
+ valid and False means invalid.
+
+ Returns:
+ weights: A float Tensor of shape [batch_size, input_size, context_size].
+ values: A float Tensor of shape [batch_size, context_size,
+ projected_dimension].
+
+ Raises:
+ ValueError: If shape of doesn't match.
+ """
+ w_batch_size, _, w_context_size = weights.shape
+ v_batch_size, v_context_size, _ = values.shape
+ m_batch_size, m_context_size = valid_mask.shape
+ if w_batch_size != v_batch_size or v_batch_size != m_batch_size:
+ raise ValueError('Please make sure the first dimension of the input'
+ ' tensors are the same.')
+
+ if w_context_size != v_context_size:
+ raise ValueError('Please make sure the third dimension of weights matches'
+ ' the second dimension of values.')
+
+ if w_context_size != m_context_size:
+ raise ValueError('Please make sure the third dimension of the weights'
+ ' matches the second dimension of the valid_mask.')
+
+ valid_mask = valid_mask[..., tf.newaxis]
+
+ # Force the invalid weights to be very negative so it won't contribute to
+ # the softmax.
+ weights += tf.transpose(
+ tf.cast(tf.math.logical_not(valid_mask), weights.dtype) *
+ _NEGATIVE_PADDING_VALUE,
+ perm=[0, 2, 1])
+
+ # Force the invalid values to be 0.
+ values *= tf.cast(valid_mask, values.dtype)
+
+ return weights, values
+
+
+def project_features(features, bottleneck_dimension, is_training,
+ layer, normalize=True):
+ """Projects features to another feature space.
+
+ Args:
+ features: A float Tensor of shape [batch_size, features_size,
+ num_features].
+ bottleneck_dimension: A int32 Tensor.
+ is_training: A boolean Tensor (affecting batch normalization).
+ layer: Contains a custom layer specific to the particular operation
+ being performed (key, value, query, features)
+ normalize: A boolean Tensor. If true, the output features will be l2
+ normalized on the last dimension.
+
+ Returns:
+ A float Tensor of shape [batch, features_size, projection_dimension].
+ """
+ shape_arr = features.shape
+ batch_size, _, num_features = shape_arr
+ features = tf.reshape(features, [-1, num_features])
+
+ projected_features = layer(features, is_training)
+
+ projected_features = tf.reshape(projected_features,
+ [batch_size, -1, bottleneck_dimension])
+
+ if normalize:
+ projected_features = tf.keras.backend.l2_normalize(projected_features,
+ axis=-1)
+
+ return projected_features
+
+
+def compute_valid_mask(num_valid_elements, num_elements):
+ """Computes mask of valid entries within padded context feature.
+
+ Args:
+ num_valid_elements: A int32 Tensor of shape [batch_size].
+ num_elements: An int32 Tensor.
+
+ Returns:
+ A boolean Tensor of the shape [batch_size, num_elements]. True means
+ valid and False means invalid.
+ """
+ batch_size = num_valid_elements.shape[0]
+ element_idxs = tf.range(num_elements, dtype=tf.int32)
+ batch_element_idxs = tf.tile(element_idxs[tf.newaxis, ...], [batch_size, 1])
+ num_valid_elements = num_valid_elements[..., tf.newaxis]
+ valid_mask = tf.less(batch_element_idxs, num_valid_elements)
+ return valid_mask
diff --git a/research/object_detection/meta_architectures/context_rcnn_lib_tf2_test.py b/research/object_detection/meta_architectures/context_rcnn_lib_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0e6b0559d7d119fb12232741fd9c9e4d4a1534d
--- /dev/null
+++ b/research/object_detection/meta_architectures/context_rcnn_lib_tf2_test.py
@@ -0,0 +1,122 @@
+# Lint as: python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for context_rcnn_lib."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+from absl.testing import parameterized
+import tensorflow.compat.v1 as tf
+
+from object_detection.meta_architectures import context_rcnn_lib_tf2 as context_rcnn_lib
+from object_detection.utils import test_case
+from object_detection.utils import tf_version
+
+_NEGATIVE_PADDING_VALUE = -100000
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class ContextRcnnLibTest(parameterized.TestCase, test_case.TestCase):
+ """Tests for the functions in context_rcnn_lib."""
+
+ def test_compute_valid_mask(self):
+ num_elements = tf.constant(3, tf.int32)
+ num_valid_elementss = tf.constant((1, 2), tf.int32)
+ valid_mask = context_rcnn_lib.compute_valid_mask(num_valid_elementss,
+ num_elements)
+ expected_valid_mask = tf.constant([[1, 0, 0], [1, 1, 0]], tf.float32)
+ self.assertAllEqual(valid_mask, expected_valid_mask)
+
+ def test_filter_weight_value(self):
+ weights = tf.ones((2, 3, 2), tf.float32) * 4
+ values = tf.ones((2, 2, 4), tf.float32)
+ valid_mask = tf.constant([[True, True], [True, False]], tf.bool)
+
+ filtered_weights, filtered_values = context_rcnn_lib.filter_weight_value(
+ weights, values, valid_mask)
+ expected_weights = tf.constant([[[4, 4], [4, 4], [4, 4]],
+ [[4, _NEGATIVE_PADDING_VALUE + 4],
+ [4, _NEGATIVE_PADDING_VALUE + 4],
+ [4, _NEGATIVE_PADDING_VALUE + 4]]])
+
+ expected_values = tf.constant([[[1, 1, 1, 1], [1, 1, 1, 1]],
+ [[1, 1, 1, 1], [0, 0, 0, 0]]])
+ self.assertAllEqual(filtered_weights, expected_weights)
+ self.assertAllEqual(filtered_values, expected_values)
+
+ # Changes the valid_mask so the results will be different.
+ valid_mask = tf.constant([[True, True], [False, False]], tf.bool)
+
+ filtered_weights, filtered_values = context_rcnn_lib.filter_weight_value(
+ weights, values, valid_mask)
+ expected_weights = tf.constant(
+ [[[4, 4], [4, 4], [4, 4]],
+ [[_NEGATIVE_PADDING_VALUE + 4, _NEGATIVE_PADDING_VALUE + 4],
+ [_NEGATIVE_PADDING_VALUE + 4, _NEGATIVE_PADDING_VALUE + 4],
+ [_NEGATIVE_PADDING_VALUE + 4, _NEGATIVE_PADDING_VALUE + 4]]])
+
+ expected_values = tf.constant([[[1, 1, 1, 1], [1, 1, 1, 1]],
+ [[0, 0, 0, 0], [0, 0, 0, 0]]])
+ self.assertAllEqual(filtered_weights, expected_weights)
+ self.assertAllEqual(filtered_values, expected_values)
+
+ @parameterized.parameters((2, True, True), (2, False, True),
+ (10, True, False), (10, False, False))
+ def test_project_features(self, projection_dimension, is_training, normalize):
+ features = tf.ones([2, 3, 4], tf.float32)
+ projected_features = context_rcnn_lib.project_features(
+ features,
+ projection_dimension,
+ is_training,
+ context_rcnn_lib.ContextProjection(projection_dimension),
+ normalize=normalize)
+
+ # Makes sure the shape is correct.
+ self.assertAllEqual(projected_features.shape, [2, 3, projection_dimension])
+
+ @parameterized.parameters(
+ (2, 10, 1),
+ (3, 10, 2),
+ (4, None, 3),
+ (5, 20, 4),
+ (7, None, 5),
+ )
+ def test_attention_block(self, bottleneck_dimension, output_dimension,
+ attention_temperature):
+ input_features = tf.ones([2 * 8, 3, 3, 3], tf.float32)
+ context_features = tf.ones([2, 20, 10], tf.float32)
+ num_proposals = tf.convert_to_tensor([6, 3])
+ attention_block = context_rcnn_lib.AttentionBlock(
+ bottleneck_dimension,
+ attention_temperature,
+ output_dimension=output_dimension,
+ is_training=False,
+ max_num_proposals=8)
+ valid_context_size = tf.random_uniform((2,),
+ minval=0,
+ maxval=10,
+ dtype=tf.int32)
+ output_features = attention_block(input_features, context_features,
+ valid_context_size, num_proposals)
+
+ # Makes sure the shape is correct.
+ self.assertAllEqual(output_features.shape,
+ [2, 8, 1, 1, (output_dimension or 3)])
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/meta_architectures/context_rcnn_meta_arch.py b/research/object_detection/meta_architectures/context_rcnn_meta_arch.py
index abe30558b01218df8999b3f0f7698e57f67f8ff2..dc7cad1e47489ea3b238a33648b5d065852c5f5c 100644
--- a/research/object_detection/meta_architectures/context_rcnn_meta_arch.py
+++ b/research/object_detection/meta_architectures/context_rcnn_meta_arch.py
@@ -25,9 +25,18 @@ from __future__ import print_function
import functools
+import tensorflow.compat.v1 as tf
+
+from object_detection.core import box_predictor
from object_detection.core import standard_fields as fields
from object_detection.meta_architectures import context_rcnn_lib
+from object_detection.meta_architectures import context_rcnn_lib_tf2
from object_detection.meta_architectures import faster_rcnn_meta_arch
+from object_detection.protos import faster_rcnn_pb2
+from object_detection.utils import ops
+from object_detection.utils import tf_version
+
+_UNINITIALIZED_FEATURE_EXTRACTOR = '__uninitialized__'
class ContextRCNNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
@@ -74,8 +83,17 @@ class ContextRCNNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
freeze_batchnorm=False,
return_raw_detections_during_predict=False,
output_final_box_features=False,
+ output_final_box_rpn_features=False,
attention_bottleneck_dimension=None,
- attention_temperature=None):
+ attention_temperature=None,
+ use_self_attention=False,
+ use_long_term_attention=True,
+ self_attention_in_sequence=False,
+ num_attention_heads=1,
+ num_attention_layers=1,
+ attention_position=(
+ faster_rcnn_pb2.AttentionPosition.POST_BOX_CLASSIFIER)
+ ):
"""ContextRCNNMetaArch Constructor.
Args:
@@ -208,11 +226,25 @@ class ContextRCNNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
boxes in the predict() method. These are decoded boxes that have not
been through postprocessing (i.e. NMS). Default False.
output_final_box_features: Whether to output final box features. If true,
- it crops the feauture map based on the final box prediction and returns
- in the dict as detection_features.
+ it crops the feature map based on the final box prediction and returns
+ it in the output dict as detection_features.
+ output_final_box_rpn_features: Whether to output rpn box features. If
+ true, it crops the rpn feature map based on the final box prediction and
+ returns it in the output dict as detection_features.
attention_bottleneck_dimension: A single integer. The bottleneck feature
dimension of the attention block.
attention_temperature: A single float. The attention temperature.
+ use_self_attention: Whether to use self-attention within the box features
+ in the current frame.
+ use_long_term_attention: Whether to use attention into the context
+ features.
+ self_attention_in_sequence: Whether self attention and long term attention
+ are in sequence or parallel.
+ num_attention_heads: The number of attention heads to use.
+ num_attention_layers: The number of attention layers to use.
+ attention_position: Whether attention should occur post rpn or post
+ box classifier. Options are specified in the faster rcnn proto,
+ default is post box classifier.
Raises:
ValueError: If `second_stage_batch_size` > `first_stage_max_proposals` at
@@ -262,13 +294,40 @@ class ContextRCNNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
freeze_batchnorm=freeze_batchnorm,
return_raw_detections_during_predict=(
return_raw_detections_during_predict),
- output_final_box_features=output_final_box_features)
+ output_final_box_features=output_final_box_features,
+ output_final_box_rpn_features=output_final_box_rpn_features)
+
+ self._attention_position = attention_position
- self._context_feature_extract_fn = functools.partial(
- context_rcnn_lib.compute_box_context_attention,
- bottleneck_dimension=attention_bottleneck_dimension,
- attention_temperature=attention_temperature,
- is_training=is_training)
+ if tf_version.is_tf1():
+ self._context_feature_extract_fn = functools.partial(
+ context_rcnn_lib._compute_box_context_attention,
+ bottleneck_dimension=attention_bottleneck_dimension,
+ attention_temperature=attention_temperature,
+ is_training=is_training,
+ max_num_proposals=self.max_num_proposals,
+ use_self_attention=use_self_attention,
+ use_long_term_attention=use_long_term_attention,
+ self_attention_in_sequence=self_attention_in_sequence,
+ num_attention_heads=num_attention_heads,
+ num_attention_layers=num_attention_layers)
+ else:
+ if use_self_attention:
+ raise NotImplementedError
+ if self_attention_in_sequence:
+ raise NotImplementedError
+ if not use_long_term_attention:
+ raise NotImplementedError
+ if num_attention_heads > 1:
+ raise NotImplementedError
+ if num_attention_layers > 1:
+ raise NotImplementedError
+
+ self._context_feature_extract_fn = context_rcnn_lib_tf2.AttentionBlock(
+ bottleneck_dimension=attention_bottleneck_dimension,
+ attention_temperature=attention_temperature,
+ is_training=is_training,
+ max_num_proposals=self.max_num_proposals)
@staticmethod
def get_side_inputs(features):
@@ -290,8 +349,8 @@ class ContextRCNNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
if (fields.InputDataFields.context_features not in features or
fields.InputDataFields.valid_context_size not in features):
raise ValueError(
- "Please make sure context_features and valid_context_size are in the "
- "features")
+ 'Please make sure context_features and valid_context_size are in the '
+ 'features')
return {
fields.InputDataFields.context_features:
@@ -300,8 +359,189 @@ class ContextRCNNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
features[fields.InputDataFields.valid_context_size]
}
+ def _predict_second_stage(self, rpn_box_encodings,
+ rpn_objectness_predictions_with_background,
+ rpn_features_to_crop, anchors, image_shape,
+ true_image_shapes, **side_inputs):
+ """Predicts the output tensors from second stage of Faster R-CNN.
+
+ Args:
+ rpn_box_encodings: 3-D float tensor of shape
+ [batch_size, num_valid_anchors, self._box_coder.code_size] containing
+ predicted boxes.
+ rpn_objectness_predictions_with_background: 2-D float tensor of shape
+ [batch_size, num_valid_anchors, 2] containing class
+ predictions (logits) for each of the anchors. Note that this
+ tensor *includes* background class predictions (at class index 0).
+ rpn_features_to_crop: A list of 4-D float32 or bfloat16 tensor with shape
+ [batch_size, height_i, width_i, depth] representing image features to
+ crop using the proposal boxes predicted by the RPN.
+ anchors: 2-D float tensor of shape
+ [num_anchors, self._box_coder.code_size].
+ image_shape: A 1D int32 tensors of size [4] containing the image shape.
+ true_image_shapes: int32 tensor of shape [batch, 3] where each row is
+ of the form [height, width, channels] indicating the shapes
+ of true images in the resized images, as resized images can be padded
+ with zeros.
+ **side_inputs: additional tensors that are required by the network.
+
+ Returns:
+ prediction_dict: a dictionary holding "raw" prediction tensors:
+ 1) refined_box_encodings: a 3-D float32 tensor with shape
+ [total_num_proposals, num_classes, self._box_coder.code_size]
+ representing predicted (final) refined box encodings, where
+ total_num_proposals=batch_size*self._max_num_proposals. If using a
+ shared box across classes the shape will instead be
+ [total_num_proposals, 1, self._box_coder.code_size].
+ 2) class_predictions_with_background: a 3-D float32 tensor with shape
+ [total_num_proposals, num_classes + 1] containing class
+ predictions (logits) for each of the anchors, where
+ total_num_proposals=batch_size*self._max_num_proposals.
+ Note that this tensor *includes* background class predictions
+ (at class index 0).
+ 3) num_proposals: An int32 tensor of shape [batch_size] representing the
+ number of proposals generated by the RPN. `num_proposals` allows us
+ to keep track of which entries are to be treated as zero paddings and
+ which are not since we always pad the number of proposals to be
+ `self.max_num_proposals` for each image.
+ 4) proposal_boxes: A float32 tensor of shape
+ [batch_size, self.max_num_proposals, 4] representing
+ decoded proposal bounding boxes in absolute coordinates.
+ 5) proposal_boxes_normalized: A float32 tensor of shape
+ [batch_size, self.max_num_proposals, 4] representing decoded proposal
+ bounding boxes in normalized coordinates. Can be used to override the
+ boxes proposed by the RPN, thus enabling one to extract features and
+ get box classification and prediction for externally selected areas
+ of the image.
+ 6) box_classifier_features: a 4-D float32/bfloat16 tensor
+ representing the features for each proposal.
+ If self._return_raw_detections_during_predict is True, the dictionary
+ will also contain:
+ 7) raw_detection_boxes: a 4-D float32 tensor with shape
+ [batch_size, self.max_num_proposals, num_classes, 4] in normalized
+ coordinates.
+ 8) raw_detection_feature_map_indices: a 3-D int32 tensor with shape
+ [batch_size, self.max_num_proposals, num_classes].
+ """
+ proposal_boxes_normalized, num_proposals = self._proposal_postprocess(
+ rpn_box_encodings, rpn_objectness_predictions_with_background, anchors,
+ image_shape, true_image_shapes)
+
+ prediction_dict = self._box_prediction(rpn_features_to_crop,
+ proposal_boxes_normalized,
+ image_shape, true_image_shapes,
+ num_proposals,
+ **side_inputs)
+ prediction_dict['num_proposals'] = num_proposals
+ return prediction_dict
+
+ def _box_prediction(self, rpn_features_to_crop, proposal_boxes_normalized,
+ image_shape, true_image_shapes, num_proposals,
+ **side_inputs):
+ """Predicts the output tensors from second stage of Faster R-CNN.
+
+ Args:
+ rpn_features_to_crop: A list 4-D float32 or bfloat16 tensor with shape
+ [batch_size, height_i, width_i, depth] representing image features to
+ crop using the proposal boxes predicted by the RPN.
+ proposal_boxes_normalized: A float tensor with shape [batch_size,
+ max_num_proposals, 4] representing the (potentially zero padded)
+ proposal boxes for all images in the batch. These boxes are represented
+ as normalized coordinates.
+ image_shape: A 1D int32 tensors of size [4] containing the image shape.
+ true_image_shapes: int32 tensor of shape [batch, 3] where each row is
+ of the form [height, width, channels] indicating the shapes
+ of true images in the resized images, as resized images can be padded
+ with zeros.
+ num_proposals: The number of valid box proposals.
+ **side_inputs: additional tensors that are required by the network.
+
+ Returns:
+ prediction_dict: a dictionary holding "raw" prediction tensors:
+ 1) refined_box_encodings: a 3-D float32 tensor with shape
+ [total_num_proposals, num_classes, self._box_coder.code_size]
+ representing predicted (final) refined box encodings, where
+ total_num_proposals=batch_size*self._max_num_proposals. If using a
+ shared box across classes the shape will instead be
+ [total_num_proposals, 1, self._box_coder.code_size].
+ 2) class_predictions_with_background: a 3-D float32 tensor with shape
+ [total_num_proposals, num_classes + 1] containing class
+ predictions (logits) for each of the anchors, where
+ total_num_proposals=batch_size*self._max_num_proposals.
+ Note that this tensor *includes* background class predictions
+ (at class index 0).
+ 3) proposal_boxes: A float32 tensor of shape
+ [batch_size, self.max_num_proposals, 4] representing
+ decoded proposal bounding boxes in absolute coordinates.
+ 4) proposal_boxes_normalized: A float32 tensor of shape
+ [batch_size, self.max_num_proposals, 4] representing decoded proposal
+ bounding boxes in normalized coordinates. Can be used to override the
+ boxes proposed by the RPN, thus enabling one to extract features and
+ get box classification and prediction for externally selected areas
+ of the image.
+ 5) box_classifier_features: a 4-D float32/bfloat16 tensor
+ representing the features for each proposal.
+ If self._return_raw_detections_during_predict is True, the dictionary
+ will also contain:
+ 6) raw_detection_boxes: a 4-D float32 tensor with shape
+ [batch_size, self.max_num_proposals, num_classes, 4] in normalized
+ coordinates.
+ 7) raw_detection_feature_map_indices: a 3-D int32 tensor with shape
+ [batch_size, self.max_num_proposals, num_classes].
+ 8) final_anchors: a 3-D float tensor of shape [batch_size,
+ self.max_num_proposals, 4] containing the reference anchors for raw
+ detection boxes in normalized coordinates.
+ """
+ flattened_proposal_feature_maps = (
+ self._compute_second_stage_input_feature_maps(
+ rpn_features_to_crop, proposal_boxes_normalized,
+ image_shape, num_proposals, **side_inputs))
+
+ box_classifier_features = self._extract_box_classifier_features(
+ flattened_proposal_feature_maps, num_proposals, **side_inputs)
+
+ if self._mask_rcnn_box_predictor.is_keras_model:
+ box_predictions = self._mask_rcnn_box_predictor(
+ [box_classifier_features],
+ prediction_stage=2)
+ else:
+ box_predictions = self._mask_rcnn_box_predictor.predict(
+ [box_classifier_features],
+ num_predictions_per_location=[1],
+ scope=self.second_stage_box_predictor_scope,
+ prediction_stage=2)
+
+ refined_box_encodings = tf.squeeze(
+ box_predictions[box_predictor.BOX_ENCODINGS],
+ axis=1, name='all_refined_box_encodings')
+ class_predictions_with_background = tf.squeeze(
+ box_predictions[box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND],
+ axis=1, name='all_class_predictions_with_background')
+
+ absolute_proposal_boxes = ops.normalized_to_image_coordinates(
+ proposal_boxes_normalized, image_shape, self._parallel_iterations)
+
+ prediction_dict = {
+ 'refined_box_encodings': tf.cast(refined_box_encodings,
+ dtype=tf.float32),
+ 'class_predictions_with_background':
+ tf.cast(class_predictions_with_background, dtype=tf.float32),
+ 'proposal_boxes': absolute_proposal_boxes,
+ 'box_classifier_features': box_classifier_features,
+ 'proposal_boxes_normalized': proposal_boxes_normalized,
+ 'final_anchors': proposal_boxes_normalized
+ }
+
+ if self._return_raw_detections_during_predict:
+ prediction_dict.update(self._raw_detections_and_feature_map_inds(
+ refined_box_encodings, absolute_proposal_boxes, true_image_shapes))
+
+ return prediction_dict
+
def _compute_second_stage_input_feature_maps(self, features_to_crop,
proposal_boxes_normalized,
+ image_shape,
+ num_proposals,
context_features,
valid_context_size):
"""Crops to a set of proposals from the feature map for a batch of images.
@@ -316,6 +556,8 @@ class ContextRCNNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
proposal_boxes_normalized: A float32 Tensor with shape [batch_size,
num_proposals, box_code_size] containing proposal boxes in normalized
coordinates.
+ image_shape: A 1D int32 tensors of size [4] containing the image shape.
+ num_proposals: The number of valid box proposals.
context_features: A float Tensor of shape [batch_size, context_size,
num_context_features].
valid_context_size: A int32 Tensor of shape [batch_size].
@@ -323,18 +565,60 @@ class ContextRCNNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
Returns:
A float32 Tensor with shape [K, new_height, new_width, depth].
"""
+ del image_shape
box_features = self._crop_and_resize_fn(
- features_to_crop, proposal_boxes_normalized,
+ features_to_crop, proposal_boxes_normalized, None,
[self._initial_crop_size, self._initial_crop_size])
- attention_features = self._context_feature_extract_fn(
- box_features=box_features,
- context_features=context_features,
- valid_context_size=valid_context_size)
+ flattened_box_features = self._flatten_first_two_dimensions(box_features)
+
+ flattened_box_features = self._maxpool_layer(flattened_box_features)
+
+ if self._attention_position == (
+ faster_rcnn_pb2.AttentionPosition.POST_RPN):
+ attention_features = self._context_feature_extract_fn(
+ box_features=flattened_box_features,
+ num_proposals=num_proposals,
+ context_features=context_features,
+ valid_context_size=valid_context_size)
+
+ # Adds box features with attention features.
+ flattened_box_features += self._flatten_first_two_dimensions(
+ attention_features)
+
+ return flattened_box_features
+
+ def _extract_box_classifier_features(
+ self, flattened_box_features, num_proposals, context_features,
+ valid_context_size,
+ attention_position=(
+ faster_rcnn_pb2.AttentionPosition.POST_BOX_CLASSIFIER)):
+ if self._feature_extractor_for_box_classifier_features == (
+ _UNINITIALIZED_FEATURE_EXTRACTOR):
+ self._feature_extractor_for_box_classifier_features = (
+ self._feature_extractor.get_box_classifier_feature_extractor_model(
+ name=self.second_stage_feature_extractor_scope))
+
+ if self._feature_extractor_for_box_classifier_features:
+ box_classifier_features = (
+ self._feature_extractor_for_box_classifier_features(
+ flattened_box_features))
+ else:
+ box_classifier_features = (
+ self._feature_extractor.extract_box_classifier_features(
+ flattened_box_features,
+ scope=self.second_stage_feature_extractor_scope))
- # Adds box features with attention features.
- box_features += attention_features
+ if self._attention_position == (
+ faster_rcnn_pb2.AttentionPosition.POST_BOX_CLASSIFIER):
+ attention_features = self._context_feature_extract_fn(
+ box_features=box_classifier_features,
+ num_proposals=num_proposals,
+ context_features=context_features,
+ valid_context_size=valid_context_size)
- flattened_feature_maps = self._flatten_first_two_dimensions(box_features)
+ # Adds box features with attention features.
+ box_classifier_features += self._flatten_first_two_dimensions(
+ attention_features)
- return self._maxpool_layer(flattened_feature_maps)
+ return box_classifier_features
diff --git a/research/object_detection/meta_architectures/context_rcnn_meta_arch_test.py b/research/object_detection/meta_architectures/context_rcnn_meta_arch_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ee8209c7d4803bb92627eedd341e8217dcd2366
--- /dev/null
+++ b/research/object_detection/meta_architectures/context_rcnn_meta_arch_test.py
@@ -0,0 +1,540 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for object_detection.meta_architectures.context_meta_arch."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+import unittest
+from unittest import mock # pylint: disable=g-importing-member
+from absl.testing import parameterized
+import tensorflow.compat.v1 as tf
+import tf_slim as slim
+
+from google.protobuf import text_format
+
+from object_detection.anchor_generators import grid_anchor_generator
+from object_detection.builders import box_predictor_builder
+from object_detection.builders import hyperparams_builder
+from object_detection.builders import post_processing_builder
+from object_detection.core import balanced_positive_negative_sampler as sampler
+from object_detection.core import losses
+from object_detection.core import post_processing
+from object_detection.core import standard_fields as fields
+from object_detection.core import target_assigner
+from object_detection.meta_architectures import context_rcnn_meta_arch
+from object_detection.meta_architectures import faster_rcnn_meta_arch
+from object_detection.protos import box_predictor_pb2
+from object_detection.protos import hyperparams_pb2
+from object_detection.protos import post_processing_pb2
+from object_detection.utils import spatial_transform_ops as spatial_ops
+from object_detection.utils import test_case
+from object_detection.utils import test_utils
+from object_detection.utils import tf_version
+
+
+class FakeFasterRCNNFeatureExtractor(
+ faster_rcnn_meta_arch.FasterRCNNFeatureExtractor):
+ """Fake feature extractor to use in tests."""
+
+ def __init__(self):
+ super(FakeFasterRCNNFeatureExtractor, self).__init__(
+ is_training=False,
+ first_stage_features_stride=32,
+ reuse_weights=None,
+ weight_decay=0.0)
+
+ def preprocess(self, resized_inputs):
+ return tf.identity(resized_inputs)
+
+ def _extract_proposal_features(self, preprocessed_inputs, scope):
+ with tf.variable_scope('mock_model'):
+ proposal_features = 0 * slim.conv2d(
+ preprocessed_inputs, num_outputs=3, kernel_size=1, scope='layer1')
+ return proposal_features, {}
+
+ def _extract_box_classifier_features(self, proposal_feature_maps, scope):
+ with tf.variable_scope('mock_model'):
+ return 0 * slim.conv2d(
+ proposal_feature_maps, num_outputs=3, kernel_size=1, scope='layer2')
+
+
+class FakeFasterRCNNKerasFeatureExtractor(
+ faster_rcnn_meta_arch.FasterRCNNKerasFeatureExtractor):
+ """Fake feature extractor to use in tests."""
+
+ def __init__(self):
+ super(FakeFasterRCNNKerasFeatureExtractor, self).__init__(
+ is_training=False, first_stage_features_stride=32, weight_decay=0.0)
+
+ def preprocess(self, resized_inputs):
+ return tf.identity(resized_inputs)
+
+ def get_proposal_feature_extractor_model(self, name):
+
+ class ProposalFeatureExtractor(tf.keras.Model):
+ """Dummy proposal feature extraction."""
+
+ def __init__(self, name):
+ super(ProposalFeatureExtractor, self).__init__(name=name)
+ self.conv = None
+
+ def build(self, input_shape):
+ self.conv = tf.keras.layers.Conv2D(
+ 3, kernel_size=1, padding='SAME', name='layer1')
+
+ def call(self, inputs):
+ return self.conv(inputs)
+
+ return ProposalFeatureExtractor(name=name)
+
+ def get_box_classifier_feature_extractor_model(self, name):
+ return tf.keras.Sequential([
+ tf.keras.layers.Conv2D(
+ 3, kernel_size=1, padding='SAME', name=name + '_layer2')
+ ])
+
+
+class ContextRCNNMetaArchTest(test_case.TestCase, parameterized.TestCase):
+
+ def _get_model(self, box_predictor, **common_kwargs):
+ return context_rcnn_meta_arch.ContextRCNNMetaArch(
+ initial_crop_size=3,
+ maxpool_kernel_size=1,
+ maxpool_stride=1,
+ second_stage_mask_rcnn_box_predictor=box_predictor,
+ attention_bottleneck_dimension=10,
+ attention_temperature=0.2,
+ **common_kwargs)
+
+ def _build_arg_scope_with_hyperparams(self, hyperparams_text_proto,
+ is_training):
+ hyperparams = hyperparams_pb2.Hyperparams()
+ text_format.Merge(hyperparams_text_proto, hyperparams)
+ return hyperparams_builder.build(hyperparams, is_training=is_training)
+
+ def _build_keras_layer_hyperparams(self, hyperparams_text_proto):
+ hyperparams = hyperparams_pb2.Hyperparams()
+ text_format.Merge(hyperparams_text_proto, hyperparams)
+ return hyperparams_builder.KerasLayerHyperparams(hyperparams)
+
+ def _get_second_stage_box_predictor_text_proto(self,
+ share_box_across_classes=False
+ ):
+ share_box_field = 'true' if share_box_across_classes else 'false'
+ box_predictor_text_proto = """
+ mask_rcnn_box_predictor {{
+ fc_hyperparams {{
+ op: FC
+ activation: NONE
+ regularizer {{
+ l2_regularizer {{
+ weight: 0.0005
+ }}
+ }}
+ initializer {{
+ variance_scaling_initializer {{
+ factor: 1.0
+ uniform: true
+ mode: FAN_AVG
+ }}
+ }}
+ }}
+ share_box_across_classes: {share_box_across_classes}
+ }}
+ """.format(share_box_across_classes=share_box_field)
+ return box_predictor_text_proto
+
+ def _get_box_classifier_features_shape(self,
+ image_size,
+ batch_size,
+ max_num_proposals,
+ initial_crop_size,
+ maxpool_stride,
+ num_features):
+ return (batch_size * max_num_proposals,
+ initial_crop_size/maxpool_stride,
+ initial_crop_size/maxpool_stride,
+ num_features)
+
+ def _get_second_stage_box_predictor(self,
+ num_classes,
+ is_training,
+ predict_masks,
+ masks_are_class_agnostic,
+ share_box_across_classes=False,
+ use_keras=False):
+ box_predictor_proto = box_predictor_pb2.BoxPredictor()
+ text_format.Merge(
+ self._get_second_stage_box_predictor_text_proto(
+ share_box_across_classes), box_predictor_proto)
+ if predict_masks:
+ text_format.Merge(
+ self._add_mask_to_second_stage_box_predictor_text_proto(
+ masks_are_class_agnostic), box_predictor_proto)
+
+ if use_keras:
+ return box_predictor_builder.build_keras(
+ hyperparams_builder.KerasLayerHyperparams,
+ inplace_batchnorm_update=False,
+ freeze_batchnorm=False,
+ box_predictor_config=box_predictor_proto,
+ num_classes=num_classes,
+ num_predictions_per_location_list=None,
+ is_training=is_training)
+ else:
+ return box_predictor_builder.build(
+ hyperparams_builder.build,
+ box_predictor_proto,
+ num_classes=num_classes,
+ is_training=is_training)
+
+ def _build_model(self,
+ is_training,
+ number_of_stages,
+ second_stage_batch_size,
+ first_stage_max_proposals=8,
+ num_classes=2,
+ hard_mining=False,
+ softmax_second_stage_classification_loss=True,
+ predict_masks=False,
+ pad_to_max_dimension=None,
+ masks_are_class_agnostic=False,
+ use_matmul_crop_and_resize=False,
+ clip_anchors_to_image=False,
+ use_matmul_gather_in_matcher=False,
+ use_static_shapes=False,
+ calibration_mapping_value=None,
+ share_box_across_classes=False,
+ return_raw_detections_during_predict=False):
+ use_keras = tf_version.is_tf2()
+ def image_resizer_fn(image, masks=None):
+ """Fake image resizer function."""
+ resized_inputs = []
+ resized_image = tf.identity(image)
+ if pad_to_max_dimension is not None:
+ resized_image = tf.image.pad_to_bounding_box(image, 0, 0,
+ pad_to_max_dimension,
+ pad_to_max_dimension)
+ resized_inputs.append(resized_image)
+ if masks is not None:
+ resized_masks = tf.identity(masks)
+ if pad_to_max_dimension is not None:
+ resized_masks = tf.image.pad_to_bounding_box(
+ tf.transpose(masks, [1, 2, 0]), 0, 0, pad_to_max_dimension,
+ pad_to_max_dimension)
+ resized_masks = tf.transpose(resized_masks, [2, 0, 1])
+ resized_inputs.append(resized_masks)
+ resized_inputs.append(tf.shape(image))
+ return resized_inputs
+
+ # anchors in this test are designed so that a subset of anchors are inside
+ # the image and a subset of anchors are outside.
+ first_stage_anchor_scales = (0.001, 0.005, 0.1)
+ first_stage_anchor_aspect_ratios = (0.5, 1.0, 2.0)
+ first_stage_anchor_strides = (1, 1)
+ first_stage_anchor_generator = grid_anchor_generator.GridAnchorGenerator(
+ first_stage_anchor_scales,
+ first_stage_anchor_aspect_ratios,
+ anchor_stride=first_stage_anchor_strides)
+ first_stage_target_assigner = target_assigner.create_target_assigner(
+ 'FasterRCNN',
+ 'proposal',
+ use_matmul_gather=use_matmul_gather_in_matcher)
+
+ if use_keras:
+ fake_feature_extractor = FakeFasterRCNNKerasFeatureExtractor()
+ else:
+ fake_feature_extractor = FakeFasterRCNNFeatureExtractor()
+
+ first_stage_box_predictor_hyperparams_text_proto = """
+ op: CONV
+ activation: RELU
+ regularizer {
+ l2_regularizer {
+ weight: 0.00004
+ }
+ }
+ initializer {
+ truncated_normal_initializer {
+ stddev: 0.03
+ }
+ }
+ """
+ if use_keras:
+ first_stage_box_predictor_arg_scope_fn = (
+ self._build_keras_layer_hyperparams(
+ first_stage_box_predictor_hyperparams_text_proto))
+ else:
+ first_stage_box_predictor_arg_scope_fn = (
+ self._build_arg_scope_with_hyperparams(
+ first_stage_box_predictor_hyperparams_text_proto, is_training))
+
+ first_stage_box_predictor_kernel_size = 3
+ first_stage_atrous_rate = 1
+ first_stage_box_predictor_depth = 512
+ first_stage_minibatch_size = 3
+ first_stage_sampler = sampler.BalancedPositiveNegativeSampler(
+ positive_fraction=0.5, is_static=use_static_shapes)
+
+ first_stage_nms_score_threshold = -1.0
+ first_stage_nms_iou_threshold = 1.0
+ first_stage_non_max_suppression_fn = functools.partial(
+ post_processing.batch_multiclass_non_max_suppression,
+ score_thresh=first_stage_nms_score_threshold,
+ iou_thresh=first_stage_nms_iou_threshold,
+ max_size_per_class=first_stage_max_proposals,
+ max_total_size=first_stage_max_proposals,
+ use_static_shapes=use_static_shapes)
+
+ first_stage_localization_loss_weight = 1.0
+ first_stage_objectness_loss_weight = 1.0
+
+ post_processing_config = post_processing_pb2.PostProcessing()
+ post_processing_text_proto = """
+ score_converter: IDENTITY
+ batch_non_max_suppression {
+ score_threshold: -20.0
+ iou_threshold: 1.0
+ max_detections_per_class: 5
+ max_total_detections: 5
+ use_static_shapes: """ + '{}'.format(use_static_shapes) + """
+ }
+ """
+ if calibration_mapping_value:
+ calibration_text_proto = """
+ calibration_config {
+ function_approximation {
+ x_y_pairs {
+ x_y_pair {
+ x: 0.0
+ y: %f
+ }
+ x_y_pair {
+ x: 1.0
+ y: %f
+ }}}}""" % (calibration_mapping_value, calibration_mapping_value)
+ post_processing_text_proto = (
+ post_processing_text_proto + ' ' + calibration_text_proto)
+ text_format.Merge(post_processing_text_proto, post_processing_config)
+ second_stage_non_max_suppression_fn, second_stage_score_conversion_fn = (
+ post_processing_builder.build(post_processing_config))
+
+ second_stage_target_assigner = target_assigner.create_target_assigner(
+ 'FasterRCNN',
+ 'detection',
+ use_matmul_gather=use_matmul_gather_in_matcher)
+ second_stage_sampler = sampler.BalancedPositiveNegativeSampler(
+ positive_fraction=1.0, is_static=use_static_shapes)
+
+ second_stage_localization_loss_weight = 1.0
+ second_stage_classification_loss_weight = 1.0
+ if softmax_second_stage_classification_loss:
+ second_stage_classification_loss = (
+ losses.WeightedSoftmaxClassificationLoss())
+ else:
+ second_stage_classification_loss = (
+ losses.WeightedSigmoidClassificationLoss())
+
+ hard_example_miner = None
+ if hard_mining:
+ hard_example_miner = losses.HardExampleMiner(
+ num_hard_examples=1,
+ iou_threshold=0.99,
+ loss_type='both',
+ cls_loss_weight=second_stage_classification_loss_weight,
+ loc_loss_weight=second_stage_localization_loss_weight,
+ max_negatives_per_positive=None)
+
+ crop_and_resize_fn = (
+ spatial_ops.multilevel_matmul_crop_and_resize
+ if use_matmul_crop_and_resize
+ else spatial_ops.multilevel_native_crop_and_resize)
+ common_kwargs = {
+ 'is_training':
+ is_training,
+ 'num_classes':
+ num_classes,
+ 'image_resizer_fn':
+ image_resizer_fn,
+ 'feature_extractor':
+ fake_feature_extractor,
+ 'number_of_stages':
+ number_of_stages,
+ 'first_stage_anchor_generator':
+ first_stage_anchor_generator,
+ 'first_stage_target_assigner':
+ first_stage_target_assigner,
+ 'first_stage_atrous_rate':
+ first_stage_atrous_rate,
+ 'first_stage_box_predictor_arg_scope_fn':
+ first_stage_box_predictor_arg_scope_fn,
+ 'first_stage_box_predictor_kernel_size':
+ first_stage_box_predictor_kernel_size,
+ 'first_stage_box_predictor_depth':
+ first_stage_box_predictor_depth,
+ 'first_stage_minibatch_size':
+ first_stage_minibatch_size,
+ 'first_stage_sampler':
+ first_stage_sampler,
+ 'first_stage_non_max_suppression_fn':
+ first_stage_non_max_suppression_fn,
+ 'first_stage_max_proposals':
+ first_stage_max_proposals,
+ 'first_stage_localization_loss_weight':
+ first_stage_localization_loss_weight,
+ 'first_stage_objectness_loss_weight':
+ first_stage_objectness_loss_weight,
+ 'second_stage_target_assigner':
+ second_stage_target_assigner,
+ 'second_stage_batch_size':
+ second_stage_batch_size,
+ 'second_stage_sampler':
+ second_stage_sampler,
+ 'second_stage_non_max_suppression_fn':
+ second_stage_non_max_suppression_fn,
+ 'second_stage_score_conversion_fn':
+ second_stage_score_conversion_fn,
+ 'second_stage_localization_loss_weight':
+ second_stage_localization_loss_weight,
+ 'second_stage_classification_loss_weight':
+ second_stage_classification_loss_weight,
+ 'second_stage_classification_loss':
+ second_stage_classification_loss,
+ 'hard_example_miner':
+ hard_example_miner,
+ 'crop_and_resize_fn':
+ crop_and_resize_fn,
+ 'clip_anchors_to_image':
+ clip_anchors_to_image,
+ 'use_static_shapes':
+ use_static_shapes,
+ 'resize_masks':
+ True,
+ 'return_raw_detections_during_predict':
+ return_raw_detections_during_predict
+ }
+
+ return self._get_model(
+ self._get_second_stage_box_predictor(
+ num_classes=num_classes,
+ is_training=is_training,
+ use_keras=use_keras,
+ predict_masks=predict_masks,
+ masks_are_class_agnostic=masks_are_class_agnostic,
+ share_box_across_classes=share_box_across_classes), **common_kwargs)
+
+ @unittest.skipIf(tf_version.is_tf2(), 'Skipping TF1.X only test.')
+ @mock.patch.object(context_rcnn_meta_arch, 'context_rcnn_lib')
+ def test_prediction_mock_tf1(self, mock_context_rcnn_lib_v1):
+ """Mocks the context_rcnn_lib_v1 module to test the prediction.
+
+ Using mock object so that we can ensure _compute_box_context_attention is
+ called in side the prediction function.
+
+ Args:
+ mock_context_rcnn_lib_v1: mock module for the context_rcnn_lib_v1.
+ """
+ model = self._build_model(
+ is_training=False,
+ number_of_stages=2,
+ second_stage_batch_size=6,
+ num_classes=42)
+ mock_tensor = tf.ones([2, 8, 3, 3, 3], tf.float32)
+
+ mock_context_rcnn_lib_v1._compute_box_context_attention.return_value = mock_tensor
+ inputs_shape = (2, 20, 20, 3)
+ inputs = tf.cast(
+ tf.random_uniform(inputs_shape, minval=0, maxval=255, dtype=tf.int32),
+ dtype=tf.float32)
+ preprocessed_inputs, true_image_shapes = model.preprocess(inputs)
+ context_features = tf.random_uniform((2, 20, 10),
+ minval=0,
+ maxval=255,
+ dtype=tf.float32)
+ valid_context_size = tf.random_uniform((2,),
+ minval=0,
+ maxval=10,
+ dtype=tf.int32)
+ features = {
+ fields.InputDataFields.context_features: context_features,
+ fields.InputDataFields.valid_context_size: valid_context_size
+ }
+
+ side_inputs = model.get_side_inputs(features)
+
+ _ = model.predict(preprocessed_inputs, true_image_shapes, **side_inputs)
+ mock_context_rcnn_lib_v1._compute_box_context_attention.assert_called_once()
+
+ @parameterized.named_parameters(
+ {'testcase_name': 'static_shapes', 'static_shapes': True},
+ {'testcase_name': 'nostatic_shapes', 'static_shapes': False},
+ )
+ def test_prediction_end_to_end(self, static_shapes):
+ """Runs prediction end to end and test the shape of the results."""
+ with test_utils.GraphContextOrNone() as g:
+ model = self._build_model(
+ is_training=False,
+ number_of_stages=2,
+ second_stage_batch_size=6,
+ use_matmul_crop_and_resize=static_shapes,
+ clip_anchors_to_image=static_shapes,
+ use_matmul_gather_in_matcher=static_shapes,
+ use_static_shapes=static_shapes,
+ num_classes=42)
+
+ def graph_fn():
+ inputs_shape = (2, 20, 20, 3)
+ inputs = tf.cast(
+ tf.random_uniform(inputs_shape, minval=0, maxval=255, dtype=tf.int32),
+ dtype=tf.float32)
+ preprocessed_inputs, true_image_shapes = model.preprocess(inputs)
+ context_features = tf.random_uniform((2, 20, 10),
+ minval=0,
+ maxval=255,
+ dtype=tf.float32)
+ valid_context_size = tf.random_uniform((2,),
+ minval=0,
+ maxval=10,
+ dtype=tf.int32)
+ features = {
+ fields.InputDataFields.context_features: context_features,
+ fields.InputDataFields.valid_context_size: valid_context_size
+ }
+
+ side_inputs = model.get_side_inputs(features)
+ prediction_dict = model.predict(preprocessed_inputs, true_image_shapes,
+ **side_inputs)
+ return (prediction_dict['rpn_box_predictor_features'],
+ prediction_dict['rpn_box_encodings'],
+ prediction_dict['refined_box_encodings'],
+ prediction_dict['proposal_boxes_normalized'],
+ prediction_dict['proposal_boxes'])
+ execute_fn = self.execute if static_shapes else self.execute_cpu
+ (rpn_box_predictor_features, rpn_box_encodings, refined_box_encodings,
+ proposal_boxes_normalized, proposal_boxes) = execute_fn(graph_fn, [],
+ graph=g)
+ self.assertAllEqual(len(rpn_box_predictor_features), 1)
+ self.assertAllEqual(rpn_box_predictor_features[0].shape, [2, 20, 20, 512])
+ self.assertAllEqual(rpn_box_encodings.shape, [2, 3600, 4])
+ self.assertAllEqual(refined_box_encodings.shape, [16, 42, 4])
+ self.assertAllEqual(proposal_boxes_normalized.shape, [2, 8, 4])
+ self.assertAllEqual(proposal_boxes.shape, [2, 8, 4])
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/meta_architectures/context_rcnn_meta_arch_tf1_test.py b/research/object_detection/meta_architectures/context_rcnn_meta_arch_tf1_test.py
deleted file mode 100644
index d80404f45c91252be4a631897fe52976b9109b13..0000000000000000000000000000000000000000
--- a/research/object_detection/meta_architectures/context_rcnn_meta_arch_tf1_test.py
+++ /dev/null
@@ -1,540 +0,0 @@
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for object_detection.meta_architectures.context_meta_arch."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import functools
-import unittest
-from unittest import mock # pylint: disable=g-importing-member
-from absl.testing import parameterized
-import tensorflow.compat.v1 as tf
-import tf_slim as slim
-
-from google.protobuf import text_format
-
-from object_detection.anchor_generators import grid_anchor_generator
-from object_detection.builders import box_predictor_builder
-from object_detection.builders import hyperparams_builder
-from object_detection.builders import post_processing_builder
-from object_detection.core import balanced_positive_negative_sampler as sampler
-from object_detection.core import losses
-from object_detection.core import post_processing
-from object_detection.core import standard_fields as fields
-from object_detection.core import target_assigner
-from object_detection.meta_architectures import context_rcnn_meta_arch
-from object_detection.meta_architectures import faster_rcnn_meta_arch
-from object_detection.protos import box_predictor_pb2
-from object_detection.protos import hyperparams_pb2
-from object_detection.protos import post_processing_pb2
-from object_detection.utils import ops
-from object_detection.utils import test_case
-from object_detection.utils import test_utils
-from object_detection.utils import tf_version
-
-
-class FakeFasterRCNNFeatureExtractor(
- faster_rcnn_meta_arch.FasterRCNNFeatureExtractor):
- """Fake feature extractor to use in tests."""
-
- def __init__(self):
- super(FakeFasterRCNNFeatureExtractor, self).__init__(
- is_training=False,
- first_stage_features_stride=32,
- reuse_weights=None,
- weight_decay=0.0)
-
- def preprocess(self, resized_inputs):
- return tf.identity(resized_inputs)
-
- def _extract_proposal_features(self, preprocessed_inputs, scope):
- with tf.variable_scope('mock_model'):
- proposal_features = 0 * slim.conv2d(
- preprocessed_inputs, num_outputs=3, kernel_size=1, scope='layer1')
- return proposal_features, {}
-
- def _extract_box_classifier_features(self, proposal_feature_maps, scope):
- with tf.variable_scope('mock_model'):
- return 0 * slim.conv2d(
- proposal_feature_maps, num_outputs=3, kernel_size=1, scope='layer2')
-
-
-class FakeFasterRCNNKerasFeatureExtractor(
- faster_rcnn_meta_arch.FasterRCNNKerasFeatureExtractor):
- """Fake feature extractor to use in tests."""
-
- def __init__(self):
- super(FakeFasterRCNNKerasFeatureExtractor, self).__init__(
- is_training=False, first_stage_features_stride=32, weight_decay=0.0)
-
- def preprocess(self, resized_inputs):
- return tf.identity(resized_inputs)
-
- def get_proposal_feature_extractor_model(self, name):
-
- class ProposalFeatureExtractor(tf.keras.Model):
- """Dummy proposal feature extraction."""
-
- def __init__(self, name):
- super(ProposalFeatureExtractor, self).__init__(name=name)
- self.conv = None
-
- def build(self, input_shape):
- self.conv = tf.keras.layers.Conv2D(
- 3, kernel_size=1, padding='SAME', name='layer1')
-
- def call(self, inputs):
- return self.conv(inputs)
-
- return ProposalFeatureExtractor(name=name)
-
- def get_box_classifier_feature_extractor_model(self, name):
- return tf.keras.Sequential([
- tf.keras.layers.Conv2D(
- 3, kernel_size=1, padding='SAME', name=name + '_layer2')
- ])
-
-
-@unittest.skipIf(tf_version.is_tf2(), 'Skipping TF1.X only test.')
-class ContextRCNNMetaArchTest(test_case.TestCase, parameterized.TestCase):
-
- def _get_model(self, box_predictor, **common_kwargs):
- return context_rcnn_meta_arch.ContextRCNNMetaArch(
- initial_crop_size=3,
- maxpool_kernel_size=1,
- maxpool_stride=1,
- second_stage_mask_rcnn_box_predictor=box_predictor,
- attention_bottleneck_dimension=10,
- attention_temperature=0.2,
- **common_kwargs)
-
- def _build_arg_scope_with_hyperparams(self, hyperparams_text_proto,
- is_training):
- hyperparams = hyperparams_pb2.Hyperparams()
- text_format.Merge(hyperparams_text_proto, hyperparams)
- return hyperparams_builder.build(hyperparams, is_training=is_training)
-
- def _build_keras_layer_hyperparams(self, hyperparams_text_proto):
- hyperparams = hyperparams_pb2.Hyperparams()
- text_format.Merge(hyperparams_text_proto, hyperparams)
- return hyperparams_builder.KerasLayerHyperparams(hyperparams)
-
- def _get_second_stage_box_predictor_text_proto(self,
- share_box_across_classes=False
- ):
- share_box_field = 'true' if share_box_across_classes else 'false'
- box_predictor_text_proto = """
- mask_rcnn_box_predictor {{
- fc_hyperparams {{
- op: FC
- activation: NONE
- regularizer {{
- l2_regularizer {{
- weight: 0.0005
- }}
- }}
- initializer {{
- variance_scaling_initializer {{
- factor: 1.0
- uniform: true
- mode: FAN_AVG
- }}
- }}
- }}
- share_box_across_classes: {share_box_across_classes}
- }}
- """.format(share_box_across_classes=share_box_field)
- return box_predictor_text_proto
-
- def _get_box_classifier_features_shape(self,
- image_size,
- batch_size,
- max_num_proposals,
- initial_crop_size,
- maxpool_stride,
- num_features):
- return (batch_size * max_num_proposals,
- initial_crop_size/maxpool_stride,
- initial_crop_size/maxpool_stride,
- num_features)
-
- def _get_second_stage_box_predictor(self,
- num_classes,
- is_training,
- predict_masks,
- masks_are_class_agnostic,
- share_box_across_classes=False,
- use_keras=False):
- box_predictor_proto = box_predictor_pb2.BoxPredictor()
- text_format.Merge(
- self._get_second_stage_box_predictor_text_proto(
- share_box_across_classes), box_predictor_proto)
- if predict_masks:
- text_format.Merge(
- self._add_mask_to_second_stage_box_predictor_text_proto(
- masks_are_class_agnostic), box_predictor_proto)
-
- if use_keras:
- return box_predictor_builder.build_keras(
- hyperparams_builder.KerasLayerHyperparams,
- inplace_batchnorm_update=False,
- freeze_batchnorm=False,
- box_predictor_config=box_predictor_proto,
- num_classes=num_classes,
- num_predictions_per_location_list=None,
- is_training=is_training)
- else:
- return box_predictor_builder.build(
- hyperparams_builder.build,
- box_predictor_proto,
- num_classes=num_classes,
- is_training=is_training)
-
- def _build_model(self,
- is_training,
- number_of_stages,
- second_stage_batch_size,
- first_stage_max_proposals=8,
- num_classes=2,
- hard_mining=False,
- softmax_second_stage_classification_loss=True,
- predict_masks=False,
- pad_to_max_dimension=None,
- masks_are_class_agnostic=False,
- use_matmul_crop_and_resize=False,
- clip_anchors_to_image=False,
- use_matmul_gather_in_matcher=False,
- use_static_shapes=False,
- calibration_mapping_value=None,
- share_box_across_classes=False,
- return_raw_detections_during_predict=False):
- use_keras = tf_version.is_tf2()
- def image_resizer_fn(image, masks=None):
- """Fake image resizer function."""
- resized_inputs = []
- resized_image = tf.identity(image)
- if pad_to_max_dimension is not None:
- resized_image = tf.image.pad_to_bounding_box(image, 0, 0,
- pad_to_max_dimension,
- pad_to_max_dimension)
- resized_inputs.append(resized_image)
- if masks is not None:
- resized_masks = tf.identity(masks)
- if pad_to_max_dimension is not None:
- resized_masks = tf.image.pad_to_bounding_box(
- tf.transpose(masks, [1, 2, 0]), 0, 0, pad_to_max_dimension,
- pad_to_max_dimension)
- resized_masks = tf.transpose(resized_masks, [2, 0, 1])
- resized_inputs.append(resized_masks)
- resized_inputs.append(tf.shape(image))
- return resized_inputs
-
- # anchors in this test are designed so that a subset of anchors are inside
- # the image and a subset of anchors are outside.
- first_stage_anchor_scales = (0.001, 0.005, 0.1)
- first_stage_anchor_aspect_ratios = (0.5, 1.0, 2.0)
- first_stage_anchor_strides = (1, 1)
- first_stage_anchor_generator = grid_anchor_generator.GridAnchorGenerator(
- first_stage_anchor_scales,
- first_stage_anchor_aspect_ratios,
- anchor_stride=first_stage_anchor_strides)
- first_stage_target_assigner = target_assigner.create_target_assigner(
- 'FasterRCNN',
- 'proposal',
- use_matmul_gather=use_matmul_gather_in_matcher)
-
- if use_keras:
- fake_feature_extractor = FakeFasterRCNNKerasFeatureExtractor()
- else:
- fake_feature_extractor = FakeFasterRCNNFeatureExtractor()
-
- first_stage_box_predictor_hyperparams_text_proto = """
- op: CONV
- activation: RELU
- regularizer {
- l2_regularizer {
- weight: 0.00004
- }
- }
- initializer {
- truncated_normal_initializer {
- stddev: 0.03
- }
- }
- """
- if use_keras:
- first_stage_box_predictor_arg_scope_fn = (
- self._build_keras_layer_hyperparams(
- first_stage_box_predictor_hyperparams_text_proto))
- else:
- first_stage_box_predictor_arg_scope_fn = (
- self._build_arg_scope_with_hyperparams(
- first_stage_box_predictor_hyperparams_text_proto, is_training))
-
- first_stage_box_predictor_kernel_size = 3
- first_stage_atrous_rate = 1
- first_stage_box_predictor_depth = 512
- first_stage_minibatch_size = 3
- first_stage_sampler = sampler.BalancedPositiveNegativeSampler(
- positive_fraction=0.5, is_static=use_static_shapes)
-
- first_stage_nms_score_threshold = -1.0
- first_stage_nms_iou_threshold = 1.0
- first_stage_max_proposals = first_stage_max_proposals
- first_stage_non_max_suppression_fn = functools.partial(
- post_processing.batch_multiclass_non_max_suppression,
- score_thresh=first_stage_nms_score_threshold,
- iou_thresh=first_stage_nms_iou_threshold,
- max_size_per_class=first_stage_max_proposals,
- max_total_size=first_stage_max_proposals,
- use_static_shapes=use_static_shapes)
-
- first_stage_localization_loss_weight = 1.0
- first_stage_objectness_loss_weight = 1.0
-
- post_processing_config = post_processing_pb2.PostProcessing()
- post_processing_text_proto = """
- score_converter: IDENTITY
- batch_non_max_suppression {
- score_threshold: -20.0
- iou_threshold: 1.0
- max_detections_per_class: 5
- max_total_detections: 5
- use_static_shapes: """ + '{}'.format(use_static_shapes) + """
- }
- """
- if calibration_mapping_value:
- calibration_text_proto = """
- calibration_config {
- function_approximation {
- x_y_pairs {
- x_y_pair {
- x: 0.0
- y: %f
- }
- x_y_pair {
- x: 1.0
- y: %f
- }}}}""" % (calibration_mapping_value, calibration_mapping_value)
- post_processing_text_proto = (
- post_processing_text_proto + ' ' + calibration_text_proto)
- text_format.Merge(post_processing_text_proto, post_processing_config)
- second_stage_non_max_suppression_fn, second_stage_score_conversion_fn = (
- post_processing_builder.build(post_processing_config))
-
- second_stage_target_assigner = target_assigner.create_target_assigner(
- 'FasterRCNN',
- 'detection',
- use_matmul_gather=use_matmul_gather_in_matcher)
- second_stage_sampler = sampler.BalancedPositiveNegativeSampler(
- positive_fraction=1.0, is_static=use_static_shapes)
-
- second_stage_localization_loss_weight = 1.0
- second_stage_classification_loss_weight = 1.0
- if softmax_second_stage_classification_loss:
- second_stage_classification_loss = (
- losses.WeightedSoftmaxClassificationLoss())
- else:
- second_stage_classification_loss = (
- losses.WeightedSigmoidClassificationLoss())
-
- hard_example_miner = None
- if hard_mining:
- hard_example_miner = losses.HardExampleMiner(
- num_hard_examples=1,
- iou_threshold=0.99,
- loss_type='both',
- cls_loss_weight=second_stage_classification_loss_weight,
- loc_loss_weight=second_stage_localization_loss_weight,
- max_negatives_per_positive=None)
-
- crop_and_resize_fn = (
- ops.matmul_crop_and_resize
- if use_matmul_crop_and_resize else ops.native_crop_and_resize)
- common_kwargs = {
- 'is_training':
- is_training,
- 'num_classes':
- num_classes,
- 'image_resizer_fn':
- image_resizer_fn,
- 'feature_extractor':
- fake_feature_extractor,
- 'number_of_stages':
- number_of_stages,
- 'first_stage_anchor_generator':
- first_stage_anchor_generator,
- 'first_stage_target_assigner':
- first_stage_target_assigner,
- 'first_stage_atrous_rate':
- first_stage_atrous_rate,
- 'first_stage_box_predictor_arg_scope_fn':
- first_stage_box_predictor_arg_scope_fn,
- 'first_stage_box_predictor_kernel_size':
- first_stage_box_predictor_kernel_size,
- 'first_stage_box_predictor_depth':
- first_stage_box_predictor_depth,
- 'first_stage_minibatch_size':
- first_stage_minibatch_size,
- 'first_stage_sampler':
- first_stage_sampler,
- 'first_stage_non_max_suppression_fn':
- first_stage_non_max_suppression_fn,
- 'first_stage_max_proposals':
- first_stage_max_proposals,
- 'first_stage_localization_loss_weight':
- first_stage_localization_loss_weight,
- 'first_stage_objectness_loss_weight':
- first_stage_objectness_loss_weight,
- 'second_stage_target_assigner':
- second_stage_target_assigner,
- 'second_stage_batch_size':
- second_stage_batch_size,
- 'second_stage_sampler':
- second_stage_sampler,
- 'second_stage_non_max_suppression_fn':
- second_stage_non_max_suppression_fn,
- 'second_stage_score_conversion_fn':
- second_stage_score_conversion_fn,
- 'second_stage_localization_loss_weight':
- second_stage_localization_loss_weight,
- 'second_stage_classification_loss_weight':
- second_stage_classification_loss_weight,
- 'second_stage_classification_loss':
- second_stage_classification_loss,
- 'hard_example_miner':
- hard_example_miner,
- 'crop_and_resize_fn':
- crop_and_resize_fn,
- 'clip_anchors_to_image':
- clip_anchors_to_image,
- 'use_static_shapes':
- use_static_shapes,
- 'resize_masks':
- True,
- 'return_raw_detections_during_predict':
- return_raw_detections_during_predict
- }
-
- return self._get_model(
- self._get_second_stage_box_predictor(
- num_classes=num_classes,
- is_training=is_training,
- use_keras=use_keras,
- predict_masks=predict_masks,
- masks_are_class_agnostic=masks_are_class_agnostic,
- share_box_across_classes=share_box_across_classes), **common_kwargs)
-
- @mock.patch.object(context_rcnn_meta_arch, 'context_rcnn_lib')
- def test_prediction_mock(self, mock_context_rcnn_lib):
- """Mocks the context_rcnn_lib module to test the prediction.
-
- Using mock object so that we can ensure compute_box_context_attention is
- called in side the prediction function.
-
- Args:
- mock_context_rcnn_lib: mock module for the context_rcnn_lib.
- """
- model = self._build_model(
- is_training=False,
- number_of_stages=2,
- second_stage_batch_size=6,
- num_classes=42)
- mock_tensor = tf.ones([2, 8, 3, 3, 3], tf.float32)
-
- mock_context_rcnn_lib.compute_box_context_attention.return_value = mock_tensor
- inputs_shape = (2, 20, 20, 3)
- inputs = tf.cast(
- tf.random_uniform(inputs_shape, minval=0, maxval=255, dtype=tf.int32),
- dtype=tf.float32)
- preprocessed_inputs, true_image_shapes = model.preprocess(inputs)
- context_features = tf.random_uniform((2, 20, 10),
- minval=0,
- maxval=255,
- dtype=tf.float32)
- valid_context_size = tf.random_uniform((2,),
- minval=0,
- maxval=10,
- dtype=tf.int32)
- features = {
- fields.InputDataFields.context_features: context_features,
- fields.InputDataFields.valid_context_size: valid_context_size
- }
-
- side_inputs = model.get_side_inputs(features)
-
- _ = model.predict(preprocessed_inputs, true_image_shapes, **side_inputs)
- mock_context_rcnn_lib.compute_box_context_attention.assert_called_once()
-
- @parameterized.named_parameters(
- {'testcase_name': 'static_shapes', 'static_shapes': True},
- {'testcase_name': 'nostatic_shapes', 'static_shapes': False},
- )
- def test_prediction_end_to_end(self, static_shapes):
- """Runs prediction end to end and test the shape of the results."""
- with test_utils.GraphContextOrNone() as g:
- model = self._build_model(
- is_training=False,
- number_of_stages=2,
- second_stage_batch_size=6,
- use_matmul_crop_and_resize=static_shapes,
- clip_anchors_to_image=static_shapes,
- use_matmul_gather_in_matcher=static_shapes,
- use_static_shapes=static_shapes,
- num_classes=42)
-
- def graph_fn():
- inputs_shape = (2, 20, 20, 3)
- inputs = tf.cast(
- tf.random_uniform(inputs_shape, minval=0, maxval=255, dtype=tf.int32),
- dtype=tf.float32)
- preprocessed_inputs, true_image_shapes = model.preprocess(inputs)
- context_features = tf.random_uniform((2, 20, 10),
- minval=0,
- maxval=255,
- dtype=tf.float32)
- valid_context_size = tf.random_uniform((2,),
- minval=0,
- maxval=10,
- dtype=tf.int32)
- features = {
- fields.InputDataFields.context_features: context_features,
- fields.InputDataFields.valid_context_size: valid_context_size
- }
-
- side_inputs = model.get_side_inputs(features)
-
- prediction_dict = model.predict(preprocessed_inputs, true_image_shapes,
- **side_inputs)
- return (prediction_dict['rpn_box_predictor_features'],
- prediction_dict['rpn_box_encodings'],
- prediction_dict['refined_box_encodings'],
- prediction_dict['proposal_boxes_normalized'],
- prediction_dict['proposal_boxes'])
- execute_fn = self.execute if static_shapes else self.execute_cpu
- (rpn_box_predictor_features, rpn_box_encodings, refined_box_encodings,
- proposal_boxes_normalized, proposal_boxes) = execute_fn(graph_fn, [],
- graph=g)
- self.assertAllEqual(rpn_box_predictor_features.shape, [2, 20, 20, 512])
- self.assertAllEqual(rpn_box_encodings.shape, [2, 3600, 4])
- self.assertAllEqual(refined_box_encodings.shape, [16, 42, 4])
- self.assertAllEqual(proposal_boxes_normalized.shape, [2, 8, 4])
- self.assertAllEqual(proposal_boxes.shape, [2, 8, 4])
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/research/object_detection/meta_architectures/deepmac_meta_arch.py b/research/object_detection/meta_architectures/deepmac_meta_arch.py
new file mode 100644
index 0000000000000000000000000000000000000000..cf689a0fcec4a9df6fd3ae8bc03021ff8090158a
--- /dev/null
+++ b/research/object_detection/meta_architectures/deepmac_meta_arch.py
@@ -0,0 +1,862 @@
+"""Deep Mask heads above CenterNet (DeepMAC) architecture.
+
+TODO(vighneshb) Add link to paper when done.
+"""
+
+import collections
+
+import numpy as np
+import tensorflow as tf
+
+from object_detection.builders import losses_builder
+from object_detection.core import box_list
+from object_detection.core import box_list_ops
+from object_detection.core import losses
+from object_detection.core import preprocessor
+from object_detection.core import standard_fields as fields
+from object_detection.meta_architectures import center_net_meta_arch
+from object_detection.models.keras_models import hourglass_network
+from object_detection.models.keras_models import resnet_v1
+from object_detection.protos import losses_pb2
+from object_detection.protos import preprocessor_pb2
+from object_detection.utils import shape_utils
+from object_detection.utils import spatial_transform_ops
+
+
+INSTANCE_EMBEDDING = 'INSTANCE_EMBEDDING'
+PIXEL_EMBEDDING = 'PIXEL_EMBEDDING'
+DEEP_MASK_ESTIMATION = 'deep_mask_estimation'
+LOSS_KEY_PREFIX = center_net_meta_arch.LOSS_KEY_PREFIX
+
+
+class DeepMACParams(
+ collections.namedtuple('DeepMACParams', [
+ 'classification_loss', 'dim', 'task_loss_weight', 'pixel_embedding_dim',
+ 'allowed_masked_classes_ids', 'mask_size', 'mask_num_subsamples',
+ 'use_xy', 'network_type', 'use_instance_embedding', 'num_init_channels',
+ 'predict_full_resolution_masks', 'postprocess_crop_size',
+ 'max_roi_jitter_ratio', 'roi_jitter_mode'
+ ])):
+ """Class holding the DeepMAC network configutration."""
+
+ __slots__ = ()
+
+ def __new__(cls, classification_loss, dim, task_loss_weight,
+ pixel_embedding_dim, allowed_masked_classes_ids, mask_size,
+ mask_num_subsamples, use_xy, network_type, use_instance_embedding,
+ num_init_channels, predict_full_resolution_masks,
+ postprocess_crop_size, max_roi_jitter_ratio,
+ roi_jitter_mode):
+ return super(DeepMACParams,
+ cls).__new__(cls, classification_loss, dim,
+ task_loss_weight, pixel_embedding_dim,
+ allowed_masked_classes_ids, mask_size,
+ mask_num_subsamples, use_xy, network_type,
+ use_instance_embedding, num_init_channels,
+ predict_full_resolution_masks,
+ postprocess_crop_size, max_roi_jitter_ratio,
+ roi_jitter_mode)
+
+
+def subsample_instances(classes, weights, boxes, masks, num_subsamples):
+ """Randomly subsamples instances to the desired number.
+
+ Args:
+ classes: [num_instances, num_classes] float tensor of one-hot encoded
+ classes.
+ weights: [num_instances] float tensor of weights of each instance.
+ boxes: [num_instances, 4] tensor of box coordinates.
+ masks: [num_instances, height, width] tensor of per-instance masks.
+ num_subsamples: int, the desired number of samples.
+
+ Returns:
+ classes: [num_subsamples, num_classes] float tensor of classes.
+ weights: [num_subsamples] float tensor of weights.
+ boxes: [num_subsamples, 4] float tensor of box coordinates.
+ masks: [num_subsamples, height, width] float tensor of per-instance masks.
+
+ """
+
+ if num_subsamples <= -1:
+ return classes, weights, boxes, masks
+
+ num_instances = tf.reduce_sum(tf.cast(weights > 0.5, tf.int32))
+
+ if num_instances <= num_subsamples:
+ return (classes[:num_subsamples], weights[:num_subsamples],
+ boxes[:num_subsamples], masks[:num_subsamples])
+
+ else:
+ random_index = tf.random.uniform([num_subsamples], 0, num_instances,
+ dtype=tf.int32)
+
+ return (tf.gather(classes, random_index), tf.gather(weights, random_index),
+ tf.gather(boxes, random_index), tf.gather(masks, random_index))
+
+
+def _get_deepmac_network_by_type(name, num_init_channels, mask_size=None):
+ """Get DeepMAC network model given a string type."""
+
+ if name.startswith('hourglass'):
+ if name == 'hourglass10':
+ return hourglass_network.hourglass_10(num_init_channels,
+ initial_downsample=False)
+ elif name == 'hourglass20':
+ return hourglass_network.hourglass_20(num_init_channels,
+ initial_downsample=False)
+ elif name == 'hourglass32':
+ return hourglass_network.hourglass_32(num_init_channels,
+ initial_downsample=False)
+ elif name == 'hourglass52':
+ return hourglass_network.hourglass_52(num_init_channels,
+ initial_downsample=False)
+ elif name == 'hourglass100':
+ return hourglass_network.hourglass_100(num_init_channels,
+ initial_downsample=False)
+ elif name == 'hourglass20_uniform_size':
+ return hourglass_network.hourglass_20_uniform_size(num_init_channels)
+
+ elif name == 'hourglass20_no_shortcut':
+ return hourglass_network.hourglass_20_no_shortcut(num_init_channels)
+
+ elif name == 'fully_connected':
+ if not mask_size:
+ raise ValueError('Mask size must be set.')
+ return FullyConnectedMaskHead(num_init_channels, mask_size)
+
+ elif name.startswith('resnet'):
+ return ResNetMaskNetwork(name, num_init_channels)
+
+ raise ValueError('Unknown network type {}'.format(name))
+
+
+def crop_masks_within_boxes(masks, boxes, output_size):
+ """Crops masks to lie tightly within the boxes.
+
+ Args:
+ masks: A [num_instances, height, width] float tensor of masks.
+ boxes: A [num_instances, 4] sized tensor of normalized bounding boxes.
+ output_size: The height and width of the output masks.
+
+ Returns:
+ masks: A [num_instances, output_size, output_size] tensor of masks which
+ are cropped to be tightly within the gives boxes and resized.
+
+ """
+ masks = spatial_transform_ops.matmul_crop_and_resize(
+ masks[:, :, :, tf.newaxis], boxes[:, tf.newaxis, :],
+ [output_size, output_size])
+ return masks[:, 0, :, :, 0]
+
+
+def resize_instance_masks(masks, shape):
+ height, width = shape
+ masks_ex = masks[:, :, :, tf.newaxis]
+ masks_ex = tf.image.resize(masks_ex, (height, width),
+ method=tf.image.ResizeMethod.BILINEAR)
+ masks = masks_ex[:, :, :, 0]
+
+ return masks
+
+
+def filter_masked_classes(masked_class_ids, classes, weights, masks):
+ """Filter out masks whose class IDs are not present in masked_class_ids.
+
+ Args:
+ masked_class_ids: A list of class IDs allowed to have masks. These class IDs
+ are 1-indexed.
+ classes: A [num_instances, num_classes] float tensor containing the one-hot
+ encoded classes.
+ weights: A [num_instances] float tensor containing the weights of each
+ sample.
+ masks: A [num_instances, height, width] tensor containing the mask per
+ instance.
+
+ Returns:
+ classes_filtered: A [num_instances, num_classes] float tensor containing the
+ one-hot encoded classes with classes not in masked_class_ids zeroed out.
+ weights_filtered: A [num_instances] float tensor containing the weights of
+ each sample with instances whose classes aren't in masked_class_ids
+ zeroed out.
+ masks_filtered: A [num_instances, height, width] tensor containing the mask
+ per instance with masks not belonging to masked_class_ids zeroed out.
+ """
+
+ if len(masked_class_ids) == 0: # pylint:disable=g-explicit-length-test
+ return classes, weights, masks
+
+ if tf.shape(classes)[0] == 0:
+ return classes, weights, masks
+
+ masked_class_ids = tf.constant(np.array(masked_class_ids, dtype=np.int32))
+ label_id_offset = 1
+ masked_class_ids -= label_id_offset
+ class_ids = tf.argmax(classes, axis=1, output_type=tf.int32)
+ matched_classes = tf.equal(
+ class_ids[:, tf.newaxis], masked_class_ids[tf.newaxis, :]
+ )
+
+ matched_classes = tf.reduce_any(matched_classes, axis=1)
+ matched_classes = tf.cast(matched_classes, tf.float32)
+
+ return (
+ classes * matched_classes[:, tf.newaxis],
+ weights * matched_classes,
+ masks * matched_classes[:, tf.newaxis, tf.newaxis]
+ )
+
+
+class ResNetMaskNetwork(tf.keras.layers.Layer):
+ """A small wrapper around ResNet blocks to predict masks."""
+
+ def __init__(self, resnet_type, num_init_channels):
+ """Creates the ResNet mask network.
+
+ Args:
+ resnet_type: A string of the for resnetN where N where N is in
+ [4, 8, 12, 16, 20]
+ num_init_channels: Number of filters in the ResNet block.
+ """
+
+ super(ResNetMaskNetwork, self).__init__()
+ nc = num_init_channels
+
+ if resnet_type == 'resnet4':
+ channel_dims = [nc * 2]
+ blocks = [2]
+ elif resnet_type == 'resnet8':
+ channel_dims = [nc * 2]
+ blocks = [4]
+ elif resnet_type == 'resnet12':
+ channel_dims = [nc * 2]
+ blocks = [6]
+ elif resnet_type == 'resnet16':
+ channel_dims = [nc * 2]
+ blocks = [8]
+ # Defined such that the channels are roughly similar to the hourglass20.
+ elif resnet_type == 'resnet20':
+ channel_dims = [nc * 2, nc * 3]
+ blocks = [8, 2]
+ else:
+ raise ValueError('Unknown resnet type "{}"'.format(resnet_type))
+
+ self.input_layer = tf.keras.layers.Conv2D(nc, 1, 1)
+
+ # Last channel has to be defined so that batch norm can initialize properly.
+ model_input = tf.keras.layers.Input([None, None, nc])
+ output = model_input
+
+ for i, (num_blocks, channels) in enumerate(zip(blocks, channel_dims)):
+ output = resnet_v1.stack_basic(output, filters=channels,
+ blocks=num_blocks, stride1=1,
+ name='resnet_mask_block_%d' % i)
+ self.model = tf.keras.Model(inputs=model_input, outputs=output)
+
+ def __call__(self, inputs):
+ return self.model(self.input_layer(inputs))
+
+
+class FullyConnectedMaskHead(tf.keras.layers.Layer):
+ """A 2 layer fully connected mask head."""
+
+ def __init__(self, num_init_channels, mask_size):
+ super(FullyConnectedMaskHead, self).__init__()
+ self.fc1 = tf.keras.layers.Dense(units=1024, activation='relu')
+ self.fc2 = tf.keras.layers.Dense(units=mask_size*mask_size)
+ self.mask_size = mask_size
+ self.num_input_channels = num_init_channels
+ self.input_layer = tf.keras.layers.Conv2D(num_init_channels, 1, 1)
+ model_input = tf.keras.layers.Input(
+ [mask_size * mask_size * num_init_channels,])
+ output = self.fc2(self.fc1(model_input))
+ self.model = tf.keras.Model(inputs=model_input, outputs=output)
+
+ def __call__(self, inputs):
+ inputs = self.input_layer(inputs)
+ inputs_shape = tf.shape(inputs)
+ num_instances = inputs_shape[0]
+ height = inputs_shape[1]
+ width = inputs_shape[2]
+ dims = inputs_shape[3]
+ flattened_inputs = tf.reshape(inputs,
+ [num_instances, height * width * dims])
+ flattened_masks = self.model(flattened_inputs)
+ return tf.reshape(flattened_masks,
+ [num_instances, self.mask_size, self.mask_size, 1])
+
+
+class MaskHeadNetwork(tf.keras.layers.Layer):
+ """Mask head class for DeepMAC."""
+
+ def __init__(self, network_type, num_init_channels=64,
+ use_instance_embedding=True, mask_size=None):
+ """Initializes the network.
+
+ Args:
+ network_type: A string denoting the kind of network we want to use
+ internally.
+ num_init_channels: int, the number of channels in the first block. The
+ number of channels in the following blocks depend on the network type
+ used.
+ use_instance_embedding: bool, if set, we concatenate the instance
+ embedding to the input while predicting the mask.
+ mask_size: int, size of the output mask. Required only with
+ `fully_connected` mask type.
+ """
+
+ super(MaskHeadNetwork, self).__init__()
+
+ self._net = _get_deepmac_network_by_type(
+ network_type, num_init_channels, mask_size)
+ self._use_instance_embedding = use_instance_embedding
+
+ self.project_out = tf.keras.layers.Conv2D(
+ filters=1, kernel_size=1, activation=None)
+
+ def __call__(self, instance_embedding, pixel_embedding, training):
+ """Returns mask logits given object center and spatial embeddings.
+
+ Args:
+ instance_embedding: A [num_instances, embedding_size] float tensor
+ representing the center emedding vector of each instance.
+ pixel_embedding: A [num_instances, height, width, pixel_embedding_size]
+ float tensor representing the per-pixel spatial embedding for each
+ instance.
+ training: boolean flag indicating training or testing mode.
+
+ Returns:
+ mask: A [num_instances, height, width] float tensor containing the mask
+ logits for each instance.
+ """
+
+ height = tf.shape(pixel_embedding)[1]
+ width = tf.shape(pixel_embedding)[2]
+
+ instance_embedding = instance_embedding[:, tf.newaxis, tf.newaxis, :]
+ instance_embedding = tf.tile(instance_embedding, [1, height, width, 1])
+
+ if self._use_instance_embedding:
+ inputs = tf.concat([pixel_embedding, instance_embedding], axis=3)
+ else:
+ inputs = pixel_embedding
+
+ out = self._net(inputs)
+ if isinstance(out, list):
+ out = out[-1]
+
+ if out.shape[-1] > 1:
+ out = self.project_out(out)
+
+ return tf.squeeze(out, axis=-1)
+
+
+def deepmac_proto_to_params(deepmac_config):
+ """Convert proto to named tuple."""
+
+ loss = losses_pb2.Loss()
+ # Add dummy localization loss to avoid the loss_builder throwing error.
+ loss.localization_loss.weighted_l2.CopyFrom(
+ losses_pb2.WeightedL2LocalizationLoss())
+ loss.classification_loss.CopyFrom(deepmac_config.classification_loss)
+ classification_loss, _, _, _, _, _, _ = (losses_builder.build(loss))
+
+ jitter_mode = preprocessor_pb2.RandomJitterBoxes.JitterMode.Name(
+ deepmac_config.jitter_mode).lower()
+
+ return DeepMACParams(
+ dim=deepmac_config.dim,
+ classification_loss=classification_loss,
+ task_loss_weight=deepmac_config.task_loss_weight,
+ pixel_embedding_dim=deepmac_config.pixel_embedding_dim,
+ allowed_masked_classes_ids=deepmac_config.allowed_masked_classes_ids,
+ mask_size=deepmac_config.mask_size,
+ mask_num_subsamples=deepmac_config.mask_num_subsamples,
+ use_xy=deepmac_config.use_xy,
+ network_type=deepmac_config.network_type,
+ use_instance_embedding=deepmac_config.use_instance_embedding,
+ num_init_channels=deepmac_config.num_init_channels,
+ predict_full_resolution_masks=
+ deepmac_config.predict_full_resolution_masks,
+ postprocess_crop_size=deepmac_config.postprocess_crop_size,
+ max_roi_jitter_ratio=deepmac_config.max_roi_jitter_ratio,
+ roi_jitter_mode=jitter_mode
+ )
+
+
+class DeepMACMetaArch(center_net_meta_arch.CenterNetMetaArch):
+ """The experimental CenterNet DeepMAC[1] model.
+
+ [1]: https://arxiv.org/abs/2104.00613
+ """
+
+ def __init__(self,
+ is_training,
+ add_summaries,
+ num_classes,
+ feature_extractor,
+ image_resizer_fn,
+ object_center_params,
+ object_detection_params,
+ deepmac_params,
+ compute_heatmap_sparse=False):
+ """Constructs the super class with object center & detection params only."""
+
+ self._deepmac_params = deepmac_params
+ super(DeepMACMetaArch, self).__init__(
+ is_training=is_training, add_summaries=add_summaries,
+ num_classes=num_classes, feature_extractor=feature_extractor,
+ image_resizer_fn=image_resizer_fn,
+ object_center_params=object_center_params,
+ object_detection_params=object_detection_params,
+ compute_heatmap_sparse=compute_heatmap_sparse)
+
+ def _construct_prediction_heads(self, num_classes, num_feature_outputs,
+ class_prediction_bias_init):
+ super_instance = super(DeepMACMetaArch, self)
+ prediction_heads = super_instance._construct_prediction_heads( # pylint:disable=protected-access
+ num_classes, num_feature_outputs, class_prediction_bias_init)
+
+ if self._deepmac_params is not None:
+ prediction_heads[INSTANCE_EMBEDDING] = [
+ center_net_meta_arch.make_prediction_net(self._deepmac_params.dim)
+ for _ in range(num_feature_outputs)
+ ]
+
+ prediction_heads[PIXEL_EMBEDDING] = [
+ center_net_meta_arch.make_prediction_net(
+ self._deepmac_params.pixel_embedding_dim)
+ for _ in range(num_feature_outputs)
+ ]
+
+ self._mask_net = MaskHeadNetwork(
+ network_type=self._deepmac_params.network_type,
+ use_instance_embedding=self._deepmac_params.use_instance_embedding,
+ num_init_channels=self._deepmac_params.num_init_channels)
+
+ return prediction_heads
+
+ def _get_mask_head_input(self, boxes, pixel_embedding):
+ """Get the input to the mask network, given bounding boxes.
+
+ Args:
+ boxes: A [num_instances, 4] float tensor containing bounding boxes in
+ normalized coordinates.
+ pixel_embedding: A [height, width, embedding_size] float tensor
+ containing spatial pixel embeddings.
+
+ Returns:
+ embedding: A [num_instances, mask_height, mask_width, embedding_size + 2]
+ float tensor containing the inputs to the mask network. For each
+ bounding box, we concatenate the normalized box coordinates to the
+ cropped pixel embeddings. If predict_full_resolution_masks is set,
+ mask_height and mask_width are the same as height and width of
+ pixel_embedding. If not, mask_height and mask_width are the same as
+ mask_size.
+ """
+
+ num_instances = tf.shape(boxes)[0]
+ mask_size = self._deepmac_params.mask_size
+
+ if self._deepmac_params.predict_full_resolution_masks:
+ num_instances = tf.shape(boxes)[0]
+ pixel_embedding = pixel_embedding[tf.newaxis, :, :, :]
+ pixel_embeddings_processed = tf.tile(pixel_embedding,
+ [num_instances, 1, 1, 1])
+ else:
+ # TODO(vighneshb) Explore multilevel_roi_align and align_corners=False.
+ pixel_embeddings_cropped = spatial_transform_ops.matmul_crop_and_resize(
+ pixel_embedding[tf.newaxis], boxes[tf.newaxis],
+ [mask_size, mask_size])
+ pixel_embeddings_processed = pixel_embeddings_cropped[0]
+
+ mask_shape = tf.shape(pixel_embeddings_processed)
+ mask_height, mask_width = mask_shape[1], mask_shape[2]
+ y_grid, x_grid = tf.meshgrid(tf.linspace(-1.0, 1.0, mask_height),
+ tf.linspace(-1.0, 1.0, mask_width),
+ indexing='ij')
+ coords = tf.stack([y_grid, x_grid], axis=2)
+ coords = coords[tf.newaxis, :, :, :]
+ coords = tf.tile(coords, [num_instances, 1, 1, 1])
+
+ if self._deepmac_params.use_xy:
+ return tf.concat([coords, pixel_embeddings_processed], axis=3)
+ else:
+ return pixel_embeddings_processed
+
+ def _get_instance_embeddings(self, boxes, instance_embedding):
+ """Return the instance embeddings from bounding box centers.
+
+ Args:
+ boxes: A [num_instances, 4] float tensor holding bounding boxes. The
+ coordinates are in normalized input space.
+ instance_embedding: A [height, width, embedding_size] float tensor
+ containing the instance embeddings.
+
+ Returns:
+ instance_embeddings: A [num_instances, embedding_size] shaped float tensor
+ containing the center embedding for each instance.
+ """
+ blist = box_list.BoxList(boxes)
+ output_height = tf.shape(instance_embedding)[0]
+ output_width = tf.shape(instance_embedding)[1]
+
+ blist_output = box_list_ops.to_absolute_coordinates(
+ blist, output_height, output_width, check_range=False)
+ (y_center_output, x_center_output,
+ _, _) = blist_output.get_center_coordinates_and_sizes()
+ center_coords_output = tf.stack([y_center_output, x_center_output], axis=1)
+ center_coords_output_int = tf.cast(center_coords_output, tf.int32)
+ center_latents = tf.gather_nd(instance_embedding, center_coords_output_int)
+
+ return center_latents
+
+ def _get_groundtruth_mask_output(self, boxes, masks):
+ """Get the expected mask output for each box.
+
+ Args:
+ boxes: A [num_instances, 4] float tensor containing bounding boxes in
+ normalized coordinates.
+ masks: A [num_instances, height, width] float tensor containing binary
+ ground truth masks.
+
+ Returns:
+ masks: If predict_full_resolution_masks is set, masks are not resized
+ and the size of this tensor is [num_instances, input_height, input_width].
+ Otherwise, returns a tensor of size [num_instances, mask_size, mask_size].
+ """
+ mask_size = self._deepmac_params.mask_size
+ if self._deepmac_params.predict_full_resolution_masks:
+ return masks
+ else:
+ cropped_masks = spatial_transform_ops.matmul_crop_and_resize(
+ masks[:, :, :, tf.newaxis], boxes[:, tf.newaxis, :],
+ [mask_size, mask_size])
+ cropped_masks = tf.stop_gradient(cropped_masks)
+ cropped_masks = tf.squeeze(cropped_masks, axis=[1, 4])
+
+ # TODO(vighneshb) should we discretize masks?
+ return cropped_masks
+
+ def _resize_logits_like_gt(self, logits, gt):
+
+ height, width = tf.shape(gt)[1], tf.shape(gt)[2]
+
+ return resize_instance_masks(logits, (height, width))
+
+ def _compute_per_instance_mask_loss(
+ self, boxes, masks, instance_embedding, pixel_embedding):
+ """Returns the mask loss per instance.
+
+ Args:
+ boxes: A [num_instances, 4] float tensor holding bounding boxes. The
+ coordinates are in normalized input space.
+ masks: A [num_instances, input_height, input_width] float tensor
+ containing the instance masks.
+ instance_embedding: A [output_height, output_width, embedding_size]
+ float tensor containing the instance embeddings.
+ pixel_embedding: optional [output_height, output_width,
+ pixel_embedding_size] float tensor containing the per-pixel embeddings.
+
+ Returns:
+ mask_loss: A [num_instances] shaped float tensor containing the
+ mask loss for each instance.
+ """
+
+ num_instances = tf.shape(boxes)[0]
+
+ if tf.keras.backend.learning_phase():
+ boxes = preprocessor.random_jitter_boxes(
+ boxes, self._deepmac_params.max_roi_jitter_ratio,
+ jitter_mode=self._deepmac_params.roi_jitter_mode)
+ mask_input = self._get_mask_head_input(
+ boxes, pixel_embedding)
+ instance_embeddings = self._get_instance_embeddings(
+ boxes, instance_embedding)
+
+ mask_logits = self._mask_net(
+ instance_embeddings, mask_input,
+ training=tf.keras.backend.learning_phase())
+ mask_gt = self._get_groundtruth_mask_output(boxes, masks)
+ mask_logits = self._resize_logits_like_gt(mask_logits, mask_gt)
+
+ mask_logits = tf.reshape(mask_logits, [num_instances, -1, 1])
+ mask_gt = tf.reshape(mask_gt, [num_instances, -1, 1])
+ loss = self._deepmac_params.classification_loss(
+ prediction_tensor=mask_logits,
+ target_tensor=mask_gt,
+ weights=tf.ones_like(mask_logits))
+
+ # TODO(vighneshb) Make this configurable via config.
+ if isinstance(self._deepmac_params.classification_loss,
+ losses.WeightedDiceClassificationLoss):
+ return tf.reduce_sum(loss, axis=1)
+ else:
+ return tf.reduce_mean(loss, axis=[1, 2])
+
+ def _compute_instance_masks_loss(self, prediction_dict):
+ """Computes the mask loss.
+
+ Args:
+ prediction_dict: dict from predict() method containing
+ INSTANCE_EMBEDDING and PIXEL_EMBEDDING prediction.
+ Both of these are lists of tensors, each of size
+ [batch_size, height, width, embedding_size].
+
+ Returns:
+ loss: float, the mask loss as a scalar.
+ """
+ gt_boxes_list = self.groundtruth_lists(fields.BoxListFields.boxes)
+ gt_weights_list = self.groundtruth_lists(fields.BoxListFields.weights)
+ gt_masks_list = self.groundtruth_lists(fields.BoxListFields.masks)
+ gt_classes_list = self.groundtruth_lists(fields.BoxListFields.classes)
+
+ allowed_masked_classes_ids = (
+ self._deepmac_params.allowed_masked_classes_ids)
+
+ total_loss = 0.0
+
+ # Iterate over multiple preidctions by backbone (for hourglass length=2)
+ for instance_pred, pixel_pred in zip(
+ prediction_dict[INSTANCE_EMBEDDING],
+ prediction_dict[PIXEL_EMBEDDING]):
+ # Iterate over samples in batch
+ # TODO(vighneshb) find out how autograph is handling this. Converting
+ # to a single op may give speed improvements
+ for i, (boxes, weights, classes, masks) in enumerate(
+ zip(gt_boxes_list, gt_weights_list, gt_classes_list, gt_masks_list)):
+
+ _, weights, masks = filter_masked_classes(allowed_masked_classes_ids,
+ classes, weights, masks)
+ num_subsample = self._deepmac_params.mask_num_subsamples
+ _, weights, boxes, masks = subsample_instances(
+ classes, weights, boxes, masks, num_subsample)
+
+ per_instance_loss = self._compute_per_instance_mask_loss(
+ boxes, masks, instance_pred[i], pixel_pred[i])
+ per_instance_loss *= weights
+
+ num_instances = tf.maximum(tf.reduce_sum(weights), 1.0)
+
+ total_loss += tf.reduce_sum(per_instance_loss) / num_instances
+
+ batch_size = len(gt_boxes_list)
+ num_predictions = len(prediction_dict[INSTANCE_EMBEDDING])
+
+ return total_loss / float(batch_size * num_predictions)
+
+ def loss(self, prediction_dict, true_image_shapes, scope=None):
+
+ losses_dict = super(DeepMACMetaArch, self).loss(
+ prediction_dict, true_image_shapes, scope)
+
+ if self._deepmac_params is not None:
+ mask_loss = self._compute_instance_masks_loss(
+ prediction_dict=prediction_dict)
+ key = LOSS_KEY_PREFIX + '/' + DEEP_MASK_ESTIMATION
+ losses_dict[key] = (
+ self._deepmac_params.task_loss_weight * mask_loss
+ )
+
+ return losses_dict
+
+ def postprocess(self, prediction_dict, true_image_shapes, **params):
+ """Produces boxes given a prediction dict returned by predict().
+
+ Args:
+ prediction_dict: a dictionary holding predicted tensors from "predict"
+ function.
+ true_image_shapes: int32 tensor of shape [batch, 3] where each row is of
+ the form [height, width, channels] indicating the shapes of true images
+ in the resized images, as resized images can be padded with zeros.
+ **params: Currently ignored.
+
+ Returns:
+ detections: a dictionary containing the following fields
+ detection_masks: (Optional) A uint8 tensor of shape [batch,
+ max_detections, mask_height, mask_width] with masks for each
+ detection. Background is specified with 0, and foreground is specified
+ with positive integers (1 for standard instance segmentation mask, and
+ 1-indexed parts for DensePose task).
+ And all other fields returned by the super class method.
+ """
+ postprocess_dict = super(DeepMACMetaArch, self).postprocess(
+ prediction_dict, true_image_shapes, **params)
+ boxes_strided = postprocess_dict['detection_boxes_strided']
+
+ if self._deepmac_params is not None:
+ masks = self._postprocess_masks(
+ boxes_strided, prediction_dict[INSTANCE_EMBEDDING][-1],
+ prediction_dict[PIXEL_EMBEDDING][-1])
+ postprocess_dict[fields.DetectionResultFields.detection_masks] = masks
+
+ return postprocess_dict
+
+ def _postprocess_masks(self, boxes_output_stride,
+ instance_embedding, pixel_embedding):
+ """Postprocess masks with the deep mask network.
+
+ Args:
+ boxes_output_stride: A [batch_size, num_instances, 4] float tensor
+ containing the batch of boxes in the absolute output space of the
+ feature extractor.
+ instance_embedding: A [batch_size, output_height, output_width,
+ embedding_size] float tensor containing instance embeddings.
+ pixel_embedding: A [batch_size, output_height, output_width,
+ pixel_embedding_size] float tensor containing the per-pixel embedding.
+
+ Returns:
+ masks: A float tensor of size [batch_size, num_instances, mask_size,
+ mask_size] containing binary per-box instance masks.
+ """
+
+ def process(elems):
+ boxes, instance_embedding, pixel_embedding = elems
+ return self._postprocess_sample(boxes, instance_embedding,
+ pixel_embedding)
+
+ max_instances = self._center_params.max_box_predictions
+ return tf.map_fn(process, [boxes_output_stride, instance_embedding,
+ pixel_embedding],
+ dtype=tf.float32, parallel_iterations=max_instances)
+
+ def _postprocess_sample(self, boxes_output_stride,
+ instance_embedding, pixel_embedding):
+ """Post process masks for a single sample.
+
+ Args:
+ boxes_output_stride: A [num_instances, 4] float tensor containing
+ bounding boxes in the absolute output space.
+ instance_embedding: A [output_height, output_width, embedding_size]
+ float tensor containing instance embeddings.
+ pixel_embedding: A [batch_size, output_height, output_width,
+ pixel_embedding_size] float tensor containing the per-pixel embedding.
+
+ Returns:
+ masks: A float tensor of size [num_instances, mask_height, mask_width]
+ containing binary per-box instance masks. If
+ predict_full_resolution_masks is set, the masks will be resized to
+ postprocess_crop_size. Otherwise, mask_height=mask_width=mask_size
+ """
+
+ height, width = (tf.shape(instance_embedding)[0],
+ tf.shape(instance_embedding)[1])
+ height, width = tf.cast(height, tf.float32), tf.cast(width, tf.float32)
+ blist = box_list.BoxList(boxes_output_stride)
+ blist = box_list_ops.to_normalized_coordinates(
+ blist, height, width, check_range=False)
+ boxes = blist.get()
+
+ mask_input = self._get_mask_head_input(boxes, pixel_embedding)
+ instance_embeddings = self._get_instance_embeddings(
+ boxes, instance_embedding)
+
+ mask_logits = self._mask_net(
+ instance_embeddings, mask_input,
+ training=tf.keras.backend.learning_phase())
+
+ # TODO(vighneshb) Explore sweeping mask thresholds.
+
+ if self._deepmac_params.predict_full_resolution_masks:
+
+ height, width = tf.shape(mask_logits)[1], tf.shape(mask_logits)[2]
+ height *= self._stride
+ width *= self._stride
+ mask_logits = resize_instance_masks(mask_logits, (height, width))
+ mask_logits = crop_masks_within_boxes(
+ mask_logits, boxes, self._deepmac_params.postprocess_crop_size)
+
+ masks_prob = tf.nn.sigmoid(mask_logits)
+
+ return masks_prob
+
+ def _transform_boxes_to_feature_coordinates(self, provided_boxes,
+ true_image_shapes,
+ resized_image_shape,
+ instance_embedding):
+ """Transforms normalzied boxes to feature map coordinates.
+
+ Args:
+ provided_boxes: A [batch, num_instances, 4] float tensor containing
+ normalized bounding boxes.
+ true_image_shapes: int32 tensor of shape [batch, 3] where each row is of
+ the form [height, width, channels] indicating the shapes of true images
+ in the resized images, as resized images can be padded with zeros.
+ resized_image_shape: A 4D int32 tensor containing shapes of the
+ preprocessed inputs (N, H, W, C).
+ instance_embedding: A [batch, output_height, output_width, embedding_size]
+ float tensor containing instance embeddings.
+
+ Returns:
+ A float tensor of size [batch, num_instances, 4] containing boxes whose
+ coordinates have been transformed to the absolute output space of the
+ feature extractor.
+ """
+ # Input boxes must be normalized.
+ shape_utils.assert_box_normalized(provided_boxes)
+
+ # Transform the provided boxes to the absolute output space of the feature
+ # extractor.
+ height, width = (tf.shape(instance_embedding)[1],
+ tf.shape(instance_embedding)[2])
+
+ resized_image_height = resized_image_shape[1]
+ resized_image_width = resized_image_shape[2]
+
+ def transform_boxes(elems):
+ boxes_per_image, true_image_shape = elems
+ blist = box_list.BoxList(boxes_per_image)
+ # First transform boxes from image space to resized image space since
+ # there may have paddings in the resized images.
+ blist = box_list_ops.scale(blist,
+ true_image_shape[0] / resized_image_height,
+ true_image_shape[1] / resized_image_width)
+ # Then transform boxes from resized image space (normalized) to the
+ # feature map space (absolute).
+ blist = box_list_ops.to_absolute_coordinates(
+ blist, height, width, check_range=False)
+ return blist.get()
+
+ return tf.map_fn(
+ transform_boxes, [provided_boxes, true_image_shapes], dtype=tf.float32)
+
+ def predict_masks_from_boxes(self, prediction_dict, true_image_shapes,
+ provided_boxes, **params):
+ """Produces masks for the provided boxes.
+
+ Args:
+ prediction_dict: a dictionary holding predicted tensors from "predict"
+ function.
+ true_image_shapes: int32 tensor of shape [batch, 3] where each row is of
+ the form [height, width, channels] indicating the shapes of true images
+ in the resized images, as resized images can be padded with zeros.
+ provided_boxes: float tensor of shape [batch, num_boxes, 4] containing
+ boxes coordinates (normalized) from which we will produce masks.
+ **params: Currently ignored.
+
+ Returns:
+ detections: a dictionary containing the following fields
+ detection_masks: (Optional) A uint8 tensor of shape [batch,
+ max_detections, mask_height, mask_width] with masks for each
+ detection. Background is specified with 0, and foreground is specified
+ with positive integers (1 for standard instance segmentation mask, and
+ 1-indexed parts for DensePose task).
+ And all other fields returned by the super class method.
+ """
+ postprocess_dict = super(DeepMACMetaArch,
+ self).postprocess(prediction_dict,
+ true_image_shapes, **params)
+
+ instance_embedding = prediction_dict[INSTANCE_EMBEDDING][-1]
+ resized_image_shapes = shape_utils.combined_static_and_dynamic_shape(
+ prediction_dict['preprocessed_inputs'])
+ boxes_strided = self._transform_boxes_to_feature_coordinates(
+ provided_boxes, true_image_shapes, resized_image_shapes,
+ instance_embedding)
+
+ if self._deepmac_params is not None:
+ masks = self._postprocess_masks(
+ boxes_strided, instance_embedding,
+ prediction_dict[PIXEL_EMBEDDING][-1])
+ postprocess_dict[fields.DetectionResultFields.detection_masks] = masks
+
+ return postprocess_dict
diff --git a/research/object_detection/meta_architectures/deepmac_meta_arch_test.py b/research/object_detection/meta_architectures/deepmac_meta_arch_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a2dd7447a65129921ebdb0535e6ffcc67fec041
--- /dev/null
+++ b/research/object_detection/meta_architectures/deepmac_meta_arch_test.py
@@ -0,0 +1,419 @@
+"""Tests for google3.third_party.tensorflow_models.object_detection.meta_architectures.deepmac_meta_arch."""
+
+import functools
+import unittest
+
+from absl.testing import parameterized
+import numpy as np
+import tensorflow as tf
+
+from object_detection.core import losses
+from object_detection.core import preprocessor
+from object_detection.meta_architectures import center_net_meta_arch
+from object_detection.meta_architectures import deepmac_meta_arch
+from object_detection.utils import tf_version
+
+
+class DummyFeatureExtractor(center_net_meta_arch.CenterNetFeatureExtractor):
+
+ def __init__(self,
+ channel_means,
+ channel_stds,
+ bgr_ordering,
+ num_feature_outputs,
+ stride):
+ self._num_feature_outputs = num_feature_outputs
+ self._stride = stride
+ super(DummyFeatureExtractor, self).__init__(
+ channel_means=channel_means, channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
+
+ def predict(self):
+ pass
+
+ def loss(self):
+ pass
+
+ def postprocess(self):
+ pass
+
+ def call(self, inputs):
+ batch_size, input_height, input_width, _ = inputs.shape
+ fake_output = tf.ones([
+ batch_size, input_height // self._stride, input_width // self._stride,
+ 64
+ ], dtype=tf.float32)
+ return [fake_output] * self._num_feature_outputs
+
+ @property
+ def out_stride(self):
+ return self._stride
+
+ @property
+ def num_feature_outputs(self):
+ return self._num_feature_outputs
+
+
+class MockMaskNet(tf.keras.layers.Layer):
+
+ def __call__(self, instance_embedding, pixel_embedding, training):
+ return tf.zeros_like(pixel_embedding[:, :, :, 0]) + 0.9
+
+
+def build_meta_arch(predict_full_resolution_masks=False, use_dice_loss=False):
+ """Builds the DeepMAC meta architecture."""
+
+ feature_extractor = DummyFeatureExtractor(
+ channel_means=(1.0, 2.0, 3.0),
+ channel_stds=(10., 20., 30.),
+ bgr_ordering=False,
+ num_feature_outputs=2,
+ stride=4)
+ image_resizer_fn = functools.partial(
+ preprocessor.resize_to_range,
+ min_dimension=128,
+ max_dimension=128,
+ pad_to_max_dimesnion=True)
+
+ object_center_params = center_net_meta_arch.ObjectCenterParams(
+ classification_loss=losses.WeightedSigmoidClassificationLoss(),
+ object_center_loss_weight=1.0,
+ min_box_overlap_iou=1.0,
+ max_box_predictions=5,
+ use_labeled_classes=False)
+
+ if use_dice_loss:
+ classification_loss = losses.WeightedDiceClassificationLoss(False)
+ else:
+ classification_loss = losses.WeightedSigmoidClassificationLoss()
+
+ deepmac_params = deepmac_meta_arch.DeepMACParams(
+ classification_loss=classification_loss,
+ dim=8,
+ task_loss_weight=1.0,
+ pixel_embedding_dim=2,
+ allowed_masked_classes_ids=[],
+ mask_size=16,
+ mask_num_subsamples=-1,
+ use_xy=True,
+ network_type='hourglass10',
+ use_instance_embedding=True,
+ num_init_channels=8,
+ predict_full_resolution_masks=predict_full_resolution_masks,
+ postprocess_crop_size=128,
+ max_roi_jitter_ratio=0.0,
+ roi_jitter_mode='random'
+ )
+
+ object_detection_params = center_net_meta_arch.ObjectDetectionParams(
+ localization_loss=losses.L1LocalizationLoss(),
+ offset_loss_weight=1.0,
+ scale_loss_weight=0.1
+ )
+
+ return deepmac_meta_arch.DeepMACMetaArch(
+ is_training=True,
+ add_summaries=False,
+ num_classes=6,
+ feature_extractor=feature_extractor,
+ object_center_params=object_center_params,
+ deepmac_params=deepmac_params,
+ object_detection_params=object_detection_params,
+ image_resizer_fn=image_resizer_fn)
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class DeepMACUtilsTest(tf.test.TestCase):
+
+ def test_subsample_trivial(self):
+ """Test subsampling masks."""
+
+ boxes = np.arange(4).reshape(4, 1) * np.ones((4, 4))
+ masks = np.arange(4).reshape(4, 1, 1) * np.ones((4, 32, 32))
+ weights = np.ones(4)
+ classes = tf.one_hot(tf.range(4), depth=4)
+
+ result = deepmac_meta_arch.subsample_instances(
+ classes, weights, boxes, masks, 4)
+ self.assertAllClose(result[0], classes)
+ self.assertAllClose(result[1], weights)
+ self.assertAllClose(result[2], boxes)
+ self.assertAllClose(result[3], masks)
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class DeepMACMetaArchTest(tf.test.TestCase):
+
+ def setUp(self): # pylint:disable=g-missing-super-call
+ self.model = build_meta_arch()
+
+ def test_mask_network(self):
+ net = deepmac_meta_arch.MaskHeadNetwork('hourglass10', 8)
+
+ out = net(tf.zeros((2, 4)), tf.zeros((2, 32, 32, 16)), training=True)
+ self.assertEqual(out.shape, (2, 32, 32))
+
+ def test_mask_network_hourglass20(self):
+ net = deepmac_meta_arch.MaskHeadNetwork('hourglass20', 8)
+
+ out = net(tf.zeros((2, 4)), tf.zeros((2, 32, 32, 16)), training=True)
+ self.assertEqual(out.shape, (2, 32, 32))
+
+ def test_mask_network_resnet(self):
+
+ net = deepmac_meta_arch.MaskHeadNetwork('resnet4')
+
+ out = net(tf.zeros((2, 4)), tf.zeros((2, 32, 32, 16)), training=True)
+ self.assertEqual(out.shape, (2, 32, 32))
+
+ def test_mask_network_resnet_tf_function(self):
+
+ net = deepmac_meta_arch.MaskHeadNetwork('resnet8')
+ call_func = tf.function(net.__call__)
+
+ out = call_func(tf.zeros((2, 4)), tf.zeros((2, 32, 32, 16)), training=True)
+ self.assertEqual(out.shape, (2, 32, 32))
+
+ def test_get_mask_head_input(self):
+
+ boxes = tf.constant([[0., 0., 0.25, 0.25], [0.75, 0.75, 1.0, 1.0]],
+ dtype=tf.float32)
+
+ pixel_embedding = np.zeros((32, 32, 4), dtype=np.float32)
+ pixel_embedding[:16, :16] = 1.0
+ pixel_embedding[16:, 16:] = 2.0
+ pixel_embedding = tf.constant(pixel_embedding)
+
+ mask_inputs = self.model._get_mask_head_input(boxes, pixel_embedding)
+ self.assertEqual(mask_inputs.shape, (2, 16, 16, 6))
+
+ y_grid, x_grid = tf.meshgrid(np.linspace(-1.0, 1.0, 16),
+ np.linspace(-1.0, 1.0, 16), indexing='ij')
+ for i in range(2):
+ mask_input = mask_inputs[i]
+ self.assertAllClose(y_grid, mask_input[:, :, 0])
+ self.assertAllClose(x_grid, mask_input[:, :, 1])
+ pixel_embedding = mask_input[:, :, 2:]
+ self.assertAllClose(np.zeros((16, 16, 4)) + i + 1, pixel_embedding)
+
+ def test_get_mask_head_input_no_crop_resize(self):
+
+ model = build_meta_arch(predict_full_resolution_masks=True)
+ boxes = tf.constant([[0., 0., 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+ dtype=tf.float32)
+
+ pixel_embedding_np = np.random.randn(32, 32, 4).astype(np.float32)
+ pixel_embedding = tf.constant(pixel_embedding_np)
+
+ mask_inputs = model._get_mask_head_input(boxes, pixel_embedding)
+ self.assertEqual(mask_inputs.shape, (2, 32, 32, 6))
+
+ y_grid, x_grid = tf.meshgrid(np.linspace(-1.0, 1.0, 32),
+ np.linspace(-1.0, 1.0, 32), indexing='ij')
+ for i in range(2):
+ mask_input = mask_inputs[i]
+ self.assertAllClose(y_grid, mask_input[:, :, 0])
+ self.assertAllClose(x_grid, mask_input[:, :, 1])
+ pixel_embedding = mask_input[:, :, 2:]
+ self.assertAllClose(pixel_embedding_np, pixel_embedding)
+
+ def test_get_instance_embeddings(self):
+
+ embeddings = np.zeros((32, 32, 2))
+ embeddings[8, 8] = 1.0
+ embeddings[24, 16] = 2.0
+ embeddings = tf.constant(embeddings)
+
+ boxes = tf.constant([[0., 0., 0.5, 0.5], [0.5, 0.0, 1.0, 1.0]])
+
+ center_embeddings = self.model._get_instance_embeddings(boxes, embeddings)
+
+ self.assertAllClose(center_embeddings, [[1.0, 1.0], [2.0, 2.0]])
+
+ def test_get_groundtruth_mask_output(self):
+
+ boxes = tf.constant([[0., 0., 0.25, 0.25], [0.75, 0.75, 1.0, 1.0]],
+ dtype=tf.float32)
+ masks = np.zeros((2, 32, 32), dtype=np.float32)
+ masks[0, :16, :16] = 0.5
+ masks[1, 16:, 16:] = 0.1
+ masks = self.model._get_groundtruth_mask_output(boxes, masks)
+ self.assertEqual(masks.shape, (2, 16, 16))
+
+ self.assertAllClose(masks[0], np.zeros((16, 16)) + 0.5)
+ self.assertAllClose(masks[1], np.zeros((16, 16)) + 0.1)
+
+ def test_get_groundtruth_mask_output_crop_resize(self):
+
+ model = build_meta_arch(predict_full_resolution_masks=True)
+ boxes = tf.constant([[0., 0., 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+ dtype=tf.float32)
+ masks = tf.ones((2, 32, 32))
+ masks = model._get_groundtruth_mask_output(boxes, masks)
+ self.assertAllClose(masks, np.ones((2, 32, 32)))
+
+ def test_per_instance_loss(self):
+
+ model = build_meta_arch()
+ model._mask_net = MockMaskNet()
+ boxes = tf.constant([[0.0, 0.0, 0.25, 0.25], [0.75, 0.75, 1.0, 1.0]])
+ masks = np.zeros((2, 32, 32), dtype=np.float32)
+ masks[0, :16, :16] = 1.0
+ masks[1, 16:, 16:] = 1.0
+ masks = tf.constant(masks)
+
+ loss = model._compute_per_instance_mask_loss(
+ boxes, masks, tf.zeros((32, 32, 2)), tf.zeros((32, 32, 2)))
+ self.assertAllClose(
+ loss, np.zeros(2) - tf.math.log(tf.nn.sigmoid(0.9)))
+
+ def test_per_instance_loss_no_crop_resize(self):
+
+ model = build_meta_arch(predict_full_resolution_masks=True)
+ model._mask_net = MockMaskNet()
+ boxes = tf.constant([[0.0, 0.0, 1.0, 1.0], [0.0, 0.0, 1.0, 1.0]])
+ masks = np.ones((2, 128, 128), dtype=np.float32)
+ masks = tf.constant(masks)
+
+ loss = model._compute_per_instance_mask_loss(
+ boxes, masks, tf.zeros((32, 32, 2)), tf.zeros((32, 32, 2)))
+ self.assertAllClose(
+ loss, np.zeros(2) - tf.math.log(tf.nn.sigmoid(0.9)))
+
+ def test_per_instance_loss_no_crop_resize_dice(self):
+
+ model = build_meta_arch(predict_full_resolution_masks=True,
+ use_dice_loss=True)
+ model._mask_net = MockMaskNet()
+ boxes = tf.constant([[0.0, 0.0, 1.0, 1.0], [0.0, 0.0, 1.0, 1.0]])
+ masks = np.ones((2, 128, 128), dtype=np.float32)
+ masks = tf.constant(masks)
+
+ loss = model._compute_per_instance_mask_loss(
+ boxes, masks, tf.zeros((32, 32, 2)), tf.zeros((32, 32, 2)))
+ pred = tf.nn.sigmoid(0.9)
+ expected = (1.0 - ((2.0 * pred) / (1.0 + pred)))
+ self.assertAllClose(loss, [expected, expected], rtol=1e-3)
+
+ def test_empty_masks(self):
+ boxes = tf.zeros([0, 4])
+ masks = tf.zeros([0, 128, 128])
+
+ loss = self.model._compute_per_instance_mask_loss(
+ boxes, masks, tf.zeros((32, 32, 2)), tf.zeros((32, 32, 2)))
+ self.assertEqual(loss.shape, (0,))
+
+ def test_postprocess(self):
+
+ model = build_meta_arch()
+ model._mask_net = MockMaskNet()
+ boxes = np.zeros((2, 3, 4), dtype=np.float32)
+ boxes[:, :, [0, 2]] = 0.0
+ boxes[:, :, [1, 3]] = 8.0
+ boxes = tf.constant(boxes)
+
+ masks = model._postprocess_masks(
+ boxes, tf.zeros((2, 32, 32, 2)), tf.zeros((2, 32, 32, 2)))
+ prob = tf.nn.sigmoid(0.9).numpy()
+ self.assertAllClose(masks, prob * np.ones((2, 3, 16, 16)))
+
+ def test_postprocess_no_crop_resize_shape(self):
+
+ model = build_meta_arch(predict_full_resolution_masks=True)
+ model._mask_net = MockMaskNet()
+ boxes = np.zeros((2, 3, 4), dtype=np.float32)
+ boxes[:, :, [0, 2]] = 0.0
+ boxes[:, :, [1, 3]] = 8.0
+ boxes = tf.constant(boxes)
+
+ masks = model._postprocess_masks(
+ boxes, tf.zeros((2, 32, 32, 2)), tf.zeros((2, 32, 32, 2)))
+ prob = tf.nn.sigmoid(0.9).numpy()
+ self.assertAllClose(masks, prob * np.ones((2, 3, 128, 128)))
+
+ def test_crop_masks_within_boxes(self):
+ masks = np.zeros((2, 32, 32))
+ masks[0, :16, :16] = 1.0
+ masks[1, 16:, 16:] = 1.0
+ boxes = tf.constant([[0.0, 0.0, 15.0 / 32, 15.0 / 32],
+ [0.5, 0.5, 1.0, 1]])
+ masks = deepmac_meta_arch.crop_masks_within_boxes(
+ masks, boxes, 128)
+ masks = (masks.numpy() > 0.0).astype(np.float32)
+ self.assertAlmostEqual(masks.sum(), 2 * 128 * 128)
+
+ def test_transform_boxes_to_feature_coordinates(self):
+ batch_size = 2
+ model = build_meta_arch()
+ model._mask_net = MockMaskNet()
+ boxes = np.zeros((batch_size, 3, 4), dtype=np.float32)
+ boxes[:, :, [0, 2]] = 0.1
+ boxes[:, :, [1, 3]] = 0.5
+ boxes = tf.constant(boxes)
+ true_image_shapes = tf.constant([
+ [64, 32, 3], # Image 1 is padded during resizing.
+ [64, 64, 3], # Image 2 is not padded.
+ ])
+ resized_image_height = 64
+ resized_image_width = 64
+ resized_image_shape = [
+ batch_size, resized_image_height, resized_image_width, 3
+ ]
+
+ feature_map_height = 32
+ feature_map_width = 32
+ instance_embedding = tf.zeros(
+ (batch_size, feature_map_height, feature_map_width, 2))
+
+ expected_boxes = np.array([
+ [ # Image 1
+ # 0.1 * (64 / resized_image_height) * feature_map_height -> 3.2
+ # 0.5 * (32 / resized_image_width) * feature_map_width -> 8.0
+ [3.2, 8., 3.2, 8.],
+ [3.2, 8., 3.2, 8.],
+ [3.2, 8., 3.2, 8.],
+ ],
+ [ # Image 2
+ # 0.1 * (64 / resized_image_height) * feature_map_height -> 3.2
+ # 0.5 * (64 / resized_image_width) * feature_map_width -> 16
+ [3.2, 16., 3.2, 16.],
+ [3.2, 16., 3.2, 16.],
+ [3.2, 16., 3.2, 16.],
+ ],
+ ])
+
+ box_strided = model._transform_boxes_to_feature_coordinates(
+ boxes, true_image_shapes, resized_image_shape, instance_embedding)
+ self.assertAllClose(box_strided, expected_boxes)
+
+ def test_fc_tf_function(self):
+
+ net = deepmac_meta_arch.MaskHeadNetwork('fully_connected', 8, mask_size=32)
+ call_func = tf.function(net.__call__)
+
+ out = call_func(tf.zeros((2, 4)), tf.zeros((2, 32, 32, 8)), training=True)
+ self.assertEqual(out.shape, (2, 32, 32))
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class FullyConnectedMaskHeadTest(tf.test.TestCase):
+
+ def test_fc_mask_head(self):
+ head = deepmac_meta_arch.FullyConnectedMaskHead(512, 16)
+ inputs = tf.random.uniform([100, 16, 16, 512])
+ output = head(inputs)
+ self.assertAllEqual([100, 16, 16, 1], output.numpy().shape)
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class ResNetMaskHeadTest(tf.test.TestCase, parameterized.TestCase):
+
+ @parameterized.parameters(['resnet4', 'resnet8', 'resnet20'])
+ def test_pass(self, name):
+ net = deepmac_meta_arch.ResNetMaskNetwork(name, 8)
+ out = net(tf.zeros((3, 32, 32, 16)))
+ self.assertEqual(out.shape[:3], (3, 32, 32))
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py b/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py
index 4f944eda6c414f83d4b1bcea1d08fd22b25f7f62..6f88b2df02d568a4da80612d92680e2f5a3ca2c7 100644
--- a/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py
+++ b/research/object_detection/meta_architectures/faster_rcnn_meta_arch.py
@@ -99,7 +99,6 @@ import functools
import tensorflow.compat.v1 as tf
import tf_slim as slim
-from object_detection.anchor_generators import grid_anchor_generator
from object_detection.builders import box_predictor_builder
from object_detection.builders import hyperparams_builder
from object_detection.core import box_list
@@ -305,7 +304,8 @@ class FasterRCNNMetaArch(model.DetectionModel):
resize_masks=True,
freeze_batchnorm=False,
return_raw_detections_during_predict=False,
- output_final_box_features=False):
+ output_final_box_features=False,
+ output_final_box_rpn_features=False):
"""FasterRCNNMetaArch Constructor.
Args:
@@ -438,8 +438,11 @@ class FasterRCNNMetaArch(model.DetectionModel):
boxes in the predict() method. These are decoded boxes that have not
been through postprocessing (i.e. NMS). Default False.
output_final_box_features: Whether to output final box features. If true,
- it crops the feauture map based on the final box prediction and returns
- in the dict as detection_features.
+ it crops the rpn feature map and passes it through box_classifier then
+ returns in the output dict as `detection_features`.
+ output_final_box_rpn_features: Whether to output rpn box features. If
+ true, it crops the rpn feature map and returns in the output dict as
+ `detection_features`.
Raises:
ValueError: If `second_stage_batch_size` > `first_stage_max_proposals` at
@@ -451,11 +454,6 @@ class FasterRCNNMetaArch(model.DetectionModel):
# in the future.
super(FasterRCNNMetaArch, self).__init__(num_classes=num_classes)
- if not isinstance(first_stage_anchor_generator,
- grid_anchor_generator.GridAnchorGenerator):
- raise ValueError('first_stage_anchor_generator must be of type '
- 'grid_anchor_generator.GridAnchorGenerator.')
-
self._is_training = is_training
self._image_resizer_fn = image_resizer_fn
self._resize_masks = resize_masks
@@ -492,9 +490,7 @@ class FasterRCNNMetaArch(model.DetectionModel):
hyperparams_builder.KerasLayerHyperparams):
num_anchors_per_location = (
self._first_stage_anchor_generator.num_anchors_per_location())
- if len(num_anchors_per_location) != 1:
- raise ValueError('anchor_generator is expected to generate anchors '
- 'corresponding to a single feature map.')
+
conv_hyperparams = (
first_stage_box_predictor_arg_scope_fn)
self._first_stage_box_predictor_first_conv = (
@@ -533,11 +529,10 @@ class FasterRCNNMetaArch(model.DetectionModel):
else:
self._first_stage_box_predictor_arg_scope_fn = (
first_stage_box_predictor_arg_scope_fn)
- def rpn_box_predictor_feature_extractor(rpn_features_to_crop):
+ def rpn_box_predictor_feature_extractor(single_rpn_features_to_crop):
with slim.arg_scope(self._first_stage_box_predictor_arg_scope_fn()):
- reuse = tf.get_variable_scope().reuse
return slim.conv2d(
- rpn_features_to_crop,
+ single_rpn_features_to_crop,
self._first_stage_box_predictor_depth,
kernel_size=[
self._first_stage_box_predictor_kernel_size,
@@ -546,7 +541,7 @@ class FasterRCNNMetaArch(model.DetectionModel):
rate=self._first_stage_atrous_rate,
activation_fn=tf.nn.relu6,
scope='Conv',
- reuse=reuse)
+ reuse=tf.AUTO_REUSE)
self._first_stage_box_predictor_first_conv = (
rpn_box_predictor_feature_extractor)
self._first_stage_box_predictor = (
@@ -613,6 +608,7 @@ class FasterRCNNMetaArch(model.DetectionModel):
self._return_raw_detections_during_predict = (
return_raw_detections_during_predict)
self._output_final_box_features = output_final_box_features
+ self._output_final_box_rpn_features = output_final_box_rpn_features
@property
def first_stage_feature_extractor_scope(self):
@@ -762,10 +758,10 @@ class FasterRCNNMetaArch(model.DetectionModel):
Returns:
prediction_dict: a dictionary holding "raw" prediction tensors:
- 1) rpn_box_predictor_features: A 4-D float32 tensor with shape
- [batch_size, height, width, depth] to be used for predicting proposal
- boxes and corresponding objectness scores.
- 2) rpn_features_to_crop: A 4-D float32 tensor with shape
+ 1) rpn_box_predictor_features: A list of 4-D float32 tensor with shape
+ [batch_size, height_i, width_j, depth] to be used for predicting
+ proposal boxes and corresponding objectness scores.
+ 2) rpn_features_to_crop: A list of 4-D float32 tensor with shape
[batch_size, height, width, depth] representing image features to crop
using the proposal boxes predicted by the RPN.
3) image_shape: a 1-D tensor of shape [4] representing the input
@@ -830,7 +826,8 @@ class FasterRCNNMetaArch(model.DetectionModel):
prediction_dict['rpn_objectness_predictions_with_background'],
prediction_dict['rpn_features_to_crop'],
prediction_dict['anchors'], prediction_dict['image_shape'],
- true_image_shapes, **side_inputs))
+ true_image_shapes,
+ **side_inputs))
if self._number_of_stages == 3:
prediction_dict = self._predict_third_stage(prediction_dict,
@@ -850,12 +847,12 @@ class FasterRCNNMetaArch(model.DetectionModel):
Returns:
prediction_dict: a dictionary holding "raw" prediction tensors:
- 1) rpn_box_predictor_features: A 4-D float32/bfloat16 tensor with shape
- [batch_size, height, width, depth] to be used for predicting proposal
- boxes and corresponding objectness scores.
- 2) rpn_features_to_crop: A 4-D float32/bfloat16 tensor with shape
- [batch_size, height, width, depth] representing image features to crop
- using the proposal boxes predicted by the RPN.
+ 1) rpn_box_predictor_features: A list of 4-D float32/bfloat16 tensor
+ with shape [batch_size, height_i, width_j, depth] to be used for
+ predicting proposal boxes and corresponding objectness scores.
+ 2) rpn_features_to_crop: A list of 4-D float32/bfloat16 tensor with
+ shape [batch_size, height, width, depth] representing image features
+ to crop using the proposal boxes predicted by the RPN.
3) image_shape: a 1-D tensor of shape [4] representing the input
image shape.
4) rpn_box_encodings: 3-D float32 tensor of shape
@@ -911,7 +908,7 @@ class FasterRCNNMetaArch(model.DetectionModel):
dtype=tf.float32),
'anchors':
anchors_boxlist.data['boxes'],
- fields.PredictionFields.feature_maps: [rpn_features_to_crop]
+ fields.PredictionFields.feature_maps: rpn_features_to_crop
}
return prediction_dict
@@ -947,9 +944,9 @@ class FasterRCNNMetaArch(model.DetectionModel):
[batch_size, num_valid_anchors, 2] containing class
predictions (logits) for each of the anchors. Note that this
tensor *includes* background class predictions (at class index 0).
- rpn_features_to_crop: A 4-D float32 or bfloat16 tensor with shape
- [batch_size, height, width, depth] representing image features to crop
- using the proposal boxes predicted by the RPN.
+ rpn_features_to_crop: A list of 4-D float32 or bfloat16 tensor with shape
+ [batch_size, height_i, width_i, depth] representing image features to
+ crop using the proposal boxes predicted by the RPN.
anchors: 2-D float tensor of shape
[num_anchors, self._box_coder.code_size].
image_shape: A 1D int32 tensors of size [4] containing the image shape.
@@ -1012,9 +1009,9 @@ class FasterRCNNMetaArch(model.DetectionModel):
"""Predicts the output tensors from second stage of Faster R-CNN.
Args:
- rpn_features_to_crop: A 4-D float32 or bfloat16 tensor with shape
- [batch_size, height, width, depth] representing image features to crop
- using the proposal boxes predicted by the RPN.
+ rpn_features_to_crop: A list 4-D float32 or bfloat16 tensor with shape
+ [batch_size, height_i, width_i, depth] representing image features to
+ crop using the proposal boxes predicted by the RPN.
proposal_boxes_normalized: A float tensor with shape [batch_size,
max_num_proposals, 4] representing the (potentially zero padded)
proposal boxes for all images in the batch. These boxes are represented
@@ -1064,10 +1061,11 @@ class FasterRCNNMetaArch(model.DetectionModel):
"""
flattened_proposal_feature_maps = (
self._compute_second_stage_input_feature_maps(
- rpn_features_to_crop, proposal_boxes_normalized, **side_inputs))
+ rpn_features_to_crop, proposal_boxes_normalized,
+ image_shape, **side_inputs))
box_classifier_features = self._extract_box_classifier_features(
- flattened_proposal_feature_maps)
+ flattened_proposal_feature_maps, **side_inputs)
if self._mask_rcnn_box_predictor.is_keras_model:
box_predictions = self._mask_rcnn_box_predictor(
@@ -1196,6 +1194,8 @@ class FasterRCNNMetaArch(model.DetectionModel):
decoded proposal bounding boxes in absolute coordinates.
5) box_classifier_features: a 4-D float32 tensor representing the
features for each proposal.
+ 6) image_shape: a 1-D tensor of shape [4] representing the input
+ image shape.
image_shapes: A 2-D int32 tensors of shape [batch_size, 3] containing
shapes of images in the batch.
@@ -1234,11 +1234,12 @@ class FasterRCNNMetaArch(model.DetectionModel):
detection_classes = detections_dict[
fields.DetectionResultFields.detection_classes]
rpn_features_to_crop = prediction_dict['rpn_features_to_crop']
+ image_shape = prediction_dict['image_shape']
batch_size = tf.shape(detection_boxes)[0]
max_detection = tf.shape(detection_boxes)[1]
flattened_detected_feature_maps = (
self._compute_second_stage_input_feature_maps(
- rpn_features_to_crop, detection_boxes))
+ rpn_features_to_crop, detection_boxes, image_shape))
curr_box_classifier_features = self._extract_box_classifier_features(
flattened_detected_feature_maps)
@@ -1302,13 +1303,13 @@ class FasterRCNNMetaArch(model.DetectionModel):
preprocessed_inputs: a [batch, height, width, channels] image tensor.
Returns:
- rpn_box_predictor_features: A 4-D float32 tensor with shape
- [batch, height, width, depth] to be used for predicting proposal boxes
- and corresponding objectness scores.
- rpn_features_to_crop: A 4-D float32 tensor with shape
+ rpn_box_predictor_features: A list of 4-D float32 tensor with shape
+ [batch, height_i, width_j, depth] to be used for predicting proposal
+ boxes and corresponding objectness scores.
+ rpn_features_to_crop: A list of 4-D float32 tensor with shape
[batch, height, width, depth] representing image features to crop using
the proposals boxes.
- anchors: A BoxList representing anchors (for the RPN) in
+ anchors: A list of BoxList representing anchors (for the RPN) in
absolute coordinates.
image_shape: A 1-D tensor representing the input image shape.
"""
@@ -1317,12 +1318,21 @@ class FasterRCNNMetaArch(model.DetectionModel):
rpn_features_to_crop, self.endpoints = self._extract_proposal_features(
preprocessed_inputs)
- feature_map_shape = tf.shape(rpn_features_to_crop)
+ # Decide if rpn_features_to_crop is a list. If not make it a list
+ if not isinstance(rpn_features_to_crop, list):
+ rpn_features_to_crop = [rpn_features_to_crop]
+
+ feature_map_shapes = []
+ rpn_box_predictor_features = []
+ for single_rpn_features_to_crop in rpn_features_to_crop:
+ single_shape = tf.shape(single_rpn_features_to_crop)
+ feature_map_shapes.append((single_shape[1], single_shape[2]))
+ single_rpn_box_predictor_features = (
+ self._first_stage_box_predictor_first_conv(
+ single_rpn_features_to_crop))
+ rpn_box_predictor_features.append(single_rpn_box_predictor_features)
anchors = box_list_ops.concatenate(
- self._first_stage_anchor_generator.generate([(feature_map_shape[1],
- feature_map_shape[2])]))
- rpn_box_predictor_features = (
- self._first_stage_box_predictor_first_conv(rpn_features_to_crop))
+ self._first_stage_anchor_generator.generate(feature_map_shapes))
return (rpn_box_predictor_features, rpn_features_to_crop,
anchors, image_shape)
@@ -1349,9 +1359,9 @@ class FasterRCNNMetaArch(model.DetectionModel):
Note resulting tensors will not have been postprocessed.
Args:
- rpn_box_predictor_features: A 4-D float32 tensor with shape
- [batch, height, width, depth] to be used for predicting proposal boxes
- and corresponding objectness scores.
+ rpn_box_predictor_features: A list of 4-D float32 tensor with shape
+ [batch, height_i, width_j, depth] to be used for predicting proposal
+ boxes and corresponding objectness scores.
Returns:
box_encodings: 3-D float tensor of shape
@@ -1369,15 +1379,13 @@ class FasterRCNNMetaArch(model.DetectionModel):
"""
num_anchors_per_location = (
self._first_stage_anchor_generator.num_anchors_per_location())
- if len(num_anchors_per_location) != 1:
- raise RuntimeError('anchor_generator is expected to generate anchors '
- 'corresponding to a single feature map.')
+
if self._first_stage_box_predictor.is_keras_model:
box_predictions = self._first_stage_box_predictor(
- [rpn_box_predictor_features])
+ rpn_box_predictor_features)
else:
box_predictions = self._first_stage_box_predictor.predict(
- [rpn_box_predictor_features],
+ rpn_box_predictor_features,
num_anchors_per_location,
scope=self.first_stage_box_predictor_scope)
@@ -1545,9 +1553,22 @@ class FasterRCNNMetaArch(model.DetectionModel):
'Please make sure rpn_features_to_crop is in the prediction_dict.'
)
detections_dict[
- 'detection_features'] = self._add_detection_features_output_node(
+ 'detection_features'] = (
+ self._add_detection_box_boxclassifier_features_output_node(
+ detections_dict[
+ fields.DetectionResultFields.detection_boxes],
+ prediction_dict['rpn_features_to_crop'],
+ prediction_dict['image_shape']))
+ if self._output_final_box_rpn_features:
+ if 'rpn_features_to_crop' not in prediction_dict:
+ raise ValueError(
+ 'Please make sure rpn_features_to_crop is in the prediction_dict.'
+ )
+ detections_dict['cropped_rpn_box_features'] = (
+ self._add_detection_box_rpn_features_output_node(
detections_dict[fields.DetectionResultFields.detection_boxes],
- prediction_dict['rpn_features_to_crop'])
+ prediction_dict['rpn_features_to_crop'],
+ prediction_dict['image_shape']))
return detections_dict
@@ -1563,8 +1584,8 @@ class FasterRCNNMetaArch(model.DetectionModel):
prediction_dict.pop(k)
return prediction_dict
- def _add_detection_features_output_node(self, detection_boxes,
- rpn_features_to_crop):
+ def _add_detection_box_boxclassifier_features_output_node(
+ self, detection_boxes, rpn_features_to_crop, image_shape):
"""Add detection features to outputs.
This function extracts box features for each box in rpn_features_to_crop.
@@ -1576,9 +1597,10 @@ class FasterRCNNMetaArch(model.DetectionModel):
Args:
detection_boxes: a 3-D float32 tensor of shape
[batch_size, max_detections, 4] which represents the bounding boxes.
- rpn_features_to_crop: A 4-D float32 tensor with shape
+ rpn_features_to_crop: A list of 4-D float32 tensor with shape
[batch, height, width, depth] representing image features to crop using
the proposals boxes.
+ image_shape: a 1-D tensor of shape [4] representing the image shape.
Returns:
detection_features: a 4-D float32 tensor of shape
@@ -1588,7 +1610,7 @@ class FasterRCNNMetaArch(model.DetectionModel):
with tf.name_scope('SecondStageDetectionFeaturesExtract'):
flattened_detected_feature_maps = (
self._compute_second_stage_input_feature_maps(
- rpn_features_to_crop, detection_boxes))
+ rpn_features_to_crop, detection_boxes, image_shape))
detection_features_unpooled = self._extract_box_classifier_features(
flattened_detected_feature_maps)
@@ -1602,6 +1624,8 @@ class FasterRCNNMetaArch(model.DetectionModel):
reshaped_detection_features_pool = tf.identity(
reshaped_detection_features_pool, 'pooled_detection_features')
+ # TODO(sbeery) add node to extract rpn features here!!
+
reshaped_detection_features = tf.reshape(
detection_features_unpooled,
[batch_size, max_detections,
@@ -1611,6 +1635,44 @@ class FasterRCNNMetaArch(model.DetectionModel):
return reshaped_detection_features
+ def _add_detection_box_rpn_features_output_node(self, detection_boxes,
+ rpn_features_to_crop,
+ image_shape):
+ """Add detection features to outputs.
+
+ This function extracts box features for each box in rpn_features_to_crop.
+ It returns the extracted box features, reshaped to
+ [batch size, max_detections, height, width, depth]
+
+ Args:
+ detection_boxes: a 3-D float32 tensor of shape
+ [batch_size, max_detections, 4] which represents the bounding boxes.
+ rpn_features_to_crop: A list of 4-D float32 tensor with shape
+ [batch, height, width, depth] representing image features to crop using
+ the proposals boxes.
+ image_shape: a 1-D tensor of shape [4] representing the image shape.
+
+ Returns:
+ detection_features: a 4-D float32 tensor of shape
+ [batch size, max_detections, height, width, depth] representing
+ cropped image features
+ """
+ with tf.name_scope('FirstStageDetectionFeaturesExtract'):
+ flattened_detected_feature_maps = (
+ self._compute_second_stage_input_feature_maps(
+ rpn_features_to_crop, detection_boxes, image_shape))
+
+ batch_size = tf.shape(detection_boxes)[0]
+ max_detections = tf.shape(detection_boxes)[1]
+ reshaped_detection_features = tf.reshape(
+ flattened_detected_feature_maps,
+ [batch_size, max_detections,
+ tf.shape(flattened_detected_feature_maps)[1],
+ tf.shape(flattened_detected_feature_maps)[2],
+ tf.shape(flattened_detected_feature_maps)[3]])
+
+ return reshaped_detection_features
+
def _postprocess_rpn(self,
rpn_box_encodings_batch,
rpn_objectness_predictions_with_background_batch,
@@ -1930,6 +1992,7 @@ class FasterRCNNMetaArch(model.DetectionModel):
def _compute_second_stage_input_feature_maps(self, features_to_crop,
proposal_boxes_normalized,
+ image_shape,
**side_inputs):
"""Crops to a set of proposals from the feature map for a batch of images.
@@ -1943,14 +2006,27 @@ class FasterRCNNMetaArch(model.DetectionModel):
proposal_boxes_normalized: A float32 tensor with shape [batch_size,
num_proposals, box_code_size] containing proposal boxes in
normalized coordinates.
+ image_shape: A 1D int32 tensors of size [4] containing the image shape.
**side_inputs: additional tensors that are required by the network.
Returns:
A float32 tensor with shape [K, new_height, new_width, depth].
"""
+ num_levels = len(features_to_crop)
+ box_levels = None
+ if num_levels != 1:
+ # If there are multiple levels to select, get the box levels
+ # unit_scale_index: num_levels-2 is chosen based on section 4.2 of
+ # https://arxiv.org/pdf/1612.03144.pdf and works best for Resnet based
+ # feature extractor.
+ box_levels = ops.fpn_feature_levels(
+ num_levels, num_levels - 2,
+ tf.sqrt(tf.cast(image_shape[1] * image_shape[2], tf.float32)) / 224.0,
+ proposal_boxes_normalized)
+
cropped_regions = self._flatten_first_two_dimensions(
self._crop_and_resize_fn(
- features_to_crop, proposal_boxes_normalized,
+ features_to_crop, proposal_boxes_normalized, box_levels,
[self._initial_crop_size, self._initial_crop_size]))
return self._maxpool_layer(cropped_regions)
@@ -2394,7 +2470,15 @@ class FasterRCNNMetaArch(model.DetectionModel):
unmatched_class_label=tf.constant(
[1] + self._num_classes * [0], dtype=tf.float32),
gt_weights_batch=groundtruth_weights_list)
-
+ if self.groundtruth_has_field(
+ fields.InputDataFields.groundtruth_labeled_classes):
+ gt_labeled_classes = self.groundtruth_lists(
+ fields.InputDataFields.groundtruth_labeled_classes)
+ gt_labeled_classes = tf.pad(
+ gt_labeled_classes, [[0, 0], [1, 0]],
+ mode='CONSTANT',
+ constant_values=1)
+ batch_cls_weights *= tf.expand_dims(gt_labeled_classes, 1)
class_predictions_with_background = tf.reshape(
class_predictions_with_background,
[batch_size, self.max_num_proposals, -1])
@@ -2517,8 +2601,8 @@ class FasterRCNNMetaArch(model.DetectionModel):
image_shape[1], image_shape[2], check_range=False).get()
flat_cropped_gt_mask = self._crop_and_resize_fn(
- tf.expand_dims(flat_gt_masks, -1),
- tf.expand_dims(flat_normalized_proposals, axis=1),
+ [tf.expand_dims(flat_gt_masks, -1)],
+ tf.expand_dims(flat_normalized_proposals, axis=1), None,
[mask_height, mask_width])
# Without stopping gradients into cropped groundtruth masks the
# performance with 100-padded groundtruth masks when batch size > 1 is
diff --git a/research/object_detection/meta_architectures/faster_rcnn_meta_arch_test.py b/research/object_detection/meta_architectures/faster_rcnn_meta_arch_test.py
index 6c830b32d58cff5756521abc5bfeecacc7118531..d935c99fad63dcdecb67b310430f97e2c51a9ed6 100644
--- a/research/object_detection/meta_architectures/faster_rcnn_meta_arch_test.py
+++ b/research/object_detection/meta_architectures/faster_rcnn_meta_arch_test.py
@@ -484,7 +484,7 @@ class FasterRCNNMetaArchTest(
'mask_predictions':
mask_predictions,
'rpn_features_to_crop':
- rpn_features_to_crop
+ [rpn_features_to_crop]
}, true_image_shapes)
self.assertIn('detection_features', detections)
return (detections['detection_boxes'], detections['detection_scores'],
diff --git a/research/object_detection/meta_architectures/faster_rcnn_meta_arch_test_lib.py b/research/object_detection/meta_architectures/faster_rcnn_meta_arch_test_lib.py
index beead134d515a1084b4ed1a57f63d601e07a02b2..d5d454de9f964933ef2f902e3687b1b6d8cc0500 100644
--- a/research/object_detection/meta_architectures/faster_rcnn_meta_arch_test_lib.py
+++ b/research/object_detection/meta_architectures/faster_rcnn_meta_arch_test_lib.py
@@ -23,6 +23,7 @@ import tensorflow.compat.v1 as tf
from google.protobuf import text_format
from object_detection.anchor_generators import grid_anchor_generator
+from object_detection.anchor_generators import multiscale_grid_anchor_generator
from object_detection.builders import box_predictor_builder
from object_detection.builders import hyperparams_builder
from object_detection.builders import post_processing_builder
@@ -34,7 +35,7 @@ from object_detection.meta_architectures import faster_rcnn_meta_arch
from object_detection.protos import box_predictor_pb2
from object_detection.protos import hyperparams_pb2
from object_detection.protos import post_processing_pb2
-from object_detection.utils import ops
+from object_detection.utils import spatial_transform_ops as spatial_ops
from object_detection.utils import test_case
from object_detection.utils import test_utils
from object_detection.utils import tf_version
@@ -76,6 +77,36 @@ class FakeFasterRCNNFeatureExtractor(
proposal_feature_maps, num_outputs=3, kernel_size=1, scope='layer2')
+class FakeFasterRCNNMultiLevelFeatureExtractor(
+ faster_rcnn_meta_arch.FasterRCNNFeatureExtractor):
+ """Fake feature extractor to use in tests."""
+
+ def __init__(self):
+ super(FakeFasterRCNNMultiLevelFeatureExtractor, self).__init__(
+ is_training=False,
+ first_stage_features_stride=32,
+ reuse_weights=None,
+ weight_decay=0.0)
+
+ def preprocess(self, resized_inputs):
+ return tf.identity(resized_inputs)
+
+ def _extract_proposal_features(self, preprocessed_inputs, scope):
+ with tf.variable_scope('mock_model'):
+ proposal_features_1 = 0 * slim.conv2d(
+ preprocessed_inputs, num_outputs=3, kernel_size=3, scope='layer1',
+ padding='VALID')
+ proposal_features_2 = 0 * slim.conv2d(
+ proposal_features_1, num_outputs=3, kernel_size=3, scope='layer2',
+ padding='VALID')
+ return [proposal_features_1, proposal_features_2], {}
+
+ def _extract_box_classifier_features(self, proposal_feature_maps, scope):
+ with tf.variable_scope('mock_model'):
+ return 0 * slim.conv2d(
+ proposal_feature_maps, num_outputs=3, kernel_size=1, scope='layer3')
+
+
class FakeFasterRCNNKerasFeatureExtractor(
faster_rcnn_meta_arch.FasterRCNNKerasFeatureExtractor):
"""Fake feature extractor to use in tests."""
@@ -112,6 +143,42 @@ class FakeFasterRCNNKerasFeatureExtractor(
3, kernel_size=1, padding='SAME', name=name + '_layer2')])
+class FakeFasterRCNNKerasMultilevelFeatureExtractor(
+ faster_rcnn_meta_arch.FasterRCNNKerasFeatureExtractor):
+ """Fake feature extractor to use in tests."""
+
+ def __init__(self):
+ super(FakeFasterRCNNKerasMultilevelFeatureExtractor, self).__init__(
+ is_training=False,
+ first_stage_features_stride=32,
+ weight_decay=0.0)
+
+ def preprocess(self, resized_inputs):
+ return tf.identity(resized_inputs)
+
+ def get_proposal_feature_extractor_model(self, name):
+
+ class ProposalFeatureExtractor(tf.keras.Model):
+ """Dummy proposal feature extraction."""
+
+ def __init__(self, name):
+ super(ProposalFeatureExtractor, self).__init__(name=name)
+ self.conv = None
+
+ def build(self, input_shape):
+ self.conv = tf.keras.layers.Conv2D(
+ 3, kernel_size=3, name='layer1')
+ self.conv_1 = tf.keras.layers.Conv2D(
+ 3, kernel_size=3, name='layer1')
+
+ def call(self, inputs):
+ output_1 = self.conv(inputs)
+ output_2 = self.conv_1(output_1)
+ return [output_1, output_2]
+
+ return ProposalFeatureExtractor(name=name)
+
+
class FasterRCNNMetaArchTestBase(test_case.TestCase, parameterized.TestCase):
"""Base class to test Faster R-CNN and R-FCN meta architectures."""
@@ -234,7 +301,8 @@ class FasterRCNNMetaArchTestBase(test_case.TestCase, parameterized.TestCase):
calibration_mapping_value=None,
share_box_across_classes=False,
return_raw_detections_during_predict=False,
- output_final_box_features=False):
+ output_final_box_features=False,
+ multi_level=False):
use_keras = tf_version.is_tf2()
def image_resizer_fn(image, masks=None):
"""Fake image resizer function."""
@@ -260,22 +328,41 @@ class FasterRCNNMetaArchTestBase(test_case.TestCase, parameterized.TestCase):
# anchors in this test are designed so that a subset of anchors are inside
# the image and a subset of anchors are outside.
- first_stage_anchor_scales = (0.001, 0.005, 0.1)
- first_stage_anchor_aspect_ratios = (0.5, 1.0, 2.0)
- first_stage_anchor_strides = (1, 1)
- first_stage_anchor_generator = grid_anchor_generator.GridAnchorGenerator(
- first_stage_anchor_scales,
- first_stage_anchor_aspect_ratios,
- anchor_stride=first_stage_anchor_strides)
+ first_stage_anchor_generator = None
+ if multi_level:
+ min_level = 0
+ max_level = 1
+ anchor_scale = 0.1
+ aspect_ratios = [1.0, 2.0, 0.5]
+ scales_per_octave = 2
+ normalize_coordinates = False
+ (first_stage_anchor_generator
+ ) = multiscale_grid_anchor_generator.MultiscaleGridAnchorGenerator(
+ min_level, max_level, anchor_scale, aspect_ratios, scales_per_octave,
+ normalize_coordinates)
+ else:
+ first_stage_anchor_scales = (0.001, 0.005, 0.1)
+ first_stage_anchor_aspect_ratios = (0.5, 1.0, 2.0)
+ first_stage_anchor_strides = (1, 1)
+ first_stage_anchor_generator = grid_anchor_generator.GridAnchorGenerator(
+ first_stage_anchor_scales,
+ first_stage_anchor_aspect_ratios,
+ anchor_stride=first_stage_anchor_strides)
first_stage_target_assigner = target_assigner.create_target_assigner(
'FasterRCNN',
'proposal',
use_matmul_gather=use_matmul_gather_in_matcher)
if use_keras:
- fake_feature_extractor = FakeFasterRCNNKerasFeatureExtractor()
+ if multi_level:
+ fake_feature_extractor = FakeFasterRCNNKerasMultilevelFeatureExtractor()
+ else:
+ fake_feature_extractor = FakeFasterRCNNKerasFeatureExtractor()
else:
- fake_feature_extractor = FakeFasterRCNNFeatureExtractor()
+ if multi_level:
+ fake_feature_extractor = FakeFasterRCNNMultiLevelFeatureExtractor()
+ else:
+ fake_feature_extractor = FakeFasterRCNNFeatureExtractor()
first_stage_box_predictor_hyperparams_text_proto = """
op: CONV
@@ -377,8 +464,9 @@ class FasterRCNNMetaArchTestBase(test_case.TestCase, parameterized.TestCase):
max_negatives_per_positive=None)
crop_and_resize_fn = (
- ops.matmul_crop_and_resize
- if use_matmul_crop_and_resize else ops.native_crop_and_resize)
+ spatial_ops.multilevel_matmul_crop_and_resize
+ if use_matmul_crop_and_resize
+ else spatial_ops.multilevel_native_crop_and_resize)
common_kwargs = {
'is_training':
is_training,
@@ -478,8 +566,8 @@ class FasterRCNNMetaArchTestBase(test_case.TestCase, parameterized.TestCase):
preprocessed_inputs, true_image_shapes = model.preprocess(images)
prediction_dict = model.predict(preprocessed_inputs, true_image_shapes)
- return (prediction_dict['rpn_box_predictor_features'],
- prediction_dict['rpn_features_to_crop'],
+ return (prediction_dict['rpn_box_predictor_features'][0],
+ prediction_dict['rpn_features_to_crop'][0],
prediction_dict['image_shape'],
prediction_dict['rpn_box_encodings'],
prediction_dict['rpn_objectness_predictions_with_background'],
@@ -528,6 +616,92 @@ class FasterRCNNMetaArchTestBase(test_case.TestCase, parameterized.TestCase):
self.assertTrue(np.all(np.less_equal(anchors[:, 2], height)))
self.assertTrue(np.all(np.less_equal(anchors[:, 3], width)))
+ @parameterized.parameters(
+ {'use_static_shapes': False},
+ {'use_static_shapes': True},
+ )
+ def test_predict_shape_in_inference_mode_first_stage_only_multi_level(
+ self, use_static_shapes):
+ batch_size = 2
+ height = 50
+ width = 52
+ input_image_shape = (batch_size, height, width, 3)
+
+ with test_utils.GraphContextOrNone() as g:
+ model = self._build_model(
+ is_training=False,
+ number_of_stages=1,
+ second_stage_batch_size=2,
+ clip_anchors_to_image=use_static_shapes,
+ use_static_shapes=use_static_shapes,
+ multi_level=True)
+ def graph_fn(images):
+ """Function to construct tf graph for the test."""
+
+ preprocessed_inputs, true_image_shapes = model.preprocess(images)
+ prediction_dict = model.predict(preprocessed_inputs, true_image_shapes)
+ return (prediction_dict['rpn_box_predictor_features'][0],
+ prediction_dict['rpn_box_predictor_features'][1],
+ prediction_dict['rpn_features_to_crop'][0],
+ prediction_dict['rpn_features_to_crop'][1],
+ prediction_dict['image_shape'],
+ prediction_dict['rpn_box_encodings'],
+ prediction_dict['rpn_objectness_predictions_with_background'],
+ prediction_dict['anchors'])
+
+ images = np.zeros(input_image_shape, dtype=np.float32)
+
+ # In inference mode, anchors are clipped to the image window, but not
+ # pruned. Since MockFasterRCNN.extract_proposal_features returns a
+ # tensor with the same shape as its input, the expected number of anchors
+ # is height * width * the number of anchors per location (i.e. 3x3).
+ expected_num_anchors = ((height-2) * (width-2) + (height-4) * (width-4)) * 6
+ expected_output_shapes = {
+ 'rpn_box_predictor_features_0': (batch_size, height-2, width-2, 512),
+ 'rpn_box_predictor_features_1': (batch_size, height-4, width-4, 512),
+ 'rpn_features_to_crop_0': (batch_size, height-2, width-2, 3),
+ 'rpn_features_to_crop_1': (batch_size, height-4, width-4, 3),
+ 'rpn_box_encodings': (batch_size, expected_num_anchors, 4),
+ 'rpn_objectness_predictions_with_background':
+ (batch_size, expected_num_anchors, 2),
+ }
+
+ if use_static_shapes:
+ expected_output_shapes['anchors'] = (expected_num_anchors, 4)
+ else:
+ expected_output_shapes['anchors'] = (18300, 4)
+
+ if use_static_shapes:
+ results = self.execute(graph_fn, [images], graph=g)
+ else:
+ results = self.execute_cpu(graph_fn, [images], graph=g)
+
+ self.assertAllEqual(results[0].shape,
+ expected_output_shapes['rpn_box_predictor_features_0'])
+ self.assertAllEqual(results[1].shape,
+ expected_output_shapes['rpn_box_predictor_features_1'])
+ self.assertAllEqual(results[2].shape,
+ expected_output_shapes['rpn_features_to_crop_0'])
+ self.assertAllEqual(results[3].shape,
+ expected_output_shapes['rpn_features_to_crop_1'])
+ self.assertAllEqual(results[4],
+ input_image_shape)
+ self.assertAllEqual(results[5].shape,
+ expected_output_shapes['rpn_box_encodings'])
+ self.assertAllEqual(
+ results[6].shape,
+ expected_output_shapes['rpn_objectness_predictions_with_background'])
+ self.assertAllEqual(results[7].shape,
+ expected_output_shapes['anchors'])
+
+ # Check that anchors are clipped to window.
+ anchors = results[5]
+ self.assertTrue(np.all(np.greater_equal(anchors, 0)))
+ self.assertTrue(np.all(np.less_equal(anchors[:, 0], height)))
+ self.assertTrue(np.all(np.less_equal(anchors[:, 1], width)))
+ self.assertTrue(np.all(np.less_equal(anchors[:, 2], height)))
+ self.assertTrue(np.all(np.less_equal(anchors[:, 3], width)))
+
def test_regularization_losses(self):
with test_utils.GraphContextOrNone() as g:
model = self._build_model(
@@ -600,9 +774,9 @@ class FasterRCNNMetaArchTestBase(test_case.TestCase, parameterized.TestCase):
def compare_results(results, expected_output_shapes):
"""Checks if the shape of the predictions are as expected."""
- self.assertAllEqual(results[0].shape,
+ self.assertAllEqual(results[0][0].shape,
expected_output_shapes['rpn_box_predictor_features'])
- self.assertAllEqual(results[1].shape,
+ self.assertAllEqual(results[1][0].shape,
expected_output_shapes['rpn_features_to_crop'])
self.assertAllEqual(results[2].shape,
expected_output_shapes['image_shape'])
@@ -745,8 +919,8 @@ class FasterRCNNMetaArchTestBase(test_case.TestCase, parameterized.TestCase):
result_tensor_dict['anchors'],
result_tensor_dict['rpn_box_encodings'],
result_tensor_dict['rpn_objectness_predictions_with_background'],
- result_tensor_dict['rpn_features_to_crop'],
- result_tensor_dict['rpn_box_predictor_features'],
+ result_tensor_dict['rpn_features_to_crop'][0],
+ result_tensor_dict['rpn_box_predictor_features'][0],
result_tensor_dict['final_anchors'],
)
diff --git a/research/object_detection/meta_architectures/rfcn_meta_arch.py b/research/object_detection/meta_architectures/rfcn_meta_arch.py
index 1228a4b90a79039aa6519ffe2d899bc80541aedc..c19dc04d6cad120140593114dccf81c5683aa306 100644
--- a/research/object_detection/meta_architectures/rfcn_meta_arch.py
+++ b/research/object_detection/meta_architectures/rfcn_meta_arch.py
@@ -84,7 +84,8 @@ class RFCNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
resize_masks=False,
freeze_batchnorm=False,
return_raw_detections_during_predict=False,
- output_final_box_features=False):
+ output_final_box_features=False,
+ output_final_box_rpn_features=False):
"""RFCNMetaArch Constructor.
Args:
@@ -194,8 +195,11 @@ class RFCNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
boxes in the predict() method. These are decoded boxes that have not
been through postprocessing (i.e. NMS). Default False.
output_final_box_features: Whether to output final box features. If true,
- it crops the feauture map based on the final box prediction and returns
- in the dict as detection_features.
+ it crops the feature map based on the final box prediction and returns
+ it in the dict as detection_features.
+ output_final_box_rpn_features: Whether to output rpn box features. If
+ true, it crops the rpn feature map based on the final box prediction and
+ returns it in the dict as detection_features.
Raises:
ValueError: If `second_stage_batch_size` > `first_stage_max_proposals`
@@ -245,7 +249,8 @@ class RFCNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
freeze_batchnorm=freeze_batchnorm,
return_raw_detections_during_predict=(
return_raw_detections_during_predict),
- output_final_box_features=output_final_box_features)
+ output_final_box_features=output_final_box_features,
+ output_final_box_rpn_features=output_final_box_rpn_features)
self._rfcn_box_predictor = second_stage_rfcn_box_predictor
@@ -265,7 +270,7 @@ class RFCNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
[batch_size, num_valid_anchors, 2] containing class
predictions (logits) for each of the anchors. Note that this
tensor *includes* background class predictions (at class index 0).
- rpn_features: A 4-D float32 tensor with shape
+ rpn_features: A list of single 4-D float32 tensor with shape
[batch_size, height, width, depth] representing image features from the
RPN.
anchors: 2-D float tensor of shape
@@ -313,6 +318,7 @@ class RFCNMetaArch(faster_rcnn_meta_arch.FasterRCNNMetaArch):
rpn_objectness_predictions_with_background,
anchors, image_shape_2d, true_image_shapes)
+ rpn_features = rpn_features[0]
box_classifier_features = (
self._extract_box_classifier_features(rpn_features))
diff --git a/research/object_detection/meta_architectures/ssd_meta_arch.py b/research/object_detection/meta_architectures/ssd_meta_arch.py
index eb1fd320d7061a72fe6fa48955b421eee3b0f96e..055e6185fd19d1e95f3a80bfee06789694d26031 100644
--- a/research/object_detection/meta_architectures/ssd_meta_arch.py
+++ b/research/object_detection/meta_architectures/ssd_meta_arch.py
@@ -1308,10 +1308,17 @@ class SSDMetaArch(model.DetectionModel):
to be used to restore Slim-based models when running Tensorflow 1.x.
Args:
- fine_tune_checkpoint_type: whether to restore from a full detection
- checkpoint (with compatible variable names) or to restore from a
- classification checkpoint for initialization prior to training.
- Valid values: `detection`, `classification`. Default 'detection'.
+ fine_tune_checkpoint_type: A string inidicating the subset of variables
+ to load. Valid values: `detection`, `classification`, `full`. Default
+ `detection`.
+ An SSD checkpoint has three parts:
+ 1) Classification Network (like ResNet)
+ 2) DeConv layers (for FPN)
+ 3) Box/Class prediction parameters
+ The parameters will be loaded using the following strategy:
+ `classification` - will load #1
+ `detection` - will load #1, #2
+ `full` - will load #1, #2, #3
Returns:
A dict mapping keys to Trackable objects (tf.Module or Checkpoint).
@@ -1325,6 +1332,10 @@ class SSDMetaArch(model.DetectionModel):
fake_model = tf.train.Checkpoint(
_feature_extractor=self._feature_extractor)
return {'model': fake_model}
+
+ elif fine_tune_checkpoint_type == 'full':
+ return {'model': self}
+
else:
raise ValueError('Not supported fine_tune_checkpoint_type: {}'.format(
fine_tune_checkpoint_type))
diff --git a/research/object_detection/meta_architectures/ssd_meta_arch_test.py b/research/object_detection/meta_architectures/ssd_meta_arch_test.py
index 585eb1778f72deae1aeee45bfbf1d18fa3af1212..7ad061e8d40bbb261bd7c51d9cf8350ecb8883be 100644
--- a/research/object_detection/meta_architectures/ssd_meta_arch_test.py
+++ b/research/object_detection/meta_architectures/ssd_meta_arch_test.py
@@ -615,7 +615,6 @@ class SsdMetaArchTest(ssd_meta_arch_test_lib.SSDMetaArchTestBase,
self.assertNotIn(six.ensure_binary('FeatureExtractor'), var)
def test_load_all_det_checkpoint_vars(self):
- # TODO(rathodv): Support TF2.X
if self.is_tf2(): return
test_graph_detection = tf.Graph()
with test_graph_detection.as_default():
@@ -634,6 +633,39 @@ class SsdMetaArchTest(ssd_meta_arch_test_lib.SSDMetaArchTestBase,
self.assertIsInstance(var_map, dict)
self.assertIn('another_variable', var_map)
+ def test_load_checkpoint_vars_tf2(self):
+
+ if not self.is_tf2():
+ self.skipTest('Not running TF2 checkpoint test with TF1.')
+
+ model, _, _, _ = self._create_model()
+ inputs_shape = [2, 2, 2, 3]
+ inputs = tf.cast(
+ tf.random_uniform(inputs_shape, minval=0, maxval=255, dtype=tf.int32),
+ dtype=tf.float32)
+ model(inputs)
+
+ detection_var_names = sorted([
+ var.name for var in model.restore_from_objects('detection')[
+ 'model']._feature_extractor.weights
+ ])
+ expected_detection_names = [
+ 'ssd_meta_arch/fake_ssd_keras_feature_extractor/mock_model/layer1/bias:0',
+ 'ssd_meta_arch/fake_ssd_keras_feature_extractor/mock_model/layer1/kernel:0'
+ ]
+ self.assertEqual(detection_var_names, expected_detection_names)
+
+ full_var_names = sorted([
+ var.name for var in
+ model.restore_from_objects('full')['model'].weights
+ ])
+
+ exepcted_full_names = ['box_predictor_var:0'] + expected_detection_names
+ self.assertEqual(exepcted_full_names, full_var_names)
+ # TODO(vighneshb) Add similar test for classification checkpoint type.
+ # TODO(vighneshb) Test loading a checkpoint from disk to verify that
+ # checkpoints are loaded correctly.
+
def test_loss_results_are_correct_with_random_example_sampling(self):
with test_utils.GraphContextOrNone() as g:
model, num_classes, _, _ = self._create_model(
diff --git a/research/object_detection/metrics/coco_evaluation.py b/research/object_detection/metrics/coco_evaluation.py
index f721bbe3a503666d938fe4233b4619c044301e09..89437bd1fb3854d0de8b9ea8ff07ce5f14b7d1aa 100644
--- a/research/object_detection/metrics/coco_evaluation.py
+++ b/research/object_detection/metrics/coco_evaluation.py
@@ -34,7 +34,9 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
def __init__(self,
categories,
include_metrics_per_category=False,
- all_metrics_per_category=False):
+ all_metrics_per_category=False,
+ skip_predictions_for_unlabeled_class=False,
+ super_categories=None):
"""Constructor.
Args:
@@ -46,6 +48,13 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
each category in per_category_ap. Be careful with setting it to true if
you have more than handful of categories, because it will pollute
your mldash.
+ skip_predictions_for_unlabeled_class: Skip predictions that do not match
+ with the labeled classes for the image.
+ super_categories: None or a python dict mapping super-category names
+ (strings) to lists of categories (corresponding to category names
+ in the label_map). Metrics are aggregated along these super-categories
+ and added to the `per_category_ap` and are associated with the name
+ `PerformanceBySuperCategory/`.
"""
super(CocoDetectionEvaluator, self).__init__(categories)
# _image_ids is a dictionary that maps unique image ids to Booleans which
@@ -58,6 +67,9 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
self._metrics = None
self._include_metrics_per_category = include_metrics_per_category
self._all_metrics_per_category = all_metrics_per_category
+ self._skip_predictions_for_unlabeled_class = skip_predictions_for_unlabeled_class
+ self._groundtruth_labeled_classes = {}
+ self._super_categories = super_categories
def clear(self):
"""Clears the state to prepare for a fresh evaluation."""
@@ -92,6 +104,10 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
numpy array of keypoint visibilities with shape [num_gt_boxes,
num_keypoints]. Integer is treated as an enum with 0=not labeled,
1=labeled but not visible and 2=labeled and visible.
+ InputDataFields.groundtruth_labeled_classes (optional): a tensor of
+ shape [num_classes + 1] containing the multi-hot tensor indicating the
+ classes that each image is labeled for. Note that the classes labels
+ are 1-indexed.
"""
if image_id in self._image_ids:
tf.logging.warning('Ignoring ground truth with image id %s since it was '
@@ -134,6 +150,19 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
self._annotation_id += groundtruth_dict[standard_fields.InputDataFields.
groundtruth_boxes].shape[0]
+ if (standard_fields.InputDataFields.groundtruth_labeled_classes
+ ) in groundtruth_dict:
+ labeled_classes = groundtruth_dict[
+ standard_fields.InputDataFields.groundtruth_labeled_classes]
+ if labeled_classes.shape != (len(self._category_id_set) + 1,):
+ raise ValueError('Invalid shape for groundtruth labeled classes: {}, '
+ 'num_categories_including_background: {}'.format(
+ labeled_classes,
+ len(self._category_id_set) + 1))
+ self._groundtruth_labeled_classes[image_id] = np.flatnonzero(
+ groundtruth_dict[standard_fields.InputDataFields
+ .groundtruth_labeled_classes] == 1).tolist()
+
# Boolean to indicate whether a detection has been added for this image.
self._image_ids[image_id] = False
@@ -173,17 +202,41 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
standard_fields.DetectionResultFields.detection_keypoints)
if detection_keypoints is not None and not detection_keypoints.shape[0]:
detection_keypoints = None
- self._detection_boxes_list.extend(
- coco_tools.ExportSingleImageDetectionBoxesToCoco(
- image_id=image_id,
- category_id_set=self._category_id_set,
- detection_boxes=detections_dict[
- standard_fields.DetectionResultFields.detection_boxes],
- detection_scores=detections_dict[
- standard_fields.DetectionResultFields.detection_scores],
- detection_classes=detections_dict[
- standard_fields.DetectionResultFields.detection_classes],
- detection_keypoints=detection_keypoints))
+
+ if self._skip_predictions_for_unlabeled_class:
+ det_classes = detections_dict[
+ standard_fields.DetectionResultFields.detection_classes]
+ num_det_boxes = det_classes.shape[0]
+ keep_box_ids = []
+ for box_id in range(num_det_boxes):
+ if det_classes[box_id] in self._groundtruth_labeled_classes[image_id]:
+ keep_box_ids.append(box_id)
+ self._detection_boxes_list.extend(
+ coco_tools.ExportSingleImageDetectionBoxesToCoco(
+ image_id=image_id,
+ category_id_set=self._category_id_set,
+ detection_boxes=detections_dict[
+ standard_fields.DetectionResultFields.detection_boxes]
+ [keep_box_ids],
+ detection_scores=detections_dict[
+ standard_fields.DetectionResultFields.detection_scores]
+ [keep_box_ids],
+ detection_classes=detections_dict[
+ standard_fields.DetectionResultFields.detection_classes]
+ [keep_box_ids],
+ detection_keypoints=detection_keypoints))
+ else:
+ self._detection_boxes_list.extend(
+ coco_tools.ExportSingleImageDetectionBoxesToCoco(
+ image_id=image_id,
+ category_id_set=self._category_id_set,
+ detection_boxes=detections_dict[
+ standard_fields.DetectionResultFields.detection_boxes],
+ detection_scores=detections_dict[
+ standard_fields.DetectionResultFields.detection_scores],
+ detection_classes=detections_dict[
+ standard_fields.DetectionResultFields.detection_classes],
+ detection_keypoints=detection_keypoints))
self._image_ids[image_id] = True
def dump_detections_to_json_file(self, json_output_path):
@@ -233,6 +286,9 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
no supercategories exist). For backward compatibility
'PerformanceByCategory' is included in the output regardless of
all_metrics_per_category.
+ If super_categories are provided, then this will additionally include
+ metrics aggregated along the super_categories with keys of the form:
+ `PerformanceBySuperCategory/`
"""
tf.logging.info('Performing evaluation on %d images.', len(self._image_ids))
groundtruth_dict = {
@@ -247,7 +303,8 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
coco_wrapped_groundtruth, coco_wrapped_detections, agnostic_mode=False)
box_metrics, box_per_category_ap = box_evaluator.ComputeMetrics(
include_metrics_per_category=self._include_metrics_per_category,
- all_metrics_per_category=self._all_metrics_per_category)
+ all_metrics_per_category=self._all_metrics_per_category,
+ super_categories=self._super_categories)
box_metrics.update(box_per_category_ap)
box_metrics = {'DetectionBoxes_'+ key: value
for key, value in iter(box_metrics.items())}
@@ -271,24 +328,20 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
None when executing eagerly, or an update_op that can be used to update
the eval metrics in `tf.estimator.EstimatorSpec`.
"""
- def update_op(
- image_id_batched,
- groundtruth_boxes_batched,
- groundtruth_classes_batched,
- groundtruth_is_crowd_batched,
- num_gt_boxes_per_image,
- detection_boxes_batched,
- detection_scores_batched,
- detection_classes_batched,
- num_det_boxes_per_image,
- is_annotated_batched):
- """Update operation for adding batch of images to Coco evaluator."""
- for (image_id, gt_box, gt_class, gt_is_crowd, num_gt_box, det_box,
- det_score, det_class, num_det_box, is_annotated) in zip(
+ def update_op(image_id_batched, groundtruth_boxes_batched,
+ groundtruth_classes_batched, groundtruth_is_crowd_batched,
+ groundtruth_labeled_classes_batched, num_gt_boxes_per_image,
+ detection_boxes_batched, detection_scores_batched,
+ detection_classes_batched, num_det_boxes_per_image,
+ is_annotated_batched):
+ """Update operation for adding batch of images to Coco evaluator."""
+ for (image_id, gt_box, gt_class, gt_is_crowd, gt_labeled_classes,
+ num_gt_box, det_box, det_score, det_class,
+ num_det_box, is_annotated) in zip(
image_id_batched, groundtruth_boxes_batched,
groundtruth_classes_batched, groundtruth_is_crowd_batched,
- num_gt_boxes_per_image,
+ groundtruth_labeled_classes_batched, num_gt_boxes_per_image,
detection_boxes_batched, detection_scores_batched,
detection_classes_batched, num_det_boxes_per_image,
is_annotated_batched):
@@ -297,7 +350,8 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
image_id, {
'groundtruth_boxes': gt_box[:num_gt_box],
'groundtruth_classes': gt_class[:num_gt_box],
- 'groundtruth_is_crowd': gt_is_crowd[:num_gt_box]
+ 'groundtruth_is_crowd': gt_is_crowd[:num_gt_box],
+ 'groundtruth_labeled_classes': gt_labeled_classes
})
self.add_single_detected_image_info(
image_id,
@@ -313,22 +367,38 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
groundtruth_classes = eval_dict[input_data_fields.groundtruth_classes]
groundtruth_is_crowd = eval_dict.get(
input_data_fields.groundtruth_is_crowd, None)
+ groundtruth_labeled_classes = eval_dict.get(
+ input_data_fields.groundtruth_labeled_classes, None)
detection_boxes = eval_dict[detection_fields.detection_boxes]
detection_scores = eval_dict[detection_fields.detection_scores]
detection_classes = eval_dict[detection_fields.detection_classes]
num_gt_boxes_per_image = eval_dict.get(
- 'num_groundtruth_boxes_per_image', None)
- num_det_boxes_per_image = eval_dict.get('num_det_boxes_per_image', None)
+ input_data_fields.num_groundtruth_boxes, None)
+ num_det_boxes_per_image = eval_dict.get(detection_fields.num_detections,
+ None)
is_annotated = eval_dict.get('is_annotated', None)
if groundtruth_is_crowd is None:
groundtruth_is_crowd = tf.zeros_like(groundtruth_classes, dtype=tf.bool)
+
+ # If groundtruth_labeled_classes is not provided, make it equal to the
+ # detection_classes. This assumes that all predictions will be kept to
+ # compute eval metrics.
+ if groundtruth_labeled_classes is None:
+ groundtruth_labeled_classes = tf.reduce_max(
+ tf.one_hot(
+ tf.cast(detection_classes, tf.int32),
+ len(self._category_id_set) + 1),
+ axis=-2)
+
if not image_id.shape.as_list():
# Apply a batch dimension to all tensors.
image_id = tf.expand_dims(image_id, 0)
groundtruth_boxes = tf.expand_dims(groundtruth_boxes, 0)
groundtruth_classes = tf.expand_dims(groundtruth_classes, 0)
groundtruth_is_crowd = tf.expand_dims(groundtruth_is_crowd, 0)
+ groundtruth_labeled_classes = tf.expand_dims(groundtruth_labeled_classes,
+ 0)
detection_boxes = tf.expand_dims(detection_boxes, 0)
detection_scores = tf.expand_dims(detection_scores, 0)
detection_classes = tf.expand_dims(detection_classes, 0)
@@ -359,16 +429,12 @@ class CocoDetectionEvaluator(object_detection_evaluation.DetectionEvaluator):
if is_annotated is None:
is_annotated = tf.ones_like(image_id, dtype=tf.bool)
- return tf.py_func(update_op, [image_id,
- groundtruth_boxes,
- groundtruth_classes,
- groundtruth_is_crowd,
- num_gt_boxes_per_image,
- detection_boxes,
- detection_scores,
- detection_classes,
- num_det_boxes_per_image,
- is_annotated], [])
+ return tf.py_func(update_op, [
+ image_id, groundtruth_boxes, groundtruth_classes, groundtruth_is_crowd,
+ groundtruth_labeled_classes, num_gt_boxes_per_image, detection_boxes,
+ detection_scores, detection_classes, num_det_boxes_per_image,
+ is_annotated
+ ], [])
def get_estimator_eval_metric_ops(self, eval_dict):
"""Returns a dictionary of eval metric ops.
@@ -894,7 +960,10 @@ class CocoKeypointEvaluator(CocoDetectionEvaluator):
class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
"""Class to evaluate COCO detection metrics."""
- def __init__(self, categories, include_metrics_per_category=False):
+ def __init__(self, categories,
+ include_metrics_per_category=False,
+ all_metrics_per_category=False,
+ super_categories=None):
"""Constructor.
Args:
@@ -902,6 +971,15 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
'id': (required) an integer id uniquely identifying this category.
'name': (required) string representing category name e.g., 'cat', 'dog'.
include_metrics_per_category: If True, include metrics for each category.
+ all_metrics_per_category: Whether to include all the summary metrics for
+ each category in per_category_ap. Be careful with setting it to true if
+ you have more than handful of categories, because it will pollute
+ your mldash.
+ super_categories: None or a python dict mapping super-category names
+ (strings) to lists of categories (corresponding to category names
+ in the label_map). Metrics are aggregated along these super-categories
+ and added to the `per_category_ap` and are associated with the name
+ `PerformanceBySuperCategory/`.
"""
super(CocoMaskEvaluator, self).__init__(categories)
self._image_id_to_mask_shape_map = {}
@@ -911,6 +989,8 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
self._category_id_set = set([cat['id'] for cat in self._categories])
self._annotation_id = 1
self._include_metrics_per_category = include_metrics_per_category
+ self._super_categories = super_categories
+ self._all_metrics_per_category = all_metrics_per_category
def clear(self):
"""Clears the state to prepare for a fresh evaluation."""
@@ -939,12 +1019,27 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
[num_boxes, image_height, image_width] containing groundtruth masks
corresponding to the boxes. The elements of the array must be in
{0, 1}.
+ InputDataFields.groundtruth_is_crowd (optional): integer numpy array of
+ shape [num_boxes] containing iscrowd flag for groundtruth boxes.
+ InputDataFields.groundtruth_area (optional): float numpy array of
+ shape [num_boxes] containing the area (in the original absolute
+ coordinates) of the annotated object.
"""
if image_id in self._image_id_to_mask_shape_map:
tf.logging.warning('Ignoring ground truth with image id %s since it was '
'previously added', image_id)
return
+ # Drop optional fields if empty tensor.
+ groundtruth_is_crowd = groundtruth_dict.get(
+ standard_fields.InputDataFields.groundtruth_is_crowd)
+ groundtruth_area = groundtruth_dict.get(
+ standard_fields.InputDataFields.groundtruth_area)
+ if groundtruth_is_crowd is not None and not groundtruth_is_crowd.shape[0]:
+ groundtruth_is_crowd = None
+ if groundtruth_area is not None and not groundtruth_area.shape[0]:
+ groundtruth_area = None
+
groundtruth_instance_masks = groundtruth_dict[
standard_fields.InputDataFields.groundtruth_instance_masks]
groundtruth_instance_masks = convert_masks_to_binary(
@@ -960,7 +1055,9 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
groundtruth_classes=groundtruth_dict[standard_fields.
InputDataFields.
groundtruth_classes],
- groundtruth_masks=groundtruth_instance_masks))
+ groundtruth_masks=groundtruth_instance_masks,
+ groundtruth_is_crowd=groundtruth_is_crowd,
+ groundtruth_area=groundtruth_area))
self._annotation_id += groundtruth_dict[standard_fields.InputDataFields.
groundtruth_boxes].shape[0]
self._image_id_to_mask_shape_map[image_id] = groundtruth_dict[
@@ -1067,6 +1164,9 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
no supercategories exist). For backward compatibility
'PerformanceByCategory' is included in the output regardless of
all_metrics_per_category.
+ If super_categories are provided, then this will additionally include
+ metrics aggregated along the super_categories with keys of the form:
+ `PerformanceBySuperCategory/`
"""
groundtruth_dict = {
'annotations': self._groundtruth_list,
@@ -1083,7 +1183,9 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
coco_wrapped_groundtruth, coco_wrapped_detection_masks,
agnostic_mode=False, iou_type='segm')
mask_metrics, mask_per_category_ap = mask_evaluator.ComputeMetrics(
- include_metrics_per_category=self._include_metrics_per_category)
+ include_metrics_per_category=self._include_metrics_per_category,
+ super_categories=self._super_categories,
+ all_metrics_per_category=self._all_metrics_per_category)
mask_metrics.update(mask_per_category_ap)
mask_metrics = {'DetectionMasks_'+ key: value
for key, value in mask_metrics.items()}
@@ -1112,18 +1214,20 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
groundtruth_instance_masks_batched,
groundtruth_is_crowd_batched, num_gt_boxes_per_image,
detection_scores_batched, detection_classes_batched,
- detection_masks_batched, num_det_boxes_per_image):
+ detection_masks_batched, num_det_boxes_per_image,
+ original_image_spatial_shape):
"""Update op for metrics."""
for (image_id, groundtruth_boxes, groundtruth_classes,
groundtruth_instance_masks, groundtruth_is_crowd, num_gt_box,
detection_scores, detection_classes,
- detection_masks, num_det_box) in zip(
+ detection_masks, num_det_box, original_image_shape) in zip(
image_id_batched, groundtruth_boxes_batched,
groundtruth_classes_batched, groundtruth_instance_masks_batched,
groundtruth_is_crowd_batched, num_gt_boxes_per_image,
detection_scores_batched, detection_classes_batched,
- detection_masks_batched, num_det_boxes_per_image):
+ detection_masks_batched, num_det_boxes_per_image,
+ original_image_spatial_shape):
self.add_single_ground_truth_image_info(
image_id, {
'groundtruth_boxes':
@@ -1131,7 +1235,10 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
'groundtruth_classes':
groundtruth_classes[:num_gt_box],
'groundtruth_instance_masks':
- groundtruth_instance_masks[:num_gt_box],
+ groundtruth_instance_masks[
+ :num_gt_box,
+ :original_image_shape[0],
+ :original_image_shape[1]],
'groundtruth_is_crowd':
groundtruth_is_crowd[:num_gt_box]
})
@@ -1139,13 +1246,18 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
image_id, {
'detection_scores': detection_scores[:num_det_box],
'detection_classes': detection_classes[:num_det_box],
- 'detection_masks': detection_masks[:num_det_box]
+ 'detection_masks': detection_masks[
+ :num_det_box,
+ :original_image_shape[0],
+ :original_image_shape[1]]
})
# Unpack items from the evaluation dictionary.
input_data_fields = standard_fields.InputDataFields
detection_fields = standard_fields.DetectionResultFields
image_id = eval_dict[input_data_fields.key]
+ original_image_spatial_shape = eval_dict[
+ input_data_fields.original_image_spatial_shape]
groundtruth_boxes = eval_dict[input_data_fields.groundtruth_boxes]
groundtruth_classes = eval_dict[input_data_fields.groundtruth_classes]
groundtruth_instance_masks = eval_dict[
@@ -1197,7 +1309,7 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
image_id, groundtruth_boxes, groundtruth_classes,
groundtruth_instance_masks, groundtruth_is_crowd,
num_gt_boxes_per_image, detection_scores, detection_classes,
- detection_masks, num_det_boxes_per_image
+ detection_masks, num_det_boxes_per_image, original_image_spatial_shape
], [])
def get_estimator_eval_metric_ops(self, eval_dict):
@@ -1224,15 +1336,15 @@ class CocoMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
metric_names = ['DetectionMasks_Precision/mAP',
'DetectionMasks_Precision/mAP@.50IOU',
'DetectionMasks_Precision/mAP@.75IOU',
- 'DetectionMasks_Precision/mAP (large)',
- 'DetectionMasks_Precision/mAP (medium)',
'DetectionMasks_Precision/mAP (small)',
+ 'DetectionMasks_Precision/mAP (medium)',
+ 'DetectionMasks_Precision/mAP (large)',
'DetectionMasks_Recall/AR@1',
'DetectionMasks_Recall/AR@10',
'DetectionMasks_Recall/AR@100',
- 'DetectionMasks_Recall/AR@100 (large)',
+ 'DetectionMasks_Recall/AR@100 (small)',
'DetectionMasks_Recall/AR@100 (medium)',
- 'DetectionMasks_Recall/AR@100 (small)']
+ 'DetectionMasks_Recall/AR@100 (large)']
if self._include_metrics_per_category:
for category_dict in self._categories:
metric_names.append('DetectionMasks_PerformanceByCategory/mAP/' +
diff --git a/research/object_detection/metrics/coco_evaluation_test.py b/research/object_detection/metrics/coco_evaluation_test.py
index 110690bf211fb9ab903c3df433e542a977c64ff0..8cfb3ee5a19ea242e625cba50841ff107e918d95 100644
--- a/research/object_detection/metrics/coco_evaluation_test.py
+++ b/research/object_detection/metrics/coco_evaluation_test.py
@@ -255,9 +255,7 @@ class CocoDetectionEvaluationTest(tf.test.TestCase):
@unittest.skipIf(tf_version.is_tf2(), 'Only Supported in TF1.X')
class CocoEvaluationPyFuncTest(tf.test.TestCase):
- def testGetOneMAPWithMatchingGroundtruthAndDetections(self):
- coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
- _get_categories_list())
+ def _MatchingGroundtruthAndDetections(self, coco_evaluator):
image_id = tf.placeholder(tf.string, shape=())
groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4))
groundtruth_classes = tf.placeholder(tf.float32, shape=(None))
@@ -330,6 +328,121 @@ class CocoEvaluationPyFuncTest(tf.test.TestCase):
self.assertFalse(coco_evaluator._detection_boxes_list)
self.assertFalse(coco_evaluator._image_ids)
+ def testGetOneMAPWithMatchingGroundtruthAndDetections(self):
+ coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
+ _get_categories_list())
+ self._MatchingGroundtruthAndDetections(coco_evaluator)
+
+ # Configured to skip unmatched detector predictions with
+ # groundtruth_labeled_classes, but reverts to fully-labeled eval since there
+ # are no groundtruth_labeled_classes set.
+ def testGetMAPWithSkipUnmatchedPredictionsIgnoreGrountruthLabeledClasses(
+ self):
+ coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
+ _get_categories_list(), skip_predictions_for_unlabeled_class=True)
+ self._MatchingGroundtruthAndDetections(coco_evaluator)
+
+ # Test skipping unmatched detector predictions with
+ # groundtruth_labeled_classes.
+ def testGetMAPWithSkipUnmatchedPredictions(self):
+ coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
+ _get_categories_list(), skip_predictions_for_unlabeled_class=True)
+ image_id = tf.placeholder(tf.string, shape=())
+ groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4))
+ groundtruth_classes = tf.placeholder(tf.float32, shape=(None))
+ groundtruth_labeled_classes = tf.placeholder(tf.float32, shape=(None))
+ detection_boxes = tf.placeholder(tf.float32, shape=(None, 4))
+ detection_scores = tf.placeholder(tf.float32, shape=(None))
+ detection_classes = tf.placeholder(tf.float32, shape=(None))
+
+ input_data_fields = standard_fields.InputDataFields
+ detection_fields = standard_fields.DetectionResultFields
+ eval_dict = {
+ input_data_fields.key:
+ image_id,
+ input_data_fields.groundtruth_boxes:
+ groundtruth_boxes,
+ input_data_fields.groundtruth_classes:
+ groundtruth_classes,
+ input_data_fields.groundtruth_labeled_classes:
+ groundtruth_labeled_classes,
+ detection_fields.detection_boxes:
+ detection_boxes,
+ detection_fields.detection_scores:
+ detection_scores,
+ detection_fields.detection_classes:
+ detection_classes
+ }
+
+ eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict)
+
+ _, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP']
+
+ with self.test_session() as sess:
+ sess.run(
+ update_op,
+ feed_dict={
+ image_id:
+ 'image1',
+ groundtruth_boxes:
+ np.array([[100., 100., 200., 200.]]),
+ groundtruth_classes:
+ np.array([1]),
+ # Only class 1 is exhaustively labeled for image1.
+ groundtruth_labeled_classes:
+ np.array([0., 1., 0., 0.]),
+ detection_boxes:
+ np.array([[100., 100., 200., 200.], [100., 100., 200.,
+ 200.]]),
+ detection_scores:
+ np.array([.8, .95]),
+ detection_classes:
+ np.array([1, 2])
+ })
+ sess.run(
+ update_op,
+ feed_dict={
+ image_id: 'image2',
+ groundtruth_boxes: np.array([[50., 50., 100., 100.]]),
+ groundtruth_classes: np.array([3]),
+ groundtruth_labeled_classes: np.array([0., 0., 0., 1.]),
+ detection_boxes: np.array([[50., 50., 100., 100.]]),
+ detection_scores: np.array([.7]),
+ detection_classes: np.array([3])
+ })
+ sess.run(
+ update_op,
+ feed_dict={
+ image_id: 'image3',
+ groundtruth_boxes: np.array([[25., 25., 50., 50.]]),
+ groundtruth_classes: np.array([2]),
+ groundtruth_labeled_classes: np.array([0., 0., 1., 0.]),
+ detection_boxes: np.array([[25., 25., 50., 50.]]),
+ detection_scores: np.array([.9]),
+ detection_classes: np.array([2])
+ })
+ metrics = {}
+ for key, (value_op, _) in eval_metric_ops.items():
+ metrics[key] = value_op
+ metrics = sess.run(metrics)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.50IOU'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.75IOU'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'],
+ 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'],
+ 1.0)
+ self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0)
+ self.assertFalse(coco_evaluator._groundtruth_list)
+ self.assertFalse(coco_evaluator._detection_boxes_list)
+ self.assertFalse(coco_evaluator._image_ids)
+
def testGetOneMAPWithMatchingGroundtruthAndDetectionsIsAnnotated(self):
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
_get_categories_list())
@@ -1443,6 +1556,41 @@ class CocoMaskEvaluationTest(tf.test.TestCase):
self.assertFalse(coco_evaluator._groundtruth_list)
self.assertFalse(coco_evaluator._detection_masks_list)
+ def testGetOneMAPWithMatchingGroundtruthAndDetectionsSkipCrowd(self):
+ """Tests computing mAP with is_crowd GT boxes skipped."""
+ coco_evaluator = coco_evaluation.CocoMaskEvaluator(
+ _get_categories_list())
+ coco_evaluator.add_single_ground_truth_image_info(
+ image_id='image1',
+ groundtruth_dict={
+ standard_fields.InputDataFields.groundtruth_boxes:
+ np.array([[100., 100., 200., 200.], [99., 99., 200., 200.]]),
+ standard_fields.InputDataFields.groundtruth_classes:
+ np.array([1, 2]),
+ standard_fields.InputDataFields.groundtruth_is_crowd:
+ np.array([0, 1]),
+ standard_fields.InputDataFields.groundtruth_instance_masks:
+ np.concatenate(
+ [np.pad(np.ones([1, 100, 100], dtype=np.uint8),
+ ((0, 0), (100, 56), (100, 56)), mode='constant'),
+ np.pad(np.ones([1, 101, 101], dtype=np.uint8),
+ ((0, 0), (99, 56), (99, 56)), mode='constant')],
+ axis=0)
+ })
+ coco_evaluator.add_single_detected_image_info(
+ image_id='image1',
+ detections_dict={
+ standard_fields.DetectionResultFields.detection_scores:
+ np.array([.8]),
+ standard_fields.DetectionResultFields.detection_classes:
+ np.array([1]),
+ standard_fields.DetectionResultFields.detection_masks:
+ np.pad(np.ones([1, 100, 100], dtype=np.uint8),
+ ((0, 0), (100, 56), (100, 56)), mode='constant')
+ })
+ metrics = coco_evaluator.evaluate()
+ self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP'], 1.0)
+
@unittest.skipIf(tf_version.is_tf2(), 'Only Supported in TF1.X')
class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
@@ -1453,6 +1601,7 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4))
groundtruth_classes = tf.placeholder(tf.float32, shape=(None))
groundtruth_masks = tf.placeholder(tf.uint8, shape=(None, None, None))
+ original_image_spatial_shape = tf.placeholder(tf.int32, shape=(None, 2))
detection_scores = tf.placeholder(tf.float32, shape=(None))
detection_classes = tf.placeholder(tf.float32, shape=(None))
detection_masks = tf.placeholder(tf.uint8, shape=(None, None, None))
@@ -1464,6 +1613,8 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
input_data_fields.groundtruth_boxes: groundtruth_boxes,
input_data_fields.groundtruth_classes: groundtruth_classes,
input_data_fields.groundtruth_instance_masks: groundtruth_masks,
+ input_data_fields.original_image_spatial_shape:
+ original_image_spatial_shape,
detection_fields.detection_scores: detection_scores,
detection_fields.detection_classes: detection_classes,
detection_fields.detection_masks: detection_masks,
@@ -1489,6 +1640,7 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
np.ones([50, 50], dtype=np.uint8), ((0, 70), (0, 70)),
mode='constant')
]),
+ original_image_spatial_shape: np.array([[120, 120]]),
detection_scores:
np.array([.9, .8]),
detection_classes:
@@ -1513,6 +1665,7 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4))
groundtruth_classes = tf.placeholder(tf.float32, shape=(None))
groundtruth_masks = tf.placeholder(tf.uint8, shape=(None, None, None))
+ original_image_spatial_shape = tf.placeholder(tf.int32, shape=(None, 2))
detection_scores = tf.placeholder(tf.float32, shape=(None))
detection_classes = tf.placeholder(tf.float32, shape=(None))
detection_masks = tf.placeholder(tf.uint8, shape=(None, None, None))
@@ -1524,6 +1677,8 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
input_data_fields.groundtruth_boxes: groundtruth_boxes,
input_data_fields.groundtruth_classes: groundtruth_classes,
input_data_fields.groundtruth_instance_masks: groundtruth_masks,
+ input_data_fields.original_image_spatial_shape:
+ original_image_spatial_shape,
detection_fields.detection_scores: detection_scores,
detection_fields.detection_classes: detection_classes,
detection_fields.detection_masks: detection_masks,
@@ -1553,6 +1708,7 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
np.ones([50, 50], dtype=np.uint8), ((0, 70), (0, 70)),
mode='constant')
]),
+ original_image_spatial_shape: np.array([[120, 120], [120, 120]]),
detection_scores:
np.array([.9, .8]),
detection_classes:
@@ -1577,6 +1733,7 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
dtype=np.uint8),
((0, 0), (10, 10), (10, 10)),
mode='constant'),
+ original_image_spatial_shape: np.array([[70, 70]]),
detection_scores: np.array([.8]),
detection_classes: np.array([1]),
detection_masks: np.pad(np.ones([1, 50, 50], dtype=np.uint8),
@@ -1592,6 +1749,7 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
dtype=np.uint8),
((0, 0), (10, 10), (10, 10)),
mode='constant'),
+ original_image_spatial_shape: np.array([[45, 45]]),
detection_scores: np.array([.8]),
detection_classes: np.array([1]),
detection_masks: np.pad(np.ones([1, 25, 25],
@@ -1630,6 +1788,7 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
groundtruth_classes = tf.placeholder(tf.float32, shape=(batch_size, None))
groundtruth_masks = tf.placeholder(
tf.uint8, shape=(batch_size, None, None, None))
+ original_image_spatial_shape = tf.placeholder(tf.int32, shape=(None, 2))
detection_scores = tf.placeholder(tf.float32, shape=(batch_size, None))
detection_classes = tf.placeholder(tf.float32, shape=(batch_size, None))
detection_masks = tf.placeholder(
@@ -1642,6 +1801,8 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
input_data_fields.groundtruth_boxes: groundtruth_boxes,
input_data_fields.groundtruth_classes: groundtruth_classes,
input_data_fields.groundtruth_instance_masks: groundtruth_masks,
+ input_data_fields.original_image_spatial_shape:
+ original_image_spatial_shape,
detection_fields.detection_scores: detection_scores,
detection_fields.detection_classes: detection_classes,
detection_fields.detection_masks: detection_masks,
@@ -1678,6 +1839,8 @@ class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
mode='constant')
],
axis=0),
+ original_image_spatial_shape: np.array(
+ [[100, 100], [100, 100], [100, 100]]),
detection_scores:
np.array([[.8], [.8], [.8]]),
detection_classes:
diff --git a/research/object_detection/metrics/coco_tools.py b/research/object_detection/metrics/coco_tools.py
index 790d5bdef23bef149e8eb1afa9cdecb9ce458e6e..b3c5a92765fa3e8d49eb0701c3284d3fdab0c6fe 100644
--- a/research/object_detection/metrics/coco_tools.py
+++ b/research/object_detection/metrics/coco_tools.py
@@ -142,6 +142,35 @@ class COCOWrapper(coco.COCO):
return results
+COCO_METRIC_NAMES_AND_INDEX = (
+ ('Precision/mAP', 0),
+ ('Precision/mAP@.50IOU', 1),
+ ('Precision/mAP@.75IOU', 2),
+ ('Precision/mAP (small)', 3),
+ ('Precision/mAP (medium)', 4),
+ ('Precision/mAP (large)', 5),
+ ('Recall/AR@1', 6),
+ ('Recall/AR@10', 7),
+ ('Recall/AR@100', 8),
+ ('Recall/AR@100 (small)', 9),
+ ('Recall/AR@100 (medium)', 10),
+ ('Recall/AR@100 (large)', 11)
+)
+
+COCO_KEYPOINT_METRIC_NAMES_AND_INDEX = (
+ ('Precision/mAP', 0),
+ ('Precision/mAP@.50IOU', 1),
+ ('Precision/mAP@.75IOU', 2),
+ ('Precision/mAP (medium)', 3),
+ ('Precision/mAP (large)', 4),
+ ('Recall/AR@1', 5),
+ ('Recall/AR@10', 6),
+ ('Recall/AR@100', 7),
+ ('Recall/AR@100 (medium)', 8),
+ ('Recall/AR@100 (large)', 9)
+)
+
+
class COCOEvalWrapper(cocoeval.COCOeval):
"""Wrapper for the pycocotools COCOeval class.
@@ -202,7 +231,8 @@ class COCOEvalWrapper(cocoeval.COCOeval):
def ComputeMetrics(self,
include_metrics_per_category=False,
- all_metrics_per_category=False):
+ all_metrics_per_category=False,
+ super_categories=None):
"""Computes detection/keypoint metrics.
Args:
@@ -211,6 +241,11 @@ class COCOEvalWrapper(cocoeval.COCOeval):
each category in per_category_ap. Be careful with setting it to true if
you have more than handful of categories, because it will pollute
your mldash.
+ super_categories: None or a python dict mapping super-category names
+ (strings) to lists of categories (corresponding to category names
+ in the label_map). Metrics are aggregated along these super-categories
+ and added to the `per_category_ap` and are associated with the name
+ `PerformanceBySuperCategory/`.
Returns:
1. summary_metrics: a dictionary holding:
@@ -240,6 +275,9 @@ class COCOEvalWrapper(cocoeval.COCOeval):
output regardless of all_metrics_per_category.
If evaluating class-agnostic mode, per_category_ap is an empty
dictionary.
+ If super_categories are provided, then this will additionally include
+ metrics aggregated along the super_categories with keys of the form:
+ `PerformanceBySuperCategory/`
Raises:
ValueError: If category_stats does not exist.
@@ -250,80 +288,71 @@ class COCOEvalWrapper(cocoeval.COCOeval):
summary_metrics = {}
if self._iou_type in ['bbox', 'segm']:
- summary_metrics = OrderedDict([('Precision/mAP', self.stats[0]),
- ('Precision/mAP@.50IOU', self.stats[1]),
- ('Precision/mAP@.75IOU', self.stats[2]),
- ('Precision/mAP (small)', self.stats[3]),
- ('Precision/mAP (medium)', self.stats[4]),
- ('Precision/mAP (large)', self.stats[5]),
- ('Recall/AR@1', self.stats[6]),
- ('Recall/AR@10', self.stats[7]),
- ('Recall/AR@100', self.stats[8]),
- ('Recall/AR@100 (small)', self.stats[9]),
- ('Recall/AR@100 (medium)', self.stats[10]),
- ('Recall/AR@100 (large)', self.stats[11])])
+ summary_metrics = OrderedDict(
+ [(name, self.stats[index]) for name, index in
+ COCO_METRIC_NAMES_AND_INDEX])
elif self._iou_type == 'keypoints':
category_id = self.GetCategoryIdList()[0]
category_name = self.GetCategory(category_id)['name']
summary_metrics = OrderedDict([])
- summary_metrics['Precision/mAP ByCategory/{}'.format(
- category_name)] = self.stats[0]
- summary_metrics['Precision/mAP@.50IOU ByCategory/{}'.format(
- category_name)] = self.stats[1]
- summary_metrics['Precision/mAP@.75IOU ByCategory/{}'.format(
- category_name)] = self.stats[2]
- summary_metrics['Precision/mAP (medium) ByCategory/{}'.format(
- category_name)] = self.stats[3]
- summary_metrics['Precision/mAP (large) ByCategory/{}'.format(
- category_name)] = self.stats[4]
- summary_metrics['Recall/AR@1 ByCategory/{}'.format(
- category_name)] = self.stats[5]
- summary_metrics['Recall/AR@10 ByCategory/{}'.format(
- category_name)] = self.stats[6]
- summary_metrics['Recall/AR@100 ByCategory/{}'.format(
- category_name)] = self.stats[7]
- summary_metrics['Recall/AR@100 (medium) ByCategory/{}'.format(
- category_name)] = self.stats[8]
- summary_metrics['Recall/AR@100 (large) ByCategory/{}'.format(
- category_name)] = self.stats[9]
+ for metric_name, index in COCO_KEYPOINT_METRIC_NAMES_AND_INDEX:
+ value = self.stats[index]
+ summary_metrics['{} ByCategory/{}'.format(
+ metric_name, category_name)] = value
if not include_metrics_per_category:
return summary_metrics, {}
if not hasattr(self, 'category_stats'):
raise ValueError('Category stats do not exist')
per_category_ap = OrderedDict([])
+ super_category_ap = OrderedDict([])
if self.GetAgnosticMode():
return summary_metrics, per_category_ap
+
+ if super_categories:
+ for key in super_categories:
+ super_category_ap['PerformanceBySuperCategory/{}'.format(key)] = 0
+
+ if all_metrics_per_category:
+ for metric_name, _ in COCO_METRIC_NAMES_AND_INDEX:
+ metric_key = '{} BySuperCategory/{}'.format(metric_name, key)
+ super_category_ap[metric_key] = 0
+
for category_index, category_id in enumerate(self.GetCategoryIdList()):
category = self.GetCategory(category_id)['name']
# Kept for backward compatilbility
per_category_ap['PerformanceByCategory/mAP/{}'.format(
category)] = self.category_stats[0][category_index]
- if all_metrics_per_category:
- per_category_ap['Precision mAP ByCategory/{}'.format(
- category)] = self.category_stats[0][category_index]
- per_category_ap['Precision mAP@.50IOU ByCategory/{}'.format(
- category)] = self.category_stats[1][category_index]
- per_category_ap['Precision mAP@.75IOU ByCategory/{}'.format(
- category)] = self.category_stats[2][category_index]
- per_category_ap['Precision mAP (small) ByCategory/{}'.format(
- category)] = self.category_stats[3][category_index]
- per_category_ap['Precision mAP (medium) ByCategory/{}'.format(
- category)] = self.category_stats[4][category_index]
- per_category_ap['Precision mAP (large) ByCategory/{}'.format(
- category)] = self.category_stats[5][category_index]
- per_category_ap['Recall AR@1 ByCategory/{}'.format(
- category)] = self.category_stats[6][category_index]
- per_category_ap['Recall AR@10 ByCategory/{}'.format(
- category)] = self.category_stats[7][category_index]
- per_category_ap['Recall AR@100 ByCategory/{}'.format(
- category)] = self.category_stats[8][category_index]
- per_category_ap['Recall AR@100 (small) ByCategory/{}'.format(
- category)] = self.category_stats[9][category_index]
- per_category_ap['Recall AR@100 (medium) ByCategory/{}'.format(
- category)] = self.category_stats[10][category_index]
- per_category_ap['Recall AR@100 (large) ByCategory/{}'.format(
- category)] = self.category_stats[11][category_index]
+ if all_metrics_per_category:
+ for metric_name, index in COCO_METRIC_NAMES_AND_INDEX:
+ metric_key = '{} ByCategory/{}'.format(metric_name, category)
+ per_category_ap[metric_key] = self.category_stats[index][
+ category_index]
+
+ if super_categories:
+ for key in super_categories:
+ if category in super_categories[key]:
+ metric_key = 'PerformanceBySuperCategory/{}'.format(key)
+ super_category_ap[metric_key] += self.category_stats[0][
+ category_index]
+ if all_metrics_per_category:
+ for metric_name, index in COCO_METRIC_NAMES_AND_INDEX:
+ metric_key = '{} BySuperCategory/{}'.format(metric_name, key)
+ super_category_ap[metric_key] += (
+ self.category_stats[index][category_index])
+
+ if super_categories:
+ for key in super_categories:
+ length = len(super_categories[key])
+ super_category_ap['PerformanceBySuperCategory/{}'.format(
+ key)] /= length
+
+ if all_metrics_per_category:
+ for metric_name, _ in COCO_METRIC_NAMES_AND_INDEX:
+ super_category_ap['{} BySuperCategory/{}'.format(
+ metric_name, key)] /= length
+
+ per_category_ap.update(super_category_ap)
return summary_metrics, per_category_ap
diff --git a/research/object_detection/metrics/lvis_evaluation.py b/research/object_detection/metrics/lvis_evaluation.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fbd6e427143e96739571ad05c54bd22b31fcc39
--- /dev/null
+++ b/research/object_detection/metrics/lvis_evaluation.py
@@ -0,0 +1,463 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class for evaluating object detections with LVIS metrics."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import json
+import re
+
+from lvis import results as lvis_results
+
+import numpy as np
+from six.moves import zip
+import tensorflow.compat.v1 as tf
+
+from object_detection.core import standard_fields as fields
+from object_detection.metrics import lvis_tools
+from object_detection.utils import object_detection_evaluation
+
+
+def convert_masks_to_binary(masks):
+ """Converts masks to 0 or 1 and uint8 type."""
+ return (masks > 0).astype(np.uint8)
+
+
+class LVISMaskEvaluator(object_detection_evaluation.DetectionEvaluator):
+ """Class to evaluate LVIS mask metrics."""
+
+ def __init__(self,
+ categories,
+ include_metrics_per_category=False,
+ export_path=None):
+ """Constructor.
+
+ Args:
+ categories: A list of dicts, each of which has the following keys -
+ 'id': (required) an integer id uniquely identifying this category.
+ 'name': (required) string representing category name e.g., 'cat', 'dog'.
+ include_metrics_per_category: Additionally include per-category metrics
+ (this option is currently unsupported).
+ export_path: Path to export detections to LVIS compatible JSON format.
+ """
+ super(LVISMaskEvaluator, self).__init__(categories)
+ self._image_ids_with_detections = set([])
+ self._groundtruth_list = []
+ self._detection_masks_list = []
+ self._category_id_set = set([cat['id'] for cat in self._categories])
+ self._annotation_id = 1
+ self._image_id_to_mask_shape_map = {}
+ self._image_id_to_verified_neg_classes = {}
+ self._image_id_to_not_exhaustive_classes = {}
+ if include_metrics_per_category:
+ raise ValueError('include_metrics_per_category not yet supported '
+ 'for LVISMaskEvaluator.')
+ self._export_path = export_path
+
+ def clear(self):
+ """Clears the state to prepare for a fresh evaluation."""
+ self._image_id_to_mask_shape_map.clear()
+ self._image_ids_with_detections.clear()
+ self._image_id_to_verified_neg_classes.clear()
+ self._image_id_to_not_exhaustive_classes.clear()
+ self._groundtruth_list = []
+ self._detection_masks_list = []
+
+ def add_single_ground_truth_image_info(self,
+ image_id,
+ groundtruth_dict):
+ """Adds groundtruth for a single image to be used for evaluation.
+
+ If the image has already been added, a warning is logged, and groundtruth is
+ ignored.
+
+ Args:
+ image_id: A unique string/integer identifier for the image.
+ groundtruth_dict: A dictionary containing -
+ InputDataFields.groundtruth_boxes: float32 numpy array of shape
+ [num_boxes, 4] containing `num_boxes` groundtruth boxes of the format
+ [ymin, xmin, ymax, xmax] in absolute image coordinates.
+ InputDataFields.groundtruth_classes: integer numpy array of shape
+ [num_boxes] containing 1-indexed groundtruth classes for the boxes.
+ InputDataFields.groundtruth_instance_masks: uint8 numpy array of shape
+ [num_masks, image_height, image_width] containing groundtruth masks.
+ The elements of the array must be in {0, 1}.
+ InputDataFields.groundtruth_verified_neg_classes: [num_classes + 1]
+ float indicator vector with values in {0, 1}. The length is
+ num_classes + 1 so as to be compatible with the 1-indexed groundtruth
+ classes.
+ InputDataFields.groundtruth_not_exhaustive_classes: [num_classes + 1]
+ float indicator vector with values in {0, 1}. The length is
+ num_classes + 1 so as to be compatible with the 1-indexed groundtruth
+ classes.
+ InputDataFields.groundtruth_area (optional): float numpy array of
+ shape [num_boxes] containing the area (in the original absolute
+ coordinates) of the annotated object.
+ Raises:
+ ValueError: if groundtruth_dict is missing a required field
+ """
+ if image_id in self._image_id_to_mask_shape_map:
+ tf.logging.warning('Ignoring ground truth with image id %s since it was '
+ 'previously added', image_id)
+ return
+ for key in [fields.InputDataFields.groundtruth_boxes,
+ fields.InputDataFields.groundtruth_classes,
+ fields.InputDataFields.groundtruth_instance_masks,
+ fields.InputDataFields.groundtruth_verified_neg_classes,
+ fields.InputDataFields.groundtruth_not_exhaustive_classes]:
+ if key not in groundtruth_dict.keys():
+ raise ValueError('groundtruth_dict missing entry: {}'.format(key))
+
+ groundtruth_instance_masks = groundtruth_dict[
+ fields.InputDataFields.groundtruth_instance_masks]
+ groundtruth_instance_masks = convert_masks_to_binary(
+ groundtruth_instance_masks)
+ verified_neg_classes_shape = groundtruth_dict[
+ fields.InputDataFields.groundtruth_verified_neg_classes].shape
+ not_exhaustive_classes_shape = groundtruth_dict[
+ fields.InputDataFields.groundtruth_not_exhaustive_classes].shape
+ if verified_neg_classes_shape != (len(self._category_id_set) + 1,):
+ raise ValueError('Invalid shape for verified_neg_classes_shape.')
+ if not_exhaustive_classes_shape != (len(self._category_id_set) + 1,):
+ raise ValueError('Invalid shape for not_exhaustive_classes_shape.')
+ self._image_id_to_verified_neg_classes[image_id] = np.flatnonzero(
+ groundtruth_dict[
+ fields.InputDataFields.groundtruth_verified_neg_classes]
+ == 1).tolist()
+ self._image_id_to_not_exhaustive_classes[image_id] = np.flatnonzero(
+ groundtruth_dict[
+ fields.InputDataFields.groundtruth_not_exhaustive_classes]
+ == 1).tolist()
+
+ # Drop optional fields if empty tensor.
+ groundtruth_area = groundtruth_dict.get(
+ fields.InputDataFields.groundtruth_area)
+ if groundtruth_area is not None and not groundtruth_area.shape[0]:
+ groundtruth_area = None
+
+ self._groundtruth_list.extend(
+ lvis_tools.ExportSingleImageGroundtruthToLVIS(
+ image_id=image_id,
+ next_annotation_id=self._annotation_id,
+ category_id_set=self._category_id_set,
+ groundtruth_boxes=groundtruth_dict[
+ fields.InputDataFields.groundtruth_boxes],
+ groundtruth_classes=groundtruth_dict[
+ fields.InputDataFields.groundtruth_classes],
+ groundtruth_masks=groundtruth_instance_masks,
+ groundtruth_area=groundtruth_area)
+ )
+
+ self._annotation_id += groundtruth_dict[fields.InputDataFields.
+ groundtruth_boxes].shape[0]
+ self._image_id_to_mask_shape_map[image_id] = groundtruth_dict[
+ fields.InputDataFields.groundtruth_instance_masks].shape
+
+ def add_single_detected_image_info(self,
+ image_id,
+ detections_dict):
+ """Adds detections for a single image to be used for evaluation.
+
+ If a detection has already been added for this image id, a warning is
+ logged, and the detection is skipped.
+
+ Args:
+ image_id: A unique string/integer identifier for the image.
+ detections_dict: A dictionary containing -
+ DetectionResultFields.detection_scores: float32 numpy array of shape
+ [num_boxes] containing detection scores for the boxes.
+ DetectionResultFields.detection_classes: integer numpy array of shape
+ [num_boxes] containing 1-indexed detection classes for the boxes.
+ DetectionResultFields.detection_masks: optional uint8 numpy array of
+ shape [num_boxes, image_height, image_width] containing instance
+ masks corresponding to the boxes. The elements of the array must be
+ in {0, 1}.
+ Raises:
+ ValueError: If groundtruth for the image_id is not available.
+ """
+ if image_id not in self._image_id_to_mask_shape_map:
+ raise ValueError('Missing groundtruth for image id: {}'.format(image_id))
+
+ if image_id in self._image_ids_with_detections:
+ tf.logging.warning('Ignoring detection with image id %s since it was '
+ 'previously added', image_id)
+ return
+
+ groundtruth_masks_shape = self._image_id_to_mask_shape_map[image_id]
+ detection_masks = detections_dict[fields.DetectionResultFields.
+ detection_masks]
+ if groundtruth_masks_shape[1:] != detection_masks.shape[1:]:
+ raise ValueError('Spatial shape of groundtruth masks and detection masks '
+ 'are incompatible: {} vs {}'.format(
+ groundtruth_masks_shape,
+ detection_masks.shape))
+ detection_masks = convert_masks_to_binary(detection_masks)
+
+ self._detection_masks_list.extend(
+ lvis_tools.ExportSingleImageDetectionMasksToLVIS(
+ image_id=image_id,
+ category_id_set=self._category_id_set,
+ detection_masks=detection_masks,
+ detection_scores=detections_dict[
+ fields.DetectionResultFields.detection_scores],
+ detection_classes=detections_dict[
+ fields.DetectionResultFields.detection_classes]))
+ self._image_ids_with_detections.update([image_id])
+
+ def evaluate(self):
+ """Evaluates the detection boxes and returns a dictionary of coco metrics.
+
+ Returns:
+ A dictionary holding
+ """
+ if self._export_path:
+ tf.logging.info('Dumping detections to json.')
+ self.dump_detections_to_json_file(self._export_path)
+ tf.logging.info('Performing evaluation on %d images.',
+ len(self._image_id_to_mask_shape_map.keys()))
+ # pylint: disable=g-complex-comprehension
+ groundtruth_dict = {
+ 'annotations': self._groundtruth_list,
+ 'images': [
+ {
+ 'id': int(image_id),
+ 'height': shape[1],
+ 'width': shape[2],
+ 'neg_category_ids':
+ self._image_id_to_verified_neg_classes[image_id],
+ 'not_exhaustive_category_ids':
+ self._image_id_to_not_exhaustive_classes[image_id]
+ } for image_id, shape in self._image_id_to_mask_shape_map.items()],
+ 'categories': self._categories
+ }
+ # pylint: enable=g-complex-comprehension
+ lvis_wrapped_groundtruth = lvis_tools.LVISWrapper(groundtruth_dict)
+ detections = lvis_results.LVISResults(lvis_wrapped_groundtruth,
+ self._detection_masks_list)
+ mask_evaluator = lvis_tools.LVISEvalWrapper(
+ lvis_wrapped_groundtruth, detections, iou_type='segm')
+ mask_metrics = mask_evaluator.ComputeMetrics()
+ mask_metrics = {'DetectionMasks_'+ key: value
+ for key, value in iter(mask_metrics.items())}
+ return mask_metrics
+
+ def add_eval_dict(self, eval_dict):
+ """Observes an evaluation result dict for a single example.
+
+ When executing eagerly, once all observations have been observed by this
+ method you can use `.evaluate()` to get the final metrics.
+
+ When using `tf.estimator.Estimator` for evaluation this function is used by
+ `get_estimator_eval_metric_ops()` to construct the metric update op.
+
+ Args:
+ eval_dict: A dictionary that holds tensors for evaluating an object
+ detection model, returned from
+ eval_util.result_dict_for_single_example().
+
+ Returns:
+ None when executing eagerly, or an update_op that can be used to update
+ the eval metrics in `tf.estimator.EstimatorSpec`.
+ """
+ def update_op(image_id_batched, groundtruth_boxes_batched,
+ groundtruth_classes_batched,
+ groundtruth_instance_masks_batched,
+ groundtruth_verified_neg_classes_batched,
+ groundtruth_not_exhaustive_classes_batched,
+ num_gt_boxes_per_image,
+ detection_scores_batched, detection_classes_batched,
+ detection_masks_batched, num_det_boxes_per_image,
+ original_image_spatial_shape):
+ """Update op for metrics."""
+
+ for (image_id, groundtruth_boxes, groundtruth_classes,
+ groundtruth_instance_masks, groundtruth_verified_neg_classes,
+ groundtruth_not_exhaustive_classes, num_gt_box,
+ detection_scores, detection_classes,
+ detection_masks, num_det_box, original_image_shape) in zip(
+ image_id_batched, groundtruth_boxes_batched,
+ groundtruth_classes_batched, groundtruth_instance_masks_batched,
+ groundtruth_verified_neg_classes_batched,
+ groundtruth_not_exhaustive_classes_batched,
+ num_gt_boxes_per_image,
+ detection_scores_batched, detection_classes_batched,
+ detection_masks_batched, num_det_boxes_per_image,
+ original_image_spatial_shape):
+ self.add_single_ground_truth_image_info(
+ image_id, {
+ input_data_fields.groundtruth_boxes:
+ groundtruth_boxes[:num_gt_box],
+ input_data_fields.groundtruth_classes:
+ groundtruth_classes[:num_gt_box],
+ input_data_fields.groundtruth_instance_masks:
+ groundtruth_instance_masks[
+ :num_gt_box,
+ :original_image_shape[0],
+ :original_image_shape[1]],
+ input_data_fields.groundtruth_verified_neg_classes:
+ groundtruth_verified_neg_classes,
+ input_data_fields.groundtruth_not_exhaustive_classes:
+ groundtruth_not_exhaustive_classes
+ })
+ self.add_single_detected_image_info(
+ image_id, {
+ 'detection_scores': detection_scores[:num_det_box],
+ 'detection_classes': detection_classes[:num_det_box],
+ 'detection_masks': detection_masks[
+ :num_det_box,
+ :original_image_shape[0],
+ :original_image_shape[1]]
+ })
+
+ # Unpack items from the evaluation dictionary.
+ input_data_fields = fields.InputDataFields
+ detection_fields = fields.DetectionResultFields
+ image_id = eval_dict[input_data_fields.key]
+ original_image_spatial_shape = eval_dict[
+ input_data_fields.original_image_spatial_shape]
+ groundtruth_boxes = eval_dict[input_data_fields.groundtruth_boxes]
+ groundtruth_classes = eval_dict[input_data_fields.groundtruth_classes]
+ groundtruth_instance_masks = eval_dict[
+ input_data_fields.groundtruth_instance_masks]
+ groundtruth_verified_neg_classes = eval_dict[
+ input_data_fields.groundtruth_verified_neg_classes]
+ groundtruth_not_exhaustive_classes = eval_dict[
+ input_data_fields.groundtruth_not_exhaustive_classes]
+
+ num_gt_boxes_per_image = eval_dict.get(
+ input_data_fields.num_groundtruth_boxes, None)
+ detection_scores = eval_dict[detection_fields.detection_scores]
+ detection_classes = eval_dict[detection_fields.detection_classes]
+ detection_masks = eval_dict[detection_fields.detection_masks]
+ num_det_boxes_per_image = eval_dict.get(detection_fields.num_detections,
+ None)
+
+ if not image_id.shape.as_list():
+ # Apply a batch dimension to all tensors.
+ image_id = tf.expand_dims(image_id, 0)
+ groundtruth_boxes = tf.expand_dims(groundtruth_boxes, 0)
+ groundtruth_classes = tf.expand_dims(groundtruth_classes, 0)
+ groundtruth_instance_masks = tf.expand_dims(groundtruth_instance_masks, 0)
+ groundtruth_verified_neg_classes = tf.expand_dims(
+ groundtruth_verified_neg_classes, 0)
+ groundtruth_not_exhaustive_classes = tf.expand_dims(
+ groundtruth_not_exhaustive_classes, 0)
+ detection_scores = tf.expand_dims(detection_scores, 0)
+ detection_classes = tf.expand_dims(detection_classes, 0)
+ detection_masks = tf.expand_dims(detection_masks, 0)
+
+ if num_gt_boxes_per_image is None:
+ num_gt_boxes_per_image = tf.shape(groundtruth_boxes)[1:2]
+ else:
+ num_gt_boxes_per_image = tf.expand_dims(num_gt_boxes_per_image, 0)
+
+ if num_det_boxes_per_image is None:
+ num_det_boxes_per_image = tf.shape(detection_scores)[1:2]
+ else:
+ num_det_boxes_per_image = tf.expand_dims(num_det_boxes_per_image, 0)
+ else:
+ if num_gt_boxes_per_image is None:
+ num_gt_boxes_per_image = tf.tile(
+ tf.shape(groundtruth_boxes)[1:2],
+ multiples=tf.shape(groundtruth_boxes)[0:1])
+ if num_det_boxes_per_image is None:
+ num_det_boxes_per_image = tf.tile(
+ tf.shape(detection_scores)[1:2],
+ multiples=tf.shape(detection_scores)[0:1])
+
+ return tf.py_func(update_op, [
+ image_id, groundtruth_boxes, groundtruth_classes,
+ groundtruth_instance_masks, groundtruth_verified_neg_classes,
+ groundtruth_not_exhaustive_classes,
+ num_gt_boxes_per_image, detection_scores, detection_classes,
+ detection_masks, num_det_boxes_per_image, original_image_spatial_shape
+ ], [])
+
+ def get_estimator_eval_metric_ops(self, eval_dict):
+ """Returns a dictionary of eval metric ops.
+
+ Note that once value_op is called, the detections and groundtruth added via
+ update_op are cleared.
+
+ Args:
+ eval_dict: A dictionary that holds tensors for evaluating object detection
+ performance. For single-image evaluation, this dictionary may be
+ produced from eval_util.result_dict_for_single_example(). If multi-image
+ evaluation, `eval_dict` should contain the fields
+ 'num_groundtruth_boxes_per_image' and 'num_det_boxes_per_image' to
+ properly unpad the tensors from the batch.
+
+ Returns:
+ a dictionary of metric names to tuple of value_op and update_op that can
+ be used as eval metric ops in tf.estimator.EstimatorSpec. Note that all
+ update ops must be run together and similarly all value ops must be run
+ together to guarantee correct behaviour.
+ """
+ update_op = self.add_eval_dict(eval_dict)
+ metric_names = ['DetectionMasks_Precision/mAP',
+ 'DetectionMasks_Precision/mAP@.50IOU',
+ 'DetectionMasks_Precision/mAP@.75IOU',
+ 'DetectionMasks_Precision/mAP (small)',
+ 'DetectionMasks_Precision/mAP (medium)',
+ 'DetectionMasks_Precision/mAP (large)',
+ 'DetectionMasks_Recall/AR@1',
+ 'DetectionMasks_Recall/AR@10',
+ 'DetectionMasks_Recall/AR@100',
+ 'DetectionMasks_Recall/AR@100 (small)',
+ 'DetectionMasks_Recall/AR@100 (medium)',
+ 'DetectionMasks_Recall/AR@100 (large)']
+ if self._include_metrics_per_category:
+ for category_dict in self._categories:
+ metric_names.append('DetectionMasks_PerformanceByCategory/mAP/' +
+ category_dict['name'])
+
+ def first_value_func():
+ self._metrics = self.evaluate()
+ self.clear()
+ return np.float32(self._metrics[metric_names[0]])
+
+ def value_func_factory(metric_name):
+ def value_func():
+ return np.float32(self._metrics[metric_name])
+ return value_func
+
+ # Ensure that the metrics are only evaluated once.
+ first_value_op = tf.py_func(first_value_func, [], tf.float32)
+ eval_metric_ops = {metric_names[0]: (first_value_op, update_op)}
+ with tf.control_dependencies([first_value_op]):
+ for metric_name in metric_names[1:]:
+ eval_metric_ops[metric_name] = (tf.py_func(
+ value_func_factory(metric_name), [], np.float32), update_op)
+ return eval_metric_ops
+
+ def dump_detections_to_json_file(self, json_output_path):
+ """Saves the detections into json_output_path in the format used by MS COCO.
+
+ Args:
+ json_output_path: String containing the output file's path. It can be also
+ None. In that case nothing will be written to the output file.
+ """
+ if json_output_path and json_output_path is not None:
+ pattern = re.compile(r'\d+\.\d{8,}')
+ def mround(match):
+ return '{:.2f}'.format(float(match.group()))
+
+ with tf.io.gfile.GFile(json_output_path, 'w') as fid:
+ json_string = json.dumps(self._detection_masks_list)
+ fid.write(re.sub(pattern, mround, json_string))
+
+ tf.logging.info('Dumping detections to output json file: %s',
+ json_output_path)
diff --git a/research/object_detection/metrics/lvis_evaluation_test.py b/research/object_detection/metrics/lvis_evaluation_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a612e5c93af03acb8823eebc010f51b585e4c41
--- /dev/null
+++ b/research/object_detection/metrics/lvis_evaluation_test.py
@@ -0,0 +1,182 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow_models.object_detection.metrics.coco_evaluation."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import unittest
+import numpy as np
+import tensorflow.compat.v1 as tf
+from object_detection.core import standard_fields as fields
+from object_detection.metrics import lvis_evaluation
+from object_detection.utils import tf_version
+
+
+def _get_categories_list():
+ return [{
+ 'id': 1,
+ 'name': 'person',
+ 'frequency': 'f'
+ }, {
+ 'id': 2,
+ 'name': 'dog',
+ 'frequency': 'c'
+ }, {
+ 'id': 3,
+ 'name': 'cat',
+ 'frequency': 'r'
+ }]
+
+
+class LvisMaskEvaluationTest(tf.test.TestCase):
+
+ def testGetOneMAPWithMatchingGroundtruthAndDetections(self):
+ """Tests that mAP is calculated correctly on GT and Detections."""
+ masks1 = np.expand_dims(np.pad(
+ np.ones([100, 100], dtype=np.uint8),
+ ((100, 56), (100, 56)), mode='constant'), axis=0)
+ masks2 = np.expand_dims(np.pad(
+ np.ones([50, 50], dtype=np.uint8),
+ ((50, 156), (50, 156)), mode='constant'), axis=0)
+ masks3 = np.expand_dims(np.pad(
+ np.ones([25, 25], dtype=np.uint8),
+ ((25, 206), (25, 206)), mode='constant'), axis=0)
+
+ lvis_evaluator = lvis_evaluation.LVISMaskEvaluator(
+ _get_categories_list())
+ lvis_evaluator.add_single_ground_truth_image_info(
+ image_id=1,
+ groundtruth_dict={
+ fields.InputDataFields.groundtruth_boxes:
+ np.array([[100., 100., 200., 200.]]),
+ fields.InputDataFields.groundtruth_classes: np.array([1]),
+ fields.InputDataFields.groundtruth_instance_masks: masks1,
+ fields.InputDataFields.groundtruth_verified_neg_classes:
+ np.array([0, 0, 0, 0]),
+ fields.InputDataFields.groundtruth_not_exhaustive_classes:
+ np.array([0, 0, 0, 0])
+ })
+ lvis_evaluator.add_single_detected_image_info(
+ image_id=1,
+ detections_dict={
+ fields.DetectionResultFields.detection_masks: masks1,
+ fields.DetectionResultFields.detection_scores:
+ np.array([.8]),
+ fields.DetectionResultFields.detection_classes:
+ np.array([1])
+ })
+ lvis_evaluator.add_single_ground_truth_image_info(
+ image_id=2,
+ groundtruth_dict={
+ fields.InputDataFields.groundtruth_boxes:
+ np.array([[50., 50., 100., 100.]]),
+ fields.InputDataFields.groundtruth_classes: np.array([1]),
+ fields.InputDataFields.groundtruth_instance_masks: masks2,
+ fields.InputDataFields.groundtruth_verified_neg_classes:
+ np.array([0, 0, 0, 0]),
+ fields.InputDataFields.groundtruth_not_exhaustive_classes:
+ np.array([0, 0, 0, 0])
+ })
+ lvis_evaluator.add_single_detected_image_info(
+ image_id=2,
+ detections_dict={
+ fields.DetectionResultFields.detection_masks: masks2,
+ fields.DetectionResultFields.detection_scores:
+ np.array([.8]),
+ fields.DetectionResultFields.detection_classes:
+ np.array([1])
+ })
+ lvis_evaluator.add_single_ground_truth_image_info(
+ image_id=3,
+ groundtruth_dict={
+ fields.InputDataFields.groundtruth_boxes:
+ np.array([[25., 25., 50., 50.]]),
+ fields.InputDataFields.groundtruth_classes: np.array([1]),
+ fields.InputDataFields.groundtruth_instance_masks: masks3,
+ fields.InputDataFields.groundtruth_verified_neg_classes:
+ np.array([0, 0, 0, 0]),
+ fields.InputDataFields.groundtruth_not_exhaustive_classes:
+ np.array([0, 0, 0, 0])
+ })
+ lvis_evaluator.add_single_detected_image_info(
+ image_id=3,
+ detections_dict={
+ fields.DetectionResultFields.detection_masks: masks3,
+ fields.DetectionResultFields.detection_scores:
+ np.array([.8]),
+ fields.DetectionResultFields.detection_classes:
+ np.array([1])
+ })
+ metrics = lvis_evaluator.evaluate()
+ self.assertAlmostEqual(metrics['DetectionMasks_AP'], 1.0)
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Only Supported in TF2.X')
+class LVISMaskEvaluationPyFuncTest(tf.test.TestCase):
+
+ def testAddEvalDict(self):
+ lvis_evaluator = lvis_evaluation.LVISMaskEvaluator(_get_categories_list())
+ image_id = tf.constant(1, dtype=tf.int32)
+ groundtruth_boxes = tf.constant(
+ np.array([[100., 100., 200., 200.], [50., 50., 100., 100.]]),
+ dtype=tf.float32)
+ groundtruth_classes = tf.constant(np.array([1, 2]), dtype=tf.float32)
+ groundtruth_masks = tf.constant(np.stack([
+ np.pad(np.ones([100, 100], dtype=np.uint8), ((10, 10), (10, 10)),
+ mode='constant'),
+ np.pad(np.ones([50, 50], dtype=np.uint8), ((0, 70), (0, 70)),
+ mode='constant')
+ ]), dtype=tf.uint8)
+ original_image_spatial_shapes = tf.constant([[120, 120], [120, 120]],
+ dtype=tf.int32)
+ groundtruth_verified_neg_classes = tf.constant(np.array([0, 0, 0, 0]),
+ dtype=tf.float32)
+ groundtruth_not_exhaustive_classes = tf.constant(np.array([0, 0, 0, 0]),
+ dtype=tf.float32)
+ detection_scores = tf.constant(np.array([.9, .8]), dtype=tf.float32)
+ detection_classes = tf.constant(np.array([2, 1]), dtype=tf.float32)
+ detection_masks = tf.constant(np.stack([
+ np.pad(np.ones([50, 50], dtype=np.uint8), ((0, 70), (0, 70)),
+ mode='constant'),
+ np.pad(np.ones([100, 100], dtype=np.uint8), ((10, 10), (10, 10)),
+ mode='constant'),
+ ]), dtype=tf.uint8)
+
+ input_data_fields = fields.InputDataFields
+ detection_fields = fields.DetectionResultFields
+ eval_dict = {
+ input_data_fields.key: image_id,
+ input_data_fields.groundtruth_boxes: groundtruth_boxes,
+ input_data_fields.groundtruth_classes: groundtruth_classes,
+ input_data_fields.groundtruth_instance_masks: groundtruth_masks,
+ input_data_fields.groundtruth_verified_neg_classes:
+ groundtruth_verified_neg_classes,
+ input_data_fields.groundtruth_not_exhaustive_classes:
+ groundtruth_not_exhaustive_classes,
+ input_data_fields.original_image_spatial_shape:
+ original_image_spatial_shapes,
+ detection_fields.detection_scores: detection_scores,
+ detection_fields.detection_classes: detection_classes,
+ detection_fields.detection_masks: detection_masks
+ }
+ lvis_evaluator.add_eval_dict(eval_dict)
+ self.assertLen(lvis_evaluator._groundtruth_list, 2)
+ self.assertLen(lvis_evaluator._detection_masks_list, 2)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/metrics/lvis_tools.py b/research/object_detection/metrics/lvis_tools.py
new file mode 100644
index 0000000000000000000000000000000000000000..86f3a234b74d7f08172c2f6dc321038d7a0425f8
--- /dev/null
+++ b/research/object_detection/metrics/lvis_tools.py
@@ -0,0 +1,260 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Wrappers for third party lvis to be used within object_detection.
+
+Usage example: given a set of images with ids in the list image_ids
+and corresponding lists of numpy arrays encoding groundtruth (boxes,
+masks and classes) and detections (masks, scores and classes), where
+elements of each list correspond to detections/annotations of a single image,
+then evaluation can be invoked as follows:
+
+ groundtruth = lvis_tools.LVISWrapper(groundtruth_dict)
+ detections = lvis_results.LVISResults(groundtruth, detections_list)
+ evaluator = lvis_tools.LVISEvalWrapper(groundtruth, detections,
+ iou_type='segm')
+ summary_metrics = evaluator.ComputeMetrics()
+
+TODO(jonathanhuang): Add support for exporting to JSON.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import logging
+
+from lvis import eval as lvis_eval
+from lvis import lvis
+import numpy as np
+from pycocotools import mask
+import six
+from six.moves import range
+
+
+def RleCompress(masks):
+ """Compresses mask using Run-length encoding provided by pycocotools.
+
+ Args:
+ masks: uint8 numpy array of shape [mask_height, mask_width] with values in
+ {0, 1}.
+
+ Returns:
+ A pycocotools Run-length encoding of the mask.
+ """
+ rle = mask.encode(np.asfortranarray(masks))
+ rle['counts'] = six.ensure_str(rle['counts'])
+ return rle
+
+
+def _ConvertBoxToCOCOFormat(box):
+ """Converts a box in [ymin, xmin, ymax, xmax] format to COCO format.
+
+ This is a utility function for converting from our internal
+ [ymin, xmin, ymax, xmax] convention to the convention used by the COCO API
+ i.e., [xmin, ymin, width, height].
+
+ Args:
+ box: a [ymin, xmin, ymax, xmax] numpy array
+
+ Returns:
+ a list of floats representing [xmin, ymin, width, height]
+ """
+ return [float(box[1]), float(box[0]), float(box[3] - box[1]),
+ float(box[2] - box[0])]
+
+
+class LVISWrapper(lvis.LVIS):
+ """Wrapper for the lvis.LVIS class."""
+
+ def __init__(self, dataset, detection_type='bbox'):
+ """LVISWrapper constructor.
+
+ See https://www.lvisdataset.org/dataset for a description of the format.
+ By default, the coco.COCO class constructor reads from a JSON file.
+ This function duplicates the same behavior but loads from a dictionary,
+ allowing us to perform evaluation without writing to external storage.
+
+ Args:
+ dataset: a dictionary holding bounding box annotations in the COCO format.
+ detection_type: type of detections being wrapped. Can be one of ['bbox',
+ 'segmentation']
+
+ Raises:
+ ValueError: if detection_type is unsupported.
+ """
+ self.logger = logging.getLogger(__name__)
+ self.logger.info('Loading annotations.')
+ self.dataset = dataset
+ self._create_index()
+
+
+class LVISEvalWrapper(lvis_eval.LVISEval):
+ """LVISEval wrapper."""
+
+ def __init__(self, groundtruth=None, detections=None, iou_type='bbox'):
+ lvis_eval.LVISEval.__init__(
+ self, groundtruth, detections, iou_type=iou_type)
+ self._iou_type = iou_type
+
+ def ComputeMetrics(self):
+ self.run()
+ summary_metrics = {}
+ summary_metrics = self.results
+ return summary_metrics
+
+
+def ExportSingleImageGroundtruthToLVIS(image_id,
+ next_annotation_id,
+ category_id_set,
+ groundtruth_boxes,
+ groundtruth_classes,
+ groundtruth_masks=None,
+ groundtruth_area=None):
+ """Export groundtruth of a single image to LVIS format.
+
+ This function converts groundtruth detection annotations represented as numpy
+ arrays to dictionaries that can be ingested by the LVIS evaluation API. Note
+ that the image_ids provided here must match the ones given to
+ ExportSingleImageDetectionMasksToLVIS. We assume that boxes, classes and masks
+ are in correspondence - that is, e.g., groundtruth_boxes[i, :], and
+ groundtruth_classes[i] are associated with the same groundtruth annotation.
+
+ In the exported result, "area" fields are always set to the area of the
+ groundtruth bounding box.
+
+ Args:
+ image_id: a unique image identifier castable to integer.
+ next_annotation_id: integer specifying the first id to use for the
+ groundtruth annotations. All annotations are assigned a continuous integer
+ id starting from this value.
+ category_id_set: A set of valid class ids. Groundtruth with classes not in
+ category_id_set are dropped.
+ groundtruth_boxes: numpy array (float32) with shape [num_gt_boxes, 4]
+ groundtruth_classes: numpy array (int) with shape [num_gt_boxes]
+ groundtruth_masks: optional uint8 numpy array of shape [num_detections,
+ image_height, image_width] containing detection_masks.
+ groundtruth_area: numpy array (float32) with shape [num_gt_boxes]. If
+ provided, then the area values (in the original absolute coordinates) will
+ be populated instead of calculated from bounding box coordinates.
+
+ Returns:
+ a list of groundtruth annotations for a single image in the COCO format.
+
+ Raises:
+ ValueError: if (1) groundtruth_boxes and groundtruth_classes do not have the
+ right lengths or (2) if each of the elements inside these lists do not
+ have the correct shapes or (3) if image_ids are not integers
+ """
+
+ if len(groundtruth_classes.shape) != 1:
+ raise ValueError('groundtruth_classes is '
+ 'expected to be of rank 1.')
+ if len(groundtruth_boxes.shape) != 2:
+ raise ValueError('groundtruth_boxes is expected to be of '
+ 'rank 2.')
+ if groundtruth_boxes.shape[1] != 4:
+ raise ValueError('groundtruth_boxes should have '
+ 'shape[1] == 4.')
+ num_boxes = groundtruth_classes.shape[0]
+ if num_boxes != groundtruth_boxes.shape[0]:
+ raise ValueError('Corresponding entries in groundtruth_classes, '
+ 'and groundtruth_boxes should have '
+ 'compatible shapes (i.e., agree on the 0th dimension).'
+ 'Classes shape: %d. Boxes shape: %d. Image ID: %s' % (
+ groundtruth_classes.shape[0],
+ groundtruth_boxes.shape[0], image_id))
+
+ groundtruth_list = []
+ for i in range(num_boxes):
+ if groundtruth_classes[i] in category_id_set:
+ if groundtruth_area is not None and groundtruth_area[i] > 0:
+ area = float(groundtruth_area[i])
+ else:
+ area = float((groundtruth_boxes[i, 2] - groundtruth_boxes[i, 0]) *
+ (groundtruth_boxes[i, 3] - groundtruth_boxes[i, 1]))
+ export_dict = {
+ 'id':
+ next_annotation_id + i,
+ 'image_id':
+ int(image_id),
+ 'category_id':
+ int(groundtruth_classes[i]),
+ 'bbox':
+ list(_ConvertBoxToCOCOFormat(groundtruth_boxes[i, :])),
+ 'area': area,
+ }
+ if groundtruth_masks is not None:
+ export_dict['segmentation'] = RleCompress(groundtruth_masks[i])
+
+ groundtruth_list.append(export_dict)
+ return groundtruth_list
+
+
+def ExportSingleImageDetectionMasksToLVIS(image_id,
+ category_id_set,
+ detection_masks,
+ detection_scores,
+ detection_classes):
+ """Export detection masks of a single image to LVIS format.
+
+ This function converts detections represented as numpy arrays to dictionaries
+ that can be ingested by the LVIS evaluation API. We assume that
+ detection_masks, detection_scores, and detection_classes are in correspondence
+ - that is: detection_masks[i, :], detection_classes[i] and detection_scores[i]
+ are associated with the same annotation.
+
+ Args:
+ image_id: unique image identifier castable to integer.
+ category_id_set: A set of valid class ids. Detections with classes not in
+ category_id_set are dropped.
+ detection_masks: uint8 numpy array of shape [num_detections, image_height,
+ image_width] containing detection_masks.
+ detection_scores: float numpy array of shape [num_detections] containing
+ scores for detection masks.
+ detection_classes: integer numpy array of shape [num_detections] containing
+ the classes for detection masks.
+
+ Returns:
+ a list of detection mask annotations for a single image in the COCO format.
+
+ Raises:
+ ValueError: if (1) detection_masks, detection_scores and detection_classes
+ do not have the right lengths or (2) if each of the elements inside these
+ lists do not have the correct shapes or (3) if image_ids are not integers.
+ """
+
+ if len(detection_classes.shape) != 1 or len(detection_scores.shape) != 1:
+ raise ValueError('All entries in detection_classes and detection_scores'
+ 'expected to be of rank 1.')
+ num_boxes = detection_classes.shape[0]
+ if not num_boxes == len(detection_masks) == detection_scores.shape[0]:
+ raise ValueError('Corresponding entries in detection_classes, '
+ 'detection_scores and detection_masks should have '
+ 'compatible lengths and shapes '
+ 'Classes length: %d. Masks length: %d. '
+ 'Scores length: %d' % (
+ detection_classes.shape[0], len(detection_masks),
+ detection_scores.shape[0]
+ ))
+ detections_list = []
+ for i in range(num_boxes):
+ if detection_classes[i] in category_id_set:
+ detections_list.append({
+ 'image_id': int(image_id),
+ 'category_id': int(detection_classes[i]),
+ 'segmentation': RleCompress(detection_masks[i]),
+ 'score': float(detection_scores[i])
+ })
+
+ return detections_list
diff --git a/research/object_detection/metrics/lvis_tools_test.py b/research/object_detection/metrics/lvis_tools_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..5a5585acda9874e034464a63e1cdfdb881254a4f
--- /dev/null
+++ b/research/object_detection/metrics/lvis_tools_test.py
@@ -0,0 +1,158 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow_model.object_detection.metrics.lvis_tools."""
+from lvis import results as lvis_results
+import numpy as np
+from pycocotools import mask
+import tensorflow.compat.v1 as tf
+from object_detection.metrics import lvis_tools
+
+
+class LVISToolsTest(tf.test.TestCase):
+
+ def setUp(self):
+ super(LVISToolsTest, self).setUp()
+ mask1 = np.pad(
+ np.ones([100, 100], dtype=np.uint8),
+ ((100, 56), (100, 56)), mode='constant')
+ mask2 = np.pad(
+ np.ones([50, 50], dtype=np.uint8),
+ ((50, 156), (50, 156)), mode='constant')
+ mask1_rle = lvis_tools.RleCompress(mask1)
+ mask2_rle = lvis_tools.RleCompress(mask2)
+ groundtruth_annotations_list = [
+ {
+ 'id': 1,
+ 'image_id': 1,
+ 'category_id': 1,
+ 'bbox': [100., 100., 100., 100.],
+ 'area': 100.**2,
+ 'segmentation': mask1_rle
+ },
+ {
+ 'id': 2,
+ 'image_id': 2,
+ 'category_id': 1,
+ 'bbox': [50., 50., 50., 50.],
+ 'area': 50.**2,
+ 'segmentation': mask2_rle
+ },
+ ]
+ image_list = [
+ {
+ 'id': 1,
+ 'neg_category_ids': [],
+ 'not_exhaustive_category_ids': [],
+ 'height': 256,
+ 'width': 256
+ },
+ {
+ 'id': 2,
+ 'neg_category_ids': [],
+ 'not_exhaustive_category_ids': [],
+ 'height': 256,
+ 'width': 256
+ }
+ ]
+ category_list = [{'id': 0, 'name': 'person', 'frequency': 'f'},
+ {'id': 1, 'name': 'cat', 'frequency': 'c'},
+ {'id': 2, 'name': 'dog', 'frequency': 'r'}]
+ self._groundtruth_dict = {
+ 'annotations': groundtruth_annotations_list,
+ 'images': image_list,
+ 'categories': category_list
+ }
+
+ self._detections_list = [
+ {
+ 'image_id': 1,
+ 'category_id': 1,
+ 'segmentation': mask1_rle,
+ 'score': .8
+ },
+ {
+ 'image_id': 2,
+ 'category_id': 1,
+ 'segmentation': mask2_rle,
+ 'score': .7
+ },
+ ]
+
+ def testLVISWrappers(self):
+ groundtruth = lvis_tools.LVISWrapper(self._groundtruth_dict)
+ detections = lvis_results.LVISResults(groundtruth, self._detections_list)
+ evaluator = lvis_tools.LVISEvalWrapper(groundtruth, detections,
+ iou_type='segm')
+ summary_metrics = evaluator.ComputeMetrics()
+ self.assertAlmostEqual(1.0, summary_metrics['AP'])
+
+ def testSingleImageDetectionMaskExport(self):
+ masks = np.array(
+ [[[1, 1,], [1, 1]],
+ [[0, 0], [0, 1]],
+ [[0, 0], [0, 0]]], dtype=np.uint8)
+ classes = np.array([1, 2, 3], dtype=np.int32)
+ scores = np.array([0.8, 0.2, 0.7], dtype=np.float32)
+ lvis_annotations = lvis_tools.ExportSingleImageDetectionMasksToLVIS(
+ image_id=1,
+ category_id_set=set([1, 2, 3]),
+ detection_classes=classes,
+ detection_scores=scores,
+ detection_masks=masks)
+ expected_counts = ['04', '31', '4']
+ for i, mask_annotation in enumerate(lvis_annotations):
+ self.assertEqual(mask_annotation['segmentation']['counts'],
+ expected_counts[i])
+ self.assertTrue(np.all(np.equal(mask.decode(
+ mask_annotation['segmentation']), masks[i])))
+ self.assertEqual(mask_annotation['image_id'], 1)
+ self.assertEqual(mask_annotation['category_id'], classes[i])
+ self.assertAlmostEqual(mask_annotation['score'], scores[i])
+
+ def testSingleImageGroundtruthExport(self):
+ masks = np.array(
+ [[[1, 1,], [1, 1]],
+ [[0, 0], [0, 1]],
+ [[0, 0], [0, 0]]], dtype=np.uint8)
+ boxes = np.array([[0, 0, 1, 1],
+ [0, 0, .5, .5],
+ [.5, .5, 1, 1]], dtype=np.float32)
+ lvis_boxes = np.array([[0, 0, 1, 1],
+ [0, 0, .5, .5],
+ [.5, .5, .5, .5]], dtype=np.float32)
+ classes = np.array([1, 2, 3], dtype=np.int32)
+ next_annotation_id = 1
+ expected_counts = ['04', '31', '4']
+
+ lvis_annotations = lvis_tools.ExportSingleImageGroundtruthToLVIS(
+ image_id=1,
+ category_id_set=set([1, 2, 3]),
+ next_annotation_id=next_annotation_id,
+ groundtruth_boxes=boxes,
+ groundtruth_classes=classes,
+ groundtruth_masks=masks)
+ for i, annotation in enumerate(lvis_annotations):
+ self.assertEqual(annotation['segmentation']['counts'],
+ expected_counts[i])
+ self.assertTrue(np.all(np.equal(mask.decode(
+ annotation['segmentation']), masks[i])))
+ self.assertTrue(np.all(np.isclose(annotation['bbox'], lvis_boxes[i])))
+ self.assertEqual(annotation['image_id'], 1)
+ self.assertEqual(annotation['category_id'], classes[i])
+ self.assertEqual(annotation['id'], i + next_annotation_id)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/metrics/oid_challenge_evaluation_utils.py b/research/object_detection/metrics/oid_challenge_evaluation_utils.py
index 844cce3e8f362c2c15403269584149878f60bc51..86746912336831c9193d9b1a25272f0fc7283592 100644
--- a/research/object_detection/metrics/oid_challenge_evaluation_utils.py
+++ b/research/object_detection/metrics/oid_challenge_evaluation_utils.py
@@ -56,10 +56,9 @@ def _decode_raw_data_into_masks_and_boxes(segments, image_widths,
"""Decods binary segmentation masks into np.arrays and boxes.
Args:
- segments: pandas Series object containing either
- None entries, or strings with
- base64, zlib compressed, COCO RLE-encoded binary masks.
- All masks are expected to be the same size.
+ segments: pandas Series object containing either None entries, or strings
+ with base64, zlib compressed, COCO RLE-encoded binary masks. All masks are
+ expected to be the same size.
image_widths: pandas Series of mask widths.
image_heights: pandas Series of mask heights.
@@ -136,15 +135,15 @@ def build_groundtruth_dictionary(data, class_label_map):
dictionary = {
standard_fields.InputDataFields.groundtruth_boxes:
- data_location[['YMin', 'XMin', 'YMax', 'XMax']].to_numpy(),
+ data_location[['YMin', 'XMin', 'YMax',
+ 'XMax']].to_numpy().astype(float),
standard_fields.InputDataFields.groundtruth_classes:
data_location['LabelName'].map(lambda x: class_label_map[x]
).to_numpy(),
standard_fields.InputDataFields.groundtruth_group_of:
data_location['IsGroupOf'].to_numpy().astype(int),
standard_fields.InputDataFields.groundtruth_image_classes:
- data_labels['LabelName'].map(lambda x: class_label_map[x]
- ).to_numpy(),
+ data_labels['LabelName'].map(lambda x: class_label_map[x]).to_numpy(),
}
if 'Mask' in data_location:
@@ -181,7 +180,7 @@ def build_predictions_dictionary(data, class_label_map):
standard_fields.DetectionResultFields.detection_classes:
data['LabelName'].map(lambda x: class_label_map[x]).to_numpy(),
standard_fields.DetectionResultFields.detection_scores:
- data['Score'].to_numpy()
+ data['Score'].to_numpy().astype(float)
}
if 'Mask' in data:
@@ -192,6 +191,6 @@ def build_predictions_dictionary(data, class_label_map):
else:
dictionary[standard_fields.DetectionResultFields.detection_boxes] = data[[
'YMin', 'XMin', 'YMax', 'XMax'
- ]].to_numpy()
+ ]].to_numpy().astype(float)
return dictionary
diff --git a/research/object_detection/model_lib.py b/research/object_detection/model_lib.py
index 1eb482d19bdd546b5f7e4ab49bde2039421ead39..111be9cb4a7db5bc4838467d080204ec00125fb5 100644
--- a/research/object_detection/model_lib.py
+++ b/research/object_detection/model_lib.py
@@ -102,10 +102,18 @@ def _prepare_groundtruth_for_eval(detection_model, class_agnostic,
'groundtruth_dp_surface_coords_list': [batch_size, num_boxes,
max_sampled_points, 4] containing the DensePose surface coordinates for
each sampled point (if provided in groundtruth).
+ 'groundtruth_track_ids_list': [batch_size, num_boxes] int32 tensor
+ with track ID for each instance (if provided in groundtruth).
'groundtruth_group_of': [batch_size, num_boxes] bool tensor indicating
group_of annotations (if provided in groundtruth).
'groundtruth_labeled_classes': [batch_size, num_classes] int64
tensor of 1-indexed classes.
+ 'groundtruth_verified_neg_classes': [batch_size, num_classes] float32
+ K-hot representation of 1-indexed classes which were verified as not
+ present in the image.
+ 'groundtruth_not_exhaustive_classes': [batch_size, num_classes] K-hot
+ representation of 1-indexed classes which don't have all of their
+ instances marked exhaustively.
class_agnostic: Boolean indicating whether detections are class agnostic.
"""
input_data_fields = fields.InputDataFields()
@@ -127,6 +135,7 @@ def _prepare_groundtruth_for_eval(detection_model, class_agnostic,
input_data_fields.groundtruth_boxes: groundtruth_boxes,
input_data_fields.groundtruth_classes: groundtruth_classes
}
+
if detection_model.groundtruth_has_field(fields.BoxListFields.masks):
groundtruth[input_data_fields.groundtruth_instance_masks] = tf.stack(
detection_model.groundtruth_lists(fields.BoxListFields.masks))
@@ -143,6 +152,15 @@ def _prepare_groundtruth_for_eval(detection_model, class_agnostic,
groundtruth[input_data_fields.groundtruth_keypoints] = tf.stack(
detection_model.groundtruth_lists(fields.BoxListFields.keypoints))
+ if detection_model.groundtruth_has_field(
+ fields.BoxListFields.keypoint_depths):
+ groundtruth[input_data_fields.groundtruth_keypoint_depths] = tf.stack(
+ detection_model.groundtruth_lists(fields.BoxListFields.keypoint_depths))
+ groundtruth[
+ input_data_fields.groundtruth_keypoint_depth_weights] = tf.stack(
+ detection_model.groundtruth_lists(
+ fields.BoxListFields.keypoint_depth_weights))
+
if detection_model.groundtruth_has_field(
fields.BoxListFields.keypoint_visibilities):
groundtruth[input_data_fields.groundtruth_keypoint_visibilities] = tf.stack(
@@ -153,24 +171,21 @@ def _prepare_groundtruth_for_eval(detection_model, class_agnostic,
groundtruth[input_data_fields.groundtruth_group_of] = tf.stack(
detection_model.groundtruth_lists(fields.BoxListFields.group_of))
+ label_id_offset_paddings = tf.constant([[0, 0], [1, 0]])
if detection_model.groundtruth_has_field(
- fields.InputDataFields.groundtruth_labeled_classes):
- labeled_classes_list = detection_model.groundtruth_lists(
- fields.InputDataFields.groundtruth_labeled_classes)
- labeled_classes = [
- tf.where(x)[:, 0] + label_id_offset for x in labeled_classes_list
- ]
- if len(labeled_classes) > 1:
- num_classes = labeled_classes_list[0].shape[0]
- padded_labeled_classes = []
- for x in labeled_classes:
- padding = num_classes - tf.shape(x)[0]
- padded_labeled_classes.append(tf.pad(x, [[0, padding]]))
- groundtruth[input_data_fields.groundtruth_labeled_classes] = tf.stack(
- padded_labeled_classes)
- else:
- groundtruth[input_data_fields.groundtruth_labeled_classes] = tf.stack(
- labeled_classes)
+ input_data_fields.groundtruth_verified_neg_classes):
+ groundtruth[input_data_fields.groundtruth_verified_neg_classes] = tf.pad(
+ tf.stack(detection_model.groundtruth_lists(
+ input_data_fields.groundtruth_verified_neg_classes)),
+ label_id_offset_paddings)
+
+ if detection_model.groundtruth_has_field(
+ input_data_fields.groundtruth_not_exhaustive_classes):
+ groundtruth[
+ input_data_fields.groundtruth_not_exhaustive_classes] = tf.pad(
+ tf.stack(detection_model.groundtruth_lists(
+ input_data_fields.groundtruth_not_exhaustive_classes)),
+ label_id_offset_paddings)
if detection_model.groundtruth_has_field(
fields.BoxListFields.densepose_num_points):
@@ -187,6 +202,19 @@ def _prepare_groundtruth_for_eval(detection_model, class_agnostic,
groundtruth[input_data_fields.groundtruth_dp_surface_coords] = tf.stack(
detection_model.groundtruth_lists(
fields.BoxListFields.densepose_surface_coords))
+
+ if detection_model.groundtruth_has_field(fields.BoxListFields.track_ids):
+ groundtruth[input_data_fields.groundtruth_track_ids] = tf.stack(
+ detection_model.groundtruth_lists(fields.BoxListFields.track_ids))
+
+ if detection_model.groundtruth_has_field(
+ input_data_fields.groundtruth_labeled_classes):
+ groundtruth[input_data_fields.groundtruth_labeled_classes] = tf.pad(
+ tf.stack(
+ detection_model.groundtruth_lists(
+ input_data_fields.groundtruth_labeled_classes)),
+ label_id_offset_paddings)
+
groundtruth[input_data_fields.num_groundtruth_boxes] = (
tf.tile([max_number_of_boxes], multiples=[groundtruth_boxes_shape[0]]))
return groundtruth
@@ -241,10 +269,13 @@ def unstack_batch(tensor_dict, unpad_groundtruth_tensors=True):
fields.InputDataFields.groundtruth_classes,
fields.InputDataFields.groundtruth_boxes,
fields.InputDataFields.groundtruth_keypoints,
+ fields.InputDataFields.groundtruth_keypoint_depths,
+ fields.InputDataFields.groundtruth_keypoint_depth_weights,
fields.InputDataFields.groundtruth_keypoint_visibilities,
fields.InputDataFields.groundtruth_dp_num_points,
fields.InputDataFields.groundtruth_dp_part_ids,
fields.InputDataFields.groundtruth_dp_surface_coords,
+ fields.InputDataFields.groundtruth_track_ids,
fields.InputDataFields.groundtruth_group_of,
fields.InputDataFields.groundtruth_difficult,
fields.InputDataFields.groundtruth_is_crowd,
@@ -291,6 +322,13 @@ def provide_groundtruth(model, labels):
gt_keypoints_list = None
if fields.InputDataFields.groundtruth_keypoints in labels:
gt_keypoints_list = labels[fields.InputDataFields.groundtruth_keypoints]
+ gt_keypoint_depths_list = None
+ gt_keypoint_depth_weights_list = None
+ if fields.InputDataFields.groundtruth_keypoint_depths in labels:
+ gt_keypoint_depths_list = (
+ labels[fields.InputDataFields.groundtruth_keypoint_depths])
+ gt_keypoint_depth_weights_list = (
+ labels[fields.InputDataFields.groundtruth_keypoint_depth_weights])
gt_keypoint_visibilities_list = None
if fields.InputDataFields.groundtruth_keypoint_visibilities in labels:
gt_keypoint_visibilities_list = labels[
@@ -307,6 +345,10 @@ def provide_groundtruth(model, labels):
if fields.InputDataFields.groundtruth_dp_surface_coords in labels:
gt_dp_surface_coords_list = labels[
fields.InputDataFields.groundtruth_dp_surface_coords]
+ gt_track_ids_list = None
+ if fields.InputDataFields.groundtruth_track_ids in labels:
+ gt_track_ids_list = labels[
+ fields.InputDataFields.groundtruth_track_ids]
gt_weights_list = None
if fields.InputDataFields.groundtruth_weights in labels:
gt_weights_list = labels[fields.InputDataFields.groundtruth_weights]
@@ -327,6 +369,14 @@ def provide_groundtruth(model, labels):
if fields.InputDataFields.groundtruth_labeled_classes in labels:
gt_labeled_classes = labels[
fields.InputDataFields.groundtruth_labeled_classes]
+ gt_verified_neg_classes = None
+ if fields.InputDataFields.groundtruth_verified_neg_classes in labels:
+ gt_verified_neg_classes = labels[
+ fields.InputDataFields.groundtruth_verified_neg_classes]
+ gt_not_exhaustive_classes = None
+ if fields.InputDataFields.groundtruth_not_exhaustive_classes in labels:
+ gt_not_exhaustive_classes = labels[
+ fields.InputDataFields.groundtruth_not_exhaustive_classes]
model.provide_groundtruth(
groundtruth_boxes_list=gt_boxes_list,
groundtruth_classes_list=gt_classes_list,
@@ -341,7 +391,12 @@ def provide_groundtruth(model, labels):
groundtruth_weights_list=gt_weights_list,
groundtruth_is_crowd_list=gt_is_crowd_list,
groundtruth_group_of_list=gt_group_of_list,
- groundtruth_area_list=gt_area_list)
+ groundtruth_area_list=gt_area_list,
+ groundtruth_track_ids_list=gt_track_ids_list,
+ groundtruth_verified_neg_classes=gt_verified_neg_classes,
+ groundtruth_not_exhaustive_classes=gt_not_exhaustive_classes,
+ groundtruth_keypoint_depths_list=gt_keypoint_depths_list,
+ groundtruth_keypoint_depth_weights_list=gt_keypoint_depth_weights_list)
def create_model_fn(detection_model_fn, configs, hparams=None, use_tpu=False,
@@ -390,8 +445,7 @@ def create_model_fn(detection_model_fn, configs, hparams=None, use_tpu=False,
from tensorflow.python.keras.engine import base_layer_utils # pylint: disable=g-import-not-at-top
# Enable v2 behavior, as `mixed_bfloat16` is only supported in TF 2.0.
base_layer_utils.enable_v2_dtype_behavior()
- tf2.keras.mixed_precision.experimental.set_policy(
- 'mixed_bfloat16')
+ tf2.keras.mixed_precision.set_global_policy('mixed_bfloat16')
detection_model = detection_model_fn(
is_training=is_training, add_summaries=(not use_tpu))
scaffold_fn = None
@@ -786,12 +840,14 @@ def create_estimator_and_inputs(run_config,
train_config=train_config,
train_input_config=train_input_config,
model_config=model_config)
- eval_input_fns = [
- create_eval_input_fn(
- eval_config=eval_config,
- eval_input_config=eval_input_config,
- model_config=model_config) for eval_input_config in eval_input_configs
- ]
+ eval_input_fns = []
+ for eval_input_config in eval_input_configs:
+ eval_input_fns.append(
+ create_eval_input_fn(
+ eval_config=eval_config,
+ eval_input_config=eval_input_config,
+ model_config=model_config))
+
eval_input_names = [
eval_input_config.name for eval_input_config in eval_input_configs
]
@@ -934,12 +990,12 @@ def _evaluate_checkpoint(estimator,
raise e
-def continuous_eval(estimator,
- model_dir,
- input_fn,
- train_steps,
- name,
- max_retries=0):
+def continuous_eval_generator(estimator,
+ model_dir,
+ input_fn,
+ train_steps,
+ name,
+ max_retries=0):
"""Perform continuous evaluation on checkpoints written to a model directory.
Args:
@@ -952,6 +1008,9 @@ def continuous_eval(estimator,
max_retries: Maximum number of times to retry the evaluation on encountering
a tf.errors.InvalidArgumentError. If negative, will always retry the
evaluation.
+
+ Yields:
+ Pair of current step and eval_results.
"""
def terminate_eval():
@@ -974,6 +1033,7 @@ def continuous_eval(estimator,
# Terminate eval job when final checkpoint is reached
current_step = int(os.path.basename(ckpt).split('-')[1])
+ yield (current_step, eval_results)
if current_step >= train_steps:
tf.logging.info(
'Evaluation finished after training step %d' % current_step)
@@ -984,6 +1044,30 @@ def continuous_eval(estimator,
'Checkpoint %s no longer exists, skipping checkpoint' % ckpt)
+def continuous_eval(estimator,
+ model_dir,
+ input_fn,
+ train_steps,
+ name,
+ max_retries=0):
+ """Performs continuous evaluation on checkpoints written to a model directory.
+
+ Args:
+ estimator: Estimator object to use for evaluation.
+ model_dir: Model directory to read checkpoints for continuous evaluation.
+ input_fn: Input function to use for evaluation.
+ train_steps: Number of training steps. This is used to infer the last
+ checkpoint and stop evaluation loop.
+ name: Namescope for eval summary.
+ max_retries: Maximum number of times to retry the evaluation on encountering
+ a tf.errors.InvalidArgumentError. If negative, will always retry the
+ evaluation.
+ """
+ for current_step, eval_results in continuous_eval_generator(
+ estimator, model_dir, input_fn, train_steps, name, max_retries):
+ tf.logging.info('Step %s, Eval results: %s', current_step, eval_results)
+
+
def populate_experiment(run_config,
hparams,
pipeline_config_path,
diff --git a/research/object_detection/model_lib_tf2_test.py b/research/object_detection/model_lib_tf2_test.py
index f65273660195752227b2bcc90dceb04184a6eb62..12330dbc7fa2f4022823c72a75878aac0d731455 100644
--- a/research/object_detection/model_lib_tf2_test.py
+++ b/research/object_detection/model_lib_tf2_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
+import json
import os
import tempfile
import unittest
@@ -26,9 +27,9 @@ import six
import tensorflow.compat.v1 as tf
import tensorflow.compat.v2 as tf2
+from object_detection import exporter_lib_v2
from object_detection import inputs
from object_detection import model_lib_v2
-from object_detection.builders import model_builder
from object_detection.core import model
from object_detection.protos import train_pb2
from object_detection.utils import config_util
@@ -69,7 +70,8 @@ def _get_config_kwarg_overrides():
return {
'train_input_path': data_path,
'eval_input_path': data_path,
- 'label_map_path': label_map_path
+ 'label_map_path': label_map_path,
+ 'train_input_reader': {'batch_size': 1}
}
@@ -90,13 +92,14 @@ class ModelLibTest(tf.test.TestCase):
config_kwarg_overrides = _get_config_kwarg_overrides()
train_steps = 2
- strategy = tf2.distribute.OneDeviceStrategy(device='/cpu:0')
+ strategy = tf2.distribute.MirroredStrategy(['/cpu:0', '/cpu:1'])
with strategy.scope():
model_lib_v2.train_loop(
new_pipeline_config_path,
model_dir=model_dir,
train_steps=train_steps,
checkpoint_every_n=1,
+ num_steps_per_iteration=1,
**config_kwarg_overrides)
model_lib_v2.eval_continuously(
@@ -145,6 +148,12 @@ class SimpleModel(model.DetectionModel):
return []
+def fake_model_builder(*_, **__):
+ return SimpleModel()
+
+FAKE_BUILDER_MAP = {'detection_model_fn_base': fake_model_builder}
+
+
@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
class ModelCheckpointTest(tf.test.TestCase):
"""Test for model checkpoint related functionality."""
@@ -153,10 +162,9 @@ class ModelCheckpointTest(tf.test.TestCase):
"""Test that only the most recent checkpoints are kept."""
strategy = tf2.distribute.OneDeviceStrategy(device='/cpu:0')
- with mock.patch.object(
- model_builder, 'build', autospec=True) as mock_builder:
- with strategy.scope():
- mock_builder.return_value = SimpleModel()
+ with mock.patch.dict(
+ model_lib_v2.MODEL_BUILD_UTIL_MAP, FAKE_BUILDER_MAP):
+
model_dir = tempfile.mkdtemp(dir=self.get_temp_dir())
pipeline_config_path = get_pipeline_config_path(MODEL_NAME_FOR_TEST)
new_pipeline_config_path = os.path.join(model_dir, 'new_pipeline.config')
@@ -167,8 +175,8 @@ class ModelCheckpointTest(tf.test.TestCase):
with strategy.scope():
model_lib_v2.train_loop(
new_pipeline_config_path, model_dir=model_dir,
- train_steps=20, checkpoint_every_n=2, checkpoint_max_to_keep=3,
- **config_kwarg_overrides
+ train_steps=5, checkpoint_every_n=2, checkpoint_max_to_keep=3,
+ num_steps_per_iteration=1, **config_kwarg_overrides
)
ckpt_files = tf.io.gfile.glob(os.path.join(model_dir, 'ckpt-*.index'))
self.assertEqual(len(ckpt_files), 3,
@@ -210,6 +218,7 @@ class CheckpointV2Test(tf.test.TestCase):
model_lib_v2.load_fine_tune_checkpoint(
self._model, self._ckpt_path, checkpoint_type='',
checkpoint_version=train_pb2.CheckpointVersion.V2,
+ run_model_on_dummy_input=True,
input_dataset=self._train_input_fn(),
unpad_groundtruth_tensors=True)
np.testing.assert_allclose(self._model.weight.numpy(), 42)
@@ -222,9 +231,46 @@ class CheckpointV2Test(tf.test.TestCase):
model_lib_v2.load_fine_tune_checkpoint(
IncompatibleModel(), self._ckpt_path, checkpoint_type='',
checkpoint_version=train_pb2.CheckpointVersion.V2,
+ run_model_on_dummy_input=True,
input_dataset=self._train_input_fn(),
unpad_groundtruth_tensors=True)
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class MetricsExportTest(tf.test.TestCase):
+
+ @classmethod
+ def setUpClass(cls): # pylint:disable=g-missing-super-call
+ tf.keras.backend.clear_session()
+
+ def test_export_metrics_json_serializable(self):
+ """Tests that Estimator and input function are constructed correctly."""
+
+ strategy = tf2.distribute.OneDeviceStrategy(device='/cpu:0')
+
+ def export(data, _):
+ json.dumps(data)
+
+ with mock.patch.dict(
+ exporter_lib_v2.INPUT_BUILDER_UTIL_MAP, FAKE_BUILDER_MAP):
+ with strategy.scope():
+ model_dir = tf.test.get_temp_dir()
+ new_pipeline_config_path = os.path.join(model_dir,
+ 'new_pipeline.config')
+ pipeline_config_path = get_pipeline_config_path(MODEL_NAME_FOR_TEST)
+ config_util.clear_fine_tune_checkpoint(pipeline_config_path,
+ new_pipeline_config_path)
+ train_steps = 2
+ with strategy.scope():
+ model_lib_v2.train_loop(
+ new_pipeline_config_path,
+ model_dir=model_dir,
+ train_steps=train_steps,
+ checkpoint_every_n=100,
+ performance_summary_exporter=export,
+ num_steps_per_iteration=1,
+ **_get_config_kwarg_overrides())
+
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/model_lib_v2.py b/research/object_detection/model_lib_v2.py
index d2d24e77ec72b654282212df52f3d0709ebce14d..ee4c1b8d78ed3063cf794db315eaddefdb116c9b 100644
--- a/research/object_detection/model_lib_v2.py
+++ b/research/object_detection/model_lib_v2.py
@@ -21,13 +21,14 @@ from __future__ import print_function
import copy
import os
import time
+import numpy as np
import tensorflow.compat.v1 as tf
+import tensorflow.compat.v2 as tf2
from object_detection import eval_util
from object_detection import inputs
from object_detection import model_lib
-from object_detection.builders import model_builder
from object_detection.builders import optimizer_builder
from object_detection.core import standard_fields as fields
from object_detection.protos import train_pb2
@@ -36,15 +37,9 @@ from object_detection.utils import label_map_util
from object_detection.utils import ops
from object_detection.utils import visualization_utils as vutils
-# pylint: disable=g-import-not-at-top
-try:
- from tensorflow.contrib import tpu as contrib_tpu
-except ImportError:
- # TF 2.0 doesn't ship with contrib.
- pass
-# pylint: enable=g-import-not-at-top
MODEL_BUILD_UTIL_MAP = model_lib.MODEL_BUILD_UTIL_MAP
+NUM_STEPS_PER_ITERATION = 100
RESTORE_MAP_ERROR_TEMPLATE = (
@@ -103,6 +98,12 @@ def _compute_losses_and_predictions_dicts(
containing group_of annotations.
labels[fields.InputDataFields.groundtruth_labeled_classes] is a float32
k-hot tensor of classes.
+ labels[fields.InputDataFields.groundtruth_track_ids] is a int32
+ tensor of track IDs.
+ labels[fields.InputDataFields.groundtruth_keypoint_depths] is a
+ float32 tensor containing keypoint depths information.
+ labels[fields.InputDataFields.groundtruth_keypoint_depth_weights] is a
+ float32 tensor containing the weights of the keypoint depth feature.
add_regularization_loss: Whether or not to include the model's
regularization loss in the losses dictionary.
@@ -117,7 +118,8 @@ def _compute_losses_and_predictions_dicts(
prediction_dict = model.predict(
preprocessed_images,
- features[fields.InputDataFields.true_image_shape])
+ features[fields.InputDataFields.true_image_shape],
+ **model.get_side_inputs(features))
prediction_dict = ops.bfloat16_to_float32_nested(prediction_dict)
losses_dict = model.loss(
@@ -142,6 +144,42 @@ def _compute_losses_and_predictions_dicts(
return losses_dict, prediction_dict
+def _ensure_model_is_built(model, input_dataset, unpad_groundtruth_tensors):
+ """Ensures that model variables are all built, by running on a dummy input.
+
+ Args:
+ model: A DetectionModel to be built.
+ input_dataset: The tf.data Dataset the model is being trained on. Needed to
+ get the shapes for the dummy loss computation.
+ unpad_groundtruth_tensors: A parameter passed to unstack_batch.
+ """
+ features, labels = iter(input_dataset).next()
+
+ @tf.function
+ def _dummy_computation_fn(features, labels):
+ model._is_training = False # pylint: disable=protected-access
+ tf.keras.backend.set_learning_phase(False)
+
+ labels = model_lib.unstack_batch(
+ labels, unpad_groundtruth_tensors=unpad_groundtruth_tensors)
+
+ return _compute_losses_and_predictions_dicts(model, features, labels)
+
+ strategy = tf.compat.v2.distribute.get_strategy()
+ if hasattr(tf.distribute.Strategy, 'run'):
+ strategy.run(
+ _dummy_computation_fn, args=(
+ features,
+ labels,
+ ))
+ else:
+ strategy.experimental_run_v2(
+ _dummy_computation_fn, args=(
+ features,
+ labels,
+ ))
+
+
# TODO(kaftan): Explore removing learning_rate from this method & returning
## The full losses dict instead of just total_loss, then doing all summaries
## saving in a utility method called by the outer training loop.
@@ -214,6 +252,12 @@ def eager_train_step(detection_model,
(v, u) are part-relative normalized surface coordinates.
labels[fields.InputDataFields.groundtruth_labeled_classes] is a float32
k-hot tensor of classes.
+ labels[fields.InputDataFields.groundtruth_track_ids] is a int32
+ tensor of track IDs.
+ labels[fields.InputDataFields.groundtruth_keypoint_depths] is a
+ float32 tensor containing keypoint depths information.
+ labels[fields.InputDataFields.groundtruth_keypoint_depth_weights] is a
+ float32 tensor containing the weights of the keypoint depth feature.
unpad_groundtruth_tensors: A parameter passed to unstack_batch.
optimizer: The training optimizer that will update the variables.
learning_rate: The learning rate tensor for the current training step.
@@ -277,7 +321,8 @@ def validate_tf_v2_checkpoint_restore_map(checkpoint_restore_map):
"""Ensure that given dict is a valid TF v2 style restore map.
Args:
- checkpoint_restore_map: A dict mapping strings to tf.keras.Model objects.
+ checkpoint_restore_map: A nested dict mapping strings to
+ tf.keras.Model objects.
Raises:
ValueError: If they keys in checkpoint_restore_map are not strings or if
@@ -289,8 +334,12 @@ def validate_tf_v2_checkpoint_restore_map(checkpoint_restore_map):
if not (isinstance(key, str) and
(isinstance(value, tf.Module)
or isinstance(value, tf.train.Checkpoint))):
- raise TypeError(RESTORE_MAP_ERROR_TEMPLATE.format(
- key.__class__.__name__, value.__class__.__name__))
+ if isinstance(key, str) and isinstance(value, dict):
+ validate_tf_v2_checkpoint_restore_map(value)
+ else:
+ raise TypeError(
+ RESTORE_MAP_ERROR_TEMPLATE.format(key.__class__.__name__,
+ value.__class__.__name__))
def is_object_based_checkpoint(checkpoint_path):
@@ -299,9 +348,9 @@ def is_object_based_checkpoint(checkpoint_path):
return '_CHECKPOINTABLE_OBJECT_GRAPH' in var_names
-def load_fine_tune_checkpoint(
- model, checkpoint_path, checkpoint_type, checkpoint_version, input_dataset,
- unpad_groundtruth_tensors):
+def load_fine_tune_checkpoint(model, checkpoint_path, checkpoint_type,
+ checkpoint_version, run_model_on_dummy_input,
+ input_dataset, unpad_groundtruth_tensors):
"""Load a fine tuning classification or detection checkpoint.
To make sure the model variables are all built, this method first executes
@@ -323,6 +372,9 @@ def load_fine_tune_checkpoint(
checkpoint_version: train_pb2.CheckpointVersion.V1 or V2 enum indicating
whether to load checkpoints in V1 style or V2 style. In this binary
we only support V2 style (object-based) checkpoints.
+ run_model_on_dummy_input: Whether to run the model on a dummy input in order
+ to ensure that all model variables have been built successfully before
+ loading the fine_tune_checkpoint.
input_dataset: The tf.data Dataset the model is being trained on. Needed
to get the shapes for the dummy loss computation.
unpad_groundtruth_tensors: A parameter passed to unstack_batch.
@@ -337,34 +389,8 @@ def load_fine_tune_checkpoint(
if checkpoint_version == train_pb2.CheckpointVersion.V1:
raise ValueError('Checkpoint version should be V2')
- features, labels = iter(input_dataset).next()
-
- @tf.function
- def _dummy_computation_fn(features, labels):
- model._is_training = False # pylint: disable=protected-access
- tf.keras.backend.set_learning_phase(False)
-
- labels = model_lib.unstack_batch(
- labels, unpad_groundtruth_tensors=unpad_groundtruth_tensors)
-
- return _compute_losses_and_predictions_dicts(
- model,
- features,
- labels)
-
- strategy = tf.compat.v2.distribute.get_strategy()
- if hasattr(tf.distribute.Strategy, 'run'):
- strategy.run(
- _dummy_computation_fn, args=(
- features,
- labels,
- ))
- else:
- strategy.experimental_run_v2(
- _dummy_computation_fn, args=(
- features,
- labels,
- ))
+ if run_model_on_dummy_input:
+ _ensure_model_is_built(model, input_dataset, unpad_groundtruth_tensors)
restore_from_objects_dict = model.restore_from_objects(
fine_tune_checkpoint_type=checkpoint_type)
@@ -415,6 +441,9 @@ def train_loop(
save_final_config=False,
checkpoint_every_n=1000,
checkpoint_max_to_keep=7,
+ record_summaries=True,
+ performance_summary_exporter=None,
+ num_steps_per_iteration=NUM_STEPS_PER_ITERATION,
**kwargs):
"""Trains a model using eager + functions.
@@ -444,6 +473,10 @@ def train_loop(
Checkpoint every n training steps.
checkpoint_max_to_keep:
int, the number of most recent checkpoints to keep in the model directory.
+ record_summaries: Boolean, whether or not to record summaries.
+ performance_summary_exporter: function for exporting performance metrics.
+ num_steps_per_iteration: int, The number of training steps to perform
+ in each iteration.
**kwargs: Additional keyword arguments for configuration override.
"""
## Parse the configs
@@ -453,6 +486,7 @@ def train_loop(
'merge_external_params_with_configs']
create_pipeline_proto_from_configs = MODEL_BUILD_UTIL_MAP[
'create_pipeline_proto_from_configs']
+ steps_per_sec_list = []
configs = get_configs_from_pipeline_file(
pipeline_config_path, config_override=config_override)
@@ -477,7 +511,7 @@ def train_loop(
train_steps = train_config.num_steps
if kwargs['use_bfloat16']:
- tf.compat.v2.keras.mixed_precision.experimental.set_policy('mixed_bfloat16')
+ tf.compat.v2.keras.mixed_precision.set_global_policy('mixed_bfloat16')
if train_config.load_all_detection_checkpoint_vars:
raise ValueError('train_pb2.load_all_detection_checkpoint_vars '
@@ -489,13 +523,15 @@ def train_loop(
# Write the as-run pipeline config to disk.
if save_final_config:
+ tf.logging.info('Saving pipeline config file to directory {}'.format(
+ model_dir))
pipeline_config_final = create_pipeline_proto_from_configs(configs)
config_util.save_pipeline_config(pipeline_config_final, model_dir)
# Build the model, optimizer, and training input
strategy = tf.compat.v2.distribute.get_strategy()
with strategy.scope():
- detection_model = model_builder.build(
+ detection_model = MODEL_BUILD_UTIL_MAP['detection_model_fn_base'](
model_config=model_config, is_training=True)
def train_dataset_fn(input_context):
@@ -520,6 +556,15 @@ def train_loop(
optimizer, (learning_rate,) = optimizer_builder.build(
train_config.optimizer, global_step=global_step)
+ # We run the detection_model on dummy inputs in order to ensure that the
+ # model and all its variables have been properly constructed. Specifically,
+ # this is currently necessary prior to (potentially) creating shadow copies
+ # of the model variables for the EMA optimizer.
+ if train_config.optimizer.use_moving_average:
+ _ensure_model_is_built(detection_model, train_input,
+ unpad_groundtruth_tensors)
+ optimizer.shadow_copy(detection_model)
+
if callable(learning_rate):
learning_rate_fn = learning_rate
else:
@@ -530,15 +575,11 @@ def train_loop(
# is the chief.
summary_writer_filepath = get_filepath(strategy,
os.path.join(model_dir, 'train'))
- summary_writer = tf.compat.v2.summary.create_file_writer(
- summary_writer_filepath)
-
- if use_tpu:
- num_steps_per_iteration = 100
+ if record_summaries:
+ summary_writer = tf.compat.v2.summary.create_file_writer(
+ summary_writer_filepath)
else:
- # TODO(b/135933080) Explore setting to 100 when GPU performance issues
- # are fixed.
- num_steps_per_iteration = 1
+ summary_writer = tf2.summary.create_noop_writer()
with summary_writer.as_default():
with strategy.scope():
@@ -546,12 +587,11 @@ def train_loop(
lambda: global_step % num_steps_per_iteration == 0):
# Load a fine-tuning checkpoint.
if train_config.fine_tune_checkpoint:
- load_fine_tune_checkpoint(detection_model,
- train_config.fine_tune_checkpoint,
- fine_tune_checkpoint_type,
- fine_tune_checkpoint_version,
- train_input,
- unpad_groundtruth_tensors)
+ load_fine_tune_checkpoint(
+ detection_model, train_config.fine_tune_checkpoint,
+ fine_tune_checkpoint_type, fine_tune_checkpoint_version,
+ train_config.run_fine_tune_checkpoint_dummy_computation,
+ train_input, unpad_groundtruth_tensors)
ckpt = tf.compat.v2.train.Checkpoint(
step=global_step, model=detection_model, optimizer=optimizer)
@@ -603,7 +643,9 @@ def train_loop(
if num_steps_per_iteration > 1:
for _ in tf.range(num_steps_per_iteration - 1):
- _sample_and_train(strategy, train_step_fn, data_iterator)
+ # Following suggestion on yaqs/5402607292645376
+ with tf.name_scope(''):
+ _sample_and_train(strategy, train_step_fn, data_iterator)
return _sample_and_train(strategy, train_step_fn, data_iterator)
@@ -623,10 +665,12 @@ def train_loop(
time_taken = time.time() - last_step_time
last_step_time = time.time()
+ steps_per_sec = num_steps_per_iteration * 1.0 / time_taken
tf.compat.v2.summary.scalar(
- 'steps_per_sec', num_steps_per_iteration * 1.0 / time_taken,
- step=global_step)
+ 'steps_per_sec', steps_per_sec, step=global_step)
+
+ steps_per_sec_list.append(steps_per_sec)
if global_step.value() - logged_step >= 100:
tf.logging.info(
@@ -645,6 +689,116 @@ def train_loop(
# training.
clean_temporary_directories(strategy, manager_dir)
clean_temporary_directories(strategy, summary_writer_filepath)
+ # TODO(pkanwar): add accuracy metrics.
+ if performance_summary_exporter is not None:
+ metrics = {
+ 'steps_per_sec': np.mean(steps_per_sec_list),
+ 'steps_per_sec_p50': np.median(steps_per_sec_list),
+ 'steps_per_sec_max': max(steps_per_sec_list),
+ 'last_batch_loss': float(loss)
+ }
+ mixed_precision = 'bf16' if kwargs['use_bfloat16'] else 'fp32'
+ performance_summary_exporter(metrics, mixed_precision)
+
+
+def prepare_eval_dict(detections, groundtruth, features):
+ """Prepares eval dictionary containing detections and groundtruth.
+
+ Takes in `detections` from the model, `groundtruth` and `features` returned
+ from the eval tf.data.dataset and creates a dictionary of tensors suitable
+ for detection eval modules.
+
+ Args:
+ detections: A dictionary of tensors returned by `model.postprocess`.
+ groundtruth: `inputs.eval_input` returns an eval dataset of (features,
+ labels) tuple. `groundtruth` must be set to `labels`.
+ Please note that:
+ * fields.InputDataFields.groundtruth_classes must be 0-indexed and
+ in its 1-hot representation.
+ * fields.InputDataFields.groundtruth_verified_neg_classes must be
+ 0-indexed and in its multi-hot repesentation.
+ * fields.InputDataFields.groundtruth_not_exhaustive_classes must be
+ 0-indexed and in its multi-hot repesentation.
+ * fields.InputDataFields.groundtruth_labeled_classes must be
+ 0-indexed and in its multi-hot repesentation.
+ features: `inputs.eval_input` returns an eval dataset of (features, labels)
+ tuple. This argument must be set to a dictionary containing the following
+ keys and their corresponding values from `features` --
+ * fields.InputDataFields.image
+ * fields.InputDataFields.original_image
+ * fields.InputDataFields.original_image_spatial_shape
+ * fields.InputDataFields.true_image_shape
+ * inputs.HASH_KEY
+
+ Returns:
+ eval_dict: A dictionary of tensors to pass to eval module.
+ class_agnostic: Whether to evaluate detection in class agnostic mode.
+ """
+
+ groundtruth_boxes = groundtruth[fields.InputDataFields.groundtruth_boxes]
+ groundtruth_boxes_shape = tf.shape(groundtruth_boxes)
+ # For class-agnostic models, groundtruth one-hot encodings collapse to all
+ # ones.
+ class_agnostic = (
+ fields.DetectionResultFields.detection_classes not in detections)
+ if class_agnostic:
+ groundtruth_classes_one_hot = tf.ones(
+ [groundtruth_boxes_shape[0], groundtruth_boxes_shape[1], 1])
+ else:
+ groundtruth_classes_one_hot = groundtruth[
+ fields.InputDataFields.groundtruth_classes]
+ label_id_offset = 1 # Applying label id offset (b/63711816)
+ groundtruth_classes = (
+ tf.argmax(groundtruth_classes_one_hot, axis=2) + label_id_offset)
+ groundtruth[fields.InputDataFields.groundtruth_classes] = groundtruth_classes
+
+ label_id_offset_paddings = tf.constant([[0, 0], [1, 0]])
+ if fields.InputDataFields.groundtruth_verified_neg_classes in groundtruth:
+ groundtruth[
+ fields.InputDataFields.groundtruth_verified_neg_classes] = tf.pad(
+ groundtruth[
+ fields.InputDataFields.groundtruth_verified_neg_classes],
+ label_id_offset_paddings)
+ if fields.InputDataFields.groundtruth_not_exhaustive_classes in groundtruth:
+ groundtruth[
+ fields.InputDataFields.groundtruth_not_exhaustive_classes] = tf.pad(
+ groundtruth[
+ fields.InputDataFields.groundtruth_not_exhaustive_classes],
+ label_id_offset_paddings)
+ if fields.InputDataFields.groundtruth_labeled_classes in groundtruth:
+ groundtruth[fields.InputDataFields.groundtruth_labeled_classes] = tf.pad(
+ groundtruth[fields.InputDataFields.groundtruth_labeled_classes],
+ label_id_offset_paddings)
+
+ use_original_images = fields.InputDataFields.original_image in features
+ if use_original_images:
+ eval_images = features[fields.InputDataFields.original_image]
+ true_image_shapes = features[fields.InputDataFields.true_image_shape][:, :3]
+ original_image_spatial_shapes = features[
+ fields.InputDataFields.original_image_spatial_shape]
+ else:
+ eval_images = features[fields.InputDataFields.image]
+ true_image_shapes = None
+ original_image_spatial_shapes = None
+
+ eval_dict = eval_util.result_dict_for_batched_example(
+ eval_images,
+ features[inputs.HASH_KEY],
+ detections,
+ groundtruth,
+ class_agnostic=class_agnostic,
+ scale_to_absolute=True,
+ original_image_spatial_shapes=original_image_spatial_shapes,
+ true_image_shapes=true_image_shapes)
+
+ return eval_dict, class_agnostic
+
+
+def concat_replica_results(tensor_dict):
+ new_tensor_dict = {}
+ for key, values in tensor_dict.items():
+ new_tensor_dict[key] = tf.concat(values, axis=0)
+ return new_tensor_dict
def eager_eval_loop(
@@ -653,7 +807,8 @@ def eager_eval_loop(
eval_dataset,
use_tpu=False,
postprocess_on_cpu=False,
- global_step=None):
+ global_step=None,
+ ):
"""Evaluate the model eagerly on the evaluation dataset.
This method will compute the evaluation metrics specified in the configs on
@@ -675,6 +830,7 @@ def eager_eval_loop(
Returns:
A dict of evaluation metrics representing the results of this evaluation.
"""
+ del postprocess_on_cpu
train_config = configs['train_config']
eval_input_config = configs['eval_input_config']
eval_config = configs['eval_config']
@@ -686,6 +842,7 @@ def eager_eval_loop(
evaluator_options = eval_util.evaluator_options_from_eval_config(
eval_config)
+ batch_size = eval_config.batch_size
class_agnostic_category_index = (
label_map_util.create_class_agnostic_category_index())
@@ -714,55 +871,29 @@ def eager_eval_loop(
# must be unpadded.
boxes_shape = (
labels[fields.InputDataFields.groundtruth_boxes].get_shape().as_list())
- unpad_groundtruth_tensors = boxes_shape[1] is not None and not use_tpu
+ unpad_groundtruth_tensors = (boxes_shape[1] is not None
+ and not use_tpu
+ and batch_size == 1)
+ groundtruth_dict = labels
labels = model_lib.unstack_batch(
labels, unpad_groundtruth_tensors=unpad_groundtruth_tensors)
losses_dict, prediction_dict = _compute_losses_and_predictions_dicts(
detection_model, features, labels, add_regularization_loss)
-
- def postprocess_wrapper(args):
- return detection_model.postprocess(args[0], args[1])
-
- # TODO(kaftan): Depending on how postprocessing will work for TPUS w/
- ## TPUStrategy, may be good to move wrapping to a utility method
- if use_tpu and postprocess_on_cpu:
- detections = contrib_tpu.outside_compilation(
- postprocess_wrapper,
- (prediction_dict, features[fields.InputDataFields.true_image_shape]))
- else:
- detections = postprocess_wrapper(
- (prediction_dict, features[fields.InputDataFields.true_image_shape]))
-
- class_agnostic = (
- fields.DetectionResultFields.detection_classes not in detections)
- # TODO(kaftan) (or anyone): move `_prepare_groundtruth_for_eval to eval_util
- ## and call this from there.
- groundtruth = model_lib._prepare_groundtruth_for_eval( # pylint: disable=protected-access
- detection_model, class_agnostic, eval_input_config.max_number_of_boxes)
- use_original_images = fields.InputDataFields.original_image in features
- if use_original_images:
- eval_images = features[fields.InputDataFields.original_image]
- true_image_shapes = tf.slice(
- features[fields.InputDataFields.true_image_shape], [0, 0], [-1, 3])
- original_image_spatial_shapes = features[
- fields.InputDataFields.original_image_spatial_shape]
- else:
- eval_images = features[fields.InputDataFields.image]
- true_image_shapes = None
- original_image_spatial_shapes = None
-
- eval_dict = eval_util.result_dict_for_batched_example(
- eval_images,
- features[inputs.HASH_KEY],
- detections,
- groundtruth,
- class_agnostic=class_agnostic,
- scale_to_absolute=True,
- original_image_spatial_shapes=original_image_spatial_shapes,
- true_image_shapes=true_image_shapes)
-
- return eval_dict, losses_dict, class_agnostic
+ prediction_dict = detection_model.postprocess(
+ prediction_dict, features[fields.InputDataFields.true_image_shape])
+ eval_features = {
+ fields.InputDataFields.image:
+ features[fields.InputDataFields.image],
+ fields.InputDataFields.original_image:
+ features[fields.InputDataFields.original_image],
+ fields.InputDataFields.original_image_spatial_shape:
+ features[fields.InputDataFields.original_image_spatial_shape],
+ fields.InputDataFields.true_image_shape:
+ features[fields.InputDataFields.true_image_shape],
+ inputs.HASH_KEY: features[inputs.HASH_KEY],
+ }
+ return losses_dict, prediction_dict, groundtruth_dict, eval_features
agnostic_categories = label_map_util.create_class_agnostic_category_index()
per_class_categories = label_map_util.create_category_index_from_labelmap(
@@ -770,9 +901,32 @@ def eager_eval_loop(
keypoint_edges = [
(kp.start, kp.end) for kp in eval_config.keypoint_edge]
- for i, (features, labels) in enumerate(eval_dataset):
- eval_dict, losses_dict, class_agnostic = compute_eval_dict(features, labels)
+ strategy = tf.compat.v2.distribute.get_strategy()
+ for i, (features, labels) in enumerate(eval_dataset):
+ try:
+ (losses_dict, prediction_dict, groundtruth_dict,
+ eval_features) = strategy.run(
+ compute_eval_dict, args=(features, labels))
+ except Exception as exc: # pylint:disable=broad-except
+ tf.logging.info('Encountered %s exception.', exc)
+ tf.logging.info('A replica probably exhausted all examples. Skipping '
+ 'pending examples on other replicas.')
+ break
+ (local_prediction_dict, local_groundtruth_dict,
+ local_eval_features) = tf.nest.map_structure(
+ strategy.experimental_local_results,
+ [prediction_dict, groundtruth_dict, eval_features])
+ local_prediction_dict = concat_replica_results(local_prediction_dict)
+ local_groundtruth_dict = concat_replica_results(local_groundtruth_dict)
+ local_eval_features = concat_replica_results(local_eval_features)
+
+ eval_dict, class_agnostic = prepare_eval_dict(local_prediction_dict,
+ local_groundtruth_dict,
+ local_eval_features)
+ for loss_key, loss_tensor in iter(losses_dict.items()):
+ losses_dict[loss_key] = strategy.reduce(tf.distribute.ReduceOp.MEAN,
+ loss_tensor, None)
if class_agnostic:
category_index = agnostic_categories
else:
@@ -782,7 +936,7 @@ def eager_eval_loop(
tf.logging.info('Finished eval step %d', i)
use_original_images = fields.InputDataFields.original_image in features
- if use_original_images and i < eval_config.num_visualizations:
+ if (use_original_images and i < eval_config.num_visualizations):
sbys_image_list = vutils.draw_side_by_side_evaluation_image(
eval_dict,
category_index=category_index,
@@ -790,21 +944,21 @@ def eager_eval_loop(
min_score_thresh=eval_config.min_score_threshold,
use_normalized_coordinates=False,
keypoint_edges=keypoint_edges or None)
- sbys_images = tf.concat(sbys_image_list, axis=0)
- tf.compat.v2.summary.image(
- name='eval_side_by_side_' + str(i),
- step=global_step,
- data=sbys_images,
- max_outputs=eval_config.num_visualizations)
- if eval_util.has_densepose(eval_dict):
- dp_image_list = vutils.draw_densepose_visualizations(
- eval_dict)
- dp_images = tf.concat(dp_image_list, axis=0)
+ for j, sbys_image in enumerate(sbys_image_list):
tf.compat.v2.summary.image(
- name='densepose_detections_' + str(i),
+ name='eval_side_by_side_{}_{}'.format(i, j),
step=global_step,
- data=dp_images,
+ data=sbys_image,
max_outputs=eval_config.num_visualizations)
+ if eval_util.has_densepose(eval_dict):
+ dp_image_list = vutils.draw_densepose_visualizations(
+ eval_dict)
+ for j, dp_image in enumerate(dp_image_list):
+ tf.compat.v2.summary.image(
+ name='densepose_detections_{}_{}'.format(i, j),
+ step=global_step,
+ data=dp_image,
+ max_outputs=eval_config.num_visualizations)
if evaluators is None:
if class_agnostic:
@@ -817,27 +971,21 @@ def eager_eval_loop(
for loss_key, loss_tensor in iter(losses_dict.items()):
if loss_key not in loss_metrics:
- loss_metrics[loss_key] = tf.keras.metrics.Mean()
- # Skip the loss with value equal or lower than 0.0 when calculating the
- # average loss since they don't usually reflect the normal loss values
- # causing spurious average loss value.
- if loss_tensor <= 0.0:
- continue
- loss_metrics[loss_key].update_state(loss_tensor)
+ loss_metrics[loss_key] = []
+ loss_metrics[loss_key].append(loss_tensor)
eval_metrics = {}
for evaluator in evaluators:
eval_metrics.update(evaluator.evaluate())
for loss_key in loss_metrics:
- eval_metrics[loss_key] = loss_metrics[loss_key].result()
+ eval_metrics[loss_key] = tf.reduce_mean(loss_metrics[loss_key])
eval_metrics = {str(k): v for k, v in eval_metrics.items()}
- tf.logging.info('Eval metrics at step %d', global_step)
+ tf.logging.info('Eval metrics at step %d', global_step.numpy())
for k in eval_metrics:
tf.compat.v2.summary.scalar(k, eval_metrics[k], step=global_step)
tf.logging.info('\t+ %s: %f', k, eval_metrics[k])
-
return eval_metrics
@@ -854,6 +1002,8 @@ def eval_continuously(
checkpoint_dir=None,
wait_interval=180,
timeout=3600,
+ eval_index=0,
+ save_final_config=False,
**kwargs):
"""Run continuous evaluation of a detection model eagerly.
@@ -883,11 +1033,16 @@ def eval_continuously(
new checkpoint.
timeout: The maximum number of seconds to wait for a checkpoint. Execution
will terminate if no new checkpoints are found after these many seconds.
-
+ eval_index: int, If given, only evaluate the dataset at the given
+ index. By default, evaluates dataset at 0'th index.
+ save_final_config: Whether to save the pipeline config file to the model
+ directory.
**kwargs: Additional keyword arguments for configuration override.
"""
get_configs_from_pipeline_file = MODEL_BUILD_UTIL_MAP[
'get_configs_from_pipeline_file']
+ create_pipeline_proto_from_configs = MODEL_BUILD_UTIL_MAP[
+ 'create_pipeline_proto_from_configs']
merge_external_params_with_configs = MODEL_BUILD_UTIL_MAP[
'merge_external_params_with_configs']
@@ -905,6 +1060,12 @@ def eval_continuously(
'Forced number of epochs for all eval validations to be 1.')
configs = merge_external_params_with_configs(
configs, None, kwargs_dict=kwargs)
+ if model_dir and save_final_config:
+ tf.logging.info('Saving pipeline config file to directory {}'.format(
+ model_dir))
+ pipeline_config_final = create_pipeline_proto_from_configs(configs)
+ config_util.save_pipeline_config(pipeline_config_final, model_dir)
+
model_config = configs['model']
train_input_config = configs['train_input_config']
eval_config = configs['eval_config']
@@ -921,39 +1082,55 @@ def eval_continuously(
eval_on_train_input_config.num_epochs = 1
if kwargs['use_bfloat16']:
- tf.compat.v2.keras.mixed_precision.experimental.set_policy('mixed_bfloat16')
+ tf.compat.v2.keras.mixed_precision.set_global_policy('mixed_bfloat16')
- detection_model = model_builder.build(
- model_config=model_config, is_training=True)
+ eval_input_config = eval_input_configs[eval_index]
+ strategy = tf.compat.v2.distribute.get_strategy()
+ with strategy.scope():
+ detection_model = MODEL_BUILD_UTIL_MAP['detection_model_fn_base'](
+ model_config=model_config, is_training=True)
- # Create the inputs.
- eval_inputs = []
- for eval_input_config in eval_input_configs:
- next_eval_input = inputs.eval_input(
- eval_config=eval_config,
- eval_input_config=eval_input_config,
- model_config=model_config,
- model=detection_model)
- eval_inputs.append((eval_input_config.name, next_eval_input))
+ eval_input = strategy.experimental_distribute_dataset(
+ inputs.eval_input(
+ eval_config=eval_config,
+ eval_input_config=eval_input_config,
+ model_config=model_config,
+ model=detection_model))
global_step = tf.compat.v2.Variable(
0, trainable=False, dtype=tf.compat.v2.dtypes.int64)
+ optimizer, _ = optimizer_builder.build(
+ configs['train_config'].optimizer, global_step=global_step)
+
for latest_checkpoint in tf.train.checkpoints_iterator(
checkpoint_dir, timeout=timeout, min_interval_secs=wait_interval):
ckpt = tf.compat.v2.train.Checkpoint(
- step=global_step, model=detection_model)
+ step=global_step, model=detection_model, optimizer=optimizer)
+
+ # We run the detection_model on dummy inputs in order to ensure that the
+ # model and all its variables have been properly constructed. Specifically,
+ # this is currently necessary prior to (potentially) creating shadow copies
+ # of the model variables for the EMA optimizer.
+ if eval_config.use_moving_averages:
+ unpad_groundtruth_tensors = (eval_config.batch_size == 1 and not use_tpu)
+ _ensure_model_is_built(detection_model, eval_input,
+ unpad_groundtruth_tensors)
+ optimizer.shadow_copy(detection_model)
ckpt.restore(latest_checkpoint).expect_partial()
- for eval_name, eval_input in eval_inputs:
- summary_writer = tf.compat.v2.summary.create_file_writer(
- os.path.join(model_dir, 'eval', eval_name))
- with summary_writer.as_default():
- eager_eval_loop(
- detection_model,
- configs,
- eval_input,
- use_tpu=use_tpu,
- postprocess_on_cpu=postprocess_on_cpu,
- global_step=global_step)
+ if eval_config.use_moving_averages:
+ optimizer.swap_weights()
+
+ summary_writer = tf.compat.v2.summary.create_file_writer(
+ os.path.join(model_dir, 'eval', eval_input_config.name))
+ with summary_writer.as_default():
+ eager_eval_loop(
+ detection_model,
+ configs,
+ eval_input,
+ use_tpu=use_tpu,
+ postprocess_on_cpu=postprocess_on_cpu,
+ global_step=global_step,
+ )
diff --git a/research/object_detection/model_main_tf2.py b/research/object_detection/model_main_tf2.py
index 82c7c7acfc52ca8a8040095560b5f001eddf3625..0cf053039ec16461fef0c1eb2f94df66fad2b70c 100644
--- a/research/object_detection/model_main_tf2.py
+++ b/research/object_detection/model_main_tf2.py
@@ -62,6 +62,11 @@ flags.DEFINE_integer(
'num_workers', 1, 'When num_workers > 1, training uses '
'MultiWorkerMirroredStrategy. When num_workers = 1 it uses '
'MirroredStrategy.')
+flags.DEFINE_integer(
+ 'checkpoint_every_n', 1000, 'Integer defining how often we checkpoint.')
+flags.DEFINE_boolean('record_summaries', True,
+ ('Whether or not to record summaries during'
+ ' training.'))
FLAGS = flags.FLAGS
@@ -100,7 +105,9 @@ def main(unused_argv):
pipeline_config_path=FLAGS.pipeline_config_path,
model_dir=FLAGS.model_dir,
train_steps=FLAGS.num_train_steps,
- use_tpu=FLAGS.use_tpu)
+ use_tpu=FLAGS.use_tpu,
+ checkpoint_every_n=FLAGS.checkpoint_every_n,
+ record_summaries=FLAGS.record_summaries)
if __name__ == '__main__':
tf.compat.v1.app.run()
diff --git a/research/object_detection/models/center_net_hourglass_feature_extractor.py b/research/object_detection/models/center_net_hourglass_feature_extractor.py
index 4761915aa5ad0023673199f2083ff355816f7bb1..785041e89d26bfc34808cd5c40486b9182db263e 100644
--- a/research/object_detection/models/center_net_hourglass_feature_extractor.py
+++ b/research/object_detection/models/center_net_hourglass_feature_extractor.py
@@ -62,13 +62,63 @@ class CenterNetHourglassFeatureExtractor(
"""Ther number of feature outputs returned by the feature extractor."""
return self._network.num_hourglasses
- def get_model(self):
- return self._network
+ @property
+ def supported_sub_model_types(self):
+ return ['detection']
+
+ def get_sub_model(self, sub_model_type):
+ if sub_model_type == 'detection':
+ return self._network
+ else:
+ ValueError('Sub model type "{}" not supported.'.format(sub_model_type))
+
+
+def hourglass_10(channel_means, channel_stds, bgr_ordering, **kwargs):
+ """The Hourglass-10 backbone for CenterNet."""
+ del kwargs
+
+ network = hourglass_network.hourglass_10(num_channels=32)
+ return CenterNetHourglassFeatureExtractor(
+ network, channel_means=channel_means, channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
+
+
+def hourglass_20(channel_means, channel_stds, bgr_ordering, **kwargs):
+ """The Hourglass-20 backbone for CenterNet."""
+ del kwargs
+
+ network = hourglass_network.hourglass_20(num_channels=48)
+ return CenterNetHourglassFeatureExtractor(
+ network, channel_means=channel_means, channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
+
+
+def hourglass_32(channel_means, channel_stds, bgr_ordering, **kwargs):
+ """The Hourglass-32 backbone for CenterNet."""
+ del kwargs
+
+ network = hourglass_network.hourglass_32(num_channels=48)
+ return CenterNetHourglassFeatureExtractor(
+ network, channel_means=channel_means, channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
+
+
+def hourglass_52(channel_means, channel_stds, bgr_ordering, **kwargs):
+ """The Hourglass-52 backbone for CenterNet."""
+ del kwargs
+
+ network = hourglass_network.hourglass_52(num_channels=64)
+ return CenterNetHourglassFeatureExtractor(
+ network, channel_means=channel_means, channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
-def hourglass_104(channel_means, channel_stds, bgr_ordering):
+def hourglass_104(channel_means, channel_stds, bgr_ordering, **kwargs):
"""The Hourglass-104 backbone for CenterNet."""
+ del kwargs
+ # TODO(vighneshb): update hourglass_104 signature to match with other
+ # hourglass networks.
network = hourglass_network.hourglass_104()
return CenterNetHourglassFeatureExtractor(
network, channel_means=channel_means, channel_stds=channel_stds,
diff --git a/research/object_detection/models/center_net_hourglass_feature_extractor_tf2_test.py b/research/object_detection/models/center_net_hourglass_feature_extractor_tf2_test.py
index 19d5cbe9843ff03d6d1499a02980a067dc305579..31c26c5ab9efddc99518e92a2320ed409d737ea3 100644
--- a/research/object_detection/models/center_net_hourglass_feature_extractor_tf2_test.py
+++ b/research/object_detection/models/center_net_hourglass_feature_extractor_tf2_test.py
@@ -30,7 +30,8 @@ class CenterNetHourglassFeatureExtractorTest(test_case.TestCase):
net = hourglass_network.HourglassNetwork(
num_stages=4, blocks_per_stage=[2, 3, 4, 5, 6],
- channel_dims=[4, 6, 8, 10, 12, 14], num_hourglasses=2)
+ input_channel_dims=4, channel_dims_per_stage=[6, 8, 10, 12, 14],
+ num_hourglasses=2)
model = hourglass.CenterNetHourglassFeatureExtractor(net)
def graph_fn():
diff --git a/research/object_detection/models/center_net_mobilenet_v2_feature_extractor.py b/research/object_detection/models/center_net_mobilenet_v2_feature_extractor.py
new file mode 100644
index 0000000000000000000000000000000000000000..63ce95d6c524c10ba08fff4e3f06d14ca36c7a91
--- /dev/null
+++ b/research/object_detection/models/center_net_mobilenet_v2_feature_extractor.py
@@ -0,0 +1,128 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""MobileNet V2[1] feature extractor for CenterNet[2] meta architecture.
+
+[1]: https://arxiv.org/abs/1801.04381
+[2]: https://arxiv.org/abs/1904.07850
+"""
+
+import tensorflow.compat.v1 as tf
+
+from object_detection.meta_architectures import center_net_meta_arch
+from object_detection.models.keras_models import mobilenet_v2 as mobilenetv2
+
+
+class CenterNetMobileNetV2FeatureExtractor(
+ center_net_meta_arch.CenterNetFeatureExtractor):
+ """The MobileNet V2 feature extractor for CenterNet."""
+
+ def __init__(self,
+ mobilenet_v2_net,
+ channel_means=(0., 0., 0.),
+ channel_stds=(1., 1., 1.),
+ bgr_ordering=False):
+ """Intializes the feature extractor.
+
+ Args:
+ mobilenet_v2_net: The underlying mobilenet_v2 network to use.
+ channel_means: A tuple of floats, denoting the mean of each channel
+ which will be subtracted from it.
+ channel_stds: A tuple of floats, denoting the standard deviation of each
+ channel. Each channel will be divided by its standard deviation value.
+ bgr_ordering: bool, if set will change the channel ordering to be in the
+ [blue, red, green] order.
+ """
+
+ super(CenterNetMobileNetV2FeatureExtractor, self).__init__(
+ channel_means=channel_means,
+ channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
+ self._network = mobilenet_v2_net
+
+ output = self._network(self._network.input)
+
+ # MobileNet by itself transforms a 224x224x3 volume into a 7x7x1280, which
+ # leads to a stride of 32. We perform upsampling to get it to a target
+ # stride of 4.
+ for num_filters in [256, 128, 64]:
+ # 1. We use a simple convolution instead of a deformable convolution
+ conv = tf.keras.layers.Conv2D(
+ filters=num_filters, kernel_size=1, strides=1, padding='same')
+ output = conv(output)
+ output = tf.keras.layers.BatchNormalization()(output)
+ output = tf.keras.layers.ReLU()(output)
+
+ # 2. We use the default initialization for the convolution layers
+ # instead of initializing it to do bilinear upsampling.
+ conv_transpose = tf.keras.layers.Conv2DTranspose(
+ filters=num_filters, kernel_size=3, strides=2, padding='same')
+ output = conv_transpose(output)
+ output = tf.keras.layers.BatchNormalization()(output)
+ output = tf.keras.layers.ReLU()(output)
+
+ self._network = tf.keras.models.Model(
+ inputs=self._network.input, outputs=output)
+
+ def preprocess(self, resized_inputs):
+ resized_inputs = super(CenterNetMobileNetV2FeatureExtractor,
+ self).preprocess(resized_inputs)
+ return tf.keras.applications.mobilenet_v2.preprocess_input(resized_inputs)
+
+ def load_feature_extractor_weights(self, path):
+ self._network.load_weights(path)
+
+ def get_base_model(self):
+ return self._network
+
+ def call(self, inputs):
+ return [self._network(inputs)]
+
+ @property
+ def out_stride(self):
+ """The stride in the output image of the network."""
+ return 4
+
+ @property
+ def num_feature_outputs(self):
+ """The number of feature outputs returned by the feature extractor."""
+ return 1
+
+ @property
+ def supported_sub_model_types(self):
+ return ['detection']
+
+ def get_sub_model(self, sub_model_type):
+ if sub_model_type == 'detection':
+ return self._network
+ else:
+ ValueError('Sub model type "{}" not supported.'.format(sub_model_type))
+
+
+def mobilenet_v2(channel_means, channel_stds, bgr_ordering,
+ depth_multiplier=1.0, **kwargs):
+ """The MobileNetV2 backbone for CenterNet."""
+ del kwargs
+
+ # We set 'is_training' to True for now.
+ network = mobilenetv2.mobilenet_v2(
+ batchnorm_training=True,
+ alpha=depth_multiplier,
+ include_top=False,
+ weights='imagenet' if depth_multiplier == 1.0 else None)
+ return CenterNetMobileNetV2FeatureExtractor(
+ network,
+ channel_means=channel_means,
+ channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
diff --git a/research/object_detection/models/center_net_mobilenet_v2_feature_extractor_tf2_test.py b/research/object_detection/models/center_net_mobilenet_v2_feature_extractor_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..5211701138d8e134bba7c2ff6b247cf19d156691
--- /dev/null
+++ b/research/object_detection/models/center_net_mobilenet_v2_feature_extractor_tf2_test.py
@@ -0,0 +1,46 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Testing mobilenet_v2 feature extractor for CenterNet."""
+import unittest
+import numpy as np
+import tensorflow.compat.v1 as tf
+
+from object_detection.models import center_net_mobilenet_v2_feature_extractor
+from object_detection.models.keras_models import mobilenet_v2
+from object_detection.utils import test_case
+from object_detection.utils import tf_version
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class CenterNetMobileNetV2FeatureExtractorTest(test_case.TestCase):
+
+ def test_center_net_mobilenet_v2_feature_extractor(self):
+
+ net = mobilenet_v2.mobilenet_v2(True, include_top=False)
+
+ model = center_net_mobilenet_v2_feature_extractor.CenterNetMobileNetV2FeatureExtractor(
+ net)
+
+ def graph_fn():
+ img = np.zeros((8, 224, 224, 3), dtype=np.float32)
+ processed_img = model.preprocess(img)
+ return model(processed_img)
+
+ outputs = self.execute(graph_fn, [])
+ self.assertEqual(outputs.shape, (8, 56, 56, 64))
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/models/center_net_mobilenet_v2_fpn_feature_extractor.py b/research/object_detection/models/center_net_mobilenet_v2_fpn_feature_extractor.py
new file mode 100644
index 0000000000000000000000000000000000000000..15ba3d85cf6160b6dddefa8116939c88cd812dc0
--- /dev/null
+++ b/research/object_detection/models/center_net_mobilenet_v2_fpn_feature_extractor.py
@@ -0,0 +1,162 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""MobileNet V2[1] + FPN[2] feature extractor for CenterNet[3] meta architecture.
+
+[1]: https://arxiv.org/abs/1801.04381
+[2]: https://arxiv.org/abs/1612.03144.
+[3]: https://arxiv.org/abs/1904.07850
+"""
+
+import tensorflow.compat.v1 as tf
+
+from object_detection.meta_architectures import center_net_meta_arch
+from object_detection.models.keras_models import mobilenet_v2 as mobilenetv2
+
+
+_MOBILENET_V2_FPN_SKIP_LAYERS = [
+ 'block_2_add', 'block_5_add', 'block_9_add', 'out_relu'
+]
+
+
+class CenterNetMobileNetV2FPNFeatureExtractor(
+ center_net_meta_arch.CenterNetFeatureExtractor):
+ """The MobileNet V2 with FPN skip layers feature extractor for CenterNet."""
+
+ def __init__(self,
+ mobilenet_v2_net,
+ channel_means=(0., 0., 0.),
+ channel_stds=(1., 1., 1.),
+ bgr_ordering=False,
+ use_separable_conv=False):
+ """Intializes the feature extractor.
+
+ Args:
+ mobilenet_v2_net: The underlying mobilenet_v2 network to use.
+ channel_means: A tuple of floats, denoting the mean of each channel
+ which will be subtracted from it.
+ channel_stds: A tuple of floats, denoting the standard deviation of each
+ channel. Each channel will be divided by its standard deviation value.
+ bgr_ordering: bool, if set will change the channel ordering to be in the
+ [blue, red, green] order.
+ use_separable_conv: If set to True, all convolutional layers in the FPN
+ network will be replaced by separable convolutions.
+ """
+
+ super(CenterNetMobileNetV2FPNFeatureExtractor, self).__init__(
+ channel_means=channel_means,
+ channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
+ self._base_model = mobilenet_v2_net
+
+ output = self._base_model(self._base_model.input)
+
+ # Add pyramid feature network on every layer that has stride 2.
+ skip_outputs = [
+ self._base_model.get_layer(skip_layer_name).output
+ for skip_layer_name in _MOBILENET_V2_FPN_SKIP_LAYERS
+ ]
+ self._fpn_model = tf.keras.models.Model(
+ inputs=self._base_model.input, outputs=skip_outputs)
+ fpn_outputs = self._fpn_model(self._base_model.input)
+
+ # Construct the top-down feature maps -- we start with an output of
+ # 7x7x1280, which we continually upsample, apply a residual on and merge.
+ # This results in a 56x56x24 output volume.
+ top_layer = fpn_outputs[-1]
+ # Use normal convolutional layer since the kernel_size is 1.
+ residual_op = tf.keras.layers.Conv2D(
+ filters=64, kernel_size=1, strides=1, padding='same')
+ top_down = residual_op(top_layer)
+
+ num_filters_list = [64, 32, 24]
+ for i, num_filters in enumerate(num_filters_list):
+ level_ind = len(num_filters_list) - 1 - i
+ # Upsample.
+ upsample_op = tf.keras.layers.UpSampling2D(2, interpolation='nearest')
+ top_down = upsample_op(top_down)
+
+ # Residual (skip-connection) from bottom-up pathway.
+ # Use normal convolutional layer since the kernel_size is 1.
+ residual_op = tf.keras.layers.Conv2D(
+ filters=num_filters, kernel_size=1, strides=1, padding='same')
+ residual = residual_op(fpn_outputs[level_ind])
+
+ # Merge.
+ top_down = top_down + residual
+ next_num_filters = num_filters_list[i + 1] if i + 1 <= 2 else 24
+ if use_separable_conv:
+ conv = tf.keras.layers.SeparableConv2D(
+ filters=next_num_filters, kernel_size=3, strides=1, padding='same')
+ else:
+ conv = tf.keras.layers.Conv2D(
+ filters=next_num_filters, kernel_size=3, strides=1, padding='same')
+ top_down = conv(top_down)
+ top_down = tf.keras.layers.BatchNormalization()(top_down)
+ top_down = tf.keras.layers.ReLU()(top_down)
+
+ output = top_down
+
+ self._feature_extractor_model = tf.keras.models.Model(
+ inputs=self._base_model.input, outputs=output)
+
+ def preprocess(self, resized_inputs):
+ resized_inputs = super(CenterNetMobileNetV2FPNFeatureExtractor,
+ self).preprocess(resized_inputs)
+ return tf.keras.applications.mobilenet_v2.preprocess_input(resized_inputs)
+
+ def load_feature_extractor_weights(self, path):
+ self._base_model.load_weights(path)
+
+ @property
+ def supported_sub_model_types(self):
+ return ['classification']
+
+ def get_sub_model(self, sub_model_type):
+ if sub_model_type == 'classification':
+ return self._base_model
+ else:
+ ValueError('Sub model type "{}" not supported.'.format(sub_model_type))
+
+ def call(self, inputs):
+ return [self._feature_extractor_model(inputs)]
+
+ @property
+ def out_stride(self):
+ """The stride in the output image of the network."""
+ return 4
+
+ @property
+ def num_feature_outputs(self):
+ """The number of feature outputs returned by the feature extractor."""
+ return 1
+
+
+def mobilenet_v2_fpn(channel_means, channel_stds, bgr_ordering,
+ use_separable_conv=False, depth_multiplier=1.0, **kwargs):
+ """The MobileNetV2+FPN backbone for CenterNet."""
+ del kwargs
+
+ # Set to batchnorm_training to True for now.
+ network = mobilenetv2.mobilenet_v2(
+ batchnorm_training=True,
+ alpha=depth_multiplier,
+ include_top=False,
+ weights='imagenet' if depth_multiplier == 1.0 else None)
+ return CenterNetMobileNetV2FPNFeatureExtractor(
+ network,
+ channel_means=channel_means,
+ channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering,
+ use_separable_conv=use_separable_conv)
diff --git a/research/object_detection/models/center_net_mobilenet_v2_fpn_feature_extractor_tf2_test.py b/research/object_detection/models/center_net_mobilenet_v2_fpn_feature_extractor_tf2_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..82e2ca02167b1882bef7260e1a5b233914add8bb
--- /dev/null
+++ b/research/object_detection/models/center_net_mobilenet_v2_fpn_feature_extractor_tf2_test.py
@@ -0,0 +1,108 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Testing mobilenet_v2+FPN feature extractor for CenterNet."""
+import unittest
+import numpy as np
+import tensorflow.compat.v1 as tf
+
+from object_detection.models import center_net_mobilenet_v2_fpn_feature_extractor
+from object_detection.utils import test_case
+from object_detection.utils import tf_version
+
+
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class CenterNetMobileNetV2FPNFeatureExtractorTest(test_case.TestCase):
+
+ def test_center_net_mobilenet_v2_fpn_feature_extractor(self):
+
+ channel_means = (0., 0., 0.)
+ channel_stds = (1., 1., 1.)
+ bgr_ordering = False
+ model = (
+ center_net_mobilenet_v2_fpn_feature_extractor.mobilenet_v2_fpn(
+ channel_means, channel_stds, bgr_ordering,
+ use_separable_conv=False))
+
+ def graph_fn():
+ img = np.zeros((8, 224, 224, 3), dtype=np.float32)
+ processed_img = model.preprocess(img)
+ return model(processed_img)
+
+ outputs = self.execute(graph_fn, [])
+ self.assertEqual(outputs.shape, (8, 56, 56, 24))
+
+ # Pull out the FPN network.
+ output = model.get_layer('model_1')
+ for layer in output.layers:
+ # All convolution layers should be normal 2D convolutions.
+ if 'conv' in layer.name:
+ self.assertIsInstance(layer, tf.keras.layers.Conv2D)
+
+ def test_center_net_mobilenet_v2_fpn_feature_extractor_sep_conv(self):
+
+ channel_means = (0., 0., 0.)
+ channel_stds = (1., 1., 1.)
+ bgr_ordering = False
+ model = (
+ center_net_mobilenet_v2_fpn_feature_extractor.mobilenet_v2_fpn(
+ channel_means, channel_stds, bgr_ordering, use_separable_conv=True))
+
+ def graph_fn():
+ img = np.zeros((8, 224, 224, 3), dtype=np.float32)
+ processed_img = model.preprocess(img)
+ return model(processed_img)
+
+ outputs = self.execute(graph_fn, [])
+ self.assertEqual(outputs.shape, (8, 56, 56, 24))
+ # Pull out the FPN network.
+ backbone = model.get_layer('model')
+ first_conv = backbone.get_layer('Conv1')
+ self.assertEqual(32, first_conv.filters)
+
+ # Pull out the FPN network.
+ output = model.get_layer('model_1')
+ for layer in output.layers:
+ # Convolution layers with kernel size not equal to (1, 1) should be
+ # separable 2D convolutions.
+ if 'conv' in layer.name and layer.kernel_size != (1, 1):
+ self.assertIsInstance(layer, tf.keras.layers.SeparableConv2D)
+
+ def test_center_net_mobilenet_v2_fpn_feature_extractor_depth_multiplier(self):
+
+ channel_means = (0., 0., 0.)
+ channel_stds = (1., 1., 1.)
+ bgr_ordering = False
+ model = (
+ center_net_mobilenet_v2_fpn_feature_extractor.mobilenet_v2_fpn(
+ channel_means, channel_stds, bgr_ordering, use_separable_conv=True,
+ depth_multiplier=2.0))
+
+ def graph_fn():
+ img = np.zeros((8, 224, 224, 3), dtype=np.float32)
+ processed_img = model.preprocess(img)
+ return model(processed_img)
+
+ outputs = self.execute(graph_fn, [])
+ self.assertEqual(outputs.shape, (8, 56, 56, 24))
+ # Pull out the FPN network.
+ backbone = model.get_layer('model')
+ first_conv = backbone.get_layer('Conv1')
+ # Note that the first layer typically has 32 filters, but this model has
+ # a depth multiplier of 2.
+ self.assertEqual(64, first_conv.filters)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/object_detection/models/center_net_resnet_feature_extractor.py b/research/object_detection/models/center_net_resnet_feature_extractor.py
index 477fa4c50ea9e0bc62b43a75c1674acfef7a183c..d8fbe7126563c3ef0d34602d86df3061fbb366a5 100644
--- a/research/object_detection/models/center_net_resnet_feature_extractor.py
+++ b/research/object_detection/models/center_net_resnet_feature_extractor.py
@@ -46,10 +46,12 @@ class CenterNetResnetFeatureExtractor(CenterNetFeatureExtractor):
channel_means=channel_means, channel_stds=channel_stds,
bgr_ordering=bgr_ordering)
if resnet_type == 'resnet_v2_101':
- self._base_model = tf.keras.applications.ResNet101V2(weights=None)
+ self._base_model = tf.keras.applications.ResNet101V2(weights=None,
+ include_top=False)
output_layer = 'conv5_block3_out'
elif resnet_type == 'resnet_v2_50':
- self._base_model = tf.keras.applications.ResNet50V2(weights=None)
+ self._base_model = tf.keras.applications.ResNet50V2(weights=None,
+ include_top=False)
output_layer = 'conv5_block3_out'
else:
raise ValueError('Unknown Resnet Model {}'.format(resnet_type))
@@ -101,10 +103,6 @@ class CenterNetResnetFeatureExtractor(CenterNetFeatureExtractor):
def load_feature_extractor_weights(self, path):
self._base_model.load_weights(path)
- def get_base_model(self):
- """Get base resnet model for inspection and testing."""
- return self._base_model
-
def call(self, inputs):
"""Returns image features extracted by the backbone.
@@ -127,9 +125,20 @@ class CenterNetResnetFeatureExtractor(CenterNetFeatureExtractor):
def out_stride(self):
return 4
+ @property
+ def supported_sub_model_types(self):
+ return ['classification']
+
+ def get_sub_model(self, sub_model_type):
+ if sub_model_type == 'classification':
+ return self._base_model
+ else:
+ ValueError('Sub model type "{}" not supported.'.format(sub_model_type))
+
-def resnet_v2_101(channel_means, channel_stds, bgr_ordering):
+def resnet_v2_101(channel_means, channel_stds, bgr_ordering, **kwargs):
"""The ResNet v2 101 feature extractor."""
+ del kwargs
return CenterNetResnetFeatureExtractor(
resnet_type='resnet_v2_101',
@@ -139,8 +148,9 @@ def resnet_v2_101(channel_means, channel_stds, bgr_ordering):
)
-def resnet_v2_50(channel_means, channel_stds, bgr_ordering):
+def resnet_v2_50(channel_means, channel_stds, bgr_ordering, **kwargs):
"""The ResNet v2 50 feature extractor."""
+ del kwargs
return CenterNetResnetFeatureExtractor(
resnet_type='resnet_v2_50',
diff --git a/research/object_detection/models/center_net_resnet_feature_extractor_tf2_test.py b/research/object_detection/models/center_net_resnet_feature_extractor_tf2_test.py
index 3429c0442053982d3d3d9502508ede3177cbf102..d8f9b22a746cbd6da862f9a37f4ef2e57f10b451 100644
--- a/research/object_detection/models/center_net_resnet_feature_extractor_tf2_test.py
+++ b/research/object_detection/models/center_net_resnet_feature_extractor_tf2_test.py
@@ -31,11 +31,11 @@ class CenterNetResnetFeatureExtractorTest(test_case.TestCase):
model = center_net_resnet_feature_extractor.\
CenterNetResnetFeatureExtractor('resnet_v2_101')
def graph_fn():
- img = np.zeros((8, 224, 224, 3), dtype=np.float32)
+ img = np.zeros((8, 512, 512, 3), dtype=np.float32)
processed_img = model.preprocess(img)
return model(processed_img)
outputs = self.execute(graph_fn, [])
- self.assertEqual(outputs.shape, (8, 56, 56, 64))
+ self.assertEqual(outputs.shape, (8, 128, 128, 64))
def test_output_size_resnet50(self):
"""Verify that shape of features returned by the backbone is correct."""
diff --git a/research/object_detection/models/center_net_resnet_v1_fpn_feature_extractor.py b/research/object_detection/models/center_net_resnet_v1_fpn_feature_extractor.py
index 842e9cf1b2e5393a6bc87df3989f173d0409de70..c78091e8d04b76bd383092078d0e08c0f05bbdb4 100644
--- a/research/object_detection/models/center_net_resnet_v1_fpn_feature_extractor.py
+++ b/research/object_detection/models/center_net_resnet_v1_fpn_feature_extractor.py
@@ -21,9 +21,14 @@
import tensorflow.compat.v1 as tf
from object_detection.meta_architectures.center_net_meta_arch import CenterNetFeatureExtractor
+from object_detection.models.keras_models import resnet_v1
_RESNET_MODEL_OUTPUT_LAYERS = {
+ 'resnet_v1_18': ['conv2_block2_out', 'conv3_block2_out',
+ 'conv4_block2_out', 'conv5_block2_out'],
+ 'resnet_v1_34': ['conv2_block3_out', 'conv3_block4_out',
+ 'conv4_block6_out', 'conv5_block3_out'],
'resnet_v1_50': ['conv2_block3_out', 'conv3_block4_out',
'conv4_block6_out', 'conv5_block3_out'],
'resnet_v1_101': ['conv2_block3_out', 'conv3_block4_out',
@@ -66,9 +71,15 @@ class CenterNetResnetV1FpnFeatureExtractor(CenterNetFeatureExtractor):
channel_means=channel_means, channel_stds=channel_stds,
bgr_ordering=bgr_ordering)
if resnet_type == 'resnet_v1_50':
- self._base_model = tf.keras.applications.ResNet50(weights=None)
+ self._base_model = tf.keras.applications.ResNet50(weights=None,
+ include_top=False)
elif resnet_type == 'resnet_v1_101':
- self._base_model = tf.keras.applications.ResNet101(weights=None)
+ self._base_model = tf.keras.applications.ResNet101(weights=None,
+ include_top=False)
+ elif resnet_type == 'resnet_v1_18':
+ self._base_model = resnet_v1.resnet_v1_18(weights=None, include_top=False)
+ elif resnet_type == 'resnet_v1_34':
+ self._base_model = resnet_v1.resnet_v1_34(weights=None, include_top=False)
else:
raise ValueError('Unknown Resnet Model {}'.format(resnet_type))
output_layers = _RESNET_MODEL_OUTPUT_LAYERS[resnet_type]
@@ -128,10 +139,6 @@ class CenterNetResnetV1FpnFeatureExtractor(CenterNetFeatureExtractor):
def load_feature_extractor_weights(self, path):
self._base_model.load_weights(path)
- def get_base_model(self):
- """Get base resnet model for inspection and testing."""
- return self._base_model
-
def call(self, inputs):
"""Returns image features extracted by the backbone.
@@ -154,9 +161,20 @@ class CenterNetResnetV1FpnFeatureExtractor(CenterNetFeatureExtractor):
def out_stride(self):
return 4
+ @property
+ def supported_sub_model_types(self):
+ return ['classification']
+
+ def get_sub_model(self, sub_model_type):
+ if sub_model_type == 'classification':
+ return self._base_model
+ else:
+ ValueError('Sub model type "{}" not supported.'.format(sub_model_type))
+
-def resnet_v1_101_fpn(channel_means, channel_stds, bgr_ordering):
+def resnet_v1_101_fpn(channel_means, channel_stds, bgr_ordering, **kwargs):
"""The ResNet v1 101 FPN feature extractor."""
+ del kwargs
return CenterNetResnetV1FpnFeatureExtractor(
resnet_type='resnet_v1_101',
@@ -166,11 +184,35 @@ def resnet_v1_101_fpn(channel_means, channel_stds, bgr_ordering):
)
-def resnet_v1_50_fpn(channel_means, channel_stds, bgr_ordering):
+def resnet_v1_50_fpn(channel_means, channel_stds, bgr_ordering, **kwargs):
"""The ResNet v1 50 FPN feature extractor."""
+ del kwargs
return CenterNetResnetV1FpnFeatureExtractor(
resnet_type='resnet_v1_50',
channel_means=channel_means,
channel_stds=channel_stds,
bgr_ordering=bgr_ordering)
+
+
+def resnet_v1_34_fpn(channel_means, channel_stds, bgr_ordering, **kwargs):
+ """The ResNet v1 34 FPN feature extractor."""
+ del kwargs
+
+ return CenterNetResnetV1FpnFeatureExtractor(
+ resnet_type='resnet_v1_34',
+ channel_means=channel_means,
+ channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering
+ )
+
+
+def resnet_v1_18_fpn(channel_means, channel_stds, bgr_ordering, **kwargs):
+ """The ResNet v1 18 FPN feature extractor."""
+ del kwargs
+
+ return CenterNetResnetV1FpnFeatureExtractor(
+ resnet_type='resnet_v1_18',
+ channel_means=channel_means,
+ channel_stds=channel_stds,
+ bgr_ordering=bgr_ordering)
diff --git a/research/object_detection/models/center_net_resnet_v1_fpn_feature_extractor_tf2_test.py b/research/object_detection/models/center_net_resnet_v1_fpn_feature_extractor_tf2_test.py
index 3f1524904f0a055e48342d09febdd7bd3ec6fb3c..2508e52f793157c9bf3b644601e7772f38511534 100644
--- a/research/object_detection/models/center_net_resnet_v1_fpn_feature_extractor_tf2_test.py
+++ b/research/object_detection/models/center_net_resnet_v1_fpn_feature_extractor_tf2_test.py
@@ -31,6 +31,8 @@ class CenterNetResnetV1FpnFeatureExtractorTest(test_case.TestCase,
@parameterized.parameters(
{'resnet_type': 'resnet_v1_50'},
{'resnet_type': 'resnet_v1_101'},
+ {'resnet_type': 'resnet_v1_18'},
+ {'resnet_type': 'resnet_v1_34'},
)
def test_correct_output_size(self, resnet_type):
"""Verify that shape of features returned by the backbone is correct."""
@@ -38,11 +40,11 @@ class CenterNetResnetV1FpnFeatureExtractorTest(test_case.TestCase,
model = center_net_resnet_v1_fpn_feature_extractor.\
CenterNetResnetV1FpnFeatureExtractor(resnet_type)
def graph_fn():
- img = np.zeros((8, 224, 224, 3), dtype=np.float32)
+ img = np.zeros((8, 512, 512, 3), dtype=np.float32)
processed_img = model.preprocess(img)
return model(processed_img)
- self.assertEqual(self.execute(graph_fn, []).shape, (8, 56, 56, 64))
+ self.assertEqual(self.execute(graph_fn, []).shape, (8, 128, 128, 64))
if __name__ == '__main__':
diff --git a/research/object_detection/models/faster_rcnn_resnet_v1_fpn_keras_feature_extractor.py b/research/object_detection/models/faster_rcnn_resnet_v1_fpn_keras_feature_extractor.py
index e3a161e01f7c2f6d13a390c039070855393b82bd..27d8844b7b765e1d195f1a40580c5b3863637b12 100644
--- a/research/object_detection/models/faster_rcnn_resnet_v1_fpn_keras_feature_extractor.py
+++ b/research/object_detection/models/faster_rcnn_resnet_v1_fpn_keras_feature_extractor.py
@@ -20,6 +20,7 @@ import tensorflow.compat.v1 as tf
from object_detection.meta_architectures import faster_rcnn_meta_arch
from object_detection.models import feature_map_generators
from object_detection.models.keras_models import resnet_v1
+from object_detection.utils import ops
_RESNET_MODEL_OUTPUT_LAYERS = {
@@ -32,6 +33,78 @@ _RESNET_MODEL_OUTPUT_LAYERS = {
}
+class _ResnetFPN(tf.keras.layers.Layer):
+ """Construct Resnet FPN layer."""
+
+ def __init__(self,
+ backbone_classifier,
+ fpn_features_generator,
+ coarse_feature_layers,
+ pad_to_multiple,
+ fpn_min_level,
+ resnet_block_names,
+ base_fpn_max_level):
+ """Constructor.
+
+ Args:
+ backbone_classifier: Classifier backbone. Should be one of 'resnet_v1_50',
+ 'resnet_v1_101', 'resnet_v1_152'.
+ fpn_features_generator: KerasFpnTopDownFeatureMaps that accepts a
+ dictionary of features and returns a ordered dictionary of fpn features.
+ coarse_feature_layers: Coarse feature layers for fpn.
+ pad_to_multiple: An integer multiple to pad input image.
+ fpn_min_level: the highest resolution feature map to use in FPN. The valid
+ values are {2, 3, 4, 5} which map to Resnet v1 layers.
+ resnet_block_names: a list of block names of resnet.
+ base_fpn_max_level: maximum level of fpn without coarse feature layers.
+ """
+ super(_ResnetFPN, self).__init__()
+ self.classification_backbone = backbone_classifier
+ self.fpn_features_generator = fpn_features_generator
+ self.coarse_feature_layers = coarse_feature_layers
+ self.pad_to_multiple = pad_to_multiple
+ self._fpn_min_level = fpn_min_level
+ self._resnet_block_names = resnet_block_names
+ self._base_fpn_max_level = base_fpn_max_level
+
+ def call(self, inputs):
+ """Create internal Resnet FPN layer.
+
+ Args:
+ inputs: A [batch, height_out, width_out, channels] float32 tensor
+ representing a batch of images.
+
+ Returns:
+ feature_maps: A list of tensors with shape [batch, height, width, depth]
+ represent extracted features.
+ """
+ inputs = ops.pad_to_multiple(inputs, self.pad_to_multiple)
+ backbone_outputs = self.classification_backbone(inputs)
+
+ feature_block_list = []
+ for level in range(self._fpn_min_level, self._base_fpn_max_level + 1):
+ feature_block_list.append('block{}'.format(level - 1))
+ feature_block_map = dict(
+ list(zip(self._resnet_block_names, backbone_outputs)))
+ fpn_input_image_features = [
+ (feature_block, feature_block_map[feature_block])
+ for feature_block in feature_block_list]
+ fpn_features = self.fpn_features_generator(fpn_input_image_features)
+
+ feature_maps = []
+ for level in range(self._fpn_min_level, self._base_fpn_max_level + 1):
+ feature_maps.append(fpn_features['top_down_block{}'.format(level-1)])
+ last_feature_map = fpn_features['top_down_block{}'.format(
+ self._base_fpn_max_level - 1)]
+
+ for coarse_feature_layers in self.coarse_feature_layers:
+ for layer in coarse_feature_layers:
+ last_feature_map = layer(last_feature_map)
+ feature_maps.append(last_feature_map)
+
+ return feature_maps
+
+
class FasterRCNNResnetV1FpnKerasFeatureExtractor(
faster_rcnn_meta_arch.FasterRCNNKerasFeatureExtractor):
"""Faster RCNN Feature Extractor using Keras-based Resnet V1 FPN features."""
@@ -42,7 +115,8 @@ class FasterRCNNResnetV1FpnKerasFeatureExtractor(
resnet_v1_base_model_name,
first_stage_features_stride,
conv_hyperparams,
- batch_norm_trainable=False,
+ batch_norm_trainable=True,
+ pad_to_multiple=32,
weight_decay=0.0,
fpn_min_level=2,
fpn_max_level=6,
@@ -60,6 +134,7 @@ class FasterRCNNResnetV1FpnKerasFeatureExtractor(
containing convolution hyperparameters for the layers added on top of
the base feature extractor.
batch_norm_trainable: See base class.
+ pad_to_multiple: An integer multiple to pad input image.
weight_decay: See base class.
fpn_min_level: the highest resolution feature map to use in FPN. The valid
values are {2, 3, 4, 5} which map to Resnet v1 layers.
@@ -93,6 +168,8 @@ class FasterRCNNResnetV1FpnKerasFeatureExtractor(
self._fpn_max_level = fpn_max_level
self._additional_layer_depth = additional_layer_depth
self._freeze_batchnorm = (not batch_norm_trainable)
+ self._pad_to_multiple = pad_to_multiple
+
self._override_base_feature_extractor_hyperparams = \
override_base_feature_extractor_hyperparams
self._resnet_block_names = ['block1', 'block2', 'block3', 'block4']
@@ -156,10 +233,7 @@ class FasterRCNNResnetV1FpnKerasFeatureExtractor(
self.classification_backbone = tf.keras.Model(
inputs=full_resnet_v1_model.inputs,
outputs=outputs)
- backbone_outputs = self.classification_backbone(
- full_resnet_v1_model.inputs)
- # construct FPN feature generator
self._base_fpn_max_level = min(self._fpn_max_level, 5)
self._num_levels = self._base_fpn_max_level + 1 - self._fpn_min_level
self._fpn_features_generator = (
@@ -171,16 +245,6 @@ class FasterRCNNResnetV1FpnKerasFeatureExtractor(
freeze_batchnorm=self._freeze_batchnorm,
name='FeatureMaps'))
- feature_block_list = []
- for level in range(self._fpn_min_level, self._base_fpn_max_level + 1):
- feature_block_list.append('block{}'.format(level - 1))
- feature_block_map = dict(
- list(zip(self._resnet_block_names, backbone_outputs)))
- fpn_input_image_features = [
- (feature_block, feature_block_map[feature_block])
- for feature_block in feature_block_list]
- fpn_features = self._fpn_features_generator(fpn_input_image_features)
-
# Construct coarse feature layers
for i in range(self._base_fpn_max_level, self._fpn_max_level):
layers = []
@@ -202,19 +266,13 @@ class FasterRCNNResnetV1FpnKerasFeatureExtractor(
name=layer_name))
self._coarse_feature_layers.append(layers)
- feature_maps = []
- for level in range(self._fpn_min_level, self._base_fpn_max_level + 1):
- feature_maps.append(fpn_features['top_down_block{}'.format(level-1)])
- last_feature_map = fpn_features['top_down_block{}'.format(
- self._base_fpn_max_level - 1)]
-
- for coarse_feature_layers in self._coarse_feature_layers:
- for layer in coarse_feature_layers:
- last_feature_map = layer(last_feature_map)
- feature_maps.append(last_feature_map)
-
- feature_extractor_model = tf.keras.models.Model(
- inputs=full_resnet_v1_model.inputs, outputs=feature_maps)
+ feature_extractor_model = _ResnetFPN(self.classification_backbone,
+ self._fpn_features_generator,
+ self._coarse_feature_layers,
+ self._pad_to_multiple,
+ self._fpn_min_level,
+ self._resnet_block_names,
+ self._base_fpn_max_level)
return feature_extractor_model
def get_box_classifier_feature_extractor_model(self, name=None):
@@ -233,16 +291,18 @@ class FasterRCNNResnetV1FpnKerasFeatureExtractor(
And returns proposal_classifier_features:
A 4-D float tensor with shape
- [batch_size * self.max_num_proposals, 1024]
+ [batch_size * self.max_num_proposals, 1, 1, 1024]
representing box classifier features for each proposal.
"""
with tf.name_scope(name):
with tf.name_scope('ResnetV1FPN'):
- # TODO(yiming): Add a batchnorm layer between two fc layers.
feature_extractor_model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(units=1024, activation='relu'),
- tf.keras.layers.Dense(units=1024, activation='relu')
+ self._conv_hyperparams.build_batch_norm(
+ training=(self._is_training and not self._freeze_batchnorm)),
+ tf.keras.layers.Dense(units=1024, activation='relu'),
+ tf.keras.layers.Reshape((1, 1, 1024))
])
return feature_extractor_model
@@ -254,8 +314,8 @@ class FasterRCNNResnet50FpnKerasFeatureExtractor(
def __init__(self,
is_training,
first_stage_features_stride=16,
+ batch_norm_trainable=True,
conv_hyperparams=None,
- batch_norm_trainable=False,
weight_decay=0.0,
fpn_min_level=2,
fpn_max_level=6,
@@ -266,8 +326,8 @@ class FasterRCNNResnet50FpnKerasFeatureExtractor(
Args:
is_training: See base class.
first_stage_features_stride: See base class.
- conv_hyperparams: See base class.
batch_norm_trainable: See base class.
+ conv_hyperparams: See base class.
weight_decay: See base class.
fpn_min_level: See base class.
fpn_max_level: See base class.
@@ -297,8 +357,8 @@ class FasterRCNNResnet101FpnKerasFeatureExtractor(
def __init__(self,
is_training,
first_stage_features_stride=16,
+ batch_norm_trainable=True,
conv_hyperparams=None,
- batch_norm_trainable=False,
weight_decay=0.0,
fpn_min_level=2,
fpn_max_level=6,
@@ -309,8 +369,8 @@ class FasterRCNNResnet101FpnKerasFeatureExtractor(
Args:
is_training: See base class.
first_stage_features_stride: See base class.
- conv_hyperparams: See base class.
batch_norm_trainable: See base class.
+ conv_hyperparams: See base class.
weight_decay: See base class.
fpn_min_level: See base class.
fpn_max_level: See base class.
@@ -339,8 +399,8 @@ class FasterRCNNResnet152FpnKerasFeatureExtractor(
def __init__(self,
is_training,
first_stage_features_stride=16,
+ batch_norm_trainable=True,
conv_hyperparams=None,
- batch_norm_trainable=False,
weight_decay=0.0,
fpn_min_level=2,
fpn_max_level=6,
@@ -351,8 +411,8 @@ class FasterRCNNResnet152FpnKerasFeatureExtractor(
Args:
is_training: See base class.
first_stage_features_stride: See base class.
- conv_hyperparams: See base class.
batch_norm_trainable: See base class.
+ conv_hyperparams: See base class.
weight_decay: See base class.
fpn_min_level: See base class.
fpn_max_level: See base class.
diff --git a/research/object_detection/models/faster_rcnn_resnet_v1_fpn_keras_feature_extractor_tf2_test.py b/research/object_detection/models/faster_rcnn_resnet_v1_fpn_keras_feature_extractor_tf2_test.py
index 5407f2cfa9aafa7ffc44eb93c1c775cd5bb2784d..d0a0813cf65e873a4109fc8bc33add099c1ab87c 100644
--- a/research/object_detection/models/faster_rcnn_resnet_v1_fpn_keras_feature_extractor_tf2_test.py
+++ b/research/object_detection/models/faster_rcnn_resnet_v1_fpn_keras_feature_extractor_tf2_test.py
@@ -91,4 +91,4 @@ class FasterRCNNResnetV1FpnKerasFeatureExtractorTest(tf.test.TestCase):
model(proposal_feature_maps))
features_shape = tf.shape(proposal_classifier_features)
- self.assertAllEqual(features_shape.numpy(), [3, 1024])
+ self.assertAllEqual(features_shape.numpy(), [3, 1, 1, 1024])
diff --git a/research/object_detection/models/feature_map_generators.py b/research/object_detection/models/feature_map_generators.py
index 87d15e968390446a4332e20e5b737e04d573d98a..f343f317d86e984cc441570a4a7681fca1be2c77 100644
--- a/research/object_detection/models/feature_map_generators.py
+++ b/research/object_detection/models/feature_map_generators.py
@@ -467,9 +467,11 @@ def multi_resolution_feature_maps(feature_map_layout, depth_multiplier,
stride=1,
scope=layer_name)
if pool_residual and pre_layer_depth == depth_fn(layer_depth):
+ if use_explicit_padding:
+ pre_layer = ops.fixed_padding(pre_layer, conv_kernel_size)
feature_map += slim.avg_pool2d(
- pre_layer, [3, 3],
- padding='SAME',
+ pre_layer, [conv_kernel_size, conv_kernel_size],
+ padding=padding,
stride=2,
scope=layer_name + '_pool')
else:
diff --git a/research/object_detection/models/keras_models/base_models/original_mobilenet_v2.py b/research/object_detection/models/keras_models/base_models/original_mobilenet_v2.py
index cf7f95724e86c422b568921b77dcf094d901d11f..42b40caf1c3de5dcd143c1d7200abf8ee16b7253 100644
--- a/research/object_detection/models/keras_models/base_models/original_mobilenet_v2.py
+++ b/research/object_detection/models/keras_models/base_models/original_mobilenet_v2.py
@@ -117,7 +117,7 @@ def _obtain_input_shape(
require_flatten):
"""Internal utility to compute/validate an ImageNet model's input shape.
- Arguments:
+ Args:
input_shape: either None (will return the default network input shape),
or a user-provided shape to be validated.
default_size: default input width/height for the model.
@@ -198,7 +198,7 @@ def preprocess_input(x):
the RGB values from [0, 255] to [-1, 1]. Note that this preprocessing
function is different from `imagenet_utils.preprocess_input()`.
- Arguments:
+ Args:
x: a 4D numpy array consists of RGB values within [0, 255].
Returns:
@@ -237,7 +237,7 @@ def mobilenet_v2(input_shape=None,
model = load_model('mobilenet.h5', custom_objects={
'relu6': mobilenet.relu6})
- Arguments:
+ Args:
input_shape: optional shape tuple, to be specified if you would
like to use a model with an input img resolution that is not
(224, 224, 3).
diff --git a/research/object_detection/models/keras_models/hourglass_network.py b/research/object_detection/models/keras_models/hourglass_network.py
index 09fb8ed4f4fb6f4b2712d8403ba1b94985ad25ad..e6e71545c401f0bd2df723581734d85969f400f7 100644
--- a/research/object_detection/models/keras_models/hourglass_network.py
+++ b/research/object_detection/models/keras_models/hourglass_network.py
@@ -174,8 +174,38 @@ class InputDownsampleBlock(tf.keras.layers.Layer):
return self.residual_block(self.conv_block(inputs))
+class InputConvBlock(tf.keras.layers.Layer):
+ """Block for the initial feature convolution.
+
+ This block is used in the hourglass network when we don't want to downsample
+ the input.
+ """
+
+ def __init__(self, out_channels_initial_conv, out_channels_residual_block):
+ """Initializes the downsample block.
+
+ Args:
+ out_channels_initial_conv: int, the desired number of output channels
+ in the initial conv layer.
+ out_channels_residual_block: int, the desired number of output channels
+ in the underlying residual block.
+ """
+
+ super(InputConvBlock, self).__init__()
+
+ self.conv_block = ConvolutionalBlock(
+ kernel_size=3, out_channels=out_channels_initial_conv, stride=1,
+ padding='valid')
+ self.residual_block = ResidualBlock(
+ out_channels=out_channels_residual_block, stride=1, skip_conv=True)
+
+ def call(self, inputs):
+ return self.residual_block(self.conv_block(inputs))
+
+
def _make_repeated_residual_blocks(out_channels, num_blocks,
- initial_stride=1, residual_channels=None):
+ initial_stride=1, residual_channels=None,
+ initial_skip_conv=False):
"""Stack Residual blocks one after the other.
Args:
@@ -184,6 +214,9 @@ def _make_repeated_residual_blocks(out_channels, num_blocks,
initial_stride: int, the stride of the initial residual block.
residual_channels: int, the desired number of output channels in the
intermediate residual blocks. If not specifed, we use out_channels.
+ initial_skip_conv: bool, if set, the first residual block uses a skip
+ convolution. This is useful when the number of channels in the input
+ are not the same as residual_channels.
Returns:
blocks: A list of residual blocks to be applied in sequence.
@@ -196,16 +229,34 @@ def _make_repeated_residual_blocks(out_channels, num_blocks,
residual_channels = out_channels
for i in range(num_blocks - 1):
+ # Only use the stride at the first block so we don't repeatedly downsample
+ # the input
stride = initial_stride if i == 0 else 1
+
+ # If the stide is more than 1, we cannot use an identity layer for the
+ # skip connection and are forced to use a conv for the skip connection.
skip_conv = stride > 1
+ if i == 0 and initial_skip_conv:
+ skip_conv = True
+
blocks.append(
ResidualBlock(out_channels=residual_channels, stride=stride,
skip_conv=skip_conv)
)
- skip_conv = residual_channels != out_channels
- blocks.append(ResidualBlock(out_channels=out_channels, skip_conv=skip_conv))
+ if num_blocks == 1:
+ # If there is only 1 block, the for loop above is not run,
+ # therefore we honor the requested stride in the last residual block
+ stride = initial_stride
+ # We are forced to use a conv in the skip connection if stride > 1
+ skip_conv = stride > 1
+ else:
+ stride = 1
+ skip_conv = residual_channels != out_channels
+
+ blocks.append(ResidualBlock(out_channels=out_channels, skip_conv=skip_conv,
+ stride=stride))
return blocks
@@ -222,7 +273,8 @@ def _apply_blocks(inputs, blocks):
class EncoderDecoderBlock(tf.keras.layers.Layer):
"""An encoder-decoder block which recursively defines the hourglass network."""
- def __init__(self, num_stages, channel_dims, blocks_per_stage):
+ def __init__(self, num_stages, channel_dims, blocks_per_stage,
+ stagewise_downsample=True, encoder_decoder_shortcut=True):
"""Initializes the encoder-decoder block.
Args:
@@ -237,6 +289,10 @@ class EncoderDecoderBlock(tf.keras.layers.Layer):
blocks_per_stage: int list, number of residual blocks to use at each
stage. `blocks_per_stage[0]` defines the number of blocks at the
current stage and `blocks_per_stage[1:]` is used at further stages.
+ stagewise_downsample: bool, whether or not to downsample before passing
+ inputs to the next stage.
+ encoder_decoder_shortcut: bool, whether or not to use shortcut
+ connections between encoder and decoder.
"""
super(EncoderDecoderBlock, self).__init__()
@@ -244,17 +300,26 @@ class EncoderDecoderBlock(tf.keras.layers.Layer):
out_channels = channel_dims[0]
out_channels_downsampled = channel_dims[1]
- self.encoder_block1 = _make_repeated_residual_blocks(
- out_channels=out_channels, num_blocks=blocks_per_stage[0],
- initial_stride=1)
+ self.encoder_decoder_shortcut = encoder_decoder_shortcut
+
+ if encoder_decoder_shortcut:
+ self.merge_features = tf.keras.layers.Add()
+ self.encoder_block1 = _make_repeated_residual_blocks(
+ out_channels=out_channels, num_blocks=blocks_per_stage[0],
+ initial_stride=1)
+
+ initial_stride = 2 if stagewise_downsample else 1
self.encoder_block2 = _make_repeated_residual_blocks(
out_channels=out_channels_downsampled,
- num_blocks=blocks_per_stage[0], initial_stride=2)
+ num_blocks=blocks_per_stage[0], initial_stride=initial_stride,
+ initial_skip_conv=out_channels != out_channels_downsampled)
if num_stages > 1:
self.inner_block = [
EncoderDecoderBlock(num_stages - 1, channel_dims[1:],
- blocks_per_stage[1:])
+ blocks_per_stage[1:],
+ stagewise_downsample=stagewise_downsample,
+ encoder_decoder_shortcut=encoder_decoder_shortcut)
]
else:
self.inner_block = _make_repeated_residual_blocks(
@@ -264,13 +329,13 @@ class EncoderDecoderBlock(tf.keras.layers.Layer):
self.decoder_block = _make_repeated_residual_blocks(
residual_channels=out_channels_downsampled,
out_channels=out_channels, num_blocks=blocks_per_stage[0])
- self.upsample = tf.keras.layers.UpSampling2D(2)
- self.merge_features = tf.keras.layers.Add()
+ self.upsample = tf.keras.layers.UpSampling2D(initial_stride)
def call(self, inputs):
- encoded_outputs = _apply_blocks(inputs, self.encoder_block1)
+ if self.encoder_decoder_shortcut:
+ encoded_outputs = _apply_blocks(inputs, self.encoder_block1)
encoded_downsampled_outputs = _apply_blocks(inputs, self.encoder_block2)
inner_block_outputs = _apply_blocks(
encoded_downsampled_outputs, self.inner_block)
@@ -278,48 +343,68 @@ class EncoderDecoderBlock(tf.keras.layers.Layer):
decoded_outputs = _apply_blocks(inner_block_outputs, self.decoder_block)
upsampled_outputs = self.upsample(decoded_outputs)
- return self.merge_features([encoded_outputs, upsampled_outputs])
+ if self.encoder_decoder_shortcut:
+ return self.merge_features([encoded_outputs, upsampled_outputs])
+ else:
+ return upsampled_outputs
class HourglassNetwork(tf.keras.Model):
"""The hourglass network."""
- def __init__(self, num_stages, channel_dims, blocks_per_stage,
- num_hourglasses):
+ def __init__(self, num_stages, input_channel_dims, channel_dims_per_stage,
+ blocks_per_stage, num_hourglasses, initial_downsample=True,
+ stagewise_downsample=True, encoder_decoder_shortcut=True):
"""Intializes the feature extractor.
Args:
num_stages: int, Number of stages in the network. At each stage we have 2
encoder and 1 decoder blocks. The second encoder block downsamples the
input.
- channel_dims: int list, the output channel dimensions of stages in
- the network. `channel_dims[0]` and `channel_dims[1]` are used to define
- the initial downsampling block. `channel_dims[1:]` is used to define
- the hourglass network(s) which follow(s).
+ input_channel_dims: int, the number of channels in the input conv blocks.
+ channel_dims_per_stage: int list, the output channel dimensions of each
+ stage in the hourglass network.
blocks_per_stage: int list, number of residual blocks to use at each
stage in the hourglass network
num_hourglasses: int, number of hourglas networks to stack
sequentially.
+ initial_downsample: bool, if set, downsamples the input by a factor of 4
+ before applying the rest of the network. Downsampling is done with a 7x7
+ convolution kernel, otherwise a 3x3 kernel is used.
+ stagewise_downsample: bool, whether or not to downsample before passing
+ inputs to the next stage.
+ encoder_decoder_shortcut: bool, whether or not to use shortcut
+ connections between encoder and decoder.
"""
super(HourglassNetwork, self).__init__()
self.num_hourglasses = num_hourglasses
- self.downsample_input = InputDownsampleBlock(
- out_channels_initial_conv=channel_dims[0],
- out_channels_residual_block=channel_dims[1]
- )
+ self.initial_downsample = initial_downsample
+ if initial_downsample:
+ self.downsample_input = InputDownsampleBlock(
+ out_channels_initial_conv=input_channel_dims,
+ out_channels_residual_block=channel_dims_per_stage[0]
+ )
+ else:
+ self.conv_input = InputConvBlock(
+ out_channels_initial_conv=input_channel_dims,
+ out_channels_residual_block=channel_dims_per_stage[0]
+ )
self.hourglass_network = []
self.output_conv = []
for _ in range(self.num_hourglasses):
self.hourglass_network.append(
EncoderDecoderBlock(
- num_stages=num_stages, channel_dims=channel_dims[1:],
- blocks_per_stage=blocks_per_stage)
+ num_stages=num_stages, channel_dims=channel_dims_per_stage,
+ blocks_per_stage=blocks_per_stage,
+ stagewise_downsample=stagewise_downsample,
+ encoder_decoder_shortcut=encoder_decoder_shortcut)
)
self.output_conv.append(
- ConvolutionalBlock(kernel_size=3, out_channels=channel_dims[1])
+ ConvolutionalBlock(kernel_size=3,
+ out_channels=channel_dims_per_stage[0])
)
self.intermediate_conv1 = []
@@ -329,21 +414,25 @@ class HourglassNetwork(tf.keras.Model):
for _ in range(self.num_hourglasses - 1):
self.intermediate_conv1.append(
ConvolutionalBlock(
- kernel_size=1, out_channels=channel_dims[1], relu=False)
+ kernel_size=1, out_channels=channel_dims_per_stage[0], relu=False)
)
self.intermediate_conv2.append(
ConvolutionalBlock(
- kernel_size=1, out_channels=channel_dims[1], relu=False)
+ kernel_size=1, out_channels=channel_dims_per_stage[0], relu=False)
)
self.intermediate_residual.append(
- ResidualBlock(out_channels=channel_dims[1])
+ ResidualBlock(out_channels=channel_dims_per_stage[0])
)
self.intermediate_relu = tf.keras.layers.ReLU()
def call(self, inputs):
- inputs = self.downsample_input(inputs)
+ if self.initial_downsample:
+ inputs = self.downsample_input(inputs)
+ else:
+ inputs = self.conv_input(inputs)
+
outputs = []
for i in range(self.num_hourglasses):
@@ -372,12 +461,164 @@ class HourglassNetwork(tf.keras.Model):
return self.num_hourglasses
+def _layer_depth(layer):
+ """Compute depth of Conv/Residual blocks or lists of them."""
+
+ if isinstance(layer, list):
+ return sum([_layer_depth(l) for l in layer])
+
+ elif isinstance(layer, ConvolutionalBlock):
+ return 1
+
+ elif isinstance(layer, ResidualBlock):
+ return 2
+
+ else:
+ raise ValueError('Unknown layer - {}'.format(layer))
+
+
+def _encoder_decoder_depth(network):
+ """Helper function to compute depth of encoder-decoder blocks."""
+
+ encoder_block2_layers = _layer_depth(network.encoder_block2)
+ decoder_block_layers = _layer_depth(network.decoder_block)
+
+ if isinstance(network.inner_block[0], EncoderDecoderBlock):
+
+ assert len(network.inner_block) == 1, 'Inner block is expected as length 1.'
+ inner_block_layers = _encoder_decoder_depth(network.inner_block[0])
+
+ return inner_block_layers + encoder_block2_layers + decoder_block_layers
+
+ elif isinstance(network.inner_block[0], ResidualBlock):
+ return (encoder_block2_layers + decoder_block_layers +
+ _layer_depth(network.inner_block))
+
+ else:
+ raise ValueError('Unknown inner block type.')
+
+
+def hourglass_depth(network):
+ """Helper function to verify depth of hourglass backbone."""
+
+ input_conv_layers = 3 # 1 ResidualBlock and 1 ConvBlock
+
+ # Only intermediate_conv2 and intermediate_residual are applied before
+ # sending inputs to the later stages.
+ intermediate_layers = (
+ _layer_depth(network.intermediate_conv2) +
+ _layer_depth(network.intermediate_residual)
+ )
+
+ # network.output_conv is applied before sending input to the later stages
+ output_layers = _layer_depth(network.output_conv)
+
+ encoder_decoder_layers = sum(_encoder_decoder_depth(net) for net in
+ network.hourglass_network)
+
+ return (input_conv_layers + encoder_decoder_layers + intermediate_layers
+ + output_layers)
+
+
def hourglass_104():
- """The Hourglass-104 backbone."""
+ """The Hourglass-104 backbone.
+
+ The architecture parameters are taken from [1].
+
+ Returns:
+ network: An HourglassNetwork object implementing the Hourglass-104
+ backbone.
+
+ [1]: https://arxiv.org/abs/1904.07850
+ """
return HourglassNetwork(
- channel_dims=[128, 256, 256, 384, 384, 384, 512],
+ input_channel_dims=128,
+ channel_dims_per_stage=[256, 256, 384, 384, 384, 512],
num_hourglasses=2,
num_stages=5,
blocks_per_stage=[2, 2, 2, 2, 2, 4],
)
+
+
+def single_stage_hourglass(input_channel_dims, channel_dims_per_stage,
+ blocks_per_stage, initial_downsample=True,
+ stagewise_downsample=True,
+ encoder_decoder_shortcut=True):
+ assert len(channel_dims_per_stage) == len(blocks_per_stage)
+
+ return HourglassNetwork(
+ input_channel_dims=input_channel_dims,
+ channel_dims_per_stage=channel_dims_per_stage,
+ num_hourglasses=1,
+ num_stages=len(channel_dims_per_stage) - 1,
+ blocks_per_stage=blocks_per_stage,
+ initial_downsample=initial_downsample,
+ stagewise_downsample=stagewise_downsample,
+ encoder_decoder_shortcut=encoder_decoder_shortcut
+ )
+
+
+def hourglass_10(num_channels, initial_downsample=True):
+ nc = num_channels
+ return single_stage_hourglass(
+ input_channel_dims=nc,
+ initial_downsample=initial_downsample,
+ blocks_per_stage=[1, 1],
+ channel_dims_per_stage=[nc * 2, nc * 2])
+
+
+def hourglass_20(num_channels, initial_downsample=True):
+ nc = num_channels
+ return single_stage_hourglass(
+ input_channel_dims=nc,
+ initial_downsample=initial_downsample,
+ blocks_per_stage=[1, 2, 2],
+ channel_dims_per_stage=[nc * 2, nc * 2, nc * 3])
+
+
+def hourglass_32(num_channels, initial_downsample=True):
+ nc = num_channels
+ return single_stage_hourglass(
+ input_channel_dims=nc,
+ initial_downsample=initial_downsample,
+ blocks_per_stage=[2, 2, 2, 2],
+ channel_dims_per_stage=[nc * 2, nc * 2, nc * 3, nc * 3])
+
+
+def hourglass_52(num_channels, initial_downsample=True):
+ nc = num_channels
+ return single_stage_hourglass(
+ input_channel_dims=nc,
+ initial_downsample=initial_downsample,
+ blocks_per_stage=[2, 2, 2, 2, 2, 4],
+ channel_dims_per_stage=[nc * 2, nc * 2, nc * 3, nc * 3, nc * 3, nc*4])
+
+
+def hourglass_100(num_channels, initial_downsample=True):
+ nc = num_channels
+ return single_stage_hourglass(
+ input_channel_dims=nc,
+ initial_downsample=initial_downsample,
+ blocks_per_stage=[4, 4, 4, 4, 4, 8],
+ channel_dims_per_stage=[nc * 2, nc * 2, nc * 3, nc * 3, nc * 3, nc*4])
+
+
+def hourglass_20_uniform_size(num_channels):
+ nc = num_channels
+ return single_stage_hourglass(
+ input_channel_dims=nc,
+ blocks_per_stage=[1, 2, 2],
+ channel_dims_per_stage=[nc * 2, nc * 2, nc * 3],
+ initial_downsample=False,
+ stagewise_downsample=False)
+
+
+def hourglass_20_no_shortcut(num_channels):
+ nc = num_channels
+ return single_stage_hourglass(
+ input_channel_dims=nc,
+ blocks_per_stage=[1, 2, 2],
+ channel_dims_per_stage=[nc * 2, nc * 2, nc * 3],
+ initial_downsample=False,
+ encoder_decoder_shortcut=False)
diff --git a/research/object_detection/models/keras_models/hourglass_network_tf2_test.py b/research/object_detection/models/keras_models/hourglass_network_tf2_test.py
index d90b950ecd4102a260643391de6a4475ed959c0f..d1813703c7c6debc049711551031800985b8431d 100644
--- a/research/object_detection/models/keras_models/hourglass_network_tf2_test.py
+++ b/research/object_detection/models/keras_models/hourglass_network_tf2_test.py
@@ -78,6 +78,12 @@ class HourglassFeatureExtractorTest(tf.test.TestCase, parameterized.TestCase):
output = layer(np.zeros((2, 32, 32, 8), dtype=np.float32))
self.assertEqual(output.shape, (2, 8, 8, 8))
+ def test_input_conv_block(self):
+ layer = hourglass.InputConvBlock(
+ out_channels_initial_conv=4, out_channels_residual_block=8)
+ output = layer(np.zeros((2, 32, 32, 8), dtype=np.float32))
+ self.assertEqual(output.shape, (2, 32, 32, 8))
+
def test_encoder_decoder_block(self):
layer = hourglass.EncoderDecoderBlock(
@@ -89,12 +95,64 @@ class HourglassFeatureExtractorTest(tf.test.TestCase, parameterized.TestCase):
def test_hourglass_feature_extractor(self):
model = hourglass.HourglassNetwork(
- num_stages=4, blocks_per_stage=[2, 3, 4, 5, 6],
- channel_dims=[4, 6, 8, 10, 12, 14], num_hourglasses=2)
+ num_stages=4, blocks_per_stage=[2, 3, 4, 5, 6], input_channel_dims=4,
+ channel_dims_per_stage=[6, 8, 10, 12, 14], num_hourglasses=2)
outputs = model(np.zeros((2, 64, 64, 3), dtype=np.float32))
self.assertEqual(outputs[0].shape, (2, 16, 16, 6))
self.assertEqual(outputs[1].shape, (2, 16, 16, 6))
+@unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+class HourglassDepthTest(tf.test.TestCase):
+
+ def test_hourglass_104(self):
+
+ net = hourglass.hourglass_104()
+ self.assertEqual(hourglass.hourglass_depth(net), 104)
+
+ def test_hourglass_10(self):
+ net = hourglass.hourglass_10(2, initial_downsample=False)
+ self.assertEqual(hourglass.hourglass_depth(net), 10)
+
+ outputs = net(tf.zeros((2, 32, 32, 3)))
+ self.assertEqual(outputs[0].shape, (2, 32, 32, 4))
+
+ def test_hourglass_20(self):
+ net = hourglass.hourglass_20(2, initial_downsample=False)
+ self.assertEqual(hourglass.hourglass_depth(net), 20)
+
+ outputs = net(tf.zeros((2, 32, 32, 3)))
+ self.assertEqual(outputs[0].shape, (2, 32, 32, 4))
+
+ def test_hourglass_32(self):
+ net = hourglass.hourglass_32(2, initial_downsample=False)
+ self.assertEqual(hourglass.hourglass_depth(net), 32)
+
+ outputs = net(tf.zeros((2, 32, 32, 3)))
+ self.assertEqual(outputs[0].shape, (2, 32, 32, 4))
+
+ def test_hourglass_52(self):
+ net = hourglass.hourglass_52(2, initial_downsample=False)
+ self.assertEqual(hourglass.hourglass_depth(net), 52)
+
+ outputs = net(tf.zeros((2, 32, 32, 3)))
+ self.assertEqual(outputs[0].shape, (2, 32, 32, 4))
+
+ def test_hourglass_20_uniform_size(self):
+ net = hourglass.hourglass_20_uniform_size(2)
+ self.assertEqual(hourglass.hourglass_depth(net), 20)
+
+ outputs = net(tf.zeros((2, 32, 32, 3)))
+ self.assertEqual(outputs[0].shape, (2, 32, 32, 4))
+
+ def test_hourglass_100(self):
+ net = hourglass.hourglass_100(2, initial_downsample=False)
+ self.assertEqual(hourglass.hourglass_depth(net), 100)
+
+ outputs = net(tf.zeros((2, 32, 32, 3)))
+ self.assertEqual(outputs[0].shape, (2, 32, 32, 4))
+
+
if __name__ == '__main__':
tf.test.main()
+
diff --git a/research/object_detection/models/keras_models/resnet_v1.py b/research/object_detection/models/keras_models/resnet_v1.py
index d5426ad6b5e499171dbd955dc9c3fe465c4b6051..62660d4a70d5887c4a5084117ae5c2e2cfc3f888 100644
--- a/research/object_detection/models/keras_models/resnet_v1.py
+++ b/research/object_detection/models/keras_models/resnet_v1.py
@@ -21,6 +21,7 @@ from __future__ import print_function
import tensorflow.compat.v1 as tf
+from tensorflow.python.keras.applications import resnet
from object_detection.core import freezable_batch_norm
from object_detection.models.keras_models import model_utils
@@ -95,11 +96,11 @@ class _LayersOverride(object):
self.regularizer = tf.keras.regularizers.l2(weight_decay)
self.initializer = tf.variance_scaling_initializer()
- def _FixedPaddingLayer(self, kernel_size, rate=1):
+ def _FixedPaddingLayer(self, kernel_size, rate=1): # pylint: disable=invalid-name
return tf.keras.layers.Lambda(
lambda x: _fixed_padding(x, kernel_size, rate))
- def Conv2D(self, filters, kernel_size, **kwargs):
+ def Conv2D(self, filters, kernel_size, **kwargs): # pylint: disable=invalid-name
"""Builds a Conv2D layer according to the current Object Detection config.
Overrides the Keras Resnet application's convolutions with ones that
@@ -141,7 +142,7 @@ class _LayersOverride(object):
else:
return tf.keras.layers.Conv2D(filters, kernel_size, **kwargs)
- def Activation(self, *args, **kwargs): # pylint: disable=unused-argument
+ def Activation(self, *args, **kwargs): # pylint: disable=unused-argument,invalid-name
"""Builds an activation layer.
Overrides the Keras application Activation layer specified by the
@@ -163,7 +164,7 @@ class _LayersOverride(object):
else:
return tf.keras.layers.Lambda(tf.nn.relu, name=name)
- def BatchNormalization(self, **kwargs):
+ def BatchNormalization(self, **kwargs): # pylint: disable=invalid-name
"""Builds a normalization layer.
Overrides the Keras application batch norm with the norm specified by the
@@ -191,7 +192,7 @@ class _LayersOverride(object):
momentum=self._default_batchnorm_momentum,
**kwargs)
- def Input(self, shape):
+ def Input(self, shape): # pylint: disable=invalid-name
"""Builds an Input layer.
Overrides the Keras application Input layer with one that uses a
@@ -219,7 +220,7 @@ class _LayersOverride(object):
input=input_tensor, shape=[None] + shape)
return model_utils.input_layer(shape, placeholder_with_default)
- def MaxPooling2D(self, pool_size, **kwargs):
+ def MaxPooling2D(self, pool_size, **kwargs): # pylint: disable=invalid-name
"""Builds a MaxPooling2D layer with default padding as 'SAME'.
This is specified by the default resnet arg_scope in slim.
@@ -237,7 +238,7 @@ class _LayersOverride(object):
# Add alias as Keras also has it.
MaxPool2D = MaxPooling2D # pylint: disable=invalid-name
- def ZeroPadding2D(self, padding, **kwargs): # pylint: disable=unused-argument
+ def ZeroPadding2D(self, padding, **kwargs): # pylint: disable=unused-argument,invalid-name
"""Replaces explicit padding in the Keras application with a no-op.
Args:
@@ -395,3 +396,146 @@ def resnet_v1_152(batchnorm_training,
return tf.keras.applications.resnet.ResNet152(
layers=layers_override, **kwargs)
# pylint: enable=invalid-name
+
+
+# The following codes are based on the existing keras ResNet model pattern:
+# google3/third_party/tensorflow/python/keras/applications/resnet.py
+def block_basic(x,
+ filters,
+ kernel_size=3,
+ stride=1,
+ conv_shortcut=False,
+ name=None):
+ """A residual block for ResNet18/34.
+
+ Args:
+ x: input tensor.
+ filters: integer, filters of the bottleneck layer.
+ kernel_size: default 3, kernel size of the bottleneck layer.
+ stride: default 1, stride of the first layer.
+ conv_shortcut: default False, use convolution shortcut if True, otherwise
+ identity shortcut.
+ name: string, block label.
+
+ Returns:
+ Output tensor for the residual block.
+ """
+ layers = tf.keras.layers
+ bn_axis = 3 if tf.keras.backend.image_data_format() == 'channels_last' else 1
+
+ preact = layers.BatchNormalization(
+ axis=bn_axis, epsilon=1.001e-5, name=name + '_preact_bn')(
+ x)
+ preact = layers.Activation('relu', name=name + '_preact_relu')(preact)
+
+ if conv_shortcut:
+ shortcut = layers.Conv2D(
+ filters, 1, strides=1, name=name + '_0_conv')(
+ preact)
+ else:
+ shortcut = layers.MaxPooling2D(1, strides=stride)(x) if stride > 1 else x
+
+ x = layers.ZeroPadding2D(
+ padding=((1, 1), (1, 1)), name=name + '_1_pad')(
+ preact)
+ x = layers.Conv2D(
+ filters, kernel_size, strides=1, use_bias=False, name=name + '_1_conv')(
+ x)
+ x = layers.BatchNormalization(
+ axis=bn_axis, epsilon=1.001e-5, name=name + '_1_bn')(
+ x)
+ x = layers.Activation('relu', name=name + '_1_relu')(x)
+
+ x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name=name + '_2_pad')(x)
+ x = layers.Conv2D(
+ filters,
+ kernel_size,
+ strides=stride,
+ use_bias=False,
+ name=name + '_2_conv')(
+ x)
+ x = layers.BatchNormalization(
+ axis=bn_axis, epsilon=1.001e-5, name=name + '_2_bn')(
+ x)
+ x = layers.Activation('relu', name=name + '_2_relu')(x)
+ x = layers.Add(name=name + '_out')([shortcut, x])
+ return x
+
+
+def stack_basic(x, filters, blocks, stride1=2, name=None):
+ """A set of stacked residual blocks for ResNet18/34.
+
+ Args:
+ x: input tensor.
+ filters: integer, filters of the bottleneck layer in a block.
+ blocks: integer, blocks in the stacked blocks.
+ stride1: default 2, stride of the first layer in the first block.
+ name: string, stack label.
+
+ Returns:
+ Output tensor for the stacked blocks.
+ """
+ x = block_basic(x, filters, conv_shortcut=True, name=name + '_block1')
+ for i in range(2, blocks):
+ x = block_basic(x, filters, name=name + '_block' + str(i))
+ x = block_basic(
+ x, filters, stride=stride1, name=name + '_block' + str(blocks))
+ return x
+
+
+def resnet_v1_18(include_top=True,
+ weights='imagenet',
+ input_tensor=None,
+ input_shape=None,
+ pooling=None,
+ classes=1000,
+ classifier_activation='softmax'):
+ """Instantiates the ResNet18 architecture."""
+
+ def stack_fn(x):
+ x = stack_basic(x, 64, 2, stride1=1, name='conv2')
+ x = stack_basic(x, 128, 2, name='conv3')
+ x = stack_basic(x, 256, 2, name='conv4')
+ return stack_basic(x, 512, 2, name='conv5')
+
+ return resnet.ResNet(
+ stack_fn,
+ True,
+ True,
+ 'resnet18',
+ include_top,
+ weights,
+ input_tensor,
+ input_shape,
+ pooling,
+ classes,
+ classifier_activation=classifier_activation)
+
+
+def resnet_v1_34(include_top=True,
+ weights='imagenet',
+ input_tensor=None,
+ input_shape=None,
+ pooling=None,
+ classes=1000,
+ classifier_activation='softmax'):
+ """Instantiates the ResNet34 architecture."""
+
+ def stack_fn(x):
+ x = stack_basic(x, 64, 3, stride1=1, name='conv2')
+ x = stack_basic(x, 128, 4, name='conv3')
+ x = stack_basic(x, 256, 6, name='conv4')
+ return stack_basic(x, 512, 3, name='conv5')
+
+ return resnet.ResNet(
+ stack_fn,
+ True,
+ True,
+ 'resnet34',
+ include_top,
+ weights,
+ input_tensor,
+ input_shape,
+ pooling,
+ classes,
+ classifier_activation=classifier_activation)
diff --git a/research/object_detection/models/keras_models/resnet_v1_tf2_test.py b/research/object_detection/models/keras_models/resnet_v1_tf2_test.py
index 71cc5f22bd994b6432957bf5b34837f829c9b8da..4566bc8ddda664fd7698f17b4951fca48612307b 100644
--- a/research/object_detection/models/keras_models/resnet_v1_tf2_test.py
+++ b/research/object_detection/models/keras_models/resnet_v1_tf2_test.py
@@ -20,12 +20,13 @@ object detection. To verify the consistency of the two models, we compare:
2. Number of global variables.
"""
import unittest
+
+from absl.testing import parameterized
import numpy as np
from six.moves import zip
import tensorflow.compat.v1 as tf
from google.protobuf import text_format
-
from object_detection.builders import hyperparams_builder
from object_detection.models.keras_models import resnet_v1
from object_detection.protos import hyperparams_pb2
@@ -180,5 +181,46 @@ class ResnetV1Test(test_case.TestCase):
self.assertEqual(len(variables), var_num)
+class ResnetShapeTest(test_case.TestCase, parameterized.TestCase):
+
+ @unittest.skipIf(tf_version.is_tf1(), 'Skipping TF2.X only test.')
+ @parameterized.parameters(
+ {
+ 'resnet_type':
+ 'resnet_v1_34',
+ 'output_layer_names': [
+ 'conv2_block3_out', 'conv3_block4_out', 'conv4_block6_out',
+ 'conv5_block3_out'
+ ]
+ }, {
+ 'resnet_type':
+ 'resnet_v1_18',
+ 'output_layer_names': [
+ 'conv2_block2_out', 'conv3_block2_out', 'conv4_block2_out',
+ 'conv5_block2_out'
+ ]
+ })
+ def test_output_shapes(self, resnet_type, output_layer_names):
+ if resnet_type == 'resnet_v1_34':
+ model = resnet_v1.resnet_v1_34(input_shape=(64, 64, 3), weights=None)
+ else:
+ model = resnet_v1.resnet_v1_18(input_shape=(64, 64, 3), weights=None)
+ outputs = [
+ model.get_layer(output_layer_name).output
+ for output_layer_name in output_layer_names
+ ]
+ resnet_model = tf.keras.models.Model(inputs=model.input, outputs=outputs)
+ outputs = resnet_model(np.zeros((2, 64, 64, 3), dtype=np.float32))
+
+ # Check the shape of 'conv2_block3_out':
+ self.assertEqual(outputs[0].shape, [2, 16, 16, 64])
+ # Check the shape of 'conv3_block4_out':
+ self.assertEqual(outputs[1].shape, [2, 8, 8, 128])
+ # Check the shape of 'conv4_block6_out':
+ self.assertEqual(outputs[2].shape, [2, 4, 4, 256])
+ # Check the shape of 'conv5_block3_out':
+ self.assertEqual(outputs[3].shape, [2, 2, 2, 512])
+
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/models/ssd_efficientnet_bifpn_feature_extractor.py b/research/object_detection/models/ssd_efficientnet_bifpn_feature_extractor.py
index 2ecf8fb01b2dba536b8d6c531a3e8becac75091c..3184a1c682dc4df68f31dd8554db846698c667e0 100644
--- a/research/object_detection/models/ssd_efficientnet_bifpn_feature_extractor.py
+++ b/research/object_detection/models/ssd_efficientnet_bifpn_feature_extractor.py
@@ -23,6 +23,7 @@ from six.moves import range
from six.moves import zip
import tensorflow.compat.v2 as tf
+from tensorflow.python.keras import backend as keras_backend
from object_detection.meta_architectures import ssd_meta_arch
from object_detection.models import bidirectional_feature_pyramid_generators as bifpn_generators
from object_detection.utils import ops
@@ -103,9 +104,10 @@ class SSDEfficientNetBiFPNKerasFeatureExtractor(
use_depthwise: unsupported by EfficientNetBiFPN, since BiFPN uses regular
convolutions when inputs to a node have a differing number of channels,
and use separable convolutions after combine operations.
- override_base_feature_extractor_hyperparams: unsupported. Whether to
- override hyperparameters of the base feature extractor with the one from
- `conv_hyperparams`.
+ override_base_feature_extractor_hyperparams: Whether to override the
+ efficientnet backbone's default weight decay with the weight decay
+ defined by `conv_hyperparams`. Note, only overriding of weight decay is
+ currently supported.
name: a string name scope to assign to the model. If 'None', Keras will
auto-generate one from the class name.
"""
@@ -129,9 +131,6 @@ class SSDEfficientNetBiFPNKerasFeatureExtractor(
raise ValueError('EfficientNetBiFPN does not support explicit padding.')
if use_depthwise:
raise ValueError('EfficientNetBiFPN does not support use_depthwise.')
- if override_base_feature_extractor_hyperparams:
- raise ValueError('EfficientNetBiFPN does not support '
- 'override_base_feature_extractor_hyperparams.')
self._bifpn_min_level = bifpn_min_level
self._bifpn_max_level = bifpn_max_level
@@ -158,9 +157,15 @@ class SSDEfficientNetBiFPNKerasFeatureExtractor(
# Initialize the EfficientNet backbone.
# Note, this is currently done in the init method rather than in the build
# method, since doing so introduces an error which is not well understood.
+ efficientnet_overrides = {'rescale_input': False}
+ if override_base_feature_extractor_hyperparams:
+ efficientnet_overrides[
+ 'weight_decay'] = conv_hyperparams.get_regularizer_weight()
+ if (conv_hyperparams.use_sync_batch_norm() and
+ keras_backend.is_tpu_strategy(tf.distribute.get_strategy())):
+ efficientnet_overrides['batch_norm'] = 'tpu'
efficientnet_base = efficientnet_model.EfficientNet.from_name(
- model_name=self._efficientnet_version,
- overrides={'rescale_input': False})
+ model_name=self._efficientnet_version, overrides=efficientnet_overrides)
outputs = [efficientnet_base.get_layer(output_layer_name).output
for output_layer_name in self._output_layer_names]
self._efficientnet = tf.keras.Model(
diff --git a/research/object_detection/packages/tf1/setup.py b/research/object_detection/packages/tf1/setup.py
index 1cd4923cbc14b9dc1ae4f2c7087cc5e22991ea69..a40a368a6f5fddbccfc13b4d76f38a49d3c1c8d3 100644
--- a/research/object_detection/packages/tf1/setup.py
+++ b/research/object_detection/packages/tf1/setup.py
@@ -3,9 +3,9 @@ import os
from setuptools import find_packages
from setuptools import setup
-REQUIRED_PACKAGES = ['apache-beam', 'pillow', 'lxml', 'matplotlib', 'Cython',
- 'contextlib2', 'tf-slim', 'six', 'pycocotools', 'scipy',
- 'pandas']
+REQUIRED_PACKAGES = ['pillow', 'lxml', 'matplotlib', 'Cython',
+ 'contextlib2', 'tf-slim', 'six', 'pycocotools', 'lvis',
+ 'scipy', 'pandas']
setup(
name='object_detection',
diff --git a/research/object_detection/packages/tf2/setup.py b/research/object_detection/packages/tf2/setup.py
index 09738ee079c8551f3403f4ea06600a3a284f42d3..3f9f0e35363cde03bee00641f3fb53ccc85c55ad 100644
--- a/research/object_detection/packages/tf2/setup.py
+++ b/research/object_detection/packages/tf2/setup.py
@@ -6,9 +6,23 @@ from setuptools import setup
# Note: adding apache-beam to required packages causes conflict with
# tf-models-offical requirements. These packages request for incompatible
# oauth2client package.
-REQUIRED_PACKAGES = ['pillow', 'lxml', 'matplotlib', 'Cython', 'contextlib2',
- 'tf-slim', 'six', 'pycocotools', 'scipy', 'pandas',
- 'tf-models-official']
+REQUIRED_PACKAGES = [
+ # Required for apache-beam with PY3
+ 'avro-python3',
+ 'apache-beam',
+ 'pillow',
+ 'lxml',
+ 'matplotlib',
+ 'Cython',
+ 'contextlib2',
+ 'tf-slim',
+ 'six',
+ 'pycocotools',
+ 'lvis',
+ 'scipy',
+ 'pandas',
+ 'tf-models-official'
+]
setup(
name='object_detection',
diff --git a/research/object_detection/predictors/convolutional_keras_box_predictor.py b/research/object_detection/predictors/convolutional_keras_box_predictor.py
index fc72fb04c2d47301b1ac5fc185ca98c6b00073c0..cdc90fca697865ee78b4b8b21ae0eca9470a9183 100644
--- a/research/object_detection/predictors/convolutional_keras_box_predictor.py
+++ b/research/object_detection/predictors/convolutional_keras_box_predictor.py
@@ -236,6 +236,7 @@ class WeightSharedConvolutionalBoxPredictor(box_predictor.KerasBoxPredictor):
apply_batch_norm=False,
share_prediction_tower=False,
use_depthwise=False,
+ apply_conv_hyperparams_pointwise=False,
name=None):
"""Constructor.
@@ -269,6 +270,10 @@ class WeightSharedConvolutionalBoxPredictor(box_predictor.KerasBoxPredictor):
prediction head, class prediction head and other heads.
use_depthwise: Whether to use depthwise separable conv2d instead of
regular conv2d.
+ apply_conv_hyperparams_pointwise: Whether to apply the conv_hyperparams to
+ the pointwise_initializer and pointwise_regularizer when using depthwise
+ separable convolutions. By default, conv_hyperparams are only applied to
+ the depthwise initializer and regularizer when use_depthwise is true.
name: A string name scope to assign to the model. If `None`, Keras
will auto-generate one from the class name.
"""
@@ -294,6 +299,7 @@ class WeightSharedConvolutionalBoxPredictor(box_predictor.KerasBoxPredictor):
self._apply_batch_norm = apply_batch_norm
self._share_prediction_tower = share_prediction_tower
self._use_depthwise = use_depthwise
+ self._apply_conv_hyperparams_pointwise = apply_conv_hyperparams_pointwise
# Additional projection layers to bring all feature maps to uniform
# channels.
@@ -344,6 +350,9 @@ class WeightSharedConvolutionalBoxPredictor(box_predictor.KerasBoxPredictor):
# so we remap the kernel_* to depthwise_* here.
kwargs['depthwise_regularizer'] = kwargs['kernel_regularizer']
kwargs['depthwise_initializer'] = kwargs['kernel_initializer']
+ if self._apply_conv_hyperparams_pointwise:
+ kwargs['pointwise_regularizer'] = kwargs['kernel_regularizer']
+ kwargs['pointwise_initializer'] = kwargs['kernel_initializer']
conv_layers.append(
tf.keras.layers.SeparableConv2D(
self._depth, [self._kernel_size, self._kernel_size],
diff --git a/research/object_detection/predictors/heads/class_head.py b/research/object_detection/predictors/heads/class_head.py
index 604859313de84a783953e67dbe47e301a740cb96..d7abc23c20cf8fab5e3686ca6335ea76298b99fd 100644
--- a/research/object_detection/predictors/heads/class_head.py
+++ b/research/object_detection/predictors/heads/class_head.py
@@ -24,6 +24,7 @@ import tensorflow.compat.v1 as tf
import tf_slim as slim
from object_detection.predictors.heads import head
+from object_detection.utils import shape_utils
class MaskRCNNClassHead(head.Head):
@@ -303,13 +304,23 @@ class WeightSharedConvolutionalClassHead(head.Head):
biases_initializer=tf.constant_initializer(
self._class_prediction_bias_init),
scope=self._scope)
- batch_size = features.get_shape().as_list()[0]
- if batch_size is None:
- batch_size = tf.shape(features)[0]
+ batch_size, height, width = shape_utils.combined_static_and_dynamic_shape(
+ features)[0:3]
+ class_predictions_with_background = tf.reshape(
+ class_predictions_with_background, [
+ batch_size, height, width, num_predictions_per_location,
+ self._num_class_slots
+ ])
class_predictions_with_background = self._score_converter_fn(
class_predictions_with_background)
if self._return_flat_predictions:
class_predictions_with_background = tf.reshape(
class_predictions_with_background,
[batch_size, -1, self._num_class_slots])
+ else:
+ class_predictions_with_background = tf.reshape(
+ class_predictions_with_background, [
+ batch_size, height, width,
+ num_predictions_per_location * self._num_class_slots
+ ])
return class_predictions_with_background
diff --git a/research/object_detection/predictors/heads/class_head_tf1_test.py b/research/object_detection/predictors/heads/class_head_tf1_test.py
index 3dc8fb120cb9a4c19ff2d595d31dc3645f6e06d0..986a383c1a782fe24c320ca76a5d11a4ac531d65 100644
--- a/research/object_detection/predictors/heads/class_head_tf1_test.py
+++ b/research/object_detection/predictors/heads/class_head_tf1_test.py
@@ -15,6 +15,7 @@
"""Tests for object_detection.predictors.heads.class_head."""
import unittest
+import numpy as np
import tensorflow.compat.v1 as tf
from google.protobuf import text_format
@@ -194,6 +195,37 @@ class WeightSharedConvolutionalClassPredictorTest(test_case.TestCase):
])
self.assertSetEqual(expected_var_names, actual_variable_set)
+ def test_softmax_score_converter(self):
+ num_class_slots = 10
+ batch_size = 2
+ height = 17
+ width = 19
+ num_predictions_per_location = 2
+ assert num_predictions_per_location != 1
+
+ def graph_fn():
+ class_prediction_head = (
+ class_head.WeightSharedConvolutionalClassHead(
+ num_class_slots=num_class_slots,
+ score_converter_fn=tf.nn.softmax))
+ image_feature = tf.random_uniform([batch_size, height, width, 1024],
+ minval=-10.0,
+ maxval=10.0,
+ dtype=tf.float32)
+ class_predictions = class_prediction_head.predict(
+ features=image_feature,
+ num_predictions_per_location=num_predictions_per_location)
+ return class_predictions
+
+ class_predictions_out = self.execute(graph_fn, [])
+ class_predictions_sum = np.sum(class_predictions_out, axis=-1)
+ num_anchors = height * width * num_predictions_per_location
+ exp_class_predictions_sum = np.ones((batch_size, num_anchors),
+ dtype=np.float32)
+ self.assertAllEqual((batch_size, num_anchors, num_class_slots),
+ class_predictions_out.shape)
+ self.assertAllClose(class_predictions_sum, exp_class_predictions_sum)
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/predictors/heads/keras_box_head.py b/research/object_detection/predictors/heads/keras_box_head.py
index b8def7fc1b01291d92ce545c8c3c29d9a24c646a..daf730646b8d8797b1600e16caeb0d533d1bcd54 100644
--- a/research/object_detection/predictors/heads/keras_box_head.py
+++ b/research/object_detection/predictors/heads/keras_box_head.py
@@ -248,6 +248,7 @@ class WeightSharedConvolutionalBoxHead(head.KerasHead):
conv_hyperparams,
kernel_size=3,
use_depthwise=False,
+ apply_conv_hyperparams_to_heads=False,
box_encodings_clip_range=None,
return_flat_predictions=True,
name=None):
@@ -262,6 +263,10 @@ class WeightSharedConvolutionalBoxHead(head.KerasHead):
kernel_size: Size of final convolution kernel.
use_depthwise: Whether to use depthwise convolutions for prediction steps.
Default is False.
+ apply_conv_hyperparams_to_heads: Whether to apply conv_hyperparams to
+ depthwise seperable convolution layers in the box and class heads. By
+ default, the conv_hyperparams are only applied to layers in the
+ predictor tower when using depthwise separable convolutions.
box_encodings_clip_range: Min and max values for clipping box_encodings.
return_flat_predictions: If true, returns flattened prediction tensor
of shape [batch, height * width * num_predictions_per_location,
@@ -282,19 +287,26 @@ class WeightSharedConvolutionalBoxHead(head.KerasHead):
self._kernel_size = kernel_size
self._num_predictions_per_location = num_predictions_per_location
self._use_depthwise = use_depthwise
+ self._apply_conv_hyperparams_to_heads = apply_conv_hyperparams_to_heads
self._box_encodings_clip_range = box_encodings_clip_range
self._return_flat_predictions = return_flat_predictions
self._box_encoder_layers = []
if self._use_depthwise:
+ kwargs = conv_hyperparams.params(use_bias=True)
+ if self._apply_conv_hyperparams_to_heads:
+ kwargs['depthwise_regularizer'] = kwargs['kernel_regularizer']
+ kwargs['depthwise_initializer'] = kwargs['kernel_initializer']
+ kwargs['pointwise_regularizer'] = kwargs['kernel_regularizer']
+ kwargs['pointwise_initializer'] = kwargs['kernel_initializer']
self._box_encoder_layers.append(
tf.keras.layers.SeparableConv2D(
num_predictions_per_location * self._box_code_size,
[self._kernel_size, self._kernel_size],
padding='SAME',
name='BoxPredictor',
- **conv_hyperparams.params(use_bias=True)))
+ **kwargs))
else:
self._box_encoder_layers.append(
tf.keras.layers.Conv2D(
diff --git a/research/object_detection/predictors/heads/keras_class_head.py b/research/object_detection/predictors/heads/keras_class_head.py
index 988ebb2ee720f5db137ade0aef9919a942a57a5b..596f951d42e6790fdf31b32271e48d6741481a54 100644
--- a/research/object_detection/predictors/heads/keras_class_head.py
+++ b/research/object_detection/predictors/heads/keras_class_head.py
@@ -22,6 +22,7 @@ All the class prediction heads have a predict function that receives the
import tensorflow.compat.v1 as tf
from object_detection.predictors.heads import head
+from object_detection.utils import shape_utils
class ConvolutionalClassHead(head.KerasHead):
@@ -250,6 +251,7 @@ class WeightSharedConvolutionalClassHead(head.KerasHead):
use_dropout=False,
dropout_keep_prob=0.8,
use_depthwise=False,
+ apply_conv_hyperparams_to_heads=False,
score_converter_fn=tf.identity,
return_flat_predictions=True,
name=None):
@@ -269,6 +271,10 @@ class WeightSharedConvolutionalClassHead(head.KerasHead):
dropout_keep_prob: Probability of keeping activiations.
use_depthwise: Whether to use depthwise convolutions for prediction
steps. Default is False.
+ apply_conv_hyperparams_to_heads: Whether to apply conv_hyperparams to
+ depthwise seperable convolution layers in the box and class heads. By
+ default, the conv_hyperparams are only applied to layers in the
+ predictor tower when using depthwise separable convolutions.
score_converter_fn: Callable elementwise nonlinearity (that takes tensors
as inputs and returns tensors).
return_flat_predictions: If true, returns flattened prediction tensor
@@ -287,11 +293,13 @@ class WeightSharedConvolutionalClassHead(head.KerasHead):
super(WeightSharedConvolutionalClassHead, self).__init__(name=name)
self._num_class_slots = num_class_slots
+ self._num_predictions_per_location = num_predictions_per_location
self._kernel_size = kernel_size
self._class_prediction_bias_init = class_prediction_bias_init
self._use_dropout = use_dropout
self._dropout_keep_prob = dropout_keep_prob
self._use_depthwise = use_depthwise
+ self._apply_conv_hyperparams_to_heads = apply_conv_hyperparams_to_heads
self._score_converter_fn = score_converter_fn
self._return_flat_predictions = return_flat_predictions
@@ -301,6 +309,12 @@ class WeightSharedConvolutionalClassHead(head.KerasHead):
self._class_predictor_layers.append(
tf.keras.layers.Dropout(rate=1.0 - self._dropout_keep_prob))
if self._use_depthwise:
+ kwargs = conv_hyperparams.params(use_bias=True)
+ if self._apply_conv_hyperparams_to_heads:
+ kwargs['depthwise_regularizer'] = kwargs['kernel_regularizer']
+ kwargs['depthwise_initializer'] = kwargs['kernel_initializer']
+ kwargs['pointwise_regularizer'] = kwargs['kernel_regularizer']
+ kwargs['pointwise_initializer'] = kwargs['kernel_initializer']
self._class_predictor_layers.append(
tf.keras.layers.SeparableConv2D(
num_predictions_per_location * self._num_class_slots,
@@ -311,7 +325,7 @@ class WeightSharedConvolutionalClassHead(head.KerasHead):
name='ClassPredictor',
bias_initializer=tf.constant_initializer(
self._class_prediction_bias_init),
- **conv_hyperparams.params(use_bias=True)))
+ **kwargs))
else:
self._class_predictor_layers.append(
tf.keras.layers.Conv2D(
@@ -339,13 +353,23 @@ class WeightSharedConvolutionalClassHead(head.KerasHead):
for layer in self._class_predictor_layers:
class_predictions_with_background = layer(
class_predictions_with_background)
- batch_size = features.get_shape().as_list()[0]
- if batch_size is None:
- batch_size = tf.shape(features)[0]
+ batch_size, height, width = shape_utils.combined_static_and_dynamic_shape(
+ features)[0:3]
+ class_predictions_with_background = tf.reshape(
+ class_predictions_with_background, [
+ batch_size, height, width, self._num_predictions_per_location,
+ self._num_class_slots
+ ])
class_predictions_with_background = self._score_converter_fn(
class_predictions_with_background)
if self._return_flat_predictions:
class_predictions_with_background = tf.reshape(
class_predictions_with_background,
[batch_size, -1, self._num_class_slots])
+ else:
+ class_predictions_with_background = tf.reshape(
+ class_predictions_with_background, [
+ batch_size, height, width,
+ self._num_predictions_per_location * self._num_class_slots
+ ])
return class_predictions_with_background
diff --git a/research/object_detection/predictors/heads/keras_class_head_tf2_test.py b/research/object_detection/predictors/heads/keras_class_head_tf2_test.py
index aa890ce522defb6ec4c97965846e8f20529bc24b..6aa240e98ed6995bd95f8502761b2a66ec6c2c7f 100644
--- a/research/object_detection/predictors/heads/keras_class_head_tf2_test.py
+++ b/research/object_detection/predictors/heads/keras_class_head_tf2_test.py
@@ -15,6 +15,7 @@
"""Tests for object_detection.predictors.heads.class_head."""
import unittest
+import numpy as np
import tensorflow.compat.v1 as tf
from google.protobuf import text_format
@@ -198,6 +199,38 @@ class WeightSharedConvolutionalKerasClassPredictorTest(test_case.TestCase):
class_prediction_head(image_feature)
self.assertEqual(len(class_prediction_head.variables), 2)
+ def test_softmax_score_converter(self):
+ num_class_slots = 10
+ batch_size = 2
+ height = 17
+ width = 19
+ num_predictions_per_location = 2
+ assert num_predictions_per_location != 1
+
+ conv_hyperparams = self._build_conv_hyperparams()
+ class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead(
+ num_class_slots=num_class_slots,
+ conv_hyperparams=conv_hyperparams,
+ num_predictions_per_location=num_predictions_per_location,
+ score_converter_fn=tf.nn.softmax)
+
+ def graph_fn():
+ image_feature = tf.random_uniform([batch_size, height, width, 1024],
+ minval=-10.0,
+ maxval=10.0,
+ dtype=tf.float32)
+ class_predictions = class_prediction_head(image_feature)
+ return class_predictions
+
+ class_predictions_out = self.execute(graph_fn, [])
+ class_predictions_sum = np.sum(class_predictions_out, axis=-1)
+ num_anchors = height * width * num_predictions_per_location
+ exp_class_predictions_sum = np.ones((batch_size, num_anchors),
+ dtype=np.float32)
+ self.assertAllEqual((batch_size, num_anchors, num_class_slots),
+ class_predictions_out.shape)
+ self.assertAllClose(class_predictions_sum, exp_class_predictions_sum)
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/protos/box_predictor.proto b/research/object_detection/protos/box_predictor.proto
index 0b0fadd7977eb799c2920adb0f79c5e535b68534..c4926502a4fe022b2ebedb78c14f3a2431aa7d1b 100644
--- a/research/object_detection/protos/box_predictor.proto
+++ b/research/object_detection/protos/box_predictor.proto
@@ -66,11 +66,23 @@ message ConvolutionalBoxPredictor {
}
// Configuration proto for weight shared convolutional box predictor.
-// Next id: 19
+// Next id: 21
message WeightSharedConvolutionalBoxPredictor {
// Hyperparameters for convolution ops used in the box predictor.
optional Hyperparams conv_hyperparams = 1;
+ // Whether the `conv_hyperparams` should apply to depthwise separable
+ // convolution layers in the box and class heads, in addition to the layers in
+ // the predictor tower. By default, the `conv_hyperparams` are only applied to
+ // layers in the predictor tower when use_depthwise is true.
+ optional bool apply_conv_hyperparams_to_heads = 19 [default = false];
+
+ // Whether the `conv_hyperparams` should apply to the `pointwise_initializer`
+ // and `pointwise_regularizer` when using depthwise separable convolutions in
+ // the prediction tower layers. By default, the `conv_hyperparams` only apply
+ // to the `depthwise_initializer` and `depthwise_regularizer`.
+ optional bool apply_conv_hyperparams_pointwise = 20 [default = false];
+
// Number of the additional conv layers before the predictor.
optional int32 num_layers_before_predictor = 4 [default = 0];
diff --git a/research/object_detection/protos/center_net.proto b/research/object_detection/protos/center_net.proto
index a4ad0beef1688af23072d473705ee0dca6052173..bff4183bb11c23ebd48ef144a3e8ad2c00d2a238 100644
--- a/research/object_detection/protos/center_net.proto
+++ b/research/object_detection/protos/center_net.proto
@@ -4,11 +4,14 @@ package object_detection.protos;
import "object_detection/protos/image_resizer.proto";
import "object_detection/protos/losses.proto";
+import "object_detection/protos/post_processing.proto";
+import "object_detection/protos/preprocessor.proto";
// Configuration for the CenterNet meta architecture from the "Objects as
// Points" paper [1]
// [1]: https://arxiv.org/abs/1904.07850
+// Next Id = 16
message CenterNet {
// Number of classes to predict.
optional int32 num_classes = 1;
@@ -19,6 +22,32 @@ message CenterNet {
// Image resizer for preprocessing the input image.
optional ImageResizer image_resizer = 3;
+ // If set, all task heads will be constructed with separable convolutions.
+ optional bool use_depthwise = 13 [default = false];
+
+ // Indicates whether or not to use the sparse version of the Op that computes
+ // the center heatmaps. The sparse version scales better with number of
+ // channels in the heatmap, but in some cases is known to cause an OOM error.
+ // TODO(b/170989061) When bug is fixed, make this the default behavior.
+ optional bool compute_heatmap_sparse = 15 [default = false];
+
+ // Parameters to determine the model architecture/layers of the prediction
+ // heads.
+ message PredictionHeadParams {
+ // The two fields: num_filters, kernel_sizes correspond to the parameters of
+ // the convolutional layers used by the prediction head. If provided, the
+ // length of the two repeated fields need to be the same and represents the
+ // number of convolutional layers.
+
+ // Corresponds to the "filters" argument in tf.keras.layers.Conv2D. If not
+ // provided, the default value [256] will be used.
+ repeated int32 num_filters = 1;
+
+ // Corresponds to the "kernel_size" argument in tf.keras.layers.Conv2D. If
+ // not provided, the default value [3] will be used.
+ repeated int32 kernel_sizes = 2;
+ }
+
// Parameters which are related to object detection task.
message ObjectDetection {
// The original fields are moved to ObjectCenterParams or deleted.
@@ -62,6 +91,18 @@ message CenterNet {
// If set, loss is only computed for the labeled classes.
optional bool use_labeled_classes = 6 [default = false];
+
+ // The keypoint weights used for calculating the location of object center.
+ // When the field is provided, the number of weights need to be the same as
+ // the number of keypoints. The object center is calculated by the weighted
+ // mean of the keypoint locations. When the field is not provided, the
+ // object center is determined by the bounding box groundtruth annotations
+ // (default behavior).
+ repeated float keypoint_weights_for_center = 7;
+
+ // Parameters to determine the architecture of the object center prediction
+ // head.
+ optional PredictionHeadParams center_head_params = 8;
}
optional ObjectCenterParams object_center_params = 5;
@@ -142,6 +183,12 @@ message CenterNet {
// the keypoint candidate.
optional string candidate_ranking_mode = 16 [default = "min_distance"];
+ // The score distance ratio offset, only used if candidate_ranking_mode is
+ // 'score_distance_ratio'. The offset is used in the maximization of score
+ // distance ratio, defined as:
+ // keypoint_score / (distance + score_distance_offset)
+ optional float score_distance_offset = 22 [default = 1.0];
+
// The radius (in the unit of output pixel) around heatmap peak to assign
// the offset targets. If set 0, then the offset target will only be
// assigned to the heatmap peak (same behavior as the original paper).
@@ -154,6 +201,46 @@ message CenterNet {
// out_height, out_width, 2 * num_keypoints] (recommended when the
// offset_peak_radius is not zero).
optional bool per_keypoint_offset = 18 [default = false];
+
+ // Indicates whether to predict the depth of each keypoints. Note that this
+ // is only supported in the single class keypoint task.
+ optional bool predict_depth = 19 [default = false];
+
+ // Indicates whether to predict depths for each keypoint channel
+ // separately. If set False, the output depth target has the shape
+ // [batch_size, out_height, out_width, 1]. If set True, the output depth
+ // target has the shape [batch_size, out_height, out_width,
+ // num_keypoints]. Recommend to set this value and "per_keypoint_offset" to
+ // both be True at the same time.
+ optional bool per_keypoint_depth = 20 [default = false];
+
+ // The weight of the keypoint depth loss.
+ optional float keypoint_depth_loss_weight = 21 [default = 1.0];
+
+ // Whether keypoints outside the image frame should be clipped back to the
+ // image boundary. If true, the keypoints that are clipped have scores set
+ // to 0.0.
+ optional bool clip_out_of_frame_keypoints = 23 [default = false];
+
+ // Whether instances should be rescored based on keypoint confidences. If
+ // False, will use the detection score (from the object center heatmap). If
+ // True, will compute new scores with:
+ // new_score = o * (1/k) sum {s_i}
+ // where o is the object score, s_i is the score for keypoint i, and k is
+ // the number of keypoints for that class.
+ optional bool rescore_instances = 24 [default = false];
+
+ // Parameters to determine the architecture of the keypoint heatmap
+ // prediction head.
+ optional PredictionHeadParams heatmap_head_params = 25;
+
+ // Parameters to determine the architecture of the keypoint offset
+ // prediction head.
+ optional PredictionHeadParams offset_head_params = 26;
+
+ // Parameters to determine the architecture of the keypoint regression
+ // prediction head.
+ optional PredictionHeadParams regress_head_params = 27;
}
repeated KeypointEstimation keypoint_estimation_task = 7;
@@ -218,6 +305,117 @@ message CenterNet {
optional float heatmap_bias_init = 8 [default = -2.19];
}
optional DensePoseEstimation densepose_estimation_task = 9;
+
+ // Parameters which are related to tracking embedding estimation task.
+ // A Simple Baseline for Multi-Object Tracking [2]
+ // [2]: https://arxiv.org/abs/2004.01888
+ message TrackEstimation {
+ // Weight of the task loss. The total loss of the model will be the
+ // summation of task losses weighted by the weights.
+ optional float task_loss_weight = 1 [default = 1.0];
+
+ // The maximun track ID of the datset.
+ optional int32 num_track_ids = 2;
+
+ // The embedding size for re-identification (ReID) task in tracking.
+ optional int32 reid_embed_size = 3 [default = 128];
+
+ // The number of (fully-connected, batch-norm, relu) layers for track ID
+ // classification head. The output dimension of each intermediate FC layer
+ // will all be 'reid_embed_size'. The last FC layer will directly project to
+ // the track ID classification space of size 'num_track_ids' without
+ // batch-norm and relu layers.
+ optional int32 num_fc_layers = 4 [default = 1];
+
+ // Classification loss configuration for ReID loss.
+ optional ClassificationLoss classification_loss = 5;
+ }
+ optional TrackEstimation track_estimation_task = 10;
+
+ // Temporal offset prediction head similar to CenterTrack.
+ // Currently our implementation adopts LSTM, different from original paper.
+ // See go/lstd-centernet for more details.
+ // Tracking Objects as Points [3]
+ // [3]: https://arxiv.org/abs/2004.01177
+ message TemporalOffsetEstimation {
+ // Weight of the task loss. The total loss of the model will be the
+ // summation of task losses weighted by the weights.
+ optional float task_loss_weight = 1 [default = 1.0];
+
+ // Localization loss configuration for offset loss.
+ optional LocalizationLoss localization_loss = 2;
+ }
+ optional TemporalOffsetEstimation temporal_offset_task = 12;
+
+
+ // Mask prediction support using DeepMAC. See https://arxiv.org/abs/2104.00613
+ message DeepMACMaskEstimation {
+ // The loss used for penalizing mask predictions.
+ optional ClassificationLoss classification_loss = 1;
+
+ // Weight of mask prediction loss
+ optional float task_loss_weight = 2 [default = 1.0];
+
+ // The dimension of the per-instance embedding.
+ optional int32 dim = 3 [default = 256];
+
+ // The dimension of the per-pixel embedding
+ optional int32 pixel_embedding_dim = 4 [default=16];
+
+ // If set, masks are only kept for classes listed here. Masks are deleted
+ // for all other classes. Note that this is only done at training time, eval
+ // behavior is unchanged.
+ repeated int32 allowed_masked_classes_ids = 5;
+
+ // The size of cropped pixel embedding that goes into the 2D mask prediction
+ // network (RoI align).
+ optional int32 mask_size = 6 [default=32];
+
+ // If set to a positive value, we subsample instances by this amount to
+ // save memory during training.
+ optional int32 mask_num_subsamples = 67[default=-1];
+
+ // Whether or not to use (x, y) coordinates as input to mask net.
+ optional bool use_xy = 8 [default=true];
+
+ // Defines the kind of architecture we want to use for mask network.
+ optional string network_type = 9 [default="hourglass52"];
+
+ // Whether or not we want to use instance embedding in mask network.
+ optional bool use_instance_embedding = 10 [default=true];
+
+ // Number of channels in the inital block of the mask prediction network.
+ optional int32 num_init_channels = 11 [default=64];
+
+ // Whether or not to predict masks at full resolution. If true, we predict
+ // masks at the resolution of the output stride. Otherwise, masks are
+ // predicted at resolution defined by mask_size
+ optional bool predict_full_resolution_masks = 12 [default=false];
+
+ // If predict_full_resolution_masks is set, this parameter controls the size
+ // of cropped masks returned by post-process. To be compatible with the rest
+ // of the API, masks are always cropped and resized according to detected
+ // boxes in postprocess.
+ optional int32 postprocess_crop_size = 13 [default=256];
+
+ // The maximum relative amount by which boxes will be jittered before
+ // RoI crop happens. The x and y coordinates of the box are jittered
+ // relative to width and height respectively.
+ optional float max_roi_jitter_ratio = 14 [default=0.0];
+
+ // The mode for jitterting box ROIs. See RandomJitterBoxes in
+ // preprocessor.proto for more details
+ optional RandomJitterBoxes.JitterMode jitter_mode = 15 [default=DEFAULT];
+ }
+
+ optional DeepMACMaskEstimation deepmac_mask_estimation = 14;
+
+ // CenterNet does not apply conventional post processing operations such as
+ // non max suppression as it applies a max-pool operator on box centers.
+ // However, in some cases we observe the need to remove duplicate predictions
+ // from CenterNet. Use this optional parameter to apply traditional non max
+ // suppression and score thresholding.
+ optional PostProcessing post_processing = 24;
}
message CenterNetFeatureExtractor {
@@ -235,4 +433,18 @@ message CenterNetFeatureExtractor {
// If set, will change channel order to be [blue, green, red]. This can be
// useful to be compatible with some pre-trained feature extractors.
optional bool bgr_ordering = 4 [default = false];
+
+ // If set, the feature upsampling layers will be constructed with
+ // separable convolutions. This is typically applied to feature pyramid
+ // network if any.
+ optional bool use_depthwise = 5 [default = false];
+
+
+ // Depth multiplier. Only valid for specific models (e.g. MobileNet). See subclasses of `CenterNetFeatureExtractor`.
+ optional float depth_multiplier = 9 [default = 1.0];
+
+ // Whether to use separable convolutions. Only valid for specific
+ // models. See subclasses of `CenterNetFeatureExtractor`.
+ optional bool use_separable_conv = 10 [default = false];
}
+
diff --git a/research/object_detection/protos/eval.proto b/research/object_detection/protos/eval.proto
index b1b99881c266da3643c4620efa55184de60e35d7..07ad0189b4e3f313597857a6813b29c0716f0a8c 100644
--- a/research/object_detection/protos/eval.proto
+++ b/research/object_detection/protos/eval.proto
@@ -3,7 +3,7 @@ syntax = "proto2";
package object_detection.protos;
// Message for configuring DetectionModel evaluation jobs (eval.py).
-// Next id - 33
+// Next id - 36
message EvalConfig {
optional uint32 batch_size = 25 [default = 1];
// Number of visualization images to generate.
@@ -82,6 +82,14 @@ message EvalConfig {
// If True, additionally include per-category metrics.
optional bool include_metrics_per_category = 24 [default = false];
+ // If true, includes all metrics per category.
+ optional bool all_metrics_per_category = 35 [default=false];
+
+ // Optional super-category definitions: keys are super-category names;
+ // values are comma-separated categories (assumed to correspond to category
+ // names (`display_name`) in the label map.
+ map super_categories = 34;
+
// Recall range within which precision should be computed.
optional float recall_lower_bound = 26 [default = 0.0];
optional float recall_upper_bound = 27 [default = 1.0];
@@ -103,6 +111,13 @@ message EvalConfig {
// visualization. An example would be human pose estimation where certain
// joints can be connected.
repeated KeypointEdge keypoint_edge = 32;
+
+ // The "groundtruth_labeled_classes" field indicates which classes have been
+ // labeled on the images. If skip_predictions_for_unlabeled_class is set,
+ // detector predictions that do not match to the groundtruth_labeled_classes
+ // will be ignored. This is useful for evaluating on test data that are not
+ // exhaustively labeled.
+ optional bool skip_predictions_for_unlabeled_class = 33 [default = false];
}
// A message to configure parameterized evaluation metric.
diff --git a/research/object_detection/protos/faster_rcnn.proto b/research/object_detection/protos/faster_rcnn.proto
index 486cc77ea8b156fb54500b0bbf7a01d4b17ac7b6..7509c6a3f5e81a756bf3e1fda1d8e07ae6f1f3ec 100644
--- a/research/object_detection/protos/faster_rcnn.proto
+++ b/research/object_detection/protos/faster_rcnn.proto
@@ -8,6 +8,7 @@ import "object_detection/protos/hyperparams.proto";
import "object_detection/protos/image_resizer.proto";
import "object_detection/protos/losses.proto";
import "object_detection/protos/post_processing.proto";
+import "object_detection/protos/fpn.proto";
// Configuration for Faster R-CNN models.
// See meta_architectures/faster_rcnn_meta_arch.py and models/model_builder.py
@@ -17,6 +18,7 @@ import "object_detection/protos/post_processing.proto";
// (or RPN) and a second stage box classifier. We thus use the prefixes
// `first_stage_` and `second_stage_` to indicate the stage to which each
// parameter pertains when relevant.
+
message FasterRcnn {
// Whether to construct only the Region Proposal Network (RPN).
optional int32 number_of_stages = 1 [default = 2];
@@ -175,17 +177,30 @@ message FasterRcnn {
// Whether to use tf.image.combined_non_max_suppression.
optional bool use_combined_nms_in_first_stage = 40 [default = false];
- // Whether to output final box feature. If true, it will crop the feature map
- // in the postprocess() method based on the final predictions.
+ // Whether to output final box feature. If true, it will crop the rpn feature
+ // map based on the final prediction boxes, then pass the crops through the
+ // box_classifier to compute the final features in the postprocess() method.
optional bool output_final_box_features = 42 [default = false];
+ // Whether to output final box rpn features. If true, it will crop the rpn
+ // feature map in the postprocess() method based on the final prediction
+ // boxes.
+ optional bool output_final_box_rpn_features = 43 [default = false];
+
// Configs for context model.
optional Context context_config = 41;
}
+// Input type format: whether inputs are TfExamples or TfSequenceExamples.
+enum AttentionPosition {
+ ATTENTION_DEFAULT = 0; // Default, currently post box classifier
+ POST_BOX_CLASSIFIER = 1; // Post box classifier
+ POST_RPN = 2; // Post RPN, pre box classifier
+}
+
message Context {
- // Configuration proto for Context .
- // Next id: 4
+ // Configuration proto for Context R-CNN.
+ // Next id: 12
// The maximum number of contextual features per-image, used for padding
optional int32 max_num_context_features = 1 [default = 2000];
@@ -198,6 +213,30 @@ message Context {
// The context feature length.
optional int32 context_feature_length = 4 [default = 2057];
+
+ // Whether to use self-attention from box proposals to themselves, TF1 only.
+ optional bool use_self_attention = 6 [default = false];
+
+ // Whether to use attention into context features, setting to false is only
+ // implemented in TF1.
+ optional bool use_long_term_attention = 7 [default = true];
+
+ // Whether the self-attention block and the long term attention block should
+ // be in sequence or parallel, ie whether the outputs of the self-attention
+ // block should be the inputs into the long term attention block (sequence)
+ // or whether the self attention block and long term attention block should
+ // happen in parallel, with outputs summed.
+ optional bool self_attention_in_sequence = 8 [default = false];
+
+ // Number of attention heads
+ optional int32 num_attention_heads = 9 [default = 1];
+
+ // Number of attention layers
+ optional int32 num_attention_layers = 11 [default = 1];
+
+ // Where the attention goes, 0 is pre-second-stage, 1 is post-second-stage
+ optional AttentionPosition attention_position = 10 [
+ default = POST_BOX_CLASSIFIER];
}
message FasterRcnnFeatureExtractor {
@@ -212,4 +251,21 @@ message FasterRcnnFeatureExtractor {
// When training with a relative large batch size (e.g. 8), it could be
// desirable to enable batch norm update.
optional bool batch_norm_trainable = 3 [default = false];
+
+ // Hyperparameters that affect the layers of feature extractor added on top
+ // of the base feature extractor.
+ optional Hyperparams conv_hyperparams = 4;
+
+ // if the value is set to true, the base feature extractor's hyperparams will
+ // be overridden with the `conv_hyperparams`.
+ optional bool override_base_feature_extractor_hyperparams = 5
+ [default = false];
+
+ // The nearest multiple to zero-pad the input height and width dimensions to.
+ // For example, if pad_to_multiple = 2, input dimensions are zero-padded
+ // until the resulting dimensions are even.
+ optional int32 pad_to_multiple = 6 [default = 32];
+
+ // Feature Pyramid Networks config.
+ optional FeaturePyramidNetworks fpn = 7;
}
diff --git a/research/object_detection/protos/fpn.proto b/research/object_detection/protos/fpn.proto
new file mode 100644
index 0000000000000000000000000000000000000000..568aa848de67a899709918de235d9939c776ec93
--- /dev/null
+++ b/research/object_detection/protos/fpn.proto
@@ -0,0 +1,50 @@
+syntax = "proto2";
+
+package object_detection.protos;
+
+// Configuration for Feature Pyramid Networks.
+message FeaturePyramidNetworks {
+ // We recommend to use multi_resolution_feature_map_generator with FPN, and
+ // the levels there must match the levels defined below for better
+ // performance.
+ // Correspondence from FPN levels to Resnet/Mobilenet V1 feature maps:
+ // FPN Level Resnet Feature Map Mobilenet-V1 Feature Map
+ // 2 Block 1 Conv2d_3_pointwise
+ // 3 Block 2 Conv2d_5_pointwise
+ // 4 Block 3 Conv2d_11_pointwise
+ // 5 Block 4 Conv2d_13_pointwise
+ // 6 Bottomup_5 bottom_up_Conv2d_14
+ // 7 Bottomup_6 bottom_up_Conv2d_15
+ // 8 Bottomup_7 bottom_up_Conv2d_16
+ // 9 Bottomup_8 bottom_up_Conv2d_17
+
+ // minimum level in feature pyramid
+ optional int32 min_level = 1 [default = 3];
+
+ // maximum level in feature pyramid
+ optional int32 max_level = 2 [default = 7];
+
+ // channel depth for additional coarse feature layers.
+ optional int32 additional_layer_depth = 3 [default = 256];
+
+}
+
+// Configuration for Bidirectional Feature Pyramid Networks.
+message BidirectionalFeaturePyramidNetworks {
+ // minimum level in the feature pyramid.
+ optional int32 min_level = 1 [default = 3];
+
+ // maximum level in the feature pyramid.
+ optional int32 max_level = 2 [default = 7];
+
+ // The number of repeated top-down bottom-up iterations for BiFPN-based
+ // feature extractors (bidirectional feature pyramid networks).
+ optional int32 num_iterations = 3;
+
+ // The number of filters (channels) to use in feature pyramid layers for
+ // BiFPN-based feature extractors (bidirectional feature pyramid networks).
+ optional int32 num_filters = 4;
+
+ // Method used to combine inputs to BiFPN nodes.
+ optional string combine_method = 5 [default = 'fast_attention'];
+}
diff --git a/research/object_detection/protos/hyperparams.proto b/research/object_detection/protos/hyperparams.proto
index e2fee247ca1303dfdbb9bdb69f187b7520c4e89c..fba7359f45fde3911c0ae547d1e6b61a5ae39d45 100644
--- a/research/object_detection/protos/hyperparams.proto
+++ b/research/object_detection/protos/hyperparams.proto
@@ -42,6 +42,8 @@ message Hyperparams {
// Note that if nothing below is selected, then no normalization is applied
// BatchNorm hyperparameters.
BatchNorm batch_norm = 5;
+ // SyncBatchNorm hyperparameters (KerasLayerHyperparams only).
+ BatchNorm sync_batch_norm = 9;
// GroupNorm hyperparameters. This is only supported on a subset of models.
// Note that the current implementation of group norm instantiated in
// tf.contrib.group.layers.group_norm() only supports fixed_size_resizer
@@ -86,6 +88,11 @@ message Initializer {
TruncatedNormalInitializer truncated_normal_initializer = 1;
VarianceScalingInitializer variance_scaling_initializer = 2;
RandomNormalInitializer random_normal_initializer = 3;
+ // Allows specifying initializers by name, as a string, which will be passed
+ // directly as an argument during layer construction. Currently, this is
+ // only supported when using KerasLayerHyperparams, and for valid Keras
+ // initializers, e.g. `glorot_uniform`, `variance_scaling`, etc.
+ string keras_initializer_by_name = 4;
}
}
diff --git a/research/object_detection/protos/input_reader.proto b/research/object_detection/protos/input_reader.proto
index 27d022532dc14fffc2b8078a500933d44ae5bf68..053bcfa559e0ba2e93ccf7dbe31e58745ea6ca3d 100644
--- a/research/object_detection/protos/input_reader.proto
+++ b/research/object_detection/protos/input_reader.proto
@@ -2,7 +2,6 @@ syntax = "proto2";
package object_detection.protos;
-import "object_detection/protos/image_resizer.proto";
// Configuration proto for defining input readers that generate Object Detection
// Examples from input sources. Input readers are expected to generate a
@@ -26,12 +25,12 @@ enum InstanceMaskType {
// Input type format: whether inputs are TfExamples or TfSequenceExamples.
enum InputType {
- INPUT_DEFAULT = 0; // Default implementation, currently TF_EXAMPLE
- TF_EXAMPLE = 1; // TfExample input
- TF_SEQUENCE_EXAMPLE = 2; // TfSequenceExample Input
+ INPUT_DEFAULT = 0; // Default implementation, currently TF_EXAMPLE
+ TF_EXAMPLE = 1; // TfExample input
+ TF_SEQUENCE_EXAMPLE = 2; // TfSequenceExample Input
}
-// Next id: 32
+// Next id: 38
message InputReader {
// Name of input reader. Typically used to describe the dataset that is read
// by this input reader.
@@ -62,6 +61,9 @@ message InputReader {
optional uint32 sample_1_of_n_examples = 22 [default = 1];
// Number of file shards to read in parallel.
+ //
+ // When sample_from_datasets_weights are configured, num_readers is applied
+ // for each dataset.
optional uint32 num_readers = 6 [default = 64];
// Number of batches to produce in parallel. If this is run on a 2x2 TPU set
@@ -91,6 +93,9 @@ message InputReader {
// Number of parallel decode ops to apply.
optional uint32 num_parallel_map_calls = 14 [default = 64, deprecated = true];
+ // Drop remainder when batch size does not divide dataset size.
+ optional bool drop_remainder = 35 [default = true];
+
// If positive, TfExampleDecoder will try to decode rasters of additional
// channels from tf.Examples.
optional int32 num_additional_channels = 18 [default = 0];
@@ -113,6 +118,9 @@ message InputReader {
// Whether to load context features from the dataset.
optional bool load_context_features = 25 [default = false];
+ // Whether to load context image ids from the dataset.
+ optional bool load_context_image_ids = 36 [default = false];
+
// Whether to load groundtruth instance masks.
optional bool load_instance_masks = 7 [default = false];
@@ -123,6 +131,12 @@ message InputReader {
// to true.
optional bool load_dense_pose = 31 [default = false];
+ // Whether to load track information.
+ optional bool load_track_id = 33 [default = false];
+
+ // Whether to load keypoint depth features.
+ optional bool load_keypoint_depth_features = 37 [default = false];
+
// Whether to use the display name when decoding examples. This is only used
// when mapping class text strings to integers.
optional bool use_display_name = 17 [default = false];
@@ -133,11 +147,47 @@ message InputReader {
// Whether input data type is tf.Examples or tf.SequenceExamples
optional InputType input_type = 30 [default = TF_EXAMPLE];
+ // Which frame to choose from the input if Sequence Example. -1 indicates
+ // random choice.
+ optional int32 frame_index = 32 [default = -1];
+
oneof input_reader {
TFRecordInputReader tf_record_input_reader = 8;
ExternalInputReader external_input_reader = 9;
}
+ // When multiple input files are configured, we can sample across them based
+ // on weights.
+ //
+ // The number of weights must match the number of input files configured.
+ //
+ // The number of input readers per dataset is num_readers, scaled relative to
+ // the dataset weight.
+ //
+ // When set, shuffling and shuffle buffer size, settings are
+ // applied individually to each dataset.
+ //
+ // Implementation follows tf.data.experimental.sample_from_datasets sampling
+ // strategy. Weights may take any value - only relative weights matter.
+ //
+ // Zero weights will result in a dataset not being sampled and no input
+ // readers spawned.
+ //
+ // Examples, assuming two input files configured:
+ //
+ // Equal weighting:
+ // sample_from_datasets_weights: 0.5
+ // sample_from_datasets_weights: 0.5
+ //
+ // 2:1 weighting:
+ // sample_from_datasets_weights: 2
+ // sample_from_datasets_weights: 1
+ //
+ // Exclude the second dataset:
+ // sample_from_datasets_weights: 1
+ // sample_from_datasets_weights: 0
+ repeated float sample_from_datasets_weights = 34;
+
// Expand labels to ancestors or descendants in the hierarchy for
// for positive and negative labels, respectively.
diff --git a/research/object_detection/protos/losses.proto b/research/object_detection/protos/losses.proto
index 2342fb24fceb68c58b9e06f4439cedd89efd933d..adb77c07555436b1d6179ffdd40062e399dd4037 100644
--- a/research/object_detection/protos/losses.proto
+++ b/research/object_detection/protos/losses.proto
@@ -70,6 +70,7 @@ message LocalizationLoss {
WeightedSmoothL1LocalizationLoss weighted_smooth_l1 = 2;
WeightedIOULocalizationLoss weighted_iou = 3;
L1LocalizationLoss l1_localization_loss = 4;
+ WeightedGIOULocalizationLoss weighted_giou = 5;
}
}
@@ -101,6 +102,10 @@ message WeightedIOULocalizationLoss {
message L1LocalizationLoss {
}
+// Generalized intersection over union location loss: 1 - GIOU
+message WeightedGIOULocalizationLoss {
+}
+
// Configuration for class prediction loss function.
message ClassificationLoss {
oneof classification_loss {
@@ -110,6 +115,7 @@ message ClassificationLoss {
BootstrappedSigmoidClassificationLoss bootstrapped_sigmoid = 3;
SigmoidFocalClassificationLoss weighted_sigmoid_focal = 4;
PenaltyReducedLogisticFocalLoss penalty_reduced_logistic_focal_loss = 6;
+ WeightedDiceClassificationLoss weighted_dice_classification_loss = 7;
}
}
@@ -217,3 +223,14 @@ message RandomExampleSampler {
// example sampling.
optional float positive_sample_fraction = 1 [default = 0.01];
}
+
+// Dice loss for training instance masks[1][2].
+// [1]: https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
+// [2]: https://arxiv.org/abs/1606.04797
+message WeightedDiceClassificationLoss {
+ // If set, we square the probabilities in the denominator term used for
+ // normalization.
+ optional bool squared_normalization = 1 [default=false];
+}
+
+
diff --git a/research/object_detection/protos/post_processing.proto b/research/object_detection/protos/post_processing.proto
index 80f75b18d1a1753ad672d0d84b8fbaf6e655adf8..314faab155bef5f72f316350ea69597e5b9bd5fd 100644
--- a/research/object_detection/protos/post_processing.proto
+++ b/research/object_detection/protos/post_processing.proto
@@ -27,10 +27,10 @@ message BatchNonMaxSuppression {
// Class-agnostic NMS function implements a class-agnostic version
// of Non Maximal Suppression where if max_classes_per_detection=k,
// 1) we keep the top-k scores for each detection and
- // 2) during NMS, each detection only uses the highest class score for sorting.
- // 3) Compared to regular NMS, the worst runtime of this version is O(N^2)
- // instead of O(KN^2) where N is the number of detections and K the number of
- // classes.
+ // 2) during NMS, each detection only uses the highest class score for
+ // sorting. 3) Compared to regular NMS, the worst runtime of this version is
+ // O(N^2) instead of O(KN^2) where N is the number of detections and K the
+ // number of classes.
optional bool use_class_agnostic_nms = 7 [default = false];
// Number of classes retained per detection in class agnostic NMS.
@@ -57,6 +57,12 @@ message BatchNonMaxSuppression {
// export models for older versions of TF.
optional bool use_hard_nms = 13 [default = false];
+ // Use cpu NMS. NMSV3/NMSV4 by default runs on GPU, which may cause OOM issue
+ // if the model is large and/or batch size is large during training.
+ // Setting this flag to false moves the nms op to CPU when OOM happens.
+ // The flag is not needed if use_hard_nms = false, as soft NMS currently
+ // runs on CPU by default.
+ optional bool use_cpu_nms = 14 [default = false];
}
// Configuration proto for post-processing predicted boxes and
diff --git a/research/object_detection/protos/preprocessor.proto b/research/object_detection/protos/preprocessor.proto
index a99be94194a1d412676e6d4d387cba53a8667c02..6afcc3e70f67d3f905ff8cee193b2cf89e174483 100644
--- a/research/object_detection/protos/preprocessor.proto
+++ b/research/object_detection/protos/preprocessor.proto
@@ -4,7 +4,7 @@ package object_detection.protos;
// Message for defining a preprocessing operation on input data.
// See: //third_party/tensorflow_models/object_detection/core/preprocessor.py
-// Next ID: 39
+// Next ID: 40
message PreprocessingStep {
oneof preprocessing_step {
NormalizeImage normalize_image = 1;
@@ -45,6 +45,7 @@ message PreprocessingStep {
RandomPatchGaussian random_patch_gaussian = 36;
RandomSquareCropByScale random_square_crop_by_scale = 37;
RandomScaleCropAndPadToSquare random_scale_crop_and_pad_to_square = 38;
+ AdjustGamma adjust_gamma = 39;
}
}
@@ -164,6 +165,18 @@ message RandomDistortColor {
// ie. If a box is [100, 200] and ratio is 0.02, the corners can move by [1, 4].
message RandomJitterBoxes {
optional float ratio = 1 [default=0.05];
+
+ enum JitterMode {
+ DEFAULT = 0;
+ EXPAND = 1;
+ SHRINK = 2;
+ }
+ // The mode of jittering
+ // EXPAND - Only expands boxes
+ // SHRINK - Only shrinks boxes
+ // DEFAULT - Jitters each box boundary independently
+ optional JitterMode jitter_mode = 2 [default=DEFAULT];
+
}
// Randomly crops the image and bounding boxes.
@@ -590,3 +603,9 @@ message RandomScaleCropAndPadToSquare {
optional float scale_min = 2 [default=0.1];
optional float scale_max = 3 [default=2.0];
}
+
+// Adjusts the gamma of the image using the specified gamma and gain values.
+message AdjustGamma {
+ optional float gamma = 1 [default=1.0];
+ optional float gain = 2 [default=1.0];
+}
diff --git a/research/object_detection/protos/ssd.proto b/research/object_detection/protos/ssd.proto
index 3fdcd99370a396d0fe1a9123edc00bbed4af0cf9..e4b6ffa18c24d4430587f40ebfb7cc6b183ee9ee 100644
--- a/research/object_detection/protos/ssd.proto
+++ b/research/object_detection/protos/ssd.proto
@@ -11,6 +11,7 @@ import "object_detection/protos/losses.proto";
import "object_detection/protos/matcher.proto";
import "object_detection/protos/post_processing.proto";
import "object_detection/protos/region_similarity_calculator.proto";
+import "object_detection/protos/fpn.proto";
// Configuration for Single Shot Detection (SSD) models.
// Next id: 27
@@ -203,50 +204,3 @@ message SsdFeatureExtractor {
}
-// Configuration for Feature Pyramid Networks.
-message FeaturePyramidNetworks {
- // We recommend to use multi_resolution_feature_map_generator with FPN, and
- // the levels there must match the levels defined below for better
- // performance.
- // Correspondence from FPN levels to Resnet/Mobilenet V1 feature maps:
- // FPN Level Resnet Feature Map Mobilenet-V1 Feature Map
- // 2 Block 1 Conv2d_3_pointwise
- // 3 Block 2 Conv2d_5_pointwise
- // 4 Block 3 Conv2d_11_pointwise
- // 5 Block 4 Conv2d_13_pointwise
- // 6 Bottomup_5 bottom_up_Conv2d_14
- // 7 Bottomup_6 bottom_up_Conv2d_15
- // 8 Bottomup_7 bottom_up_Conv2d_16
- // 9 Bottomup_8 bottom_up_Conv2d_17
-
- // minimum level in feature pyramid
- optional int32 min_level = 1 [default = 3];
-
- // maximum level in feature pyramid
- optional int32 max_level = 2 [default = 7];
-
- // channel depth for additional coarse feature layers.
- optional int32 additional_layer_depth = 3 [default = 256];
-
-}
-
-// Configuration for Bidirectional Feature Pyramid Networks.
-message BidirectionalFeaturePyramidNetworks {
- // minimum level in the feature pyramid.
- optional int32 min_level = 1 [default = 3];
-
- // maximum level in the feature pyramid.
- optional int32 max_level = 2 [default = 7];
-
- // The number of repeated top-down bottom-up iterations for BiFPN-based
- // feature extractors (bidirectional feature pyramid networks).
- optional int32 num_iterations = 3;
-
- // The number of filters (channels) to use in feature pyramid layers for
- // BiFPN-based feature extractors (bidirectional feature pyramid networks).
- optional int32 num_filters = 4;
-
- // Method used to combine inputs to BiFPN nodes.
- optional string combine_method = 5 [default = 'fast_attention'];
-}
-
diff --git a/research/object_detection/protos/string_int_label_map.proto b/research/object_detection/protos/string_int_label_map.proto
index c9095a9d73cf51a669c7b106ad34fb8cc0a24b43..d77dd92af8534fd0b25ed0ff2d6e2faa24d03b84 100644
--- a/research/object_detection/protos/string_int_label_map.proto
+++ b/research/object_detection/protos/string_int_label_map.proto
@@ -6,6 +6,14 @@ syntax = "proto2";
package object_detection.protos;
+// LVIS frequency:
+enum LVISFrequency {
+ UNSPECIFIED = 0;
+ FREQUENT = 1;
+ COMMON = 2;
+ RARE = 3;
+}
+
message StringIntLabelMapItem {
// String name. The most common practice is to set this to a MID or synsets
// id.
@@ -38,6 +46,10 @@ message StringIntLabelMapItem {
// current element. Value should correspond to another label id element.
repeated int32 ancestor_ids = 5;
repeated int32 descendant_ids = 6;
+
+ // LVIS specific label map fields
+ optional LVISFrequency frequency = 7;
+ optional int32 instance_count = 8;
};
message StringIntLabelMap {
diff --git a/research/object_detection/protos/train.proto b/research/object_detection/protos/train.proto
index 62d326cdf67c7329ddaa22250a4f2734a4f43066..14a63404b460db3830aa045cb43ba6bbaf5df070 100644
--- a/research/object_detection/protos/train.proto
+++ b/research/object_detection/protos/train.proto
@@ -14,7 +14,7 @@ enum CheckpointVersion {
// Message for configuring DetectionModel training jobs (train.py).
-// Next id: 30
+// Next id: 31
message TrainConfig {
// Effective batch size to use for training.
// For TPU (or sync SGD jobs), the batch size per core (or GPU) is going to be
@@ -40,9 +40,12 @@ message TrainConfig {
// extractor variables trained outside of object detection.
optional string fine_tune_checkpoint = 7 [default=""];
- // Type of checkpoint to restore variables from, e.g. 'classification' or
- // 'detection'. Provides extensibility to from_detection_checkpoint.
- // Typically used to load feature extractor variables from trained models.
+ // Type of checkpoint to restore variables from, e.g. 'classification'
+ // 'detection', `fine_tune`, `full`. Controls which variables are restored
+ // from the pre-trained checkpoint. For meta architecture specific valid
+ // values of this parameter, see the restore_map (TF1) or
+ // restore_from_object (TF2) function documentation in the
+ // /meta_architectures/*meta_arch.py files
optional string fine_tune_checkpoint_type = 22 [default=""];
// Either "v1" or "v2". If v1, restores the checkpoint using the tensorflow
@@ -60,9 +63,21 @@ message TrainConfig {
// Whether to load all checkpoint vars that match model variable names and
// sizes. This option is only available if `from_detection_checkpoint` is
// True. This option is *not* supported for TF2 --- setting it to true
- // will raise an error.
+ // will raise an error. Instead, set fine_tune_checkpoint_type: 'full'.
optional bool load_all_detection_checkpoint_vars = 19 [default = false];
+ // Whether to run dummy computation when loading a `fine_tune_checkpoint`.
+ // This option is true by default since it is often necessary to run the model
+ // on a dummy input before loading a `fine_tune_checkpoint`, in order to
+ // ensure that all the model variables have alread been built successfully.
+ // Some meta architectures, like CenterNet, do not require dummy computation
+ // to successfully load all checkpoint variables, and in these cases this
+ // flag may be set to false to reduce startup time and memory consumption.
+ // Note, this flag only affects dummy computation when loading a
+ // `fine_tune_checkpoint`, e.g. it does not affect the dummy computation that
+ // is run when creating shadow copies of model variables when using EMA.
+ optional bool run_fine_tune_checkpoint_dummy_computation = 30 [default=true];
+
// Number of steps to train the DetectionModel for. If 0, will train the model
// indefinitely.
optional uint32 num_steps = 9 [default=0];
diff --git a/research/object_detection/test_images/image3.jpg b/research/object_detection/test_images/image3.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..44a7b0322689a172ca10005bb27df1bc0e590b69
Binary files /dev/null and b/research/object_detection/test_images/image3.jpg differ
diff --git a/research/object_detection/test_images/image_info.txt b/research/object_detection/test_images/image_info.txt
index 6f805cbcd27405940398f24f2a1a4538e197e108..656af904f46462cb04097f89568342927798826a 100644
--- a/research/object_detection/test_images/image_info.txt
+++ b/research/object_detection/test_images/image_info.txt
@@ -3,4 +3,4 @@ Image provenance:
image1.jpg: https://commons.wikimedia.org/wiki/File:Baegle_dwa.jpg
image2.jpg: Michael Miley,
https://www.flickr.com/photos/mike_miley/4678754542/in/photolist-88rQHL-88oBVp-88oC2B-88rS6J-88rSqm-88oBLv-88oBC4
-
+image3.jpg: Chris Briggs, https://unsplash.com/photos/ILBrHd6PFJA
diff --git a/research/object_detection/utils/config_util.py b/research/object_detection/utils/config_util.py
index 662d42e1305538534e1cb6671086e4faa6cdf00c..05f7f7ef063809f56a4a0af6084f4dbca5e8af89 100644
--- a/research/object_detection/utils/config_util.py
+++ b/research/object_detection/utils/config_util.py
@@ -19,9 +19,9 @@ from __future__ import division
from __future__ import print_function
import os
-import tensorflow.compat.v1 as tf
from google.protobuf import text_format
+import tensorflow.compat.v1 as tf
from tensorflow.python.lib.io import file_io
@@ -621,6 +621,24 @@ def _maybe_update_config_with_key_value(configs, key, value):
value)
elif field_name == "num_classes":
_update_num_classes(configs["model"], value)
+ elif field_name == "sample_from_datasets_weights":
+ _update_sample_from_datasets_weights(configs["train_input_config"], value)
+ elif field_name == "peak_max_pool_kernel_size":
+ _update_peak_max_pool_kernel_size(configs["model"], value)
+ elif field_name == "candidate_search_scale":
+ _update_candidate_search_scale(configs["model"], value)
+ elif field_name == "candidate_ranking_mode":
+ _update_candidate_ranking_mode(configs["model"], value)
+ elif field_name == "score_distance_offset":
+ _update_score_distance_offset(configs["model"], value)
+ elif field_name == "box_scale":
+ _update_box_scale(configs["model"], value)
+ elif field_name == "keypoint_candidate_score_threshold":
+ _update_keypoint_candidate_score_threshold(configs["model"], value)
+ elif field_name == "rescore_instances":
+ _update_rescore_instances(configs["model"], value)
+ elif field_name == "unmatched_keypoint_score":
+ _update_unmatched_keypoint_score(configs["model"], value)
else:
return False
return True
@@ -1024,7 +1042,7 @@ def _update_retain_original_image_additional_channels(
retain_original_image_additional_channels)
-def remove_unecessary_ema(variables_to_restore, no_ema_collection=None):
+def remove_unnecessary_ema(variables_to_restore, no_ema_collection=None):
"""Remap and Remove EMA variable that are not created during training.
ExponentialMovingAverage.variables_to_restore() returns a map of EMA names
@@ -1036,9 +1054,8 @@ def remove_unecessary_ema(variables_to_restore, no_ema_collection=None):
}
This function takes care of the extra ExponentialMovingAverage variables
that get created during eval but aren't available in the checkpoint, by
- remapping the key to the shallow copy of the variable itself, and remove
- the entry of its EMA from the variables to restore. An example resulting
- dictionary would look like:
+ remapping the key to the variable itself, and remove the entry of its EMA from
+ the variables to restore. An example resulting dictionary would look like:
{
conv/batchnorm/gamma: conv/batchnorm/gamma,
conv_4/conv2d_params: conv_4/conv2d_params,
@@ -1057,14 +1074,15 @@ def remove_unecessary_ema(variables_to_restore, no_ema_collection=None):
if no_ema_collection is None:
return variables_to_restore
+ restore_map = {}
for key in variables_to_restore:
- if "ExponentialMovingAverage" in key:
- for name in no_ema_collection:
- if name in key:
- variables_to_restore[key.replace("/ExponentialMovingAverage",
- "")] = variables_to_restore[key]
- del variables_to_restore[key]
- return variables_to_restore
+ if ("ExponentialMovingAverage" in key
+ and any([name in key for name in no_ema_collection])):
+ new_key = key.replace("/ExponentialMovingAverage", "")
+ else:
+ new_key = key
+ restore_map[new_key] = variables_to_restore[key]
+ return restore_map
def _update_num_classes(model_config, num_classes):
@@ -1073,3 +1091,126 @@ def _update_num_classes(model_config, num_classes):
model_config.faster_rcnn.num_classes = num_classes
if meta_architecture == "ssd":
model_config.ssd.num_classes = num_classes
+
+
+def _update_sample_from_datasets_weights(input_reader_config, weights):
+ """Updated sample_from_datasets_weights with overrides."""
+ if len(weights) != len(input_reader_config.sample_from_datasets_weights):
+ raise ValueError(
+ "sample_from_datasets_weights override has a different number of values"
+ " ({}) than the configured dataset weights ({})."
+ .format(
+ len(input_reader_config.sample_from_datasets_weights),
+ len(weights)))
+
+ del input_reader_config.sample_from_datasets_weights[:]
+ input_reader_config.sample_from_datasets_weights.extend(weights)
+
+
+def _update_peak_max_pool_kernel_size(model_config, kernel_size):
+ """Updates the max pool kernel size (NMS) for keypoints in CenterNet."""
+ meta_architecture = model_config.WhichOneof("model")
+ if meta_architecture == "center_net":
+ if len(model_config.center_net.keypoint_estimation_task) == 1:
+ kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+ kpt_estimation_task.peak_max_pool_kernel_size = kernel_size
+ else:
+ tf.logging.warning("Ignoring config override key for "
+ "peak_max_pool_kernel_size since there are multiple "
+ "keypoint estimation tasks")
+
+
+def _update_candidate_search_scale(model_config, search_scale):
+ """Updates the keypoint candidate search scale in CenterNet."""
+ meta_architecture = model_config.WhichOneof("model")
+ if meta_architecture == "center_net":
+ if len(model_config.center_net.keypoint_estimation_task) == 1:
+ kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+ kpt_estimation_task.candidate_search_scale = search_scale
+ else:
+ tf.logging.warning("Ignoring config override key for "
+ "candidate_search_scale since there are multiple "
+ "keypoint estimation tasks")
+
+
+def _update_candidate_ranking_mode(model_config, mode):
+ """Updates how keypoints are snapped to candidates in CenterNet."""
+ if mode not in ("min_distance", "score_distance_ratio"):
+ raise ValueError("Attempting to set the keypoint candidate ranking mode "
+ "to {}, but the only options are 'min_distance' and "
+ "'score_distance_ratio'.".format(mode))
+ meta_architecture = model_config.WhichOneof("model")
+ if meta_architecture == "center_net":
+ if len(model_config.center_net.keypoint_estimation_task) == 1:
+ kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+ kpt_estimation_task.candidate_ranking_mode = mode
+ else:
+ tf.logging.warning("Ignoring config override key for "
+ "candidate_ranking_mode since there are multiple "
+ "keypoint estimation tasks")
+
+
+def _update_score_distance_offset(model_config, offset):
+ """Updates the keypoint candidate selection metric. See CenterNet proto."""
+ meta_architecture = model_config.WhichOneof("model")
+ if meta_architecture == "center_net":
+ if len(model_config.center_net.keypoint_estimation_task) == 1:
+ kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+ kpt_estimation_task.score_distance_offset = offset
+ else:
+ tf.logging.warning("Ignoring config override key for "
+ "score_distance_offset since there are multiple "
+ "keypoint estimation tasks")
+
+
+def _update_box_scale(model_config, box_scale):
+ """Updates the keypoint candidate search region. See CenterNet proto."""
+ meta_architecture = model_config.WhichOneof("model")
+ if meta_architecture == "center_net":
+ if len(model_config.center_net.keypoint_estimation_task) == 1:
+ kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+ kpt_estimation_task.box_scale = box_scale
+ else:
+ tf.logging.warning("Ignoring config override key for box_scale since "
+ "there are multiple keypoint estimation tasks")
+
+
+def _update_keypoint_candidate_score_threshold(model_config, threshold):
+ """Updates the keypoint candidate score threshold. See CenterNet proto."""
+ meta_architecture = model_config.WhichOneof("model")
+ if meta_architecture == "center_net":
+ if len(model_config.center_net.keypoint_estimation_task) == 1:
+ kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+ kpt_estimation_task.keypoint_candidate_score_threshold = threshold
+ else:
+ tf.logging.warning("Ignoring config override key for "
+ "keypoint_candidate_score_threshold since there are "
+ "multiple keypoint estimation tasks")
+
+
+def _update_rescore_instances(model_config, should_rescore):
+ """Updates whether boxes should be rescored based on keypoint confidences."""
+ if isinstance(should_rescore, str):
+ should_rescore = True if should_rescore == "True" else False
+ meta_architecture = model_config.WhichOneof("model")
+ if meta_architecture == "center_net":
+ if len(model_config.center_net.keypoint_estimation_task) == 1:
+ kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+ kpt_estimation_task.rescore_instances = should_rescore
+ else:
+ tf.logging.warning("Ignoring config override key for "
+ "rescore_instances since there are multiple keypoint "
+ "estimation tasks")
+
+
+def _update_unmatched_keypoint_score(model_config, score):
+ meta_architecture = model_config.WhichOneof("model")
+ if meta_architecture == "center_net":
+ if len(model_config.center_net.keypoint_estimation_task) == 1:
+ kpt_estimation_task = model_config.center_net.keypoint_estimation_task[0]
+ kpt_estimation_task.unmatched_keypoint_score = score
+ else:
+ tf.logging.warning("Ignoring config override key for "
+ "unmatched_keypoint_score since there are multiple "
+ "keypoint estimation tasks")
+
diff --git a/research/object_detection/utils/config_util_test.py b/research/object_detection/utils/config_util_test.py
index f36970c11078b222710427b46ffd502be608c109..196685e53ecd9bcb17a4786d80eb5297f4f41cee 100644
--- a/research/object_detection/utils/config_util_test.py
+++ b/research/object_detection/utils/config_util_test.py
@@ -377,6 +377,45 @@ class ConfigUtilTest(tf.test.TestCase):
new_batch_size = configs["train_config"].batch_size
self.assertEqual(10, new_batch_size)
+ @unittest.skipIf(tf_version.is_tf2(), "Skipping TF1.X only test.")
+ def testOverwriteSampleFromDatasetWeights(self):
+ """Tests config override for sample_from_datasets_weights."""
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ pipeline_config.train_input_reader.sample_from_datasets_weights.extend(
+ [1, 2])
+ pipeline_config_path = os.path.join(self.get_temp_dir(), "pipeline.config")
+ _write_config(pipeline_config, pipeline_config_path)
+
+ # Override parameters:
+ configs = config_util.get_configs_from_pipeline_file(pipeline_config_path)
+ hparams = contrib_training.HParams(sample_from_datasets_weights=[0.5, 0.5])
+ configs = config_util.merge_external_params_with_configs(configs, hparams)
+
+ # Ensure that the parameters have the overridden values:
+ self.assertListEqual(
+ [0.5, 0.5],
+ list(configs["train_input_config"].sample_from_datasets_weights))
+
+ @unittest.skipIf(tf_version.is_tf2(), "Skipping TF1.X only test.")
+ def testOverwriteSampleFromDatasetWeightsWrongLength(self):
+ """Tests config override for sample_from_datasets_weights."""
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ pipeline_config.train_input_reader.sample_from_datasets_weights.extend(
+ [1, 2])
+ pipeline_config_path = os.path.join(self.get_temp_dir(), "pipeline.config")
+ _write_config(pipeline_config, pipeline_config_path)
+
+ # Try to override parameter with too many weights:
+ configs = config_util.get_configs_from_pipeline_file(pipeline_config_path)
+ hparams = contrib_training.HParams(
+ sample_from_datasets_weights=[0.5, 0.5, 0.5])
+ with self.assertRaises(
+ ValueError,
+ msg="sample_from_datasets_weights override has a different number of"
+ " values (3) than the configured dataset weights (2)."
+ ):
+ config_util.merge_external_params_with_configs(configs, hparams)
+
@unittest.skipIf(tf_version.is_tf2(), "Skipping TF1.X only test.")
def testKeyValueOverrideBadKey(self):
"""Tests that overwriting with a bad key causes an exception."""
@@ -946,7 +985,7 @@ class ConfigUtilTest(tf.test.TestCase):
self.assertEqual(config_util.get_number_of_classes(configs["model"]), 2)
- def testRemoveUnecessaryEma(self):
+ def testRemoveUnnecessaryEma(self):
input_dict = {
"expanded_conv_10/project/act_quant/min":
1,
@@ -977,7 +1016,68 @@ class ConfigUtilTest(tf.test.TestCase):
self.assertEqual(
output_dict,
- config_util.remove_unecessary_ema(input_dict, no_ema_collection))
+ config_util.remove_unnecessary_ema(input_dict, no_ema_collection))
+
+ def testUpdateRescoreInstances(self):
+ pipeline_config_path = os.path.join(self.get_temp_dir(), "pipeline.config")
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ kpt_task = pipeline_config.model.center_net.keypoint_estimation_task.add()
+ kpt_task.rescore_instances = True
+
+ _write_config(pipeline_config, pipeline_config_path)
+
+ configs = config_util.get_configs_from_pipeline_file(pipeline_config_path)
+ cn_config = configs["model"].center_net
+ self.assertEqual(
+ True, cn_config.keypoint_estimation_task[0].rescore_instances)
+
+ config_util.merge_external_params_with_configs(
+ configs, kwargs_dict={"rescore_instances": False})
+ cn_config = configs["model"].center_net
+ self.assertEqual(
+ False, cn_config.keypoint_estimation_task[0].rescore_instances)
+
+ def testUpdateRescoreInstancesWithBooleanString(self):
+ pipeline_config_path = os.path.join(self.get_temp_dir(), "pipeline.config")
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ kpt_task = pipeline_config.model.center_net.keypoint_estimation_task.add()
+ kpt_task.rescore_instances = True
+
+ _write_config(pipeline_config, pipeline_config_path)
+
+ configs = config_util.get_configs_from_pipeline_file(pipeline_config_path)
+ cn_config = configs["model"].center_net
+ self.assertEqual(
+ True, cn_config.keypoint_estimation_task[0].rescore_instances)
+
+ config_util.merge_external_params_with_configs(
+ configs, kwargs_dict={"rescore_instances": "False"})
+ cn_config = configs["model"].center_net
+ self.assertEqual(
+ False, cn_config.keypoint_estimation_task[0].rescore_instances)
+
+ def testUpdateRescoreInstancesWithMultipleTasks(self):
+ pipeline_config_path = os.path.join(self.get_temp_dir(), "pipeline.config")
+ pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+ kpt_task = pipeline_config.model.center_net.keypoint_estimation_task.add()
+ kpt_task.rescore_instances = True
+ kpt_task = pipeline_config.model.center_net.keypoint_estimation_task.add()
+ kpt_task.rescore_instances = True
+
+ _write_config(pipeline_config, pipeline_config_path)
+
+ configs = config_util.get_configs_from_pipeline_file(pipeline_config_path)
+ cn_config = configs["model"].center_net
+ self.assertEqual(
+ True, cn_config.keypoint_estimation_task[0].rescore_instances)
+
+ config_util.merge_external_params_with_configs(
+ configs, kwargs_dict={"rescore_instances": False})
+ cn_config = configs["model"].center_net
+ self.assertEqual(
+ True, cn_config.keypoint_estimation_task[0].rescore_instances)
+ self.assertEqual(
+ True, cn_config.keypoint_estimation_task[1].rescore_instances)
if __name__ == "__main__":
diff --git a/research/object_detection/utils/dataset_util.py b/research/object_detection/utils/dataset_util.py
index 65e7f7feeac0b4015b6fb31833f8d0590e1095ce..1cf007de8da7b8ae9b99b289f46c14a4238cc1d4 100644
--- a/research/object_detection/utils/dataset_util.py
+++ b/research/object_detection/utils/dataset_util.py
@@ -38,6 +38,10 @@ def bytes_list_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
+def float_feature(value):
+ return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
+
+
def float_list_feature(value):
return tf.train.Feature(float_list=tf.train.FloatList(value=value))
diff --git a/research/object_detection/utils/label_map_util.py b/research/object_detection/utils/label_map_util.py
index 37c823a8d39cd89801f1f3d7b957d31ede2f6c06..ecf7d82fbf873a7758f6f7c534d1f86a4bd60767 100644
--- a/research/object_detection/utils/label_map_util.py
+++ b/research/object_detection/utils/label_map_util.py
@@ -130,6 +130,18 @@ def convert_label_map_to_categories(label_map,
if item.id not in list_of_ids_already_added:
list_of_ids_already_added.append(item.id)
category = {'id': item.id, 'name': name}
+ if item.HasField('frequency'):
+ if item.frequency == string_int_label_map_pb2.LVISFrequency.Value(
+ 'FREQUENT'):
+ category['frequency'] = 'f'
+ elif item.frequency == string_int_label_map_pb2.LVISFrequency.Value(
+ 'COMMON'):
+ category['frequency'] = 'c'
+ elif item.frequency == string_int_label_map_pb2.LVISFrequency.Value(
+ 'RARE'):
+ category['frequency'] = 'r'
+ if item.HasField('instance_count'):
+ category['instance_count'] = item.instance_count
if item.keypoints:
keypoints = {}
list_of_keypoint_ids = []
diff --git a/research/object_detection/utils/label_map_util_test.py b/research/object_detection/utils/label_map_util_test.py
index 969f3258baf6eadaf9017dc06a8aa1f188eb51b9..cd5bb4169c1e9ffb5375080b37ead6f74d458adb 100644
--- a/research/object_detection/utils/label_map_util_test.py
+++ b/research/object_detection/utils/label_map_util_test.py
@@ -201,7 +201,7 @@ class LabelMapUtilTest(tf.test.TestCase):
name:'n00007846'
}
"""
- text_format.Merge(label_map_string, label_map_proto)
+ text_format.Parse(label_map_string, label_map_proto)
categories = label_map_util.convert_label_map_to_categories(
label_map_proto, max_num_classes=3)
self.assertListEqual([{
@@ -227,19 +227,61 @@ class LabelMapUtilTest(tf.test.TestCase):
}]
self.assertListEqual(expected_categories_list, categories)
+ def test_convert_label_map_to_categories_lvis_frequency_and_counts(self):
+ label_map_proto = string_int_label_map_pb2.StringIntLabelMap()
+ label_map_string = """
+ item {
+ id:1
+ name:'person'
+ frequency: FREQUENT
+ instance_count: 1000
+ }
+ item {
+ id:2
+ name:'dog'
+ frequency: COMMON
+ instance_count: 100
+ }
+ item {
+ id:3
+ name:'cat'
+ frequency: RARE
+ instance_count: 10
+ }
+ """
+ text_format.Parse(label_map_string, label_map_proto)
+ categories = label_map_util.convert_label_map_to_categories(
+ label_map_proto, max_num_classes=3)
+ self.assertListEqual([{
+ 'id': 1,
+ 'name': u'person',
+ 'frequency': 'f',
+ 'instance_count': 1000
+ }, {
+ 'id': 2,
+ 'name': u'dog',
+ 'frequency': 'c',
+ 'instance_count': 100
+ }, {
+ 'id': 3,
+ 'name': u'cat',
+ 'frequency': 'r',
+ 'instance_count': 10
+ }], categories)
+
def test_convert_label_map_to_categories(self):
label_map_proto = self._generate_label_map(num_classes=4)
categories = label_map_util.convert_label_map_to_categories(
label_map_proto, max_num_classes=3)
expected_categories_list = [{
'name': u'1',
- 'id': 1
+ 'id': 1,
}, {
'name': u'2',
- 'id': 2
+ 'id': 2,
}, {
'name': u'3',
- 'id': 3
+ 'id': 3,
}]
self.assertListEqual(expected_categories_list, categories)
@@ -259,7 +301,7 @@ class LabelMapUtilTest(tf.test.TestCase):
}
"""
label_map_proto = string_int_label_map_pb2.StringIntLabelMap()
- text_format.Merge(label_map_str, label_map_proto)
+ text_format.Parse(label_map_str, label_map_proto)
categories = label_map_util.convert_label_map_to_categories(
label_map_proto, max_num_classes=1)
self.assertEqual('person', categories[0]['name'])
@@ -291,7 +333,7 @@ class LabelMapUtilTest(tf.test.TestCase):
}
"""
label_map_proto = string_int_label_map_pb2.StringIntLabelMap()
- text_format.Merge(label_map_str, label_map_proto)
+ text_format.Parse(label_map_str, label_map_proto)
with self.assertRaises(ValueError):
label_map_util.convert_label_map_to_categories(
label_map_proto, max_num_classes=2)
diff --git a/research/object_detection/utils/learning_schedules.py b/research/object_detection/utils/learning_schedules.py
index 167be22f70e8627cac86d84c905a0b0e15588ad6..678cda6bdf8609bebcfa55fcde889d8dac09b174 100644
--- a/research/object_detection/utils/learning_schedules.py
+++ b/research/object_detection/utils/learning_schedules.py
@@ -23,6 +23,14 @@ from six.moves import zip
import tensorflow.compat.v1 as tf
+def _learning_rate_return_value(eager_decay_rate):
+ """Helper function to return proper learning rate based on tf version."""
+ if tf.executing_eagerly():
+ return eager_decay_rate
+ else:
+ return eager_decay_rate()
+
+
def exponential_decay_with_burnin(global_step,
learning_rate_base,
learning_rate_decay_steps,
@@ -76,10 +84,65 @@ def exponential_decay_with_burnin(global_step,
tf.constant(burnin_learning_rate),
post_burnin_learning_rate), min_learning_rate, name='learning_rate')
- if tf.executing_eagerly():
- return eager_decay_rate
- else:
- return eager_decay_rate()
+ return _learning_rate_return_value(eager_decay_rate)
+
+
+def exponential_decay_with_warmup(global_step,
+ learning_rate_base,
+ learning_rate_decay_steps,
+ learning_rate_decay_factor,
+ warmup_learning_rate=0.0,
+ warmup_steps=0,
+ min_learning_rate=0.0,
+ staircase=True):
+ """Exponential decay schedule with warm up period.
+
+ Args:
+ global_step: int tensor representing global step.
+ learning_rate_base: base learning rate.
+ learning_rate_decay_steps: steps to take between decaying the learning rate.
+ Note that this includes the number of burn-in steps.
+ learning_rate_decay_factor: multiplicative factor by which to decay learning
+ rate.
+ warmup_learning_rate: initial learning rate during warmup period.
+ warmup_steps: number of steps to use warmup learning rate.
+ min_learning_rate: the minimum learning rate.
+ staircase: whether use staircase decay.
+
+ Returns:
+ If executing eagerly:
+ returns a no-arg callable that outputs the (scalar)
+ float tensor learning rate given the current value of global_step.
+ If in a graph:
+ immediately returns a (scalar) float tensor representing learning rate.
+ """
+
+ def eager_decay_rate():
+ """Callable to compute the learning rate."""
+ post_warmup_learning_rate = tf.train.exponential_decay(
+ learning_rate_base,
+ global_step - warmup_steps,
+ learning_rate_decay_steps,
+ learning_rate_decay_factor,
+ staircase=staircase)
+ if callable(post_warmup_learning_rate):
+ post_warmup_learning_rate = post_warmup_learning_rate()
+
+ if learning_rate_base < warmup_learning_rate:
+ raise ValueError('learning_rate_base must be larger or equal to '
+ 'warmup_learning_rate.')
+ slope = (learning_rate_base - warmup_learning_rate) / warmup_steps
+ warmup_rate = slope * tf.cast(global_step,
+ tf.float32) + warmup_learning_rate
+ learning_rate = tf.where(
+ tf.less(tf.cast(global_step, tf.int32), tf.constant(warmup_steps)),
+ warmup_rate,
+ tf.maximum(post_warmup_learning_rate, min_learning_rate),
+ name='learning_rate')
+
+ return learning_rate
+
+ return _learning_rate_return_value(eager_decay_rate)
def cosine_decay_with_warmup(global_step,
@@ -142,10 +205,7 @@ def cosine_decay_with_warmup(global_step,
return tf.where(global_step > total_steps, 0.0, learning_rate,
name='learning_rate')
- if tf.executing_eagerly():
- return eager_decay_rate
- else:
- return eager_decay_rate()
+ return _learning_rate_return_value(eager_decay_rate)
def manual_stepping(global_step, boundaries, rates, warmup=False):
@@ -212,7 +272,5 @@ def manual_stepping(global_step, boundaries, rates, warmup=False):
[0] * num_boundaries))
return tf.reduce_sum(rates * tf.one_hot(rate_index, depth=num_boundaries),
name='learning_rate')
- if tf.executing_eagerly():
- return eager_decay_rate
- else:
- return eager_decay_rate()
+
+ return _learning_rate_return_value(eager_decay_rate)
diff --git a/research/object_detection/utils/learning_schedules_test.py b/research/object_detection/utils/learning_schedules_test.py
index 2b6012d111c702a8cfd9f767b6f6fc2819f36667..d7a1c9fc4a92a7779e1bb9410bbe956c0c4eeb01 100644
--- a/research/object_detection/utils/learning_schedules_test.py
+++ b/research/object_detection/utils/learning_schedules_test.py
@@ -50,6 +50,28 @@ class LearningSchedulesTest(test_case.TestCase):
exp_rates = [.5, .5, 1, 1, 1, .1, .1, .1, .05]
self.assertAllClose(output_rates, exp_rates, rtol=1e-4)
+ def testExponentialDecayWithWarmup(self):
+ def graph_fn(global_step):
+ learning_rate_base = 1.0
+ learning_rate_decay_steps = 3
+ learning_rate_decay_factor = .1
+ warmup_learning_rate = .5
+ warmup_steps = 2
+ min_learning_rate = .05
+ learning_rate = learning_schedules.exponential_decay_with_warmup(
+ global_step, learning_rate_base, learning_rate_decay_steps,
+ learning_rate_decay_factor, warmup_learning_rate, warmup_steps,
+ min_learning_rate)
+ assert learning_rate.op.name.endswith('learning_rate')
+ return (learning_rate,)
+
+ output_rates = [
+ self.execute(graph_fn, [np.array(i).astype(np.int64)]) for i in range(9)
+ ]
+
+ exp_rates = [.5, .75, 1, 1, 1, .1, .1, .1, .05]
+ self.assertAllClose(output_rates, exp_rates, rtol=1e-4)
+
def testCosineDecayWithWarmup(self):
def graph_fn(global_step):
learning_rate_base = 1.0
diff --git a/research/object_detection/utils/object_detection_evaluation.py b/research/object_detection/utils/object_detection_evaluation.py
index e0b51101205ce7c813cced02a453e863c3d4e90f..89a8af022aaf5cba6833a5ff3229f58a6ba618c4 100644
--- a/research/object_detection/utils/object_detection_evaluation.py
+++ b/research/object_detection/utils/object_detection_evaluation.py
@@ -159,7 +159,9 @@ class ObjectDetectionEvaluator(DetectionEvaluator):
metric_prefix=None,
use_weighted_mean_ap=False,
evaluate_masks=False,
- group_of_weight=0.0):
+ group_of_weight=0.0,
+ nms_iou_threshold=1.0,
+ nms_max_output_boxes=10000):
"""Constructor.
Args:
@@ -187,6 +189,8 @@ class ObjectDetectionEvaluator(DetectionEvaluator):
matching_iou_threshold, weight group_of_weight is added to true
positives. Consequently, if no detection falls within a group-of box,
weight group_of_weight is added to false negatives.
+ nms_iou_threshold: NMS IoU threashold.
+ nms_max_output_boxes: maximal number of boxes after NMS.
Raises:
ValueError: If the category ids are not 1-indexed.
@@ -202,6 +206,8 @@ class ObjectDetectionEvaluator(DetectionEvaluator):
self._label_id_offset = 1
self._evaluate_masks = evaluate_masks
self._group_of_weight = group_of_weight
+ self._nms_iou_threshold = nms_iou_threshold
+ self._nms_max_output_boxes = nms_max_output_boxes
self._evaluation = ObjectDetectionEvaluation(
num_groundtruth_classes=self._num_classes,
matching_iou_threshold=self._matching_iou_threshold,
@@ -209,7 +215,9 @@ class ObjectDetectionEvaluator(DetectionEvaluator):
recall_upper_bound=self._recall_upper_bound,
use_weighted_mean_ap=self._use_weighted_mean_ap,
label_id_offset=self._label_id_offset,
- group_of_weight=self._group_of_weight)
+ group_of_weight=self._group_of_weight,
+ nms_iou_threshold=self._nms_iou_threshold,
+ nms_max_output_boxes=self._nms_max_output_boxes)
self._image_ids = set([])
self._evaluate_corlocs = evaluate_corlocs
self._evaluate_precision_recall = evaluate_precision_recall
@@ -246,7 +254,7 @@ class ObjectDetectionEvaluator(DetectionEvaluator):
"""
for image_id in image_ids:
if image_id in self._image_ids:
- raise ValueError('Image with id {} already added.'.format(image_id))
+ logging.warning('Image with id %s already added.', image_id)
self._evaluation.merge_internal_state(state_tuple)
@@ -313,7 +321,7 @@ class ObjectDetectionEvaluator(DetectionEvaluator):
raise error if instance masks are not in groundtruth dictionary.
"""
if image_id in self._image_ids:
- raise ValueError('Image with id {} already added.'.format(image_id))
+ logging.warning('Image with id %s already added.', image_id)
groundtruth_classes = (
groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes] -
@@ -454,7 +462,10 @@ class ObjectDetectionEvaluator(DetectionEvaluator):
num_groundtruth_classes=self._num_classes,
matching_iou_threshold=self._matching_iou_threshold,
use_weighted_mean_ap=self._use_weighted_mean_ap,
- label_id_offset=self._label_id_offset)
+ label_id_offset=self._label_id_offset,
+ nms_iou_threshold=self._nms_iou_threshold,
+ nms_max_output_boxes=self._nms_max_output_boxes,
+ )
self._image_ids.clear()
def add_eval_dict(self, eval_dict):
@@ -549,13 +560,19 @@ class ObjectDetectionEvaluator(DetectionEvaluator):
class PascalDetectionEvaluator(ObjectDetectionEvaluator):
"""A class to evaluate detections using PASCAL metrics."""
- def __init__(self, categories, matching_iou_threshold=0.5):
+ def __init__(self,
+ categories,
+ matching_iou_threshold=0.5,
+ nms_iou_threshold=1.0,
+ nms_max_output_boxes=10000):
super(PascalDetectionEvaluator, self).__init__(
categories,
matching_iou_threshold=matching_iou_threshold,
evaluate_corlocs=False,
metric_prefix='PascalBoxes',
- use_weighted_mean_ap=False)
+ use_weighted_mean_ap=False,
+ nms_iou_threshold=nms_iou_threshold,
+ nms_max_output_boxes=nms_max_output_boxes)
class WeightedPascalDetectionEvaluator(ObjectDetectionEvaluator):
@@ -712,7 +729,7 @@ class OpenImagesDetectionEvaluator(ObjectDetectionEvaluator):
ValueError: On adding groundtruth for an image more than once.
"""
if image_id in self._image_ids:
- raise ValueError('Image with id {} already added.'.format(image_id))
+ logging.warning('Image with id %s already added.', image_id)
groundtruth_classes = (
groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes] -
diff --git a/research/object_detection/utils/object_detection_evaluation_test.py b/research/object_detection/utils/object_detection_evaluation_test.py
index ff399ed4bad4d4872eb135685789678237f0e0b1..4e61ee4ff2169759bb62dc5c457339cba9b6ad8c 100644
--- a/research/object_detection/utils/object_detection_evaluation_test.py
+++ b/research/object_detection/utils/object_detection_evaluation_test.py
@@ -524,30 +524,6 @@ class PascalEvaluationTest(tf.test.TestCase):
pascal_evaluator.clear()
self.assertFalse(pascal_evaluator._image_ids)
- def test_value_error_on_duplicate_images(self):
- categories = [{'id': 1, 'name': 'cat'},
- {'id': 2, 'name': 'dog'},
- {'id': 3, 'name': 'elephant'}]
- # Add groundtruth
- pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator(
- categories)
- image_key1 = 'img1'
- groundtruth_boxes1 = np.array([[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]],
- dtype=float)
- groundtruth_class_labels1 = np.array([1, 3, 1], dtype=int)
- pascal_evaluator.add_single_ground_truth_image_info(
- image_key1,
- {standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes1,
- standard_fields.InputDataFields.groundtruth_classes:
- groundtruth_class_labels1})
- with self.assertRaises(ValueError):
- pascal_evaluator.add_single_ground_truth_image_info(
- image_key1,
- {standard_fields.InputDataFields.groundtruth_boxes:
- groundtruth_boxes1,
- standard_fields.InputDataFields.groundtruth_classes:
- groundtruth_class_labels1})
-
class WeightedPascalEvaluationTest(tf.test.TestCase):
@@ -659,28 +635,6 @@ class WeightedPascalEvaluationTest(tf.test.TestCase):
self.wp_eval.clear()
self.assertFalse(self.wp_eval._image_ids)
- def test_value_error_on_duplicate_images(self):
- # Add groundtruth
- self.wp_eval = (
- object_detection_evaluation.WeightedPascalDetectionEvaluator(
- self.categories))
- image_key1 = 'img1'
- groundtruth_boxes1 = np.array([[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]],
- dtype=float)
- groundtruth_class_labels1 = np.array([1, 3, 1], dtype=int)
- self.wp_eval.add_single_ground_truth_image_info(
- image_key1,
- {standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes1,
- standard_fields.InputDataFields.groundtruth_classes:
- groundtruth_class_labels1})
- with self.assertRaises(ValueError):
- self.wp_eval.add_single_ground_truth_image_info(
- image_key1,
- {standard_fields.InputDataFields.groundtruth_boxes:
- groundtruth_boxes1,
- standard_fields.InputDataFields.groundtruth_classes:
- groundtruth_class_labels1})
-
class PrecisionAtRecallEvaluationTest(tf.test.TestCase):
@@ -807,31 +761,6 @@ class PrecisionAtRecallEvaluationTest(tf.test.TestCase):
self.wp_eval.clear()
self.assertFalse(self.wp_eval._image_ids)
- def test_value_error_on_duplicate_images(self):
- # Add groundtruth
- self.wp_eval = (
- object_detection_evaluation.PrecisionAtRecallDetectionEvaluator(
- self.categories, recall_lower_bound=0.0, recall_upper_bound=0.5))
- image_key1 = 'img1'
- groundtruth_boxes1 = np.array([[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]],
- dtype=float)
- groundtruth_class_labels1 = np.array([1, 3, 1], dtype=int)
- self.wp_eval.add_single_ground_truth_image_info(
- image_key1, {
- standard_fields.InputDataFields.groundtruth_boxes:
- groundtruth_boxes1,
- standard_fields.InputDataFields.groundtruth_classes:
- groundtruth_class_labels1
- })
- with self.assertRaises(ValueError):
- self.wp_eval.add_single_ground_truth_image_info(
- image_key1, {
- standard_fields.InputDataFields.groundtruth_boxes:
- groundtruth_boxes1,
- standard_fields.InputDataFields.groundtruth_classes:
- groundtruth_class_labels1
- })
-
class ObjectDetectionEvaluationTest(tf.test.TestCase):
diff --git a/research/object_detection/utils/ops.py b/research/object_detection/utils/ops.py
index f345fa3889c58a4c0ede56a07f3b9fdb731faf98..e1897933cd866313f556ead13ec0a5f186af7ecd 100644
--- a/research/object_detection/utils/ops.py
+++ b/research/object_detection/utils/ops.py
@@ -216,13 +216,13 @@ def pad_to_multiple(tensor, multiple):
height_pad = tf.zeros([
batch_size, padded_tensor_height - tensor_height, tensor_width,
tensor_depth
- ])
+ ], dtype=tensor.dtype)
tensor = tf.concat([tensor, height_pad], 1)
if padded_tensor_width != tensor_width:
width_pad = tf.zeros([
batch_size, padded_tensor_height, padded_tensor_width - tensor_width,
tensor_depth
- ])
+ ], dtype=tensor.dtype)
tensor = tf.concat([tensor, width_pad], 2)
return tensor
@@ -1134,3 +1134,57 @@ def decode_image(tensor_dict):
tensor_dict[fields.InputDataFields.image], channels=3)
tensor_dict[fields.InputDataFields.image].set_shape([None, None, 3])
return tensor_dict
+
+
+def giou(boxes1, boxes2):
+ """Computes generalized IOU between two tensors.
+
+ Each box should be represented as [ymin, xmin, ymax, xmax].
+
+ Args:
+ boxes1: a tensor with shape [num_boxes, 4]
+ boxes2: a tensor with shape [num_boxes, 4]
+
+ Returns:
+ a tensor of shape [num_boxes] containing GIoUs
+
+ """
+ pred_ymin, pred_xmin, pred_ymax, pred_xmax = tf.unstack(boxes1, axis=1)
+ gt_ymin, gt_xmin, gt_ymax, gt_xmax = tf.unstack(boxes2, axis=1)
+
+ gt_area = (gt_ymax - gt_ymin) * (gt_xmax - gt_xmin)
+ pred_area = (pred_ymax - pred_ymin) * (pred_xmax - pred_xmin)
+
+ x1_i = tf.maximum(pred_xmin, gt_xmin)
+ x2_i = tf.minimum(pred_xmax, gt_xmax)
+ y1_i = tf.maximum(pred_ymin, gt_ymin)
+ y2_i = tf.minimum(pred_ymax, gt_ymax)
+ intersection_area = tf.maximum(0.0, y2_i - y1_i) * tf.maximum(0.0,
+ x2_i - x1_i)
+
+ x1_c = tf.minimum(pred_xmin, gt_xmin)
+ x2_c = tf.maximum(pred_xmax, gt_xmax)
+ y1_c = tf.minimum(pred_ymin, gt_ymin)
+ y2_c = tf.maximum(pred_ymax, gt_ymax)
+ hull_area = (y2_c - y1_c) * (x2_c - x1_c)
+
+ union_area = gt_area + pred_area - intersection_area
+ iou = tf.where(tf.equal(union_area, 0.0),
+ tf.zeros_like(union_area), intersection_area / union_area)
+ giou_ = iou - tf.where(hull_area > 0.0,
+ (hull_area - union_area) / hull_area, iou)
+ return giou_
+
+
+def center_to_corner_coordinate(input_tensor):
+ """Converts input boxes from center to corner representation."""
+ reshaped_encodings = tf.reshape(input_tensor, [-1, 4])
+ ycenter = tf.gather(reshaped_encodings, [0], axis=1)
+ xcenter = tf.gather(reshaped_encodings, [1], axis=1)
+ h = tf.gather(reshaped_encodings, [2], axis=1)
+ w = tf.gather(reshaped_encodings, [3], axis=1)
+ ymin = ycenter - h / 2.
+ xmin = xcenter - w / 2.
+ ymax = ycenter + h / 2.
+ xmax = xcenter + w / 2.
+ return tf.squeeze(tf.stack([ymin, xmin, ymax, xmax], axis=1))
diff --git a/research/object_detection/utils/ops_test.py b/research/object_detection/utils/ops_test.py
index c5252d644ccd90a4518ad32d32656100c6f3c4f9..9a7ded91d5422a27f8ecea1a50441d562bb591c3 100644
--- a/research/object_detection/utils/ops_test.py
+++ b/research/object_detection/utils/ops_test.py
@@ -1635,5 +1635,119 @@ class TestGatherWithPaddingValues(test_case.TestCase):
+
+class TestGIoU(test_case.TestCase):
+
+ def test_giou_with_no_overlap(self):
+ expected_giou_tensor = [
+ 0, -1/3, -3/4, 0, -98/100
+ ]
+
+ def graph_fn():
+ boxes1 = tf.constant([[3, 4, 5, 6], [3, 3, 5, 5],
+ [0, 0, 0, 0], [3, 3, 5, 5],
+ [9, 9, 10, 10]],
+ dtype=tf.float32)
+ boxes2 = tf.constant([[3, 2, 5, 4], [3, 7, 5, 9],
+ [5, 5, 10, 10], [3, 5, 5, 7],
+ [0, 0, 1, 1]], dtype=tf.float32)
+
+ giou = ops.giou(boxes1, boxes2)
+ self.assertEqual(giou.dtype, tf.float32)
+
+ return giou
+
+ giou = self.execute(graph_fn, [])
+ self.assertAllClose(expected_giou_tensor, giou)
+
+ def test_giou_with_overlaps(self):
+ expected_giou_tensor = [
+ 1/25, 1/4, 1/3, 1/7 - 2/9
+ ]
+
+ def graph_fn():
+ boxes1 = tf.constant([[2, 1, 7, 6], [2, 2, 4, 4],
+ [2, 2, 4, 4], [2, 2, 4, 4]],
+ dtype=tf.float32)
+ boxes2 = tf.constant([[4, 3, 5, 4], [3, 3, 4, 4],
+ [2, 3, 4, 5], [3, 3, 5, 5]], dtype=tf.float32)
+
+ giou = ops.giou(boxes1, boxes2)
+ self.assertEqual(giou.dtype, tf.float32)
+
+ return giou
+
+ giou = self.execute(graph_fn, [])
+ self.assertAllClose(expected_giou_tensor, giou)
+
+ def test_giou_with_perfect_overlap(self):
+ expected_giou_tensor = [1]
+
+ def graph_fn():
+ boxes1 = tf.constant([[3, 3, 5, 5]], dtype=tf.float32)
+ boxes2 = tf.constant([[3, 3, 5, 5]], dtype=tf.float32)
+
+ giou = ops.giou(boxes1, boxes2)
+ self.assertEqual(giou.dtype, tf.float32)
+
+ return giou
+
+ giou = self.execute(graph_fn, [])
+ self.assertAllClose(expected_giou_tensor, giou)
+
+ def test_giou_with_zero_area_boxes(self):
+ expected_giou_tensor = [0]
+
+ def graph_fn():
+ boxes1 = tf.constant([[1, 1, 1, 1]], dtype=tf.float32)
+ boxes2 = tf.constant([[1, 1, 1, 1]], dtype=tf.float32)
+
+ giou = ops.giou(boxes1, boxes2)
+ self.assertEqual(giou.dtype, tf.float32)
+
+ return giou
+
+ giou = self.execute(graph_fn, [])
+ self.assertAllClose(expected_giou_tensor, giou)
+
+ def test_giou_different_with_l1_same(self):
+ expected_giou_tensor = [
+ 2/3, 3/5
+ ]
+
+ def graph_fn():
+ boxes1 = tf.constant([[3, 3, 5, 5], [3, 3, 5, 5]], dtype=tf.float32)
+ boxes2 = tf.constant([[3, 2.5, 5, 5.5], [3, 2.5, 5, 4.5]],
+ dtype=tf.float32)
+
+ giou = ops.giou(boxes1, boxes2)
+ self.assertEqual(giou.dtype, tf.float32)
+
+ return giou
+
+ giou = self.execute(graph_fn, [])
+ self.assertAllClose(expected_giou_tensor, giou)
+
+
+class TestCoordinateConversion(test_case.TestCase):
+
+ def test_coord_conv(self):
+ expected_box_tensor = [
+ [0.5, 0.5, 5.5, 5.5], [2, 1, 4, 7], [0, 0, 0, 0]
+ ]
+
+ def graph_fn():
+ boxes = tf.constant([[3, 3, 5, 5], [3, 4, 2, 6], [0, 0, 0, 0]],
+ dtype=tf.float32)
+
+ converted = ops.center_to_corner_coordinate(boxes)
+ self.assertEqual(converted.dtype, tf.float32)
+
+ return converted
+
+ converted = self.execute(graph_fn, [])
+ self.assertAllClose(expected_box_tensor, converted)
+
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/utils/spatial_transform_ops.py b/research/object_detection/utils/spatial_transform_ops.py
index 95aaf967984172c6ef49a5c9de98b5a8ed0cdc0b..1880dffea1a4a6ddd21c60aac31dc1fda0a9ff30 100644
--- a/research/object_detection/utils/spatial_transform_ops.py
+++ b/research/object_detection/utils/spatial_transform_ops.py
@@ -411,6 +411,56 @@ def multilevel_roi_align(features, boxes, box_levels, output_size,
return features_per_box
+def multilevel_native_crop_and_resize(images, boxes, box_levels,
+ crop_size, scope=None):
+ """Multilevel native crop and resize.
+
+ Same as `multilevel_matmul_crop_and_resize` but uses tf.image.crop_and_resize.
+
+ Args:
+ images: A list of 4-D tensor of shape
+ [batch, image_height, image_width, depth] representing features of
+ different size.
+ boxes: A `Tensor` of type `float32`.
+ A 3-D tensor of shape `[batch, num_boxes, 4]`. The boxes are specified in
+ normalized coordinates and are of the form `[y1, x1, y2, x2]`. A
+ normalized coordinate value of `y` is mapped to the image coordinate at
+ `y * (image_height - 1)`, so as the `[0, 1]` interval of normalized image
+ height is mapped to `[0, image_height - 1] in image height coordinates.
+ We do allow y1 > y2, in which case the sampled crop is an up-down flipped
+ version of the original image. The width dimension is treated similarly.
+ Normalized coordinates outside the `[0, 1]` range are allowed, in which
+ case we use `extrapolation_value` to extrapolate the input image values.
+ box_levels: A 2-D tensor of shape [batch, num_boxes] representing the level
+ of the box.
+ crop_size: A list of two integers `[crop_height, crop_width]`. All
+ cropped image patches are resized to this size. The aspect ratio of the
+ image content is not preserved. Both `crop_height` and `crop_width` need
+ to be positive.
+ scope: A name for the operation (optional).
+
+ Returns:
+ A 5-D float tensor of shape `[batch, num_boxes, crop_height, crop_width,
+ depth]`
+ """
+ if box_levels is None:
+ return native_crop_and_resize(images[0], boxes, crop_size, scope)
+ with tf.name_scope('MultiLevelNativeCropAndResize'):
+ cropped_feature_list = []
+ for level, image in enumerate(images):
+ # For each level, crop the feature according to all boxes
+ # set the cropped feature not at this level to 0 tensor.
+ # Consider more efficient way of computing cropped features.
+ cropped = native_crop_and_resize(image, boxes, crop_size, scope)
+ cond = tf.tile(
+ tf.equal(box_levels, level)[:, :, tf.newaxis],
+ [1, 1] + [tf.math.reduce_prod(cropped.shape.as_list()[2:])])
+ cond = tf.reshape(cond, cropped.shape)
+ cropped_final = tf.where(cond, cropped, tf.zeros_like(cropped))
+ cropped_feature_list.append(cropped_final)
+ return tf.math.reduce_sum(cropped_feature_list, axis=0)
+
+
def native_crop_and_resize(image, boxes, crop_size, scope=None):
"""Same as `matmul_crop_and_resize` but uses tf.image.crop_and_resize."""
def get_box_inds(proposals):
@@ -431,6 +481,50 @@ def native_crop_and_resize(image, boxes, crop_size, scope=None):
return tf.reshape(cropped_regions, final_shape)
+def multilevel_matmul_crop_and_resize(images, boxes, box_levels, crop_size,
+ extrapolation_value=0.0, scope=None):
+ """Multilevel matmul crop and resize.
+
+ Same as `matmul_crop_and_resize` but crop images according to box levels.
+
+ Args:
+ images: A list of 4-D tensor of shape
+ [batch, image_height, image_width, depth] representing features of
+ different size.
+ boxes: A `Tensor` of type `float32` or 'bfloat16'.
+ A 3-D tensor of shape `[batch, num_boxes, 4]`. The boxes are specified in
+ normalized coordinates and are of the form `[y1, x1, y2, x2]`. A
+ normalized coordinate value of `y` is mapped to the image coordinate at
+ `y * (image_height - 1)`, so as the `[0, 1]` interval of normalized image
+ height is mapped to `[0, image_height - 1] in image height coordinates.
+ We do allow y1 > y2, in which case the sampled crop is an up-down flipped
+ version of the original image. The width dimension is treated similarly.
+ Normalized coordinates outside the `[0, 1]` range are allowed, in which
+ case we use `extrapolation_value` to extrapolate the input image values.
+ box_levels: A 2-D tensor of shape [batch, num_boxes] representing the level
+ of the box.
+ crop_size: A list of two integers `[crop_height, crop_width]`. All
+ cropped image patches are resized to this size. The aspect ratio of the
+ image content is not preserved. Both `crop_height` and `crop_width` need
+ to be positive.
+ extrapolation_value: A float value to use for extrapolation.
+ scope: A name for the operation (optional).
+
+ Returns:
+ A 5-D float tensor of shape `[batch, num_boxes, crop_height, crop_width,
+ depth]`
+ """
+ with tf.name_scope(scope, 'MultiLevelMatMulCropAndResize'):
+ if box_levels is None:
+ box_levels = tf.zeros(tf.shape(boxes)[:2], dtype=tf.int32)
+ return multilevel_roi_align(images,
+ boxes,
+ box_levels,
+ crop_size,
+ align_corners=True,
+ extrapolation_value=extrapolation_value)
+
+
def matmul_crop_and_resize(image, boxes, crop_size, extrapolation_value=0.0,
scope=None):
"""Matrix multiplication based implementation of the crop and resize op.
diff --git a/research/object_detection/utils/spatial_transform_ops_test.py b/research/object_detection/utils/spatial_transform_ops_test.py
index d18456640673cf736ee152b19291ed91226fcd31..2261f078ed2bfac9adf1e147a8ea15bef900c5f2 100644
--- a/research/object_detection/utils/spatial_transform_ops_test.py
+++ b/research/object_detection/utils/spatial_transform_ops_test.py
@@ -512,6 +512,38 @@ class MatMulCropAndResizeTest(test_case.TestCase):
crop_output = self.execute(graph_fn, [image, boxes])
self.assertAllClose(crop_output, expected_output)
+ def testMultilevelMatMulCropAndResize(self):
+
+ def graph_fn(image1, image2, boxes, box_levels):
+ return spatial_ops.multilevel_matmul_crop_and_resize([image1, image2],
+ boxes,
+ box_levels,
+ crop_size=[2, 2])
+
+ image = [np.array([[[[1, 0], [2, 0], [3, 0]],
+ [[4, 0], [5, 0], [6, 0]],
+ [[7, 0], [8, 0], [9, 0]]],
+ [[[1, 0], [2, 0], [3, 0]],
+ [[4, 0], [5, 0], [6, 0]],
+ [[7, 0], [8, 0], [9, 0]]]], dtype=np.float32),
+ np.array([[[[1, 0], [2, 1], [3, 2]],
+ [[4, 3], [5, 4], [6, 5]],
+ [[7, 6], [8, 7], [9, 8]]],
+ [[[1, 0], [2, 1], [3, 2]],
+ [[4, 3], [5, 4], [6, 5]],
+ [[7, 6], [8, 7], [9, 8]]]], dtype=np.float32)]
+ boxes = np.array([[[1, 1, 0, 0],
+ [.5, .5, 0, 0]],
+ [[0, 0, 1, 1],
+ [0, 0, .5, .5]]], dtype=np.float32)
+ box_levels = np.array([[0, 1], [1, 1]], dtype=np.int32)
+ expected_output = [[[[[9, 0], [7, 0]], [[3, 0], [1, 0]]],
+ [[[5, 4], [4, 3]], [[2, 1], [1, 0]]]],
+ [[[[1, 0], [3, 2]], [[7, 6], [9, 8]]],
+ [[[1, 0], [2, 1]], [[4, 3], [5, 4]]]]]
+ crop_output = self.execute(graph_fn, image + [boxes, box_levels])
+ self.assertAllClose(crop_output, expected_output)
+
class NativeCropAndResizeTest(test_case.TestCase):
@@ -537,6 +569,35 @@ class NativeCropAndResizeTest(test_case.TestCase):
crop_output = self.execute_cpu(graph_fn, [image, boxes])
self.assertAllClose(crop_output, expected_output)
+ def testMultilevelBatchCropAndResize3x3To2x2_2Channels(self):
+
+ def graph_fn(image1, image2, boxes, box_levels):
+ return spatial_ops.multilevel_native_crop_and_resize([image1, image2],
+ boxes,
+ box_levels,
+ crop_size=[2, 2])
+ image = [np.array([[[[1, 0], [2, 1], [3, 2]],
+ [[4, 3], [5, 4], [6, 5]],
+ [[7, 6], [8, 7], [9, 8]]],
+ [[[1, 0], [2, 1], [3, 2]],
+ [[4, 3], [5, 4], [6, 5]],
+ [[7, 6], [8, 7], [9, 8]]]], dtype=np.float32),
+ np.array([[[[1, 0], [2, 1]],
+ [[4, 3], [5, 4]]],
+ [[[1, 0], [2, 1]],
+ [[4, 3], [5, 4]]]], dtype=np.float32)]
+ boxes = np.array([[[0, 0, 1, 1],
+ [0, 0, .5, .5]],
+ [[1, 1, 0, 0],
+ [.5, .5, 0, 0]]], dtype=np.float32)
+ box_levels = np.array([[0, 1], [0, 0]], dtype=np.float32)
+ expected_output = [[[[[1, 0], [3, 2]], [[7, 6], [9, 8]]],
+ [[[1, 0], [1.5, 0.5]], [[2.5, 1.5], [3, 2]]]],
+ [[[[9, 8], [7, 6]], [[3, 2], [1, 0]]],
+ [[[5, 4], [4, 3]], [[2, 1], [1, 0]]]]]
+ crop_output = self.execute_cpu(graph_fn, image + [boxes, box_levels])
+ self.assertAllClose(crop_output, expected_output)
+
if __name__ == '__main__':
tf.test.main()
diff --git a/research/object_detection/utils/target_assigner_utils.py b/research/object_detection/utils/target_assigner_utils.py
index 0aa26a47ed75ac918a82aaee184fa2bb0dfa7127..7ac61e8a84d387b1e4c4a139ffcd9d5c336f19dd 100644
--- a/research/object_detection/utils/target_assigner_utils.py
+++ b/research/object_detection/utils/target_assigner_utils.py
@@ -41,13 +41,88 @@ def image_shape_to_grids(height, width):
return (y_grid, x_grid)
+def _coordinates_to_heatmap_dense(y_grid, x_grid, y_coordinates, x_coordinates,
+ sigma, channel_onehot, channel_weights=None):
+ """Dense version of coordinates to heatmap that uses an outer product."""
+ num_instances, num_channels = (
+ shape_utils.combined_static_and_dynamic_shape(channel_onehot))
+
+ x_grid = tf.expand_dims(x_grid, 2)
+ y_grid = tf.expand_dims(y_grid, 2)
+ # The raw center coordinates in the output space.
+ x_diff = x_grid - tf.math.floor(x_coordinates)
+ y_diff = y_grid - tf.math.floor(y_coordinates)
+ squared_distance = x_diff**2 + y_diff**2
+
+ gaussian_map = tf.exp(-squared_distance / (2 * sigma * sigma))
+
+ reshaped_gaussian_map = tf.expand_dims(gaussian_map, axis=-1)
+ reshaped_channel_onehot = tf.reshape(channel_onehot,
+ (1, 1, num_instances, num_channels))
+ gaussian_per_box_per_class_map = (
+ reshaped_gaussian_map * reshaped_channel_onehot)
+
+ if channel_weights is not None:
+ reshaped_weights = tf.reshape(channel_weights, (1, 1, num_instances, 1))
+ gaussian_per_box_per_class_map *= reshaped_weights
+
+ # Take maximum along the "instance" dimension so that all per-instance
+ # heatmaps of the same class are merged together.
+ heatmap = tf.reduce_max(gaussian_per_box_per_class_map, axis=2)
+
+ # Maximum of an empty tensor is -inf, the following is to avoid that.
+ heatmap = tf.maximum(heatmap, 0)
+
+ return tf.stop_gradient(heatmap)
+
+
+def _coordinates_to_heatmap_sparse(y_grid, x_grid, y_coordinates, x_coordinates,
+ sigma, channel_onehot, channel_weights=None):
+ """Sparse version of coordinates to heatmap using tf.scatter."""
+
+ if not hasattr(tf, 'tensor_scatter_nd_max'):
+ raise RuntimeError(
+ ('Please upgrade tensowflow to use `tensor_scatter_nd_max` or set '
+ 'compute_heatmap_sparse=False'))
+ _, num_channels = (
+ shape_utils.combined_static_and_dynamic_shape(channel_onehot))
+
+ height, width = shape_utils.combined_static_and_dynamic_shape(y_grid)
+ x_grid = tf.expand_dims(x_grid, 2)
+ y_grid = tf.expand_dims(y_grid, 2)
+ # The raw center coordinates in the output space.
+ x_diff = x_grid - tf.math.floor(x_coordinates)
+ y_diff = y_grid - tf.math.floor(y_coordinates)
+ squared_distance = x_diff**2 + y_diff**2
+
+ gaussian_map = tf.exp(-squared_distance / (2 * sigma * sigma))
+
+ if channel_weights is not None:
+ gaussian_map = gaussian_map * channel_weights[tf.newaxis, tf.newaxis, :]
+
+ channel_indices = tf.argmax(channel_onehot, axis=1)
+
+ channel_indices = channel_indices[:, tf.newaxis]
+ heatmap_init = tf.zeros((num_channels, height, width))
+
+ gaussian_map = tf.transpose(gaussian_map, (2, 0, 1))
+ heatmap = tf.tensor_scatter_nd_max(
+ heatmap_init, channel_indices, gaussian_map)
+
+ # Maximum of an empty tensor is -inf, the following is to avoid that.
+ heatmap = tf.maximum(heatmap, 0)
+
+ return tf.stop_gradient(tf.transpose(heatmap, (1, 2, 0)))
+
+
def coordinates_to_heatmap(y_grid,
x_grid,
y_coordinates,
x_coordinates,
sigma,
channel_onehot,
- channel_weights=None):
+ channel_weights=None,
+ sparse=False):
"""Returns the heatmap targets from a set of point coordinates.
This function maps a set of point coordinates to the output heatmap image
@@ -71,41 +146,23 @@ def coordinates_to_heatmap(y_grid,
representing the one-hot encoded channel labels for each point.
channel_weights: A 1D tensor with shape [num_instances] corresponding to the
weight of each instance.
+ sparse: bool, indicating whether or not to use the sparse implementation
+ of the function. The sparse version scales better with number of channels,
+ but in some cases is known to cause OOM error. See (b/170989061).
Returns:
heatmap: A tensor of size [height, width, num_channels] representing the
heatmap. Output (height, width) match the dimensions of the input grids.
"""
- num_instances, num_channels = (
- shape_utils.combined_static_and_dynamic_shape(channel_onehot))
-
- x_grid = tf.expand_dims(x_grid, 2)
- y_grid = tf.expand_dims(y_grid, 2)
- # The raw center coordinates in the output space.
- x_diff = x_grid - tf.math.floor(x_coordinates)
- y_diff = y_grid - tf.math.floor(y_coordinates)
- squared_distance = x_diff**2 + y_diff**2
- gaussian_map = tf.exp(-squared_distance / (2 * sigma * sigma))
-
- reshaped_gaussian_map = tf.expand_dims(gaussian_map, axis=-1)
- reshaped_channel_onehot = tf.reshape(channel_onehot,
- (1, 1, num_instances, num_channels))
- gaussian_per_box_per_class_map = (
- reshaped_gaussian_map * reshaped_channel_onehot)
-
- if channel_weights is not None:
- reshaped_weights = tf.reshape(channel_weights, (1, 1, num_instances, 1))
- gaussian_per_box_per_class_map *= reshaped_weights
-
- # Take maximum along the "instance" dimension so that all per-instance
- # heatmaps of the same class are merged together.
- heatmap = tf.reduce_max(gaussian_per_box_per_class_map, axis=2)
-
- # Maximum of an empty tensor is -inf, the following is to avoid that.
- heatmap = tf.maximum(heatmap, 0)
-
- return heatmap
+ if sparse:
+ return _coordinates_to_heatmap_sparse(
+ y_grid, x_grid, y_coordinates, x_coordinates, sigma, channel_onehot,
+ channel_weights)
+ else:
+ return _coordinates_to_heatmap_dense(
+ y_grid, x_grid, y_coordinates, x_coordinates, sigma, channel_onehot,
+ channel_weights)
def compute_floor_offsets_with_indices(y_source,
diff --git a/research/object_detection/utils/target_assigner_utils_test.py b/research/object_detection/utils/target_assigner_utils_test.py
index f663445324d7ee648130018b522fdcbaaeb74d54..ef0f3420e01b84b35ecd785a176e35d8c8871cfb 100644
--- a/research/object_detection/utils/target_assigner_utils_test.py
+++ b/research/object_detection/utils/target_assigner_utils_test.py
@@ -14,6 +14,7 @@
# ==============================================================================
"""Tests for utils.target_assigner_utils."""
+from absl.testing import parameterized
import numpy as np
import tensorflow.compat.v1 as tf
@@ -21,7 +22,7 @@ from object_detection.utils import target_assigner_utils as ta_utils
from object_detection.utils import test_case
-class TargetUtilTest(test_case.TestCase):
+class TargetUtilTest(parameterized.TestCase, test_case.TestCase):
def test_image_shape_to_grids(self):
def graph_fn():
@@ -36,7 +37,11 @@ class TargetUtilTest(test_case.TestCase):
np.testing.assert_array_equal(y_grid, expected_y_grid)
np.testing.assert_array_equal(x_grid, expected_x_grid)
- def test_coordinates_to_heatmap(self):
+ @parameterized.parameters((False,), (True,))
+ def test_coordinates_to_heatmap(self, sparse):
+ if not hasattr(tf, 'tensor_scatter_nd_max'):
+ self.skipTest('Cannot test function due to old TF version.')
+
def graph_fn():
(y_grid, x_grid) = ta_utils.image_shape_to_grids(height=3, width=5)
y_coordinates = tf.constant([1.5, 0.5], dtype=tf.float32)
@@ -46,7 +51,8 @@ class TargetUtilTest(test_case.TestCase):
channel_weights = tf.constant([1, 1], dtype=tf.float32)
heatmap = ta_utils.coordinates_to_heatmap(y_grid, x_grid, y_coordinates,
x_coordinates, sigma,
- channel_onehot, channel_weights)
+ channel_onehot,
+ channel_weights, sparse=sparse)
return heatmap
heatmap = self.execute(graph_fn, [])
diff --git a/research/object_detection/utils/test_utils.py b/research/object_detection/utils/test_utils.py
index 666a29adbad262054e039c30fb9deb52e66ac665..5cefe3f8a14ce51924adceb3524cfa15afa432af 100644
--- a/research/object_detection/utils/test_utils.py
+++ b/research/object_detection/utils/test_utils.py
@@ -101,6 +101,10 @@ class MockKerasBoxPredictor(box_predictor.KerasBoxPredictor):
is_training, num_classes, False, False)
self._add_background_class = add_background_class
+ # Dummy variable so that box predictor registers some variables.
+ self._dummy_var = tf.Variable(0.0, trainable=True,
+ name='box_predictor_var')
+
def _predict(self, image_features, **kwargs):
image_feature = image_features[0]
combined_feature_shape = shape_utils.combined_static_and_dynamic_shape(
diff --git a/research/object_detection/utils/visualization_utils.py b/research/object_detection/utils/visualization_utils.py
index 756d98e30ba71349685e04f3b2376b6ad6b76fc8..b5532a1d29b60d02efc9215e3685a273730a58ee 100644
--- a/research/object_detection/utils/visualization_utils.py
+++ b/research/object_detection/utils/visualization_utils.py
@@ -664,6 +664,10 @@ def draw_side_by_side_evaluation_image(eval_dict,
key != input_data_fields.image_additional_channels):
eval_dict[key] = tf.expand_dims(eval_dict[key], 0)
+ num_gt_boxes = [-1] * eval_dict[input_data_fields.original_image].shape[0]
+ if input_data_fields.num_groundtruth_boxes in eval_dict:
+ num_gt_boxes = tf.cast(eval_dict[input_data_fields.num_groundtruth_boxes],
+ tf.int32)
for indx in range(eval_dict[input_data_fields.original_image].shape[0]):
instance_masks = None
if detection_fields.detection_masks in eval_dict:
@@ -680,8 +684,10 @@ def draw_side_by_side_evaluation_image(eval_dict,
keypoint_scores = tf.expand_dims(
eval_dict[detection_fields.detection_keypoint_scores][indx], axis=0)
else:
- keypoint_scores = tf.cast(keypoint_ops.set_keypoint_visibilities(
- keypoints), dtype=tf.float32)
+ keypoint_scores = tf.expand_dims(tf.cast(
+ keypoint_ops.set_keypoint_visibilities(
+ eval_dict[detection_fields.detection_keypoints][indx]),
+ dtype=tf.float32), axis=0)
groundtruth_instance_masks = None
if input_data_fields.groundtruth_instance_masks in eval_dict:
@@ -699,10 +705,10 @@ def draw_side_by_side_evaluation_image(eval_dict,
groundtruth_keypoint_scores = tf.expand_dims(
tf.cast(eval_dict[gt_kpt_vis_fld][indx], dtype=tf.float32), axis=0)
else:
- groundtruth_keypoint_scores = tf.cast(
+ groundtruth_keypoint_scores = tf.expand_dims(tf.cast(
keypoint_ops.set_keypoint_visibilities(
- groundtruth_keypoints), dtype=tf.float32)
-
+ eval_dict[input_data_fields.groundtruth_keypoints][indx]),
+ dtype=tf.float32), axis=0)
images_with_detections = draw_bounding_boxes_on_image_tensors(
tf.expand_dims(
eval_dict[input_data_fields.original_image][indx], axis=0),
@@ -725,16 +731,23 @@ def draw_side_by_side_evaluation_image(eval_dict,
max_boxes_to_draw=max_boxes_to_draw,
min_score_thresh=min_score_thresh,
use_normalized_coordinates=use_normalized_coordinates)
+ num_gt_boxes_i = num_gt_boxes[indx]
images_with_groundtruth = draw_bounding_boxes_on_image_tensors(
tf.expand_dims(
- eval_dict[input_data_fields.original_image][indx], axis=0),
+ eval_dict[input_data_fields.original_image][indx],
+ axis=0),
tf.expand_dims(
- eval_dict[input_data_fields.groundtruth_boxes][indx], axis=0),
+ eval_dict[input_data_fields.groundtruth_boxes][indx]
+ [:num_gt_boxes_i],
+ axis=0),
tf.expand_dims(
- eval_dict[input_data_fields.groundtruth_classes][indx], axis=0),
+ eval_dict[input_data_fields.groundtruth_classes][indx]
+ [:num_gt_boxes_i],
+ axis=0),
tf.expand_dims(
tf.ones_like(
- eval_dict[input_data_fields.groundtruth_classes][indx],
+ eval_dict[input_data_fields.groundtruth_classes][indx]
+ [:num_gt_boxes_i],
dtype=tf.float32),
axis=0),
category_index,
@@ -760,13 +773,17 @@ def draw_side_by_side_evaluation_image(eval_dict,
eval_dict[input_data_fields.image_additional_channels][indx],
axis=0),
tf.expand_dims(
- eval_dict[input_data_fields.groundtruth_boxes][indx], axis=0),
+ eval_dict[input_data_fields.groundtruth_boxes][indx]
+ [:num_gt_boxes_i],
+ axis=0),
tf.expand_dims(
- eval_dict[input_data_fields.groundtruth_classes][indx],
+ eval_dict[input_data_fields.groundtruth_classes][indx]
+ [:num_gt_boxes_i],
axis=0),
tf.expand_dims(
tf.ones_like(
- eval_dict[input_data_fields.groundtruth_classes][indx],
+ eval_dict[input_data_fields.groundtruth_classes][indx]
+ [num_gt_boxes_i],
dtype=tf.float32),
axis=0),
category_index,
@@ -1098,6 +1115,7 @@ def visualize_boxes_and_labels_on_image_array(
min_score_thresh=.5,
agnostic_mode=False,
line_thickness=4,
+ mask_alpha=.4,
groundtruth_box_visualization_color='black',
skip_boxes=False,
skip_scores=False,
@@ -1143,6 +1161,7 @@ def visualize_boxes_and_labels_on_image_array(
class-agnostic mode or not. This mode will display scores but ignore
classes.
line_thickness: integer (default: 4) controlling line width of the boxes.
+ mask_alpha: transparency value between 0 and 1 (default: 0.4).
groundtruth_box_visualization_color: box color for visualizing groundtruth
boxes
skip_boxes: whether to skip the drawing of bounding boxes.
@@ -1218,7 +1237,8 @@ def visualize_boxes_and_labels_on_image_array(
draw_mask_on_image_array(
image,
box_to_instance_masks_map[box],
- color=color
+ color=color,
+ alpha=mask_alpha
)
if instance_boundaries is not None:
draw_mask_on_image_array(
diff --git a/research/ptn/.gitignore b/research/ptn/.gitignore
deleted file mode 100644
index 8479374e96149a2e772046da042820dae34ba305..0000000000000000000000000000000000000000
--- a/research/ptn/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-bazel
-.idea
-bazel-bin
-bazel-out
-bazel-genfiles
-bazel-ptn
-bazel-testlogs
-*.pyc
diff --git a/research/ptn/BUILD b/research/ptn/BUILD
deleted file mode 100644
index f08c6172c4e899de77d3c10dc8e9d61519e750a8..0000000000000000000000000000000000000000
--- a/research/ptn/BUILD
+++ /dev/null
@@ -1,94 +0,0 @@
-py_library(
- name = "input_generator",
- srcs = ["input_generator.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "losses",
- srcs = ["losses.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "metrics",
- srcs = ["metrics.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "utils",
- srcs = ["utils.py"],
- deps = [
- ],
-)
-
-# Defines the Rotator model here
-py_library(
- name = "model_rotator",
- srcs = ["model_rotator.py"],
- deps = [
- ":input_generator",
- ":losses",
- ":metrics",
- ":utils",
- "//nets:deeprotator_factory",
- ],
-)
-
-# Defines the Im2vox model here
-py_library(
- name = "model_voxel_generation",
- srcs = ["model_voxel_generation.py"],
- deps = [
- ":input_generator",
- "//nets:im2vox_factory",
- ],
-)
-
-py_library(
- name = "model_ptn",
- srcs = ["model_ptn.py"],
- deps = [
- ":losses",
- ":metrics",
- ":model_voxel_generation",
- ":utils",
- "//nets:im2vox_factory",
- ],
-)
-
-py_binary(
- name = "train_ptn",
- srcs = ["train_ptn.py"],
- deps = [
- ":model_ptn",
- ],
-)
-
-py_binary(
- name = "eval_ptn",
- srcs = ["eval_ptn.py"],
- deps = [
- ":model_ptn",
- ],
-)
-
-py_binary(
- name = "pretrain_rotator",
- srcs = ["pretrain_rotator.py"],
- deps = [
- ":model_rotator",
- ],
-)
-
-py_binary(
- name = "eval_rotator",
- srcs = ["eval_rotator.py"],
- deps = [
- ":model_rotator",
- ],
-)
diff --git a/research/ptn/README.md b/research/ptn/README.md
deleted file mode 100644
index e9558cd097f7c5f8dd30321222d777172cf5276d..0000000000000000000000000000000000000000
--- a/research/ptn/README.md
+++ /dev/null
@@ -1,75 +0,0 @@
-
-
-
-
-# Perspective Transformer Nets
-
-## Introduction
-This is the TensorFlow implementation for the NIPS 2016 work ["Perspective Transformer Nets: Learning Single-View 3D Object Reconstrution without 3D Supervision"](https://papers.nips.cc/paper/6206-perspective-transformer-nets-learning-single-view-3d-object-reconstruction-without-3d-supervision.pdf)
-
-Re-implemented by Xinchen Yan, Arkanath Pathak, Jasmine Hsu, Honglak Lee
-
-Reference: [Orginal implementation in Torch](https://github.com/xcyan/nips16_PTN)
-
-## How to run this code
-
-This implementation is ready to be run locally or ["distributed across multiple machines/tasks"](https://www.tensorflow.org/deploy/distributed).
-You will need to set the task number flag for each task when running in a distributed fashion.
-Please refer to the original paper for parameter explanations and training details.
-
-### Installation
-* TensorFlow
- * This code requires the latest open-source TensorFlow that you will need to build manually.
- The [documentation](https://www.tensorflow.org/install/install_sources) provides the steps required for that.
-* Bazel
- * Follow the instructions [here](http://bazel.build/docs/install.html).
- * Alternately, Download bazel from
- [https://github.com/bazelbuild/bazel/releases](https://github.com/bazelbuild/bazel/releases)
- for your system configuration.
- * Check for the bazel version using this command: bazel version
-* matplotlib
- * Follow the instructions [here](https://matplotlib.org/users/installing.html).
- * You can use a package repository like pip.
-* scikit-image
- * Follow the instructions [here](http://scikit-image.org/docs/dev/install.html).
- * You can use a package repository like pip.
-* PIL
- * Install from [here](https://pypi.python.org/pypi/Pillow/2.2.1).
-
-### Dataset
-
-This code requires the dataset to be in *tfrecords* format with the following features:
-* image
- * Flattened list of image (float representations) for each view point.
-* mask
- * Flattened list of image masks (float representations) for each view point.
-* vox
- * Flattened list of voxels (float representations) for the object.
- * This is needed for using vox loss and for prediction comparison.
-
-You can download the ShapeNet Dataset in tfrecords format from [here](https://drive.google.com/file/d/0B12XukcbU7T7OHQ4MGh6d25qQlk)*.
-
-* Disclaimer: This data is hosted personally by Arkanath Pathak for non-commercial research purposes. Please cite the [ShapeNet paper](https://arxiv.org/pdf/1512.03012.pdf) in your works when using ShapeNet for non-commercial research purposes.
-
-### Pretraining: pretrain_rotator.py for each RNN step
-$ bazel run -c opt :pretrain_rotator -- --step_size={} --init_model={}
-
-Pass the init_model as the checkpoint path for the last step trained model.
-You'll also need to set the inp_dir flag to where your data resides.
-
-### Training: train_ptn.py with last pretrained model.
-$ bazel run -c opt :train_ptn -- --init_model={}
-
-### Example TensorBoard Visualizations
-
-To compare the visualizations make sure to set the model_name flag different for each parametric setting:
-
-This code adds summaries for each loss. For instance, these are the losses we encountered in the distributed pretraining for ShapeNet Chair Dataset with 10 workers and 16 parameter servers:
-
-
-You can expect such images after fine tuning the training as "grid_vis" under **Image** summaries in TensorBoard:
-
-Here the third and fifth columns are the predicted masks and voxels respectively, alongside their ground truth values.
-
-A similar image for when trained on all ShapeNet Categories (Voxel visualizations might be skewed):
-
diff --git a/research/ptn/WORKSPACE b/research/ptn/WORKSPACE
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/ptn/eval_ptn.py b/research/ptn/eval_ptn.py
deleted file mode 100644
index 2f8dd96b1938452083253832a586eaf525d5e072..0000000000000000000000000000000000000000
--- a/research/ptn/eval_ptn.py
+++ /dev/null
@@ -1,132 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contains evaluation plan for the Im2vox model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import tensorflow as tf
-from tensorflow import app
-
-import model_ptn
-
-flags = tf.app.flags
-slim = tf.contrib.slim
-
-flags.DEFINE_string('inp_dir',
- '',
- 'Directory path containing the input data (tfrecords).')
-flags.DEFINE_string(
- 'dataset_name', 'shapenet_chair',
- 'Dataset name that is to be used for training and evaluation.')
-flags.DEFINE_integer('z_dim', 512, '')
-flags.DEFINE_integer('f_dim', 64, '')
-flags.DEFINE_integer('fc_dim', 1024, '')
-flags.DEFINE_integer('num_views', 24, 'Num of viewpoints in the input data.')
-flags.DEFINE_integer('image_size', 64,
- 'Input images dimension (pixels) - width & height.')
-flags.DEFINE_integer('vox_size', 32, 'Voxel prediction dimension.')
-flags.DEFINE_integer('step_size', 24, '')
-flags.DEFINE_integer('batch_size', 1, 'Batch size while training.')
-flags.DEFINE_float('focal_length', 0.866, '')
-flags.DEFINE_float('focal_range', 1.732, '')
-flags.DEFINE_string('encoder_name', 'ptn_encoder',
- 'Name of the encoder network being used.')
-flags.DEFINE_string('decoder_name', 'ptn_vox_decoder',
- 'Name of the decoder network being used.')
-flags.DEFINE_string('projector_name', 'ptn_projector',
- 'Name of the projector network being used.')
-# Save options
-flags.DEFINE_string('checkpoint_dir', '/tmp/ptn/eval/',
- 'Directory path for saving trained models and other data.')
-flags.DEFINE_string('model_name', 'ptn_proj',
- 'Name of the model used in naming the TF job. Must be different for each run.')
-flags.DEFINE_string('eval_set', 'val', 'Data partition to form evaluation on.')
-# Optimization
-flags.DEFINE_float('proj_weight', 10, 'Weighting factor for projection loss.')
-flags.DEFINE_float('volume_weight', 0, 'Weighting factor for volume loss.')
-flags.DEFINE_float('viewpoint_weight', 1,
- 'Weighting factor for viewpoint loss.')
-flags.DEFINE_float('learning_rate', 0.0001, 'Learning rate.')
-flags.DEFINE_float('weight_decay', 0.001, '')
-flags.DEFINE_float('clip_gradient_norm', 0, '')
-# Summary
-flags.DEFINE_integer('save_summaries_secs', 15, '')
-flags.DEFINE_integer('eval_interval_secs', 60 * 5, '')
-# Distribution
-flags.DEFINE_string('master', '', '')
-
-FLAGS = flags.FLAGS
-
-
-def main(argv=()):
- del argv # Unused.
- eval_dir = os.path.join(FLAGS.checkpoint_dir, FLAGS.model_name, 'train')
- log_dir = os.path.join(FLAGS.checkpoint_dir, FLAGS.model_name,
- 'eval_%s' % FLAGS.eval_set)
- if not os.path.exists(eval_dir):
- os.makedirs(eval_dir)
- if not os.path.exists(log_dir):
- os.makedirs(log_dir)
- g = tf.Graph()
-
- with g.as_default():
- eval_params = FLAGS
- eval_params.batch_size = 1
- eval_params.step_size = FLAGS.num_views
- ###########
- ## model ##
- ###########
- model = model_ptn.model_PTN(eval_params)
- ##########
- ## data ##
- ##########
- eval_data = model.get_inputs(
- FLAGS.inp_dir,
- FLAGS.dataset_name,
- eval_params.eval_set,
- eval_params.batch_size,
- eval_params.image_size,
- eval_params.vox_size,
- is_training=False)
- inputs = model.preprocess_with_all_views(eval_data)
- ##############
- ## model_fn ##
- ##############
- model_fn = model.get_model_fn(is_training=False, run_projection=False)
- outputs = model_fn(inputs)
- #############
- ## metrics ##
- #############
- names_to_values, names_to_updates = model.get_metrics(inputs, outputs)
- del names_to_values
- ################
- ## evaluation ##
- ################
- num_batches = eval_data['num_samples']
- slim.evaluation.evaluation_loop(
- master=FLAGS.master,
- checkpoint_dir=eval_dir,
- logdir=log_dir,
- num_evals=num_batches,
- eval_op=names_to_updates.values(),
- eval_interval_secs=FLAGS.eval_interval_secs)
-
-
-if __name__ == '__main__':
- app.run()
diff --git a/research/ptn/eval_rotator.py b/research/ptn/eval_rotator.py
deleted file mode 100644
index b7fcf0fe4ab2b98754ffbc0d75efa64828db6e25..0000000000000000000000000000000000000000
--- a/research/ptn/eval_rotator.py
+++ /dev/null
@@ -1,126 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contains evaluation plan for the Rotator model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import tensorflow as tf
-from tensorflow import app
-
-import model_rotator as model
-
-flags = tf.app.flags
-slim = tf.contrib.slim
-
-flags.DEFINE_string('inp_dir',
- '',
- 'Directory path containing the input data (tfrecords).')
-flags.DEFINE_string(
- 'dataset_name', 'shapenet_chair',
- 'Dataset name that is to be used for training and evaluation.')
-flags.DEFINE_integer('z_dim', 512, '')
-flags.DEFINE_integer('a_dim', 3, '')
-flags.DEFINE_integer('f_dim', 64, '')
-flags.DEFINE_integer('fc_dim', 1024, '')
-flags.DEFINE_integer('num_views', 24, 'Num of viewpoints in the input data.')
-flags.DEFINE_integer('image_size', 64,
- 'Input images dimension (pixels) - width & height.')
-flags.DEFINE_integer('step_size', 24, '')
-flags.DEFINE_integer('batch_size', 2, '')
-flags.DEFINE_string('encoder_name', 'ptn_encoder',
- 'Name of the encoder network being used.')
-flags.DEFINE_string('decoder_name', 'ptn_im_decoder',
- 'Name of the decoder network being used.')
-flags.DEFINE_string('rotator_name', 'ptn_rotator',
- 'Name of the rotator network being used.')
-# Save options
-flags.DEFINE_string('checkpoint_dir', '/tmp/ptn_train/',
- 'Directory path for saving trained models and other data.')
-flags.DEFINE_string('model_name', 'ptn_proj',
- 'Name of the model used in naming the TF job. Must be different for each run.')
-# Optimization
-flags.DEFINE_float('image_weight', 10, '')
-flags.DEFINE_float('mask_weight', 1, '')
-flags.DEFINE_float('learning_rate', 0.0001, 'Learning rate.')
-flags.DEFINE_float('weight_decay', 0.001, '')
-flags.DEFINE_float('clip_gradient_norm', 0, '')
-# Summary
-flags.DEFINE_integer('save_summaries_secs', 15, '')
-flags.DEFINE_integer('eval_interval_secs', 60 * 5, '')
-# Scheduling
-flags.DEFINE_string('master', '', '')
-
-FLAGS = flags.FLAGS
-
-
-def main(argv=()):
- del argv # Unused.
- eval_dir = os.path.join(FLAGS.checkpoint_dir,
- FLAGS.model_name, 'train')
- log_dir = os.path.join(FLAGS.checkpoint_dir,
- FLAGS.model_name, 'eval')
-
- if not os.path.exists(eval_dir):
- os.makedirs(eval_dir)
- if not os.path.exists(log_dir):
- os.makedirs(log_dir)
- g = tf.Graph()
-
- if FLAGS.step_size < FLAGS.num_views:
- raise ValueError('Impossible step_size, must not be less than num_views.')
-
- g = tf.Graph()
- with g.as_default():
- ##########
- ## data ##
- ##########
- val_data = model.get_inputs(
- FLAGS.inp_dir,
- FLAGS.dataset_name,
- 'val',
- FLAGS.batch_size,
- FLAGS.image_size,
- is_training=False)
- inputs = model.preprocess(val_data, FLAGS.step_size)
- ###########
- ## model ##
- ###########
- model_fn = model.get_model_fn(FLAGS, is_training=False)
- outputs = model_fn(inputs)
- #############
- ## metrics ##
- #############
- names_to_values, names_to_updates = model.get_metrics(
- inputs, outputs, FLAGS)
- del names_to_values
- ################
- ## evaluation ##
- ################
- num_batches = int(val_data['num_samples'] / FLAGS.batch_size)
- slim.evaluation.evaluation_loop(
- master=FLAGS.master,
- checkpoint_dir=eval_dir,
- logdir=log_dir,
- num_evals=num_batches,
- eval_op=names_to_updates.values(),
- eval_interval_secs=FLAGS.eval_interval_secs)
-
-
-if __name__ == '__main__':
- app.run()
diff --git a/research/ptn/input_generator.py b/research/ptn/input_generator.py
deleted file mode 100644
index 7047d6483030b8b6ad0e68897121822030f84769..0000000000000000000000000000000000000000
--- a/research/ptn/input_generator.py
+++ /dev/null
@@ -1,130 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Provides dataset dictionaries as used in our network models."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import tensorflow as tf
-import tensorflow.contrib.slim as slim
-
-from tensorflow.contrib.slim.python.slim.data import dataset
-from tensorflow.contrib.slim.python.slim.data import dataset_data_provider
-from tensorflow.contrib.slim.python.slim.data import tfexample_decoder
-
-_ITEMS_TO_DESCRIPTIONS = {
- 'image': 'Images',
- 'mask': 'Masks',
- 'vox': 'Voxels'
-}
-
-
-def _get_split(file_pattern, num_samples, num_views, image_size, vox_size):
- """Get dataset.Dataset for the given dataset file pattern and properties."""
-
- # A dictionary from TF-Example keys to tf.FixedLenFeature instance.
- keys_to_features = {
- 'image': tf.FixedLenFeature(
- shape=[num_views, image_size, image_size, 3],
- dtype=tf.float32, default_value=None),
- 'mask': tf.FixedLenFeature(
- shape=[num_views, image_size, image_size, 1],
- dtype=tf.float32, default_value=None),
- 'vox': tf.FixedLenFeature(
- shape=[vox_size, vox_size, vox_size, 1],
- dtype=tf.float32, default_value=None),
- }
-
- items_to_handler = {
- 'image': tfexample_decoder.Tensor(
- 'image', shape=[num_views, image_size, image_size, 3]),
- 'mask': tfexample_decoder.Tensor(
- 'mask', shape=[num_views, image_size, image_size, 1]),
- 'vox': tfexample_decoder.Tensor(
- 'vox', shape=[vox_size, vox_size, vox_size, 1])
- }
-
- decoder = tfexample_decoder.TFExampleDecoder(
- keys_to_features, items_to_handler)
-
- return dataset.Dataset(
- data_sources=file_pattern,
- reader=tf.TFRecordReader,
- decoder=decoder,
- num_samples=num_samples,
- items_to_descriptions=_ITEMS_TO_DESCRIPTIONS)
-
-
-def get(dataset_dir,
- dataset_name,
- split_name,
- shuffle=True,
- num_readers=1,
- common_queue_capacity=64,
- common_queue_min=50):
- """Provides input data for a specified dataset and split."""
-
- dataset_to_kwargs = {
- 'shapenet_chair': {
- 'file_pattern': '03001627_%s.tfrecords' % split_name,
- 'num_views': 24,
- 'image_size': 64,
- 'vox_size': 32,
- }, 'shapenet_all': {
- 'file_pattern': '*_%s.tfrecords' % split_name,
- 'num_views': 24,
- 'image_size': 64,
- 'vox_size': 32,
- },
- }
-
- split_sizes = {
- 'shapenet_chair': {
- 'train': 4744,
- 'val': 678,
- 'test': 1356,
- },
- 'shapenet_all': {
- 'train': 30643,
- 'val': 4378,
- 'test': 8762,
- }
- }
-
- kwargs = dataset_to_kwargs[dataset_name]
- kwargs['file_pattern'] = os.path.join(dataset_dir, kwargs['file_pattern'])
- kwargs['num_samples'] = split_sizes[dataset_name][split_name]
-
- dataset_split = _get_split(**kwargs)
- data_provider = dataset_data_provider.DatasetDataProvider(
- dataset_split,
- num_readers=num_readers,
- common_queue_capacity=common_queue_capacity,
- common_queue_min=common_queue_min,
- shuffle=shuffle)
-
- inputs = {
- 'num_samples': dataset_split.num_samples,
- }
-
- [image, mask, vox] = data_provider.get(['image', 'mask', 'vox'])
- inputs['image'] = image
- inputs['mask'] = mask
- inputs['voxel'] = vox
-
- return inputs
diff --git a/research/ptn/losses.py b/research/ptn/losses.py
deleted file mode 100644
index 53cc28847a32af88b718fbb9e53ca287b48a8b65..0000000000000000000000000000000000000000
--- a/research/ptn/losses.py
+++ /dev/null
@@ -1,178 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Defines the various loss functions in use by the PTN model."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-
-def add_rotator_image_loss(inputs, outputs, step_size, weight_scale):
- """Computes the image loss of deep rotator model.
-
- Args:
- inputs: Input dictionary to the model containing keys
- such as `images_k'.
- outputs: Output dictionary returned by the model containing keys
- such as `images_k'.
- step_size: A scalar representing the number of recurrent
- steps (number of repeated out-of-plane rotations)
- in the deep rotator network (int).
- weight_scale: A reweighting factor applied over the image loss (float).
-
- Returns:
- A `Tensor' scalar that returns averaged L2 loss
- (divided by batch_size and step_size) between the
- ground-truth images (RGB) and predicted images (tf.float32).
-
- """
- batch_size = tf.shape(inputs['images_0'])[0]
- image_loss = 0
- for k in range(1, step_size + 1):
- image_loss += tf.nn.l2_loss(
- inputs['images_%d' % k] - outputs['images_%d' % k])
-
- image_loss /= tf.to_float(step_size * batch_size)
- slim.summaries.add_scalar_summary(
- image_loss, 'image_loss', prefix='losses')
- image_loss *= weight_scale
- return image_loss
-
-
-def add_rotator_mask_loss(inputs, outputs, step_size, weight_scale):
- """Computes the mask loss of deep rotator model.
-
- Args:
- inputs: Input dictionary to the model containing keys
- such as `masks_k'.
- outputs: Output dictionary returned by the model containing
- keys such as `masks_k'.
- step_size: A scalar representing the number of recurrent
- steps (number of repeated out-of-plane rotations)
- in the deep rotator network (int).
- weight_scale: A reweighting factor applied over the mask loss (float).
-
- Returns:
- A `Tensor' that returns averaged L2 loss
- (divided by batch_size and step_size) between the ground-truth masks
- (object silhouettes) and predicted masks (tf.float32).
-
- """
- batch_size = tf.shape(inputs['images_0'])[0]
- mask_loss = 0
- for k in range(1, step_size + 1):
- mask_loss += tf.nn.l2_loss(
- inputs['masks_%d' % k] - outputs['masks_%d' % k])
-
- mask_loss /= tf.to_float(step_size * batch_size)
- slim.summaries.add_scalar_summary(
- mask_loss, 'mask_loss', prefix='losses')
- mask_loss *= weight_scale
- return mask_loss
-
-
-def add_volume_proj_loss(inputs, outputs, num_views, weight_scale):
- """Computes the projection loss of voxel generation model.
-
- Args:
- inputs: Input dictionary to the model containing keys such as
- `images_1'.
- outputs: Output dictionary returned by the model containing keys
- such as `masks_k' and ``projs_k'.
- num_views: A integer scalar represents the total number of
- viewpoints for each of the object (int).
- weight_scale: A reweighting factor applied over the projection loss (float).
-
- Returns:
- A `Tensor' that returns the averaged L2 loss
- (divided by batch_size and num_views) between the ground-truth
- masks (object silhouettes) and predicted masks (tf.float32).
-
- """
- batch_size = tf.shape(inputs['images_1'])[0]
- proj_loss = 0
- for k in range(num_views):
- proj_loss += tf.nn.l2_loss(
- outputs['masks_%d' % (k + 1)] - outputs['projs_%d' % (k + 1)])
- proj_loss /= tf.to_float(num_views * batch_size)
- slim.summaries.add_scalar_summary(
- proj_loss, 'proj_loss', prefix='losses')
- proj_loss *= weight_scale
- return proj_loss
-
-
-def add_volume_loss(inputs, outputs, num_views, weight_scale):
- """Computes the volume loss of voxel generation model.
-
- Args:
- inputs: Input dictionary to the model containing keys such as
- `images_1' and `voxels'.
- outputs: Output dictionary returned by the model containing keys
- such as `voxels_k'.
- num_views: A scalar representing the total number of
- viewpoints for each object (int).
- weight_scale: A reweighting factor applied over the volume
- loss (tf.float32).
-
- Returns:
- A `Tensor' that returns the averaged L2 loss
- (divided by batch_size and num_views) between the ground-truth
- volumes and predicted volumes (tf.float32).
-
- """
- batch_size = tf.shape(inputs['images_1'])[0]
- vol_loss = 0
- for k in range(num_views):
- vol_loss += tf.nn.l2_loss(
- inputs['voxels'] - outputs['voxels_%d' % (k + 1)])
- vol_loss /= tf.to_float(num_views * batch_size)
- slim.summaries.add_scalar_summary(
- vol_loss, 'vol_loss', prefix='losses')
- vol_loss *= weight_scale
- return vol_loss
-
-
-def regularization_loss(scopes, params):
- """Computes the weight decay as regularization during training.
-
- Args:
- scopes: A list of different components of the model such as
- ``encoder'', ``decoder'' and ``projector''.
- params: Parameters of the model.
-
- Returns:
- Regularization loss (tf.float32).
- """
-
- reg_loss = tf.zeros(dtype=tf.float32, shape=[])
- if params.weight_decay > 0:
- is_trainable = lambda x: x in tf.trainable_variables()
- is_weights = lambda x: 'weights' in x.name
- for scope in scopes:
- scope_vars = filter(is_trainable,
- tf.contrib.framework.get_model_variables(scope))
- scope_vars = filter(is_weights, scope_vars)
- if scope_vars:
- reg_loss += tf.add_n([tf.nn.l2_loss(var) for var in scope_vars])
-
- slim.summaries.add_scalar_summary(
- reg_loss, 'reg_loss', prefix='losses')
- reg_loss *= params.weight_decay
- return reg_loss
diff --git a/research/ptn/metrics.py b/research/ptn/metrics.py
deleted file mode 100644
index 5f31dd5fd5a67af3f5cc94de88493c25750e413c..0000000000000000000000000000000000000000
--- a/research/ptn/metrics.py
+++ /dev/null
@@ -1,111 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Provides metrics used by PTN."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from six.moves import xrange
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-
-def add_image_pred_metrics(
- inputs, outputs, num_views, upscale_factor):
- """Computes the image prediction metrics.
-
- Args:
- inputs: Input dictionary of the deep rotator model (model_rotator.py).
- outputs: Output dictionary of the deep rotator model (model_rotator.py).
- num_views: An integer scalar representing the total number
- of different viewpoints for each object in the dataset.
- upscale_factor: A float scalar representing the number of pixels
- per image (num_channels x image_height x image_width).
-
- Returns:
- names_to_values: A dictionary representing the current value
- of the metric.
- names_to_updates: A dictionary representing the operation
- that accumulates the error from a batch of data.
- """
- names_to_values = dict()
- names_to_updates = dict()
- for k in xrange(num_views):
- tmp_value, tmp_update = tf.contrib.metrics.streaming_mean_squared_error(
- outputs['images_%d' % (k + 1)], inputs['images_%d' % (k + 1)])
- name = 'image_pred/rnn_%d' % (k + 1)
- names_to_values.update({name: tmp_value * upscale_factor})
- names_to_updates.update({name: tmp_update})
- return names_to_values, names_to_updates
-
-
-def add_mask_pred_metrics(
- inputs, outputs, num_views, upscale_factor):
- """Computes the mask prediction metrics.
-
- Args:
- inputs: Input dictionary of the deep rotator model (model_rotator.py).
- outputs: Output dictionary of the deep rotator model (model_rotator.py).
- num_views: An integer scalar representing the total number
- of different viewpoints for each object in the dataset.
- upscale_factor: A float scalar representing the number of pixels
- per image (num_channels x image_height x image_width).
-
- Returns:
- names_to_values: A dictionary representing the current value
- of the metric.
- names_to_updates: A dictionary representing the operation
- that accumulates the error from a batch of data.
-
- """
- names_to_values = dict()
- names_to_updates = dict()
- for k in xrange(num_views):
- tmp_value, tmp_update = tf.contrib.metrics.streaming_mean_squared_error(
- outputs['masks_%d' % (k + 1)], inputs['masks_%d' % (k + 1)])
- name = 'mask_pred/rnn_%d' % (k + 1)
- names_to_values.update({name: tmp_value * upscale_factor})
- names_to_updates.update({name: tmp_update})
- return names_to_values, names_to_updates
-
-
-def add_volume_iou_metrics(inputs, outputs):
- """Computes the per-instance volume IOU.
-
- Args:
- inputs: Input dictionary of the voxel generation model.
- outputs: Output dictionary returned by the voxel generation model.
-
- Returns:
- names_to_values: metrics->values (dict).
- names_to_updates: metrics->ops (dict).
-
- """
- names_to_values = dict()
- names_to_updates = dict()
- labels = tf.greater_equal(inputs['voxels'], 0.5)
- predictions = tf.greater_equal(outputs['voxels_1'], 0.5)
- labels = (2 - tf.to_int32(labels)) - 1
- predictions = (3 - tf.to_int32(predictions) * 2) - 1
- tmp_values, tmp_updates = tf.metrics.mean_iou(
- labels=labels,
- predictions=predictions,
- num_classes=3)
- names_to_values['volume_iou'] = tmp_values * 3.0
- names_to_updates['volume_iou'] = tmp_updates
- return names_to_values, names_to_updates
diff --git a/research/ptn/model_ptn.py b/research/ptn/model_ptn.py
deleted file mode 100644
index cc0fc4fa38e5479d78307e764ae490e124e34ab7..0000000000000000000000000000000000000000
--- a/research/ptn/model_ptn.py
+++ /dev/null
@@ -1,232 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Implementations for Im2Vox PTN (NIPS16) model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import losses
-import metrics
-import model_voxel_generation
-import utils
-from nets import im2vox_factory
-
-slim = tf.contrib.slim
-
-
-class model_PTN(model_voxel_generation.Im2Vox): # pylint:disable=invalid-name
- """Inherits the generic Im2Vox model class and implements the functions."""
-
- def __init__(self, params):
- super(model_PTN, self).__init__(params)
-
- # For testing, this selects all views in input
- def preprocess_with_all_views(self, raw_inputs):
- (quantity, num_views) = raw_inputs['images'].get_shape().as_list()[:2]
-
- inputs = dict()
- inputs['voxels'] = []
- inputs['images_1'] = []
- for k in xrange(num_views):
- inputs['matrix_%d' % (k + 1)] = []
- inputs['matrix_1'] = []
- for n in xrange(quantity):
- for k in xrange(num_views):
- inputs['images_1'].append(raw_inputs['images'][n, k, :, :, :])
- inputs['voxels'].append(raw_inputs['voxels'][n, :, :, :, :])
- tf_matrix = self.get_transform_matrix(k)
- inputs['matrix_%d' % (k + 1)].append(tf_matrix)
-
- inputs['images_1'] = tf.stack(inputs['images_1'])
- inputs['voxels'] = tf.stack(inputs['voxels'])
- for k in xrange(num_views):
- inputs['matrix_%d' % (k + 1)] = tf.stack(inputs['matrix_%d' % (k + 1)])
-
- return inputs
-
- def get_model_fn(self, is_training=True, reuse=False, run_projection=True):
- return im2vox_factory.get(self._params, is_training, reuse, run_projection)
-
- def get_regularization_loss(self, scopes):
- return losses.regularization_loss(scopes, self._params)
-
- def get_loss(self, inputs, outputs):
- """Computes the loss used for PTN paper (projection + volume loss)."""
- g_loss = tf.zeros(dtype=tf.float32, shape=[])
-
- if self._params.proj_weight:
- g_loss += losses.add_volume_proj_loss(
- inputs, outputs, self._params.step_size, self._params.proj_weight)
-
- if self._params.volume_weight:
- g_loss += losses.add_volume_loss(inputs, outputs, 1,
- self._params.volume_weight)
-
- slim.summaries.add_scalar_summary(g_loss, 'im2vox_loss', prefix='losses')
-
- return g_loss
-
- def get_metrics(self, inputs, outputs):
- """Aggregate the metrics for voxel generation model.
-
- Args:
- inputs: Input dictionary of the voxel generation model.
- outputs: Output dictionary returned by the voxel generation model.
-
- Returns:
- names_to_values: metrics->values (dict).
- names_to_updates: metrics->ops (dict).
- """
- names_to_values = dict()
- names_to_updates = dict()
-
- tmp_values, tmp_updates = metrics.add_volume_iou_metrics(inputs, outputs)
-
- names_to_values.update(tmp_values)
- names_to_updates.update(tmp_updates)
-
- for name, value in names_to_values.iteritems():
- slim.summaries.add_scalar_summary(
- value, name, prefix='eval', print_summary=True)
-
- return names_to_values, names_to_updates
-
- def write_disk_grid(self,
- global_step,
- log_dir,
- input_images,
- gt_projs,
- pred_projs,
- input_voxels=None,
- output_voxels=None):
- """Function called by TF to save the prediction periodically."""
- summary_freq = self._params.save_every
-
- def write_grid(input_images, gt_projs, pred_projs, global_step,
- input_voxels, output_voxels):
- """Native python function to call for writing images to files."""
- grid = _build_image_grid(
- input_images,
- gt_projs,
- pred_projs,
- input_voxels=input_voxels,
- output_voxels=output_voxels)
-
- if global_step % summary_freq == 0:
- img_path = os.path.join(log_dir, '%s.jpg' % str(global_step))
- utils.save_image(grid, img_path)
- return grid
-
- save_op = tf.py_func(write_grid, [
- input_images, gt_projs, pred_projs, global_step, input_voxels,
- output_voxels
- ], [tf.uint8], 'write_grid')[0]
- slim.summaries.add_image_summary(
- tf.expand_dims(save_op, axis=0), name='grid_vis')
- return save_op
-
- def get_transform_matrix(self, view_out):
- """Get the 4x4 Perspective Transfromation matrix used for PTN."""
- num_views = self._params.num_views
- focal_length = self._params.focal_length
- focal_range = self._params.focal_range
- phi = 30
- theta_interval = 360.0 / num_views
- theta = theta_interval * view_out
-
- # pylint: disable=invalid-name
- camera_matrix = np.zeros((4, 4), dtype=np.float32)
- intrinsic_matrix = np.eye(4, dtype=np.float32)
- extrinsic_matrix = np.eye(4, dtype=np.float32)
-
- sin_phi = np.sin(float(phi) / 180.0 * np.pi)
- cos_phi = np.cos(float(phi) / 180.0 * np.pi)
- sin_theta = np.sin(float(-theta) / 180.0 * np.pi)
- cos_theta = np.cos(float(-theta) / 180.0 * np.pi)
-
- rotation_azimuth = np.zeros((3, 3), dtype=np.float32)
- rotation_azimuth[0, 0] = cos_theta
- rotation_azimuth[2, 2] = cos_theta
- rotation_azimuth[0, 2] = -sin_theta
- rotation_azimuth[2, 0] = sin_theta
- rotation_azimuth[1, 1] = 1.0
-
- ## rotation axis -- x
- rotation_elevation = np.zeros((3, 3), dtype=np.float32)
- rotation_elevation[0, 0] = cos_phi
- rotation_elevation[0, 1] = sin_phi
- rotation_elevation[1, 0] = -sin_phi
- rotation_elevation[1, 1] = cos_phi
- rotation_elevation[2, 2] = 1.0
-
- rotation_matrix = np.matmul(rotation_azimuth, rotation_elevation)
- displacement = np.zeros((3, 1), dtype=np.float32)
- displacement[0, 0] = float(focal_length) + float(focal_range) / 2.0
- displacement = np.matmul(rotation_matrix, displacement)
-
- extrinsic_matrix[0:3, 0:3] = rotation_matrix
- extrinsic_matrix[0:3, 3:4] = -displacement
-
- intrinsic_matrix[2, 2] = 1.0 / float(focal_length)
- intrinsic_matrix[1, 1] = 1.0 / float(focal_length)
-
- camera_matrix = np.matmul(extrinsic_matrix, intrinsic_matrix)
- return camera_matrix
-
-
-def _build_image_grid(input_images,
- gt_projs,
- pred_projs,
- input_voxels,
- output_voxels,
- vis_size=128):
- """Builds a grid image by concatenating the input images."""
- quantity = input_images.shape[0]
-
- for row in xrange(int(quantity / 3)):
- for col in xrange(3):
- index = row * 3 + col
- input_img_ = utils.resize_image(input_images[index, :, :, :], vis_size,
- vis_size)
- gt_proj_ = utils.resize_image(gt_projs[index, :, :, :], vis_size,
- vis_size)
- pred_proj_ = utils.resize_image(pred_projs[index, :, :, :], vis_size,
- vis_size)
- gt_voxel_vis = utils.resize_image(
- utils.display_voxel(input_voxels[index, :, :, :, 0]), vis_size,
- vis_size)
- pred_voxel_vis = utils.resize_image(
- utils.display_voxel(output_voxels[index, :, :, :, 0]), vis_size,
- vis_size)
- if col == 0:
- tmp_ = np.concatenate(
- [input_img_, gt_proj_, pred_proj_, gt_voxel_vis, pred_voxel_vis], 1)
- else:
- tmp_ = np.concatenate([
- tmp_, input_img_, gt_proj_, pred_proj_, gt_voxel_vis, pred_voxel_vis
- ], 1)
- if row == 0:
- out_grid = tmp_
- else:
- out_grid = np.concatenate([out_grid, tmp_], 0)
-
- return out_grid
diff --git a/research/ptn/model_rotator.py b/research/ptn/model_rotator.py
deleted file mode 100644
index 28860bc1025f78b5413b84d0c2bbf632874e0853..0000000000000000000000000000000000000000
--- a/research/ptn/model_rotator.py
+++ /dev/null
@@ -1,266 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Helper functions for pretraining (rotator) as described in PTN paper."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import input_generator
-import losses
-import metrics
-import utils
-from nets import deeprotator_factory
-
-slim = tf.contrib.slim
-
-
-def _get_data_from_provider(inputs, batch_size, split_name):
- """Returns dictionary of batch input data processed by tf.train.batch."""
- images, masks = tf.train.batch(
- [inputs['image'], inputs['mask']],
- batch_size=batch_size,
- num_threads=8,
- capacity=8 * batch_size,
- name='batching_queues/%s' % (split_name))
-
- outputs = dict()
- outputs['images'] = images
- outputs['masks'] = masks
- outputs['num_samples'] = inputs['num_samples']
-
- return outputs
-
-
-def get_inputs(dataset_dir, dataset_name, split_name, batch_size, image_size,
- is_training):
- """Loads the given dataset and split."""
- del image_size # Unused
- with tf.variable_scope('data_loading_%s/%s' % (dataset_name, split_name)):
- common_queue_min = 50
- common_queue_capacity = 256
- num_readers = 4
-
- inputs = input_generator.get(
- dataset_dir,
- dataset_name,
- split_name,
- shuffle=is_training,
- num_readers=num_readers,
- common_queue_min=common_queue_min,
- common_queue_capacity=common_queue_capacity)
-
- return _get_data_from_provider(inputs, batch_size, split_name)
-
-
-def preprocess(raw_inputs, step_size):
- """Selects the subset of viewpoints to train on."""
- shp = raw_inputs['images'].get_shape().as_list()
- quantity = shp[0]
- num_views = shp[1]
- image_size = shp[2]
- del image_size # Unused
-
- batch_rot = np.zeros((quantity, 3), dtype=np.float32)
- inputs = dict()
- for n in xrange(step_size + 1):
- inputs['images_%d' % n] = []
- inputs['masks_%d' % n] = []
-
- for n in xrange(quantity):
- view_in = np.random.randint(0, num_views)
- rng_rot = np.random.randint(0, 2)
- if step_size == 1:
- rng_rot = np.random.randint(0, 3)
-
- delta = 0
- if rng_rot == 0:
- delta = -1
- batch_rot[n, 2] = 1
- elif rng_rot == 1:
- delta = 1
- batch_rot[n, 0] = 1
- else:
- delta = 0
- batch_rot[n, 1] = 1
-
- inputs['images_0'].append(raw_inputs['images'][n, view_in, :, :, :])
- inputs['masks_0'].append(raw_inputs['masks'][n, view_in, :, :, :])
-
- view_out = view_in
- for k in xrange(1, step_size + 1):
- view_out += delta
- if view_out >= num_views:
- view_out = 0
- if view_out < 0:
- view_out = num_views - 1
-
- inputs['images_%d' % k].append(raw_inputs['images'][n, view_out, :, :, :])
- inputs['masks_%d' % k].append(raw_inputs['masks'][n, view_out, :, :, :])
-
- for n in xrange(step_size + 1):
- inputs['images_%d' % n] = tf.stack(inputs['images_%d' % n])
- inputs['masks_%d' % n] = tf.stack(inputs['masks_%d' % n])
-
- inputs['actions'] = tf.constant(batch_rot, dtype=tf.float32)
- return inputs
-
-
-def get_init_fn(scopes, params):
- """Initialization assignment operator function used while training."""
- if not params.init_model:
- return None
-
- is_trainable = lambda x: x in tf.trainable_variables()
- var_list = []
- for scope in scopes:
- var_list.extend(
- filter(is_trainable, tf.contrib.framework.get_model_variables(scope)))
-
- init_assign_op, init_feed_dict = slim.assign_from_checkpoint(
- params.init_model, var_list)
-
- def init_assign_function(sess):
- sess.run(init_assign_op, init_feed_dict)
-
- return init_assign_function
-
-
-def get_model_fn(params, is_training, reuse=False):
- return deeprotator_factory.get(params, is_training, reuse)
-
-
-def get_regularization_loss(scopes, params):
- return losses.regularization_loss(scopes, params)
-
-
-def get_loss(inputs, outputs, params):
- """Computes the rotator loss."""
- g_loss = tf.zeros(dtype=tf.float32, shape=[])
-
- if hasattr(params, 'image_weight'):
- g_loss += losses.add_rotator_image_loss(inputs, outputs, params.step_size,
- params.image_weight)
-
- if hasattr(params, 'mask_weight'):
- g_loss += losses.add_rotator_mask_loss(inputs, outputs, params.step_size,
- params.mask_weight)
-
- slim.summaries.add_scalar_summary(
- g_loss, 'rotator_loss', prefix='losses')
-
- return g_loss
-
-
-def get_train_op_for_scope(loss, optimizer, scopes, params):
- """Train operation function for the given scope used file training."""
- is_trainable = lambda x: x in tf.trainable_variables()
-
- var_list = []
- update_ops = []
-
- for scope in scopes:
- var_list.extend(
- filter(is_trainable, tf.contrib.framework.get_model_variables(scope)))
- update_ops.extend(tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope))
-
- return slim.learning.create_train_op(
- loss,
- optimizer,
- update_ops=update_ops,
- variables_to_train=var_list,
- clip_gradient_norm=params.clip_gradient_norm)
-
-
-def get_metrics(inputs, outputs, params):
- """Aggregate the metrics for rotator model.
-
- Args:
- inputs: Input dictionary of the rotator model.
- outputs: Output dictionary returned by the rotator model.
- params: Hyperparameters of the rotator model.
-
- Returns:
- names_to_values: metrics->values (dict).
- names_to_updates: metrics->ops (dict).
- """
- names_to_values = dict()
- names_to_updates = dict()
-
- tmp_values, tmp_updates = metrics.add_image_pred_metrics(
- inputs, outputs, params.num_views, 3*params.image_size**2)
- names_to_values.update(tmp_values)
- names_to_updates.update(tmp_updates)
-
- tmp_values, tmp_updates = metrics.add_mask_pred_metrics(
- inputs, outputs, params.num_views, params.image_size**2)
- names_to_values.update(tmp_values)
- names_to_updates.update(tmp_updates)
-
- for name, value in names_to_values.iteritems():
- slim.summaries.add_scalar_summary(
- value, name, prefix='eval', print_summary=True)
-
- return names_to_values, names_to_updates
-
-
-def write_disk_grid(global_step, summary_freq, log_dir, input_images,
- output_images, pred_images, pred_masks):
- """Function called by TF to save the prediction periodically."""
-
- def write_grid(grid, global_step):
- """Native python function to call for writing images to files."""
- if global_step % summary_freq == 0:
- img_path = os.path.join(log_dir, '%s.jpg' % str(global_step))
- utils.save_image(grid, img_path)
- return 0
-
- grid = _build_image_grid(input_images, output_images, pred_images, pred_masks)
- slim.summaries.add_image_summary(
- tf.expand_dims(grid, axis=0), name='grid_vis')
- save_op = tf.py_func(write_grid, [grid, global_step], [tf.int64],
- 'write_grid')[0]
- return save_op
-
-
-def _build_image_grid(input_images, output_images, pred_images, pred_masks):
- """Builds a grid image by concatenating the input images."""
- quantity = input_images.get_shape().as_list()[0]
-
- for row in xrange(int(quantity / 4)):
- for col in xrange(4):
- index = row * 4 + col
- input_img_ = input_images[index, :, :, :]
- output_img_ = output_images[index, :, :, :]
- pred_img_ = pred_images[index, :, :, :]
- pred_mask_ = tf.tile(pred_masks[index, :, :, :], [1, 1, 3])
- if col == 0:
- tmp_ = tf.concat([input_img_, output_img_, pred_img_, pred_mask_],
- 1) ## to the right
- else:
- tmp_ = tf.concat([tmp_, input_img_, output_img_, pred_img_, pred_mask_],
- 1)
- if row == 0:
- out_grid = tmp_
- else:
- out_grid = tf.concat([out_grid, tmp_], 0)
-
- return out_grid
diff --git a/research/ptn/model_voxel_generation.py b/research/ptn/model_voxel_generation.py
deleted file mode 100644
index 0c8fc84669a7fccc0f3a33ecb4c847162d99521e..0000000000000000000000000000000000000000
--- a/research/ptn/model_voxel_generation.py
+++ /dev/null
@@ -1,222 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Base class for voxel generation model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import abc
-import os
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-import input_generator
-import utils
-
-slim = tf.contrib.slim
-
-
-class Im2Vox(object):
- """Defines the voxel generation model."""
-
- __metaclass__ = abc.ABCMeta
-
- def __init__(self, params):
- self._params = params
-
- @abc.abstractmethod
- def get_metrics(self, inputs, outputs):
- """Gets dictionaries from metrics to value `Tensors` & update `Tensors`."""
- pass
-
- @abc.abstractmethod
- def get_loss(self, inputs, outputs):
- pass
-
- @abc.abstractmethod
- def get_regularization_loss(self, scopes):
- pass
-
- def set_params(self, params):
- self._params = params
-
- def get_inputs(self,
- dataset_dir,
- dataset_name,
- split_name,
- batch_size,
- image_size,
- vox_size,
- is_training=True):
- """Loads data for a specified dataset and split."""
- del image_size, vox_size
- with tf.variable_scope('data_loading_%s/%s' % (dataset_name, split_name)):
- common_queue_min = 64
- common_queue_capacity = 256
- num_readers = 4
-
- inputs = input_generator.get(
- dataset_dir,
- dataset_name,
- split_name,
- shuffle=is_training,
- num_readers=num_readers,
- common_queue_min=common_queue_min,
- common_queue_capacity=common_queue_capacity)
-
- images, voxels = tf.train.batch(
- [inputs['image'], inputs['voxel']],
- batch_size=batch_size,
- num_threads=8,
- capacity=8 * batch_size,
- name='batching_queues/%s/%s' % (dataset_name, split_name))
-
- outputs = dict()
- outputs['images'] = images
- outputs['voxels'] = voxels
- outputs['num_samples'] = inputs['num_samples']
-
- return outputs
-
- def preprocess(self, raw_inputs, step_size):
- """Selects the subset of viewpoints to train on."""
- (quantity, num_views) = raw_inputs['images'].get_shape().as_list()[:2]
-
- inputs = dict()
- inputs['voxels'] = raw_inputs['voxels']
-
- for k in xrange(step_size):
- inputs['images_%d' % (k + 1)] = []
- inputs['matrix_%d' % (k + 1)] = []
-
- for n in xrange(quantity):
- selected_views = np.random.choice(num_views, step_size, replace=False)
- for k in xrange(step_size):
- view_selected = selected_views[k]
- inputs['images_%d' %
- (k + 1)].append(raw_inputs['images'][n, view_selected, :, :, :])
- tf_matrix = self.get_transform_matrix(view_selected)
- inputs['matrix_%d' % (k + 1)].append(tf_matrix)
-
- for k in xrange(step_size):
- inputs['images_%d' % (k + 1)] = tf.stack(inputs['images_%d' % (k + 1)])
- inputs['matrix_%d' % (k + 1)] = tf.stack(inputs['matrix_%d' % (k + 1)])
-
- return inputs
-
- def get_init_fn(self, scopes):
- """Initialization assignment operator function used while training."""
- if not self._params.init_model:
- return None
-
- is_trainable = lambda x: x in tf.trainable_variables()
- var_list = []
- for scope in scopes:
- var_list.extend(
- filter(is_trainable, tf.contrib.framework.get_model_variables(scope)))
-
- init_assign_op, init_feed_dict = slim.assign_from_checkpoint(
- self._params.init_model, var_list)
-
- def init_assign_function(sess):
- sess.run(init_assign_op, init_feed_dict)
-
- return init_assign_function
-
- def get_train_op_for_scope(self, loss, optimizer, scopes):
- """Train operation function for the given scope used file training."""
- is_trainable = lambda x: x in tf.trainable_variables()
-
- var_list = []
- update_ops = []
-
- for scope in scopes:
- var_list.extend(
- filter(is_trainable, tf.contrib.framework.get_model_variables(scope)))
- update_ops.extend(tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope))
-
- return slim.learning.create_train_op(
- loss,
- optimizer,
- update_ops=update_ops,
- variables_to_train=var_list,
- clip_gradient_norm=self._params.clip_gradient_norm)
-
- def write_disk_grid(self,
- global_step,
- log_dir,
- input_images,
- gt_projs,
- pred_projs,
- pred_voxels=None):
- """Function called by TF to save the prediction periodically."""
- summary_freq = self._params.save_every
-
- def write_grid(input_images, gt_projs, pred_projs, pred_voxels,
- global_step):
- """Native python function to call for writing images to files."""
- grid = _build_image_grid(input_images, gt_projs, pred_projs, pred_voxels)
-
- if global_step % summary_freq == 0:
- img_path = os.path.join(log_dir, '%s.jpg' % str(global_step))
- utils.save_image(grid, img_path)
- with open(
- os.path.join(log_dir, 'pred_voxels_%s' % str(global_step)),
- 'w') as fout:
- np.save(fout, pred_voxels)
- with open(
- os.path.join(log_dir, 'input_images_%s' % str(global_step)),
- 'w') as fout:
- np.save(fout, input_images)
-
- return grid
-
- py_func_args = [
- input_images, gt_projs, pred_projs, pred_voxels, global_step
- ]
- save_grid_op = tf.py_func(write_grid, py_func_args, [tf.uint8],
- 'wrtie_grid')[0]
- slim.summaries.add_image_summary(
- tf.expand_dims(save_grid_op, axis=0), name='grid_vis')
- return save_grid_op
-
-
-def _build_image_grid(input_images, gt_projs, pred_projs, pred_voxels):
- """Build the visualization grid with py_func."""
- quantity, img_height, img_width = input_images.shape[:3]
- for row in xrange(int(quantity / 3)):
- for col in xrange(3):
- index = row * 3 + col
- input_img_ = input_images[index, :, :, :]
- gt_proj_ = gt_projs[index, :, :, :]
- pred_proj_ = pred_projs[index, :, :, :]
- pred_voxel_ = utils.display_voxel(pred_voxels[index, :, :, :, 0])
- pred_voxel_ = utils.resize_image(pred_voxel_, img_height, img_width)
- if col == 0:
- tmp_ = np.concatenate([input_img_, gt_proj_, pred_proj_, pred_voxel_],
- 1)
- else:
- tmp_ = np.concatenate(
- [tmp_, input_img_, gt_proj_, pred_proj_, pred_voxel_], 1)
- if row == 0:
- out_grid = tmp_
- else:
- out_grid = np.concatenate([out_grid, tmp_], 0)
-
- out_grid = out_grid.astype(np.uint8)
- return out_grid
diff --git a/research/ptn/nets/BUILD b/research/ptn/nets/BUILD
deleted file mode 100644
index 987499341ef25c8e19d532fdb14d3a9e842d0909..0000000000000000000000000000000000000000
--- a/research/ptn/nets/BUILD
+++ /dev/null
@@ -1,64 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-py_library(
- name = "deeprotator_factory",
- srcs = ["deeprotator_factory.py"],
- deps = [
- ":ptn_encoder",
- ":ptn_im_decoder",
- ":ptn_rotator",
- ],
-)
-
-py_library(
- name = "im2vox_factory",
- srcs = ["im2vox_factory.py"],
- deps = [
- ":perspective_projector",
- ":ptn_encoder",
- ":ptn_vox_decoder",
- ],
-)
-
-py_library(
- name = "perspective_projector",
- srcs = ["perspective_projector.py"],
- deps = [
- ":perspective_transform",
- ],
-)
-
-py_library(
- name = "perspective_transform",
- srcs = ["perspective_transform.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "ptn_encoder",
- srcs = ["ptn_encoder.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "ptn_im_decoder",
- srcs = ["ptn_im_decoder.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "ptn_rotator",
- srcs = ["ptn_rotator.py"],
- deps = [
- ],
-)
-
-py_library(
- name = "ptn_vox_decoder",
- srcs = ["ptn_vox_decoder.py"],
- deps = [
- ],
-)
diff --git a/research/ptn/nets/deeprotator_factory.py b/research/ptn/nets/deeprotator_factory.py
deleted file mode 100644
index e16170c41b7daae4a00068183cd0a4056ff279c3..0000000000000000000000000000000000000000
--- a/research/ptn/nets/deeprotator_factory.py
+++ /dev/null
@@ -1,91 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Factory module for different encoder/decoder network models."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from nets import ptn_encoder
-from nets import ptn_im_decoder
-from nets import ptn_rotator
-
-_NAME_TO_NETS = {
- 'ptn_encoder': ptn_encoder,
- 'ptn_rotator': ptn_rotator,
- 'ptn_im_decoder': ptn_im_decoder,
-}
-
-
-def _get_network(name):
- """Gets a single network component."""
-
- if name not in _NAME_TO_NETS:
- raise ValueError('Network name [%s] not recognized.' % name)
- return _NAME_TO_NETS[name].model
-
-
-def get(params, is_training=False, reuse=False):
- """Factory function to retrieve a network model.
-
- Args:
- params: Different parameters used througout ptn, typically FLAGS (dict)
- is_training: Set to True if while training (boolean)
- reuse: Set as True if either using a pre-trained model or
- in the training loop while the graph has already been built (boolean)
- Returns:
- Model function for network (inputs to outputs)
- """
-
- def model(inputs):
- """Model function corresponding to a specific network architecture."""
- outputs = {}
-
- # First, build the encoder.
- encoder_fn = _get_network(params.encoder_name)
- with tf.variable_scope('encoder', reuse=reuse):
- # Produces id/pose units
- features = encoder_fn(inputs['images_0'], params, is_training)
- outputs['ids'] = features['ids']
- outputs['poses_0'] = features['poses']
-
- # Second, build the rotator and decoder.
- rotator_fn = _get_network(params.rotator_name)
- with tf.variable_scope('rotator', reuse=reuse):
- outputs['poses_1'] = rotator_fn(outputs['poses_0'], inputs['actions'],
- params, is_training)
- decoder_fn = _get_network(params.decoder_name)
- with tf.variable_scope('decoder', reuse=reuse):
- dec_output = decoder_fn(outputs['ids'], outputs['poses_1'], params,
- is_training)
- outputs['images_1'] = dec_output['images']
- outputs['masks_1'] = dec_output['masks']
-
- # Third, build the recurrent connection
- for k in range(1, params.step_size):
- with tf.variable_scope('rotator', reuse=True):
- outputs['poses_%d' % (k + 1)] = rotator_fn(
- outputs['poses_%d' % k], inputs['actions'], params, is_training)
- with tf.variable_scope('decoder', reuse=True):
- dec_output = decoder_fn(outputs['ids'], outputs['poses_%d' % (k + 1)],
- params, is_training)
- outputs['images_%d' % (k + 1)] = dec_output['images']
- outputs['masks_%d' % (k + 1)] = dec_output['masks']
-
- return outputs
-
- return model
diff --git a/research/ptn/nets/im2vox_factory.py b/research/ptn/nets/im2vox_factory.py
deleted file mode 100644
index c54b96c24a56a8fa796cbc4f881a78cfbf41b86d..0000000000000000000000000000000000000000
--- a/research/ptn/nets/im2vox_factory.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Factory module for getting the complete image to voxel generation network."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from nets import perspective_projector
-from nets import ptn_encoder
-from nets import ptn_vox_decoder
-
-_NAME_TO_NETS = {
- 'ptn_encoder': ptn_encoder,
- 'ptn_vox_decoder': ptn_vox_decoder,
- 'perspective_projector': perspective_projector,
-}
-
-
-def _get_network(name):
- """Gets a single encoder/decoder network model."""
-
- if name not in _NAME_TO_NETS:
- raise ValueError('Network name [%s] not recognized.' % name)
- return _NAME_TO_NETS[name].model
-
-
-def get(params, is_training=False, reuse=False, run_projection=True):
- """Factory function to get the training/pretraining im->vox model (NIPS16).
-
- Args:
- params: Different parameters used througout ptn, typically FLAGS (dict).
- is_training: Set to True if while training (boolean).
- reuse: Set as True if sharing variables with a model that has already
- been built (boolean).
- run_projection: Set as False if not interested in mask and projection
- images. Useful in evaluation routine (boolean).
- Returns:
- Model function for network (inputs to outputs).
- """
- def model(inputs):
- """Model function corresponding to a specific network architecture."""
- outputs = {}
-
- # First, build the encoder
- encoder_fn = _get_network(params.encoder_name)
- with tf.variable_scope('encoder', reuse=reuse):
- # Produces id/pose units
- enc_outputs = encoder_fn(inputs['images_1'], params, is_training)
- outputs['ids_1'] = enc_outputs['ids']
-
- # Second, build the decoder and projector
- decoder_fn = _get_network(params.decoder_name)
- with tf.variable_scope('decoder', reuse=reuse):
- outputs['voxels_1'] = decoder_fn(outputs['ids_1'], params, is_training)
- if run_projection:
- projector_fn = _get_network(params.projector_name)
- with tf.variable_scope('projector', reuse=reuse):
- outputs['projs_1'] = projector_fn(
- outputs['voxels_1'], inputs['matrix_1'], params, is_training)
- # Infer the ground-truth mask
- with tf.variable_scope('oracle', reuse=reuse):
- outputs['masks_1'] = projector_fn(inputs['voxels'], inputs['matrix_1'],
- params, False)
-
- # Third, build the entire graph (bundled strategy described in PTN paper)
- for k in range(1, params.step_size):
- with tf.variable_scope('projector', reuse=True):
- outputs['projs_%d' % (k + 1)] = projector_fn(
- outputs['voxels_1'], inputs['matrix_%d' %
- (k + 1)], params, is_training)
- with tf.variable_scope('oracle', reuse=True):
- outputs['masks_%d' % (k + 1)] = projector_fn(
- inputs['voxels'], inputs['matrix_%d' % (k + 1)], params, False)
-
- return outputs
-
- return model
diff --git a/research/ptn/nets/perspective_projector.py b/research/ptn/nets/perspective_projector.py
deleted file mode 100644
index 38c7df86b203884327d4c4eda5f02b7fc1b16323..0000000000000000000000000000000000000000
--- a/research/ptn/nets/perspective_projector.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""3D->2D projector model as used in PTN (NIPS16)."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-from nets import perspective_transform
-
-
-def model(voxels, transform_matrix, params, is_training):
- """Model transforming the 3D voxels into 2D projections.
-
- Args:
- voxels: A tensor of size [batch, depth, height, width, channel]
- representing the input of projection layer (tf.float32).
- transform_matrix: A tensor of size [batch, 16] representing
- the flattened 4-by-4 matrix for transformation (tf.float32).
- params: Model parameters (dict).
- is_training: Set to True if while training (boolean).
-
- Returns:
- A transformed tensor (tf.float32)
-
- """
- del is_training # Doesn't make a difference for projector
- # Rearrangement (batch, z, y, x, channel) --> (batch, y, z, x, channel).
- # By the standard, projection happens along z-axis but the voxels
- # are stored in a different way. So we need to switch the y and z
- # axis for transformation operation.
- voxels = tf.transpose(voxels, [0, 2, 1, 3, 4])
- z_near = params.focal_length
- z_far = params.focal_length + params.focal_range
- transformed_voxels = perspective_transform.transformer(
- voxels, transform_matrix, [params.vox_size] * 3, z_near, z_far)
- views = tf.reduce_max(transformed_voxels, [1])
- views = tf.reverse(views, [1])
- return views
diff --git a/research/ptn/nets/perspective_transform.py b/research/ptn/nets/perspective_transform.py
deleted file mode 100644
index 1c01f15f21d4f6f36c4638d0b8ad916e895aa0b2..0000000000000000000000000000000000000000
--- a/research/ptn/nets/perspective_transform.py
+++ /dev/null
@@ -1,278 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Perspective Transformer Layer Implementation.
-
-Transform the volume based on 4 x 4 perspective projection matrix.
-
-Reference:
-(1) "Perspective Transformer Nets: Perspective Transformer Nets:
-Learning Single-View 3D Object Reconstruction without 3D Supervision."
-Xinchen Yan, Jimei Yang, Ersin Yumer, Yijie Guo, Honglak Lee. In NIPS 2016
-https://papers.nips.cc/paper/6206-perspective-transformer-nets-learning-single-view-3d-object-reconstruction-without-3d-supervision.pdf
-
-(2) Official implementation in Torch: https://github.com/xcyan/ptnbhwd
-
-(3) 2D Transformer implementation in TF:
-github.com/tensorflow/models/tree/master/research/transformer
-
-"""
-
-import tensorflow as tf
-
-
-def transformer(voxels,
- theta,
- out_size,
- z_near,
- z_far,
- name='PerspectiveTransformer'):
- """Perspective Transformer Layer.
-
- Args:
- voxels: A tensor of size [num_batch, depth, height, width, num_channels].
- It is the output of a deconv/upsampling conv network (tf.float32).
- theta: A tensor of size [num_batch, 16].
- It is the inverse camera transformation matrix (tf.float32).
- out_size: A tuple representing the size of output of
- transformer layer (float).
- z_near: A number representing the near clipping plane (float).
- z_far: A number representing the far clipping plane (float).
-
- Returns:
- A transformed tensor (tf.float32).
-
- """
- def _repeat(x, n_repeats):
- with tf.variable_scope('_repeat'):
- rep = tf.transpose(
- tf.expand_dims(tf.ones(shape=tf.stack([
- n_repeats,
- ])), 1), [1, 0])
- rep = tf.to_int32(rep)
- x = tf.matmul(tf.reshape(x, (-1, 1)), rep)
- return tf.reshape(x, [-1])
-
- def _interpolate(im, x, y, z, out_size):
- """Bilinear interploation layer.
-
- Args:
- im: A 5D tensor of size [num_batch, depth, height, width, num_channels].
- It is the input volume for the transformation layer (tf.float32).
- x: A tensor of size [num_batch, out_depth, out_height, out_width]
- representing the inverse coordinate mapping for x (tf.float32).
- y: A tensor of size [num_batch, out_depth, out_height, out_width]
- representing the inverse coordinate mapping for y (tf.float32).
- z: A tensor of size [num_batch, out_depth, out_height, out_width]
- representing the inverse coordinate mapping for z (tf.float32).
- out_size: A tuple representing the output size of transformation layer
- (float).
-
- Returns:
- A transformed tensor (tf.float32).
-
- """
- with tf.variable_scope('_interpolate'):
- num_batch = im.get_shape().as_list()[0]
- depth = im.get_shape().as_list()[1]
- height = im.get_shape().as_list()[2]
- width = im.get_shape().as_list()[3]
- channels = im.get_shape().as_list()[4]
-
- x = tf.to_float(x)
- y = tf.to_float(y)
- z = tf.to_float(z)
- depth_f = tf.to_float(depth)
- height_f = tf.to_float(height)
- width_f = tf.to_float(width)
- # Number of disparity interpolated.
- out_depth = out_size[0]
- out_height = out_size[1]
- out_width = out_size[2]
- zero = tf.zeros([], dtype='int32')
- # 0 <= z < depth, 0 <= y < height & 0 <= x < width.
- max_z = tf.to_int32(tf.shape(im)[1] - 1)
- max_y = tf.to_int32(tf.shape(im)[2] - 1)
- max_x = tf.to_int32(tf.shape(im)[3] - 1)
-
- # Converts scale indices from [-1, 1] to [0, width/height/depth].
- x = (x + 1.0) * (width_f) / 2.0
- y = (y + 1.0) * (height_f) / 2.0
- z = (z + 1.0) * (depth_f) / 2.0
-
- x0 = tf.to_int32(tf.floor(x))
- x1 = x0 + 1
- y0 = tf.to_int32(tf.floor(y))
- y1 = y0 + 1
- z0 = tf.to_int32(tf.floor(z))
- z1 = z0 + 1
-
- x0_clip = tf.clip_by_value(x0, zero, max_x)
- x1_clip = tf.clip_by_value(x1, zero, max_x)
- y0_clip = tf.clip_by_value(y0, zero, max_y)
- y1_clip = tf.clip_by_value(y1, zero, max_y)
- z0_clip = tf.clip_by_value(z0, zero, max_z)
- z1_clip = tf.clip_by_value(z1, zero, max_z)
- dim3 = width
- dim2 = width * height
- dim1 = width * height * depth
- base = _repeat(
- tf.range(num_batch) * dim1, out_depth * out_height * out_width)
- base_z0_y0 = base + z0_clip * dim2 + y0_clip * dim3
- base_z0_y1 = base + z0_clip * dim2 + y1_clip * dim3
- base_z1_y0 = base + z1_clip * dim2 + y0_clip * dim3
- base_z1_y1 = base + z1_clip * dim2 + y1_clip * dim3
-
- idx_z0_y0_x0 = base_z0_y0 + x0_clip
- idx_z0_y0_x1 = base_z0_y0 + x1_clip
- idx_z0_y1_x0 = base_z0_y1 + x0_clip
- idx_z0_y1_x1 = base_z0_y1 + x1_clip
- idx_z1_y0_x0 = base_z1_y0 + x0_clip
- idx_z1_y0_x1 = base_z1_y0 + x1_clip
- idx_z1_y1_x0 = base_z1_y1 + x0_clip
- idx_z1_y1_x1 = base_z1_y1 + x1_clip
-
- # Use indices to lookup pixels in the flat image and restore
- # channels dim
- im_flat = tf.reshape(im, tf.stack([-1, channels]))
- im_flat = tf.to_float(im_flat)
- i_z0_y0_x0 = tf.gather(im_flat, idx_z0_y0_x0)
- i_z0_y0_x1 = tf.gather(im_flat, idx_z0_y0_x1)
- i_z0_y1_x0 = tf.gather(im_flat, idx_z0_y1_x0)
- i_z0_y1_x1 = tf.gather(im_flat, idx_z0_y1_x1)
- i_z1_y0_x0 = tf.gather(im_flat, idx_z1_y0_x0)
- i_z1_y0_x1 = tf.gather(im_flat, idx_z1_y0_x1)
- i_z1_y1_x0 = tf.gather(im_flat, idx_z1_y1_x0)
- i_z1_y1_x1 = tf.gather(im_flat, idx_z1_y1_x1)
-
- # Finally calculate interpolated values.
- x0_f = tf.to_float(x0)
- x1_f = tf.to_float(x1)
- y0_f = tf.to_float(y0)
- y1_f = tf.to_float(y1)
- z0_f = tf.to_float(z0)
- z1_f = tf.to_float(z1)
- # Check the out-of-boundary case.
- x0_valid = tf.to_float(
- tf.less_equal(x0, max_x) & tf.greater_equal(x0, 0))
- x1_valid = tf.to_float(
- tf.less_equal(x1, max_x) & tf.greater_equal(x1, 0))
- y0_valid = tf.to_float(
- tf.less_equal(y0, max_y) & tf.greater_equal(y0, 0))
- y1_valid = tf.to_float(
- tf.less_equal(y1, max_y) & tf.greater_equal(y1, 0))
- z0_valid = tf.to_float(
- tf.less_equal(z0, max_z) & tf.greater_equal(z0, 0))
- z1_valid = tf.to_float(
- tf.less_equal(z1, max_z) & tf.greater_equal(z1, 0))
-
- w_z0_y0_x0 = tf.expand_dims(((x1_f - x) * (y1_f - y) *
- (z1_f - z) * x1_valid * y1_valid * z1_valid),
- 1)
- w_z0_y0_x1 = tf.expand_dims(((x - x0_f) * (y1_f - y) *
- (z1_f - z) * x0_valid * y1_valid * z1_valid),
- 1)
- w_z0_y1_x0 = tf.expand_dims(((x1_f - x) * (y - y0_f) *
- (z1_f - z) * x1_valid * y0_valid * z1_valid),
- 1)
- w_z0_y1_x1 = tf.expand_dims(((x - x0_f) * (y - y0_f) *
- (z1_f - z) * x0_valid * y0_valid * z1_valid),
- 1)
- w_z1_y0_x0 = tf.expand_dims(((x1_f - x) * (y1_f - y) *
- (z - z0_f) * x1_valid * y1_valid * z0_valid),
- 1)
- w_z1_y0_x1 = tf.expand_dims(((x - x0_f) * (y1_f - y) *
- (z - z0_f) * x0_valid * y1_valid * z0_valid),
- 1)
- w_z1_y1_x0 = tf.expand_dims(((x1_f - x) * (y - y0_f) *
- (z - z0_f) * x1_valid * y0_valid * z0_valid),
- 1)
- w_z1_y1_x1 = tf.expand_dims(((x - x0_f) * (y - y0_f) *
- (z - z0_f) * x0_valid * y0_valid * z0_valid),
- 1)
-
- output = tf.add_n([
- w_z0_y0_x0 * i_z0_y0_x0, w_z0_y0_x1 * i_z0_y0_x1,
- w_z0_y1_x0 * i_z0_y1_x0, w_z0_y1_x1 * i_z0_y1_x1,
- w_z1_y0_x0 * i_z1_y0_x0, w_z1_y0_x1 * i_z1_y0_x1,
- w_z1_y1_x0 * i_z1_y1_x0, w_z1_y1_x1 * i_z1_y1_x1
- ])
- return output
-
- def _meshgrid(depth, height, width, z_near, z_far):
- with tf.variable_scope('_meshgrid'):
- x_t = tf.reshape(
- tf.tile(tf.linspace(-1.0, 1.0, width), [height * depth]),
- [depth, height, width])
- y_t = tf.reshape(
- tf.tile(tf.linspace(-1.0, 1.0, height), [width * depth]),
- [depth, width, height])
- y_t = tf.transpose(y_t, [0, 2, 1])
- sample_grid = tf.tile(
- tf.linspace(float(z_near), float(z_far), depth), [width * height])
- z_t = tf.reshape(sample_grid, [height, width, depth])
- z_t = tf.transpose(z_t, [2, 0, 1])
-
- z_t = 1 / z_t
- d_t = 1 / z_t
- x_t /= z_t
- y_t /= z_t
-
- x_t_flat = tf.reshape(x_t, (1, -1))
- y_t_flat = tf.reshape(y_t, (1, -1))
- d_t_flat = tf.reshape(d_t, (1, -1))
-
- ones = tf.ones_like(x_t_flat)
- grid = tf.concat([d_t_flat, y_t_flat, x_t_flat, ones], 0)
- return grid
-
- def _transform(theta, input_dim, out_size, z_near, z_far):
- with tf.variable_scope('_transform'):
- num_batch = input_dim.get_shape().as_list()[0]
- num_channels = input_dim.get_shape().as_list()[4]
- theta = tf.reshape(theta, (-1, 4, 4))
- theta = tf.cast(theta, 'float32')
-
- out_depth = out_size[0]
- out_height = out_size[1]
- out_width = out_size[2]
- grid = _meshgrid(out_depth, out_height, out_width, z_near, z_far)
- grid = tf.expand_dims(grid, 0)
- grid = tf.reshape(grid, [-1])
- grid = tf.tile(grid, tf.stack([num_batch]))
- grid = tf.reshape(grid, tf.stack([num_batch, 4, -1]))
-
- # Transform A x (x_t', y_t', 1, d_t)^T -> (x_s, y_s, z_s, 1).
- t_g = tf.matmul(theta, grid)
- z_s = tf.slice(t_g, [0, 0, 0], [-1, 1, -1])
- y_s = tf.slice(t_g, [0, 1, 0], [-1, 1, -1])
- x_s = tf.slice(t_g, [0, 2, 0], [-1, 1, -1])
-
- z_s_flat = tf.reshape(z_s, [-1])
- y_s_flat = tf.reshape(y_s, [-1])
- x_s_flat = tf.reshape(x_s, [-1])
-
- input_transformed = _interpolate(input_dim, x_s_flat, y_s_flat, z_s_flat,
- out_size)
-
- output = tf.reshape(
- input_transformed,
- tf.stack([num_batch, out_depth, out_height, out_width, num_channels]))
-
- return output
-
- with tf.variable_scope(name):
- output = _transform(theta, voxels, out_size, z_near, z_far)
- return output
diff --git a/research/ptn/nets/ptn_encoder.py b/research/ptn/nets/ptn_encoder.py
deleted file mode 100644
index ede556834e6ea42e9e0a266bc0d525a679924077..0000000000000000000000000000000000000000
--- a/research/ptn/nets/ptn_encoder.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Training/Pretraining encoder as used in PTN (NIPS16)."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-
-def _preprocess(images):
- return images * 2 - 1
-
-
-def model(images, params, is_training):
- """Model encoding the images into view-invariant embedding."""
- del is_training # Unused
- image_size = images.get_shape().as_list()[1]
- f_dim = params.f_dim
- fc_dim = params.fc_dim
- z_dim = params.z_dim
- outputs = dict()
-
- images = _preprocess(images)
- with slim.arg_scope(
- [slim.conv2d, slim.fully_connected],
- weights_initializer=tf.truncated_normal_initializer(stddev=0.02, seed=1)):
- h0 = slim.conv2d(images, f_dim, [5, 5], stride=2, activation_fn=tf.nn.relu)
- h1 = slim.conv2d(h0, f_dim * 2, [5, 5], stride=2, activation_fn=tf.nn.relu)
- h2 = slim.conv2d(h1, f_dim * 4, [5, 5], stride=2, activation_fn=tf.nn.relu)
- # Reshape layer
- s8 = image_size // 8
- h2 = tf.reshape(h2, [-1, s8 * s8 * f_dim * 4])
- h3 = slim.fully_connected(h2, fc_dim, activation_fn=tf.nn.relu)
- h4 = slim.fully_connected(h3, fc_dim, activation_fn=tf.nn.relu)
-
- outputs['ids'] = slim.fully_connected(h4, z_dim, activation_fn=tf.nn.relu)
- outputs['poses'] = slim.fully_connected(h4, z_dim, activation_fn=tf.nn.relu)
- return outputs
diff --git a/research/ptn/nets/ptn_im_decoder.py b/research/ptn/nets/ptn_im_decoder.py
deleted file mode 100644
index 8ee512e878d549c2d16f13f58dee8786e129e751..0000000000000000000000000000000000000000
--- a/research/ptn/nets/ptn_im_decoder.py
+++ /dev/null
@@ -1,81 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Image/Mask decoder used while pretraining the network."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-_FEATURE_MAP_SIZE = 8
-
-
-def _postprocess_im(images):
- """Performs post-processing for the images returned from conv net.
-
- Transforms the value from [-1, 1] to [0, 1].
- """
- return (images + 1) * 0.5
-
-
-def model(identities, poses, params, is_training):
- """Decoder model to get image and mask from latent embedding."""
- del is_training
- f_dim = params.f_dim
- fc_dim = params.fc_dim
-
- outputs = dict()
-
- with slim.arg_scope(
- [slim.fully_connected, slim.conv2d_transpose],
- weights_initializer=tf.truncated_normal_initializer(stddev=0.02, seed=1)):
- # Concatenate the identity and pose units
- h0 = tf.concat([identities, poses], 1)
- h0 = slim.fully_connected(h0, fc_dim, activation_fn=tf.nn.relu)
- h1 = slim.fully_connected(h0, fc_dim, activation_fn=tf.nn.relu)
-
- # Mask decoder
- dec_m0 = slim.fully_connected(
- h1, (_FEATURE_MAP_SIZE**2) * f_dim * 2, activation_fn=tf.nn.relu)
- dec_m0 = tf.reshape(
- dec_m0, [-1, _FEATURE_MAP_SIZE, _FEATURE_MAP_SIZE, f_dim * 2])
-
- dec_m1 = slim.conv2d_transpose(
- dec_m0, f_dim, [5, 5], stride=2, activation_fn=tf.nn.relu)
- dec_m2 = slim.conv2d_transpose(
- dec_m1, int(f_dim / 2), [5, 5], stride=2, activation_fn=tf.nn.relu)
- dec_m3 = slim.conv2d_transpose(
- dec_m2, 1, [5, 5], stride=2, activation_fn=tf.nn.sigmoid)
-
- # Image decoder
- dec_i0 = slim.fully_connected(
- h1, (_FEATURE_MAP_SIZE**2) * f_dim * 4, activation_fn=tf.nn.relu)
- dec_i0 = tf.reshape(
- dec_i0, [-1, _FEATURE_MAP_SIZE, _FEATURE_MAP_SIZE, f_dim * 4])
-
- dec_i1 = slim.conv2d_transpose(
- dec_i0, f_dim * 2, [5, 5], stride=2, activation_fn=tf.nn.relu)
- dec_i2 = slim.conv2d_transpose(
- dec_i1, f_dim * 2, [5, 5], stride=2, activation_fn=tf.nn.relu)
- dec_i3 = slim.conv2d_transpose(
- dec_i2, 3, [5, 5], stride=2, activation_fn=tf.nn.tanh)
-
- outputs = dict()
- outputs['images'] = _postprocess_im(dec_i3)
- outputs['masks'] = dec_m3
- return outputs
diff --git a/research/ptn/nets/ptn_rotator.py b/research/ptn/nets/ptn_rotator.py
deleted file mode 100644
index 2cc73bb8dfe2edb624d9fa56ecd95347fbf0cf3f..0000000000000000000000000000000000000000
--- a/research/ptn/nets/ptn_rotator.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Creates rotator network model.
-
-This model performs the out-of-plane rotations given input image and action.
-The action is either no-op, rotate clockwise or rotate counter-clockwise.
-
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-
-def bilinear(input_x, input_y, output_size):
- """Define the bilinear transformation layer."""
- shape_x = input_x.get_shape().as_list()
- shape_y = input_y.get_shape().as_list()
-
- weights_initializer = tf.truncated_normal_initializer(stddev=0.02,
- seed=1)
- biases_initializer = tf.constant_initializer(0.0)
-
- matrix = tf.get_variable("Matrix", [shape_x[1], shape_y[1], output_size],
- tf.float32, initializer=weights_initializer)
- bias = tf.get_variable("Bias", [output_size],
- initializer=biases_initializer)
- # Add to GraphKeys.MODEL_VARIABLES
- tf.contrib.framework.add_model_variable(matrix)
- tf.contrib.framework.add_model_variable(bias)
- # Define the transformation
- h0 = tf.matmul(input_x, tf.reshape(matrix,
- [shape_x[1], shape_y[1]*output_size]))
- h0 = tf.reshape(h0, [-1, shape_y[1], output_size])
- h1 = tf.tile(tf.reshape(input_y, [-1, shape_y[1], 1]),
- [1, 1, output_size])
- h1 = tf.multiply(h0, h1)
- return tf.reduce_sum(h1, 1) + bias
-
-
-def model(poses, actions, params, is_training):
- """Model for performing rotation."""
- del is_training # Unused
- return bilinear(poses, actions, params.z_dim)
diff --git a/research/ptn/nets/ptn_vox_decoder.py b/research/ptn/nets/ptn_vox_decoder.py
deleted file mode 100644
index 87ea27fa2bfcffc6fd9b9292686096e06f36e6a6..0000000000000000000000000000000000000000
--- a/research/ptn/nets/ptn_vox_decoder.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Training decoder as used in PTN (NIPS16)."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-slim = tf.contrib.slim
-
-
-@tf.contrib.framework.add_arg_scope
-def conv3d_transpose(inputs,
- num_outputs,
- kernel_size,
- stride=1,
- padding='SAME',
- activation_fn=tf.nn.relu,
- weights_initializer=tf.contrib.layers.xavier_initializer(),
- biases_initializer=tf.zeros_initializer(),
- reuse=None,
- trainable=True,
- scope=None):
- """Wrapper for conv3d_transpose layer.
-
- This function wraps the tf.conv3d_transpose with basic non-linearity.
- Tt creates a variable called `weights`, representing the kernel,
- that is convoled with the input. A second varibale called `biases'
- is added to the result of operation.
- """
- with tf.variable_scope(
- scope, 'Conv3d_transpose', [inputs], reuse=reuse):
- dtype = inputs.dtype.base_dtype
- kernel_d, kernel_h, kernel_w = kernel_size[0:3]
- num_filters_in = inputs.get_shape()[4]
-
- weights_shape = [kernel_d, kernel_h, kernel_w, num_outputs, num_filters_in]
- weights = tf.get_variable('weights',
- shape=weights_shape,
- dtype=dtype,
- initializer=weights_initializer,
- trainable=trainable)
- tf.contrib.framework.add_model_variable(weights)
-
- input_shape = inputs.get_shape().as_list()
- batch_size = input_shape[0]
- depth = input_shape[1]
- height = input_shape[2]
- width = input_shape[3]
-
- def get_deconv_dim(dim_size, stride_size):
- # Only support padding='SAME'.
- if isinstance(dim_size, tf.Tensor):
- dim_size = tf.multiply(dim_size, stride_size)
- elif dim_size is not None:
- dim_size *= stride_size
- return dim_size
-
- out_depth = get_deconv_dim(depth, stride)
- out_height = get_deconv_dim(height, stride)
- out_width = get_deconv_dim(width, stride)
-
- out_shape = [batch_size, out_depth, out_height, out_width, num_outputs]
- outputs = tf.nn.conv3d_transpose(inputs, weights, out_shape,
- [1, stride, stride, stride, 1],
- padding=padding)
-
- outputs.set_shape(out_shape)
-
- if biases_initializer is not None:
- biases = tf.get_variable('biases',
- shape=[num_outputs,],
- dtype=dtype,
- initializer=biases_initializer,
- trainable=trainable)
- tf.contrib.framework.add_model_variable(biases)
- outputs = tf.nn.bias_add(outputs, biases)
-
- if activation_fn:
- outputs = activation_fn(outputs)
- return outputs
-
-
-def model(identities, params, is_training):
- """Model transforming embedding to voxels."""
- del is_training # Unused
- f_dim = params.f_dim
-
- # Please refer to the original implementation: github.com/xcyan/nips16_PTN
- # In TF replication, we use a slightly different architecture.
- with slim.arg_scope(
- [slim.fully_connected, conv3d_transpose],
- weights_initializer=tf.truncated_normal_initializer(stddev=0.02, seed=1)):
- h0 = slim.fully_connected(
- identities, 4 * 4 * 4 * f_dim * 8, activation_fn=tf.nn.relu)
- h1 = tf.reshape(h0, [-1, 4, 4, 4, f_dim * 8])
- h1 = conv3d_transpose(
- h1, f_dim * 4, [4, 4, 4], stride=2, activation_fn=tf.nn.relu)
- h2 = conv3d_transpose(
- h1, int(f_dim * 3 / 2), [5, 5, 5], stride=2, activation_fn=tf.nn.relu)
- h3 = conv3d_transpose(
- h2, 1, [6, 6, 6], stride=2, activation_fn=tf.nn.sigmoid)
- return h3
diff --git a/research/ptn/pretrain_rotator.py b/research/ptn/pretrain_rotator.py
deleted file mode 100644
index 6307f2d4f6ad4341105de7e265411d10e55f61c9..0000000000000000000000000000000000000000
--- a/research/ptn/pretrain_rotator.py
+++ /dev/null
@@ -1,236 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contains training plan for the Rotator model (Pretraining in NIPS16)."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import numpy as np
-from six.moves import xrange
-import tensorflow as tf
-
-from tensorflow import app
-
-import model_rotator as model
-
-flags = tf.app.flags
-slim = tf.contrib.slim
-
-flags.DEFINE_string('inp_dir', '',
- 'Directory path containing the input data (tfrecords).')
-flags.DEFINE_string(
- 'dataset_name', 'shapenet_chair',
- 'Dataset name that is to be used for training and evaluation.')
-flags.DEFINE_integer('z_dim', 512, '')
-flags.DEFINE_integer('a_dim', 3, '')
-flags.DEFINE_integer('f_dim', 64, '')
-flags.DEFINE_integer('fc_dim', 1024, '')
-flags.DEFINE_integer('num_views', 24, 'Num of viewpoints in the input data.')
-flags.DEFINE_integer('image_size', 64,
- 'Input images dimension (pixels) - width & height.')
-flags.DEFINE_integer('step_size', 1, 'Steps to take for rotation in pretraining.')
-flags.DEFINE_integer('batch_size', 32, 'Batch size for training.')
-flags.DEFINE_string('encoder_name', 'ptn_encoder',
- 'Name of the encoder network being used.')
-flags.DEFINE_string('decoder_name', 'ptn_im_decoder',
- 'Name of the decoder network being used.')
-flags.DEFINE_string('rotator_name', 'ptn_rotator',
- 'Name of the rotator network being used.')
-# Save options
-flags.DEFINE_string('checkpoint_dir', '/tmp/ptn_train/',
- 'Directory path for saving trained models and other data.')
-flags.DEFINE_string('model_name', 'deeprotator_pretrain',
- 'Name of the model used in naming the TF job. Must be different for each run.')
-flags.DEFINE_string('init_model', None,
- 'Checkpoint path of the model to initialize with.')
-flags.DEFINE_integer('save_every', 1000,
- 'Average period of steps after which we save a model.')
-# Optimization
-flags.DEFINE_float('image_weight', 10, 'Weighting factor for image loss.')
-flags.DEFINE_float('mask_weight', 1, 'Weighting factor for mask loss.')
-flags.DEFINE_float('learning_rate', 0.0001, 'Learning rate.')
-flags.DEFINE_float('weight_decay', 0.001, 'Weight decay parameter while training.')
-flags.DEFINE_float('clip_gradient_norm', 0, 'Gradient clim norm, leave 0 if no gradient clipping.')
-flags.DEFINE_integer('max_number_of_steps', 320000, 'Maximum number of steps for training.')
-# Summary
-flags.DEFINE_integer('save_summaries_secs', 15, 'Seconds interval for dumping TF summaries.')
-flags.DEFINE_integer('save_interval_secs', 60 * 5, 'Seconds interval to save models.')
-# Distribution
-flags.DEFINE_string('master', '', 'The address of the tensorflow master if running distributed.')
-flags.DEFINE_bool('sync_replicas', False, 'Whether to sync gradients between replicas for optimizer.')
-flags.DEFINE_integer('worker_replicas', 1, 'Number of worker replicas (train tasks).')
-flags.DEFINE_integer('backup_workers', 0, 'Number of backup workers.')
-flags.DEFINE_integer('ps_tasks', 0, 'Number of ps tasks.')
-flags.DEFINE_integer('task', 0,
- 'Task identifier flag to be set for each task running in distributed manner. Task number 0 '
- 'will be chosen as the chief.')
-
-FLAGS = flags.FLAGS
-
-
-def main(_):
- train_dir = os.path.join(FLAGS.checkpoint_dir, FLAGS.model_name, 'train')
- save_image_dir = os.path.join(train_dir, 'images')
- if not os.path.exists(train_dir):
- os.makedirs(train_dir)
- if not os.path.exists(save_image_dir):
- os.makedirs(save_image_dir)
-
- g = tf.Graph()
- with g.as_default():
- with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
- global_step = slim.get_or_create_global_step()
- ##########
- ## data ##
- ##########
- train_data = model.get_inputs(
- FLAGS.inp_dir,
- FLAGS.dataset_name,
- 'train',
- FLAGS.batch_size,
- FLAGS.image_size,
- is_training=True)
- inputs = model.preprocess(train_data, FLAGS.step_size)
- ###########
- ## model ##
- ###########
- model_fn = model.get_model_fn(FLAGS, is_training=True)
- outputs = model_fn(inputs)
- ##########
- ## loss ##
- ##########
- task_loss = model.get_loss(inputs, outputs, FLAGS)
- regularization_loss = model.get_regularization_loss(
- ['encoder', 'rotator', 'decoder'], FLAGS)
- loss = task_loss + regularization_loss
- ###############
- ## optimizer ##
- ###############
- optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
- if FLAGS.sync_replicas:
- optimizer = tf.train.SyncReplicasOptimizer(
- optimizer,
- replicas_to_aggregate=FLAGS.workers_replicas - FLAGS.backup_workers,
- total_num_replicas=FLAGS.worker_replicas)
-
- ##############
- ## train_op ##
- ##############
- train_op = model.get_train_op_for_scope(
- loss, optimizer, ['encoder', 'rotator', 'decoder'], FLAGS)
- ###########
- ## saver ##
- ###########
- saver = tf.train.Saver(max_to_keep=np.minimum(5,
- FLAGS.worker_replicas + 1))
-
- if FLAGS.task == 0:
- val_data = model.get_inputs(
- FLAGS.inp_dir,
- FLAGS.dataset_name,
- 'val',
- FLAGS.batch_size,
- FLAGS.image_size,
- is_training=False)
- val_inputs = model.preprocess(val_data, FLAGS.step_size)
- # Note: don't compute loss here
- reused_model_fn = model.get_model_fn(
- FLAGS, is_training=False, reuse=True)
- val_outputs = reused_model_fn(val_inputs)
- with tf.device(tf.DeviceSpec(device_type='CPU')):
- if FLAGS.step_size == 1:
- vis_input_images = val_inputs['images_0'] * 255.0
- vis_output_images = val_inputs['images_1'] * 255.0
- vis_pred_images = val_outputs['images_1'] * 255.0
- vis_pred_masks = (val_outputs['masks_1'] * (-1) + 1) * 255.0
- else:
- rep_times = int(np.ceil(32.0 / float(FLAGS.step_size)))
- vis_list_1 = []
- vis_list_2 = []
- vis_list_3 = []
- vis_list_4 = []
- for j in xrange(rep_times):
- for k in xrange(FLAGS.step_size):
- vis_input_image = val_inputs['images_0'][j],
- vis_output_image = val_inputs['images_%d' % (k + 1)][j]
- vis_pred_image = val_outputs['images_%d' % (k + 1)][j]
- vis_pred_mask = val_outputs['masks_%d' % (k + 1)][j]
- vis_list_1.append(tf.expand_dims(vis_input_image, 0))
- vis_list_2.append(tf.expand_dims(vis_output_image, 0))
- vis_list_3.append(tf.expand_dims(vis_pred_image, 0))
- vis_list_4.append(tf.expand_dims(vis_pred_mask, 0))
-
- vis_list_1 = tf.reshape(
- tf.stack(vis_list_1), [
- rep_times * FLAGS.step_size, FLAGS.image_size,
- FLAGS.image_size, 3
- ])
- vis_list_2 = tf.reshape(
- tf.stack(vis_list_2), [
- rep_times * FLAGS.step_size, FLAGS.image_size,
- FLAGS.image_size, 3
- ])
- vis_list_3 = tf.reshape(
- tf.stack(vis_list_3), [
- rep_times * FLAGS.step_size, FLAGS.image_size,
- FLAGS.image_size, 3
- ])
- vis_list_4 = tf.reshape(
- tf.stack(vis_list_4), [
- rep_times * FLAGS.step_size, FLAGS.image_size,
- FLAGS.image_size, 1
- ])
-
- vis_input_images = vis_list_1 * 255.0
- vis_output_images = vis_list_2 * 255.0
- vis_pred_images = vis_list_3 * 255.0
- vis_pred_masks = (vis_list_4 * (-1) + 1) * 255.0
-
- write_disk_op = model.write_disk_grid(
- global_step=global_step,
- summary_freq=FLAGS.save_every,
- log_dir=save_image_dir,
- input_images=vis_input_images,
- output_images=vis_output_images,
- pred_images=vis_pred_images,
- pred_masks=vis_pred_masks)
- with tf.control_dependencies([write_disk_op]):
- train_op = tf.identity(train_op)
-
- #############
- ## init_fn ##
- #############
- init_fn = model.get_init_fn(['encoder, ' 'rotator', 'decoder'], FLAGS)
-
- ##############
- ## training ##
- ##############
- slim.learning.train(
- train_op=train_op,
- logdir=train_dir,
- init_fn=init_fn,
- master=FLAGS.master,
- is_chief=(FLAGS.task == 0),
- number_of_steps=FLAGS.max_number_of_steps,
- saver=saver,
- save_summaries_secs=FLAGS.save_summaries_secs,
- save_interval_secs=FLAGS.save_interval_secs)
-
-
-if __name__ == '__main__':
- app.run()
diff --git a/research/ptn/train_ptn.py b/research/ptn/train_ptn.py
deleted file mode 100644
index 1b42245d4c2d7fc445e275aa4e933e89e6108699..0000000000000000000000000000000000000000
--- a/research/ptn/train_ptn.py
+++ /dev/null
@@ -1,230 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Contains training plan for the Im2vox model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-import numpy as np
-import tensorflow as tf
-
-from tensorflow import app
-
-import model_ptn
-
-flags = tf.app.flags
-slim = tf.contrib.slim
-
-flags.DEFINE_string('inp_dir',
- '',
- 'Directory path containing the input data (tfrecords).')
-flags.DEFINE_string(
- 'dataset_name', 'shapenet_chair',
- 'Dataset name that is to be used for training and evaluation.')
-flags.DEFINE_integer('z_dim', 512, '')
-flags.DEFINE_integer('f_dim', 64, '')
-flags.DEFINE_integer('fc_dim', 1024, '')
-flags.DEFINE_integer('num_views', 24, 'Num of viewpoints in the input data.')
-flags.DEFINE_integer('image_size', 64,
- 'Input images dimension (pixels) - width & height.')
-flags.DEFINE_integer('vox_size', 32, 'Voxel prediction dimension.')
-flags.DEFINE_integer('step_size', 24, 'Steps to take in rotation to fetch viewpoints.')
-flags.DEFINE_integer('batch_size', 6, 'Batch size while training.')
-flags.DEFINE_float('focal_length', 0.866, 'Focal length parameter used in perspective projection.')
-flags.DEFINE_float('focal_range', 1.732, 'Focal length parameter used in perspective projection.')
-flags.DEFINE_string('encoder_name', 'ptn_encoder',
- 'Name of the encoder network being used.')
-flags.DEFINE_string('decoder_name', 'ptn_vox_decoder',
- 'Name of the decoder network being used.')
-flags.DEFINE_string('projector_name', 'perspective_projector',
- 'Name of the projector network being used.')
-# Save options
-flags.DEFINE_string('checkpoint_dir', '/tmp/ptn_train/',
- 'Directory path for saving trained models and other data.')
-flags.DEFINE_string('model_name', 'ptn_finetune',
- 'Name of the model used in naming the TF job. Must be different for each run.')
-flags.DEFINE_string('init_model', None,
- 'Checkpoint path of the model to initialize with.')
-flags.DEFINE_integer('save_every', 1000,
- 'Average period of steps after which we save a model.')
-# Optimization
-flags.DEFINE_float('proj_weight', 10, 'Weighting factor for projection loss.')
-flags.DEFINE_float('volume_weight', 0, 'Weighting factor for volume loss.')
-flags.DEFINE_float('viewpoint_weight', 1, 'Weighting factor for viewpoint loss.')
-flags.DEFINE_float('learning_rate', 0.0001, 'Learning rate.')
-flags.DEFINE_float('weight_decay', 0.001, 'Weight decay parameter while training.')
-flags.DEFINE_float('clip_gradient_norm', 0, 'Gradient clim norm, leave 0 if no gradient clipping.')
-flags.DEFINE_integer('max_number_of_steps', 10000, 'Maximum number of steps for training.')
-# Summary
-flags.DEFINE_integer('save_summaries_secs', 15, 'Seconds interval for dumping TF summaries.')
-flags.DEFINE_integer('save_interval_secs', 60 * 5, 'Seconds interval to save models.')
-
-# Scheduling
-flags.DEFINE_string('master', '', 'The address of the tensorflow master')
-flags.DEFINE_bool('sync_replicas', False, 'Whether to sync gradients between replicas for optimizer.')
-flags.DEFINE_integer('worker_replicas', 1, 'Number of worker replicas (train tasks).')
-flags.DEFINE_integer('backup_workers', 0, 'Number of backup workers.')
-flags.DEFINE_integer('ps_tasks', 0, 'Number of ps tasks.')
-flags.DEFINE_integer('task', 0,
- 'Task identifier flag to be set for each task running in distributed manner. Task number 0 '
- 'will be chosen as the chief.')
-
-FLAGS = flags.FLAGS
-
-
-def main(_):
- train_dir = os.path.join(FLAGS.checkpoint_dir, FLAGS.model_name, 'train')
- save_image_dir = os.path.join(train_dir, 'images')
- if not os.path.exists(train_dir):
- os.makedirs(train_dir)
- if not os.path.exists(save_image_dir):
- os.makedirs(save_image_dir)
-
- g = tf.Graph()
- with g.as_default():
- with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
- global_step = slim.get_or_create_global_step()
- ###########
- ## model ##
- ###########
- model = model_ptn.model_PTN(FLAGS)
- ##########
- ## data ##
- ##########
- train_data = model.get_inputs(
- FLAGS.inp_dir,
- FLAGS.dataset_name,
- 'train',
- FLAGS.batch_size,
- FLAGS.image_size,
- FLAGS.vox_size,
- is_training=True)
- inputs = model.preprocess(train_data, FLAGS.step_size)
- ##############
- ## model_fn ##
- ##############
- model_fn = model.get_model_fn(
- is_training=True, reuse=False, run_projection=True)
- outputs = model_fn(inputs)
- ##################
- ## train_scopes ##
- ##################
- if FLAGS.init_model:
- train_scopes = ['decoder']
- init_scopes = ['encoder']
- else:
- train_scopes = ['encoder', 'decoder']
-
- ##########
- ## loss ##
- ##########
- task_loss = model.get_loss(inputs, outputs)
-
- regularization_loss = model.get_regularization_loss(train_scopes)
- loss = task_loss + regularization_loss
- ###############
- ## optimizer ##
- ###############
- optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
- if FLAGS.sync_replicas:
- optimizer = tf.train.SyncReplicasOptimizer(
- optimizer,
- replicas_to_aggregate=FLAGS.workers_replicas - FLAGS.backup_workers,
- total_num_replicas=FLAGS.worker_replicas)
-
- ##############
- ## train_op ##
- ##############
- train_op = model.get_train_op_for_scope(loss, optimizer, train_scopes)
- ###########
- ## saver ##
- ###########
- saver = tf.train.Saver(max_to_keep=np.minimum(5,
- FLAGS.worker_replicas + 1))
-
- if FLAGS.task == 0:
- params = FLAGS
- params.batch_size = params.num_views
- params.step_size = 1
- model.set_params(params)
- val_data = model.get_inputs(
- params.inp_dir,
- params.dataset_name,
- 'val',
- params.batch_size,
- params.image_size,
- params.vox_size,
- is_training=False)
- val_inputs = model.preprocess(val_data, params.step_size)
- # Note: don't compute loss here
- reused_model_fn = model.get_model_fn(is_training=False, reuse=True)
- val_outputs = reused_model_fn(val_inputs)
-
- with tf.device(tf.DeviceSpec(device_type='CPU')):
- vis_input_images = val_inputs['images_1'] * 255.0
- vis_gt_projs = (val_outputs['masks_1'] * (-1) + 1) * 255.0
- vis_pred_projs = (val_outputs['projs_1'] * (-1) + 1) * 255.0
-
- vis_gt_projs = tf.concat([vis_gt_projs] * 3, axis=3)
- vis_pred_projs = tf.concat([vis_pred_projs] * 3, axis=3)
- # rescale
- new_size = [FLAGS.image_size] * 2
- vis_gt_projs = tf.image.resize_nearest_neighbor(
- vis_gt_projs, new_size)
- vis_pred_projs = tf.image.resize_nearest_neighbor(
- vis_pred_projs, new_size)
- # flip
- # vis_gt_projs = utils.image_flipud(vis_gt_projs)
- # vis_pred_projs = utils.image_flipud(vis_pred_projs)
- # vis_gt_projs is of shape [batch, height, width, channels]
- write_disk_op = model.write_disk_grid(
- global_step=global_step,
- log_dir=save_image_dir,
- input_images=vis_input_images,
- gt_projs=vis_gt_projs,
- pred_projs=vis_pred_projs,
- input_voxels=val_inputs['voxels'],
- output_voxels=val_outputs['voxels_1'])
- with tf.control_dependencies([write_disk_op]):
- train_op = tf.identity(train_op)
-
- #############
- ## init_fn ##
- #############
- if FLAGS.init_model:
- init_fn = model.get_init_fn(init_scopes)
- else:
- init_fn = None
-
- ##############
- ## training ##
- ##############
- slim.learning.train(
- train_op=train_op,
- logdir=train_dir,
- init_fn=init_fn,
- master=FLAGS.master,
- is_chief=(FLAGS.task == 0),
- number_of_steps=FLAGS.max_number_of_steps,
- saver=saver,
- save_summaries_secs=FLAGS.save_summaries_secs,
- save_interval_secs=FLAGS.save_interval_secs)
-
-
-if __name__ == '__main__':
- app.run()
diff --git a/research/ptn/utils.py b/research/ptn/utils.py
deleted file mode 100644
index adf71731edb78740c6716d7abddfd77b557aaecd..0000000000000000000000000000000000000000
--- a/research/ptn/utils.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utility functions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import StringIO
-import matplotlib
-matplotlib.use('Agg')
-from matplotlib import pylab as p
-# axes3d is being used implictly for visualization.
-from mpl_toolkits.mplot3d import axes3d as p3 # pylint:disable=unused-import
-import numpy as np
-from PIL import Image
-from skimage import measure
-from six.moves import xrange
-
-import tensorflow as tf
-
-
-def save_image(inp_array, image_file):
- """Function that dumps the image to disk."""
- inp_array = np.clip(inp_array, 0, 255).astype(np.uint8)
- image = Image.fromarray(inp_array)
- buf = StringIO.StringIO()
- image.save(buf, format='JPEG')
- with open(image_file, 'w') as f:
- f.write(buf.getvalue())
- return None
-
-
-def image_flipud(images):
- """Function that flip (up-down) the np image."""
- quantity = images.get_shape().as_list()[0]
- image_list = []
- for k in xrange(quantity):
- image_list.append(tf.image.flip_up_down(images[k, :, :, :]))
- outputs = tf.stack(image_list)
- return outputs
-
-
-def resize_image(inp_array, new_height, new_width):
- """Function that resize the np image."""
- inp_array = np.clip(inp_array, 0, 255).astype(np.uint8)
- image = Image.fromarray(inp_array)
- # Reverse order
- image = image.resize((new_width, new_height))
- return np.array(image)
-
-
-def display_voxel(points, vis_size=128):
- """Function to display 3D voxel."""
- try:
- data = visualize_voxel_spectral(points, vis_size)
- except ValueError:
- data = visualize_voxel_scatter(points, vis_size)
- return data
-
-
-def visualize_voxel_spectral(points, vis_size=128):
- """Function to visualize voxel (spectral)."""
- points = np.rint(points)
- points = np.swapaxes(points, 0, 2)
- fig = p.figure(figsize=(1, 1), dpi=vis_size)
- verts, faces = measure.marching_cubes_classic(points, 0, spacing=(0.1, 0.1, 0.1))
- ax = fig.add_subplot(111, projection='3d')
- ax.plot_trisurf(
- verts[:, 0], verts[:, 1], faces, verts[:, 2], cmap='Spectral_r', lw=0.1)
- ax.set_axis_off()
- fig.tight_layout(pad=0)
- fig.canvas.draw()
- data = np.fromstring(
- fig.canvas.tostring_rgb(), dtype=np.uint8, sep='').reshape(
- vis_size, vis_size, 3)
- p.close('all')
- return data
-
-
-def visualize_voxel_scatter(points, vis_size=128):
- """Function to visualize voxel (scatter)."""
- points = np.rint(points)
- points = np.swapaxes(points, 0, 2)
- fig = p.figure(figsize=(1, 1), dpi=vis_size)
- ax = fig.add_subplot(111, projection='3d')
- x = []
- y = []
- z = []
- (x_dimension, y_dimension, z_dimension) = points.shape
- for i in range(x_dimension):
- for j in range(y_dimension):
- for k in range(z_dimension):
- if points[i, j, k]:
- x.append(i)
- y.append(j)
- z.append(k)
- ax.scatter3D(x, y, z)
- ax.set_axis_off()
- fig.tight_layout(pad=0)
- fig.canvas.draw()
- data = np.fromstring(
- fig.canvas.tostring_rgb(), dtype=np.uint8, sep='').reshape(
- vis_size, vis_size, 3)
- p.close('all')
- return data
diff --git a/research/qa_kg/README.md b/research/qa_kg/README.md
deleted file mode 100644
index 7224ac8f8f5ff2f6c0003c5bbd9ac1717fe7addf..0000000000000000000000000000000000000000
--- a/research/qa_kg/README.md
+++ /dev/null
@@ -1,83 +0,0 @@
-
-
-
-
-# Module networks for question answering on knowledge graph
-
-This code repository contains a TensorFlow model for question answering on
-knowledge graph with end-to-end module networks. The original paper describing
-end-to-end module networks is as follows.
-
-R. Hu, J. Andreas, M. Rohrbach, T. Darrell, K. Saenko, *Learning to Reason:
-End-to-End Module Networks for Visual Question Answering*. in arXiv preprint
-arXiv:1704.05526, 2017. ([PDF](https://arxiv.org/pdf/1704.05526.pdf))
-
-```
-@article{hu2017learning,
- title={Learning to Reason: End-to-End Module Networks for Visual Question Answering},
- author={Hu, Ronghang and Andreas, Jacob and Rohrbach, Marcus and Darrell, Trevor and Saenko, Kate},
- journal={arXiv preprint arXiv:1704.05526},
- year={2017}
-}
-```
-
-The code in this repository is based on the original
-[implementation](https://github.com/ronghanghu/n2nmn) for this paper.
-
-## Requirements
-
-1. Install TensorFlow 1.0.0. Follow the [official
- guide](https://www.tensorflow.org/install/). Please note that newer or older
- versions of TensorFlow may fail to work due to incompatibility with
- TensorFlow Fold.
-2. Install TensorFlow Fold. Follow the
- [setup instructions](https://github.com/tensorflow/fold/blob/master/tensorflow_fold/g3doc/setup.md).
- TensorFlow Fold only supports Linux platform. We have not tested
- the code on other platforms.
-
-## Data
-
-1. Download the [MetaQA dataset](https://goo.gl/f3AmcY). Click the button
- `MetaQA` and then click `Download` in the drop-down list. Extract the zip
- file after downloading completed. Read the documents there for dataset
- details.
-2. Move the `MetaQA` folder to the root directory of this repository.
-
-## How to use this code
-
-We provide an experiment folder `exp_1_hop`, which applies the implemented model
-to the 1-hop vanilla dataset in MetaQA. More experiment folders are coming soon.
-
-Currently, we provide code for training with ground truth layout, and testing
-the saved model. Configurations can be modified in `config.py`. They can also be
-set via command line parameters.
-
-To train the model:
-
-```
-python exp_1_hop/train_gt_layout.py
-```
-
-To test the saved model (need to provide the snapshot name):
-
-```
-python exp_1_hop/test.py --snapshot_name 00010000
-```
-
-## Model introduction
-
-1. In this model, we store the knowledge graph in a key-value based memory. For
- each knowledge graph edge (subject, relation, object), we use the (subject,
- relation) as the key and the object as the value.
-2. All entities and relations are embedded as fixed-dimension vectors. These
- embeddings are also end-to-end learned.
-3. Neural modules can separately operate on either the key side or the value
- side.
-4. The attention is shared between keys and corresponding values.
-5. The answer output is based on the attention-weighted sum over keys or
- values, depending on the output module.
-
-## Contact
-Authors: Yuyu Zhang, Xin Pan
-
-Pull requests and issues: @yuyuz
diff --git a/research/qa_kg/exp_1_hop/config.py b/research/qa_kg/exp_1_hop/config.py
deleted file mode 100644
index 95d8cf5f512243279365ad99c9f582c945d6c8e2..0000000000000000000000000000000000000000
--- a/research/qa_kg/exp_1_hop/config.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import argparse
-import os
-
-
-def str2bool(v):
- return v.lower() in ('true', '1')
-
-
-def add_argument_group(name):
- arg = parser.add_argument_group(name)
- arg_lists.append(arg)
- return arg
-
-
-def get_config():
- config, unparsed = parser.parse_known_args()
- return config, unparsed
-
-
-arg_lists = []
-parser = argparse.ArgumentParser()
-work_dir = os.path.abspath(os.path.join(__file__, '../../'))
-
-net_arg = add_argument_group('Network')
-net_arg.add_argument('--lstm_dim', type=int, default=128)
-net_arg.add_argument('--num_layers', type=int, default=1)
-net_arg.add_argument('--embed_dim_txt', type=int, default=128)
-net_arg.add_argument('--embed_dim_nmn', type=int, default=128)
-net_arg.add_argument(
- '--T_encoder', type=int, default=0) # will be updated when reading data
-net_arg.add_argument('--T_decoder', type=int, default=5)
-
-train_arg = add_argument_group('Training')
-train_arg.add_argument('--train_tag', type=str, default='n2nmn')
-train_arg.add_argument('--batch_size', type=int, default=128)
-train_arg.add_argument('--max_iter', type=int, default=1000000)
-train_arg.add_argument('--weight_decay', type=float, default=1e-5)
-train_arg.add_argument('--baseline_decay', type=float, default=0.99)
-train_arg.add_argument('--max_grad_norm', type=float, default=10)
-train_arg.add_argument('--random_seed', type=int, default=123)
-
-data_arg = add_argument_group('Data')
-data_path = work_dir + '/MetaQA/'
-data_arg.add_argument('--KB_file', type=str, default=data_path + 'kb.txt')
-data_arg.add_argument(
- '--data_dir', type=str, default=data_path + '1-hop/vanilla/')
-data_arg.add_argument('--train_data_file', type=str, default='qa_train.txt')
-data_arg.add_argument('--dev_data_file', type=str, default='qa_dev.txt')
-data_arg.add_argument('--test_data_file', type=str, default='qa_test.txt')
-
-exp_arg = add_argument_group('Experiment')
-exp_path = work_dir + '/exp_1_hop/'
-exp_arg.add_argument('--exp_dir', type=str, default=exp_path)
-
-log_arg = add_argument_group('Log')
-log_arg.add_argument('--log_dir', type=str, default='logs')
-log_arg.add_argument('--log_interval', type=int, default=1000)
-log_arg.add_argument('--num_log_samples', type=int, default=3)
-log_arg.add_argument(
- '--log_level', type=str, default='INFO', choices=['INFO', 'DEBUG', 'WARN'])
-
-io_arg = add_argument_group('IO')
-io_arg.add_argument('--model_dir', type=str, default='model')
-io_arg.add_argument('--snapshot_interval', type=int, default=1000)
-io_arg.add_argument('--output_dir', type=str, default='output')
diff --git a/research/qa_kg/exp_1_hop/test.py b/research/qa_kg/exp_1_hop/test.py
deleted file mode 100644
index 2937c0d582a54f6de2702a4f741963e0fe2b2f72..0000000000000000000000000000000000000000
--- a/research/qa_kg/exp_1_hop/test.py
+++ /dev/null
@@ -1,135 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import os
-import sys
-sys.path.append(os.path.abspath(os.path.join(__file__, '../../')))
-import numpy as np
-import tensorflow as tf
-from config import get_config
-from model_n2nmn.assembler import Assembler
-from model_n2nmn.model import Model
-from util.data_reader import DataReader
-from util.data_reader import SampleBuilder
-from util.misc import prepare_dirs_and_logger
-
-FLAGS = tf.flags.FLAGS
-tf.flags.DEFINE_string('snapshot_name', '00001000', 'snapshot file name')
-
-
-def main(_):
- config = prepare_dirs_and_logger(config_raw)
-
- rng = np.random.RandomState(config.random_seed)
- tf.set_random_seed(config.random_seed)
- config.rng = rng
-
- config.module_names = ['_key_find', '_key_filter', '_val_desc', '']
- config.gt_layout_tokens = ['_key_find', '_key_filter', '_val_desc', '']
- assembler = Assembler(config)
-
- sample_builder = SampleBuilder(config)
- config = sample_builder.config # update T_encoder according to data
- data_test = sample_builder.data_all['test']
- data_reader_test = DataReader(
- config, data_test, assembler, shuffle=False, one_pass=True)
-
- num_vocab_txt = len(sample_builder.dict_all)
- num_vocab_nmn = len(assembler.module_names)
- num_choices = len(sample_builder.dict_all)
-
- # Network inputs
- text_seq_batch = tf.placeholder(tf.int32, [None, None])
- seq_len_batch = tf.placeholder(tf.int32, [None])
-
- # The model
- model = Model(
- config,
- sample_builder.kb,
- text_seq_batch,
- seq_len_batch,
- num_vocab_txt=num_vocab_txt,
- num_vocab_nmn=num_vocab_nmn,
- EOS_idx=assembler.EOS_idx,
- num_choices=num_choices,
- decoder_sampling=False)
- compiler = model.compiler
- scores = model.scores
-
- sess = tf.Session()
- sess.run(tf.global_variables_initializer())
- snapshot_file = os.path.join(config.model_dir, FLAGS.snapshot_name)
- tf.logging.info('Snapshot file: %s' % snapshot_file)
-
- snapshot_saver = tf.train.Saver()
- snapshot_saver.restore(sess, snapshot_file)
-
- # Evaluation metrics
- num_questions = len(data_test.Y)
- tf.logging.info('# of test questions: %d' % num_questions)
-
- answer_correct = 0
- layout_correct = 0
- layout_valid = 0
- for batch in data_reader_test.batches():
- # set up input and output tensors
- h = sess.partial_run_setup(
- fetches=[model.predicted_tokens, scores],
- feeds=[text_seq_batch, seq_len_batch, compiler.loom_input_tensor])
-
- # Part 1: Generate module layout
- tokens = sess.partial_run(
- h,
- fetches=model.predicted_tokens,
- feed_dict={
- text_seq_batch: batch['input_seq_batch'],
- seq_len_batch: batch['seq_len_batch']
- })
-
- # Compute accuracy of the predicted layout
- gt_tokens = batch['gt_layout_batch']
- layout_correct += np.sum(
- np.all(
- np.logical_or(tokens == gt_tokens, gt_tokens == assembler.EOS_idx),
- axis=0))
-
- # Assemble the layout tokens into network structure
- expr_list, expr_validity_array = assembler.assemble(tokens)
- layout_valid += np.sum(expr_validity_array)
- labels = batch['ans_label_batch']
- # Build TensorFlow Fold input for NMN
- expr_feed = compiler.build_feed_dict(expr_list)
-
- # Part 2: Run NMN and learning steps
- scores_val = sess.partial_run(h, scores, feed_dict=expr_feed)
-
- # Compute accuracy
- predictions = np.argmax(scores_val, axis=1)
- answer_correct += np.sum(
- np.logical_and(expr_validity_array, predictions == labels))
-
- answer_accuracy = answer_correct * 1.0 / num_questions
- layout_accuracy = layout_correct * 1.0 / num_questions
- layout_validity = layout_valid * 1.0 / num_questions
-
- tf.logging.info('test answer accuracy = %f, '
- 'test layout accuracy = %f, '
- 'test layout validity = %f' %
- (answer_accuracy, layout_accuracy, layout_validity))
-
-
-if __name__ == '__main__':
- config_raw, unparsed = get_config()
- tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/research/qa_kg/exp_1_hop/train_gt_layout.py b/research/qa_kg/exp_1_hop/train_gt_layout.py
deleted file mode 100644
index 02bafc428afc4d1f2b39d6bd56e6098d4b1b8ca7..0000000000000000000000000000000000000000
--- a/research/qa_kg/exp_1_hop/train_gt_layout.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import os
-import sys
-sys.path.append(os.path.abspath(os.path.join(__file__, '../../')))
-import numpy as np
-import tensorflow as tf
-from config import get_config
-from model_n2nmn.assembler import Assembler
-from model_n2nmn.model import Model
-from util.data_reader import DataReader
-from util.data_reader import SampleBuilder
-from util.misc import prepare_dirs_and_logger
-from util.misc import save_config
-from util.misc import show_all_variables
-
-
-def main(_):
- config = prepare_dirs_and_logger(config_raw)
- save_config(config)
-
- rng = np.random.RandomState(config.random_seed)
- tf.set_random_seed(config.random_seed)
- config.rng = rng
-
- config.module_names = ['_key_find', '_key_filter', '_val_desc', '']
- config.gt_layout_tokens = ['_key_find', '_key_filter', '_val_desc', '']
- assembler = Assembler(config)
-
- sample_builder = SampleBuilder(config)
- config = sample_builder.config # update T_encoder according to data
- data_train = sample_builder.data_all['train']
- data_reader_train = DataReader(
- config, data_train, assembler, shuffle=True, one_pass=False)
-
- num_vocab_txt = len(sample_builder.dict_all)
- num_vocab_nmn = len(assembler.module_names)
- num_choices = len(sample_builder.dict_all)
-
- # Network inputs
- text_seq_batch = tf.placeholder(tf.int32, [None, None])
- seq_len_batch = tf.placeholder(tf.int32, [None])
- ans_label_batch = tf.placeholder(tf.int32, [None])
- use_gt_layout = tf.constant(True, dtype=tf.bool)
- gt_layout_batch = tf.placeholder(tf.int32, [None, None])
-
- # The model for training
- model = Model(
- config,
- sample_builder.kb,
- text_seq_batch,
- seq_len_batch,
- num_vocab_txt=num_vocab_txt,
- num_vocab_nmn=num_vocab_nmn,
- EOS_idx=assembler.EOS_idx,
- num_choices=num_choices,
- decoder_sampling=True,
- use_gt_layout=use_gt_layout,
- gt_layout_batch=gt_layout_batch)
- compiler = model.compiler
- scores = model.scores
- log_seq_prob = model.log_seq_prob
-
- # Loss function
- softmax_loss_per_sample = tf.nn.sparse_softmax_cross_entropy_with_logits(
- logits=scores, labels=ans_label_batch)
- # The final per-sample loss, which is loss for valid expr
- # and invalid_expr_loss for invalid expr
- final_loss_per_sample = softmax_loss_per_sample # All exprs are valid
-
- avg_sample_loss = tf.reduce_mean(final_loss_per_sample)
- seq_likelihood_loss = tf.reduce_mean(-log_seq_prob)
-
- total_training_loss = seq_likelihood_loss + avg_sample_loss
- total_loss = total_training_loss + config.weight_decay * model.l2_reg
-
- # Train with Adam optimizer
- solver = tf.train.AdamOptimizer()
- gradients = solver.compute_gradients(total_loss)
-
- # Clip gradient by L2 norm
- gradients = [(tf.clip_by_norm(g, config.max_grad_norm), v)
- for g, v in gradients]
- solver_op = solver.apply_gradients(gradients)
-
- # Training operation
- with tf.control_dependencies([solver_op]):
- train_step = tf.constant(0)
-
- # Write summary to TensorBoard
- log_writer = tf.summary.FileWriter(config.log_dir, tf.get_default_graph())
-
- loss_ph = tf.placeholder(tf.float32, [])
- entropy_ph = tf.placeholder(tf.float32, [])
- accuracy_ph = tf.placeholder(tf.float32, [])
- summary_train = [
- tf.summary.scalar('avg_sample_loss', loss_ph),
- tf.summary.scalar('entropy', entropy_ph),
- tf.summary.scalar('avg_accuracy', accuracy_ph)
- ]
- log_step_train = tf.summary.merge(summary_train)
-
- # Training
- sess = tf.Session()
- sess.run(tf.global_variables_initializer())
- snapshot_saver = tf.train.Saver(max_to_keep=None) # keep all snapshots
- show_all_variables()
-
- avg_accuracy = 0
- accuracy_decay = 0.99
- for n_iter, batch in enumerate(data_reader_train.batches()):
- if n_iter >= config.max_iter:
- break
-
- # set up input and output tensors
- h = sess.partial_run_setup(
- fetches=[
- model.predicted_tokens, model.entropy_reg, scores, avg_sample_loss,
- train_step
- ],
- feeds=[
- text_seq_batch, seq_len_batch, gt_layout_batch,
- compiler.loom_input_tensor, ans_label_batch
- ])
-
- # Part 1: Generate module layout
- tokens, entropy_reg_val = sess.partial_run(
- h,
- fetches=(model.predicted_tokens, model.entropy_reg),
- feed_dict={
- text_seq_batch: batch['input_seq_batch'],
- seq_len_batch: batch['seq_len_batch'],
- gt_layout_batch: batch['gt_layout_batch']
- })
- # Assemble the layout tokens into network structure
- expr_list, expr_validity_array = assembler.assemble(tokens)
- # all exprs should be valid (since they are ground-truth)
- assert np.all(expr_validity_array)
- labels = batch['ans_label_batch']
- # Build TensorFlow Fold input for NMN
- expr_feed = compiler.build_feed_dict(expr_list)
- expr_feed[ans_label_batch] = labels
-
- # Part 2: Run NMN and learning steps
- scores_val, avg_sample_loss_val, _ = sess.partial_run(
- h, fetches=(scores, avg_sample_loss, train_step), feed_dict=expr_feed)
-
- # Compute accuracy
- predictions = np.argmax(scores_val, axis=1)
- accuracy = np.mean(
- np.logical_and(expr_validity_array, predictions == labels))
- avg_accuracy += (1 - accuracy_decay) * (accuracy - avg_accuracy)
-
- # Add to TensorBoard summary
- if (n_iter + 1) % config.log_interval == 0:
- tf.logging.info('iter = %d\n\t'
- 'loss = %f, accuracy (cur) = %f, '
- 'accuracy (avg) = %f, entropy = %f' %
- (n_iter + 1, avg_sample_loss_val, accuracy, avg_accuracy,
- -entropy_reg_val))
- summary = sess.run(
- fetches=log_step_train,
- feed_dict={
- loss_ph: avg_sample_loss_val,
- entropy_ph: -entropy_reg_val,
- accuracy_ph: avg_accuracy
- })
- log_writer.add_summary(summary, n_iter + 1)
-
- # Save snapshot
- if (n_iter + 1) % config.snapshot_interval == 0:
- snapshot_file = os.path.join(config.model_dir, '%08d' % (n_iter + 1))
- snapshot_saver.save(sess, snapshot_file, write_meta_graph=False)
- tf.logging.info('Snapshot saved to %s' % snapshot_file)
-
- tf.logging.info('Run finished.')
-
-
-if __name__ == '__main__':
- config_raw, unparsed = get_config()
- tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/research/qa_kg/model_n2nmn/__init__.py b/research/qa_kg/model_n2nmn/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/qa_kg/model_n2nmn/assembler.py b/research/qa_kg/model_n2nmn/assembler.py
deleted file mode 100644
index f5839f6f49d7d30774195749405d43ed014a0049..0000000000000000000000000000000000000000
--- a/research/qa_kg/model_n2nmn/assembler.py
+++ /dev/null
@@ -1,145 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import numpy as np
-
-# the number of attention input to each module
-_module_input_num = {
- '_key_find': 0,
- '_key_filter': 1,
- '_val_desc': 1}
-_module_output_type = {
- '_key_find': 'att',
- '_key_filter': 'att',
- '_val_desc': 'ans'
-}
-
-INVALID_EXPR = 'INVALID_EXPR'
-
-
-class Assembler:
-
- def __init__(self, config):
- # read the module list, and record the index of each module and
- self.module_names = config.module_names
- # find the index of
- for n_s in range(len(self.module_names)):
- if self.module_names[n_s] == '':
- self.EOS_idx = n_s
- break
- # build a dictionary from module name to token index
- self.name2idx_dict = {
- name: n_s
- for n_s, name in enumerate(self.module_names)
- }
-
- def module_list2tokens(self, module_list, max_len=None):
- layout_tokens = [self.name2idx_dict[name] for name in module_list]
- if max_len is not None:
- if len(module_list) >= max_len:
- raise ValueError('Not enough time steps to add ')
- layout_tokens += [self.EOS_idx] * (max_len - len(module_list))
- return layout_tokens
-
- def _layout_tokens2str(self, layout_tokens):
- return ' '.join([self.module_names[idx] for idx in layout_tokens])
-
- def _invalid_expr(self, layout_tokens, error_str):
- return {
- 'module': INVALID_EXPR,
- 'expr_str': self._layout_tokens2str(layout_tokens),
- 'error': error_str
- }
-
- def _assemble_layout_tokens(self, layout_tokens, batch_idx):
- # Every module takes a time_idx as the index from LSTM hidden states
- # (even if it doesn't need it, like _and), and different arity of
- # attention inputs. The output type can be either attention or answer
- #
- # The final assembled expression for each instance is as follows:
- # expr_type :=
- # {'module': '_find', 'output_type': 'att', 'time_idx': idx}
- # | {'module': '_relocate', 'output_type': 'att', 'time_idx': idx,
- # 'inputs_0': }
- # | {'module': '_and', 'output_type': 'att', 'time_idx': idx,
- # 'inputs_0': , 'inputs_1': )}
- # | {'module': '_describe', 'output_type': 'ans', 'time_idx': idx,
- # 'inputs_0': }
- # | {'module': INVALID_EXPR, 'expr_str': '...', 'error': '...',
- # 'assembly_loss': } (for invalid expressions)
- #
-
- # A valid layout must contain . Assembly fails if it doesn't.
- if not np.any(layout_tokens == self.EOS_idx):
- return self._invalid_expr(layout_tokens, 'cannot find ')
-
- # Decoding Reverse Polish Notation with a stack
- decoding_stack = []
- for t in range(len(layout_tokens)):
- # decode a module/operation
- module_idx = layout_tokens[t]
- if module_idx == self.EOS_idx:
- break
- module_name = self.module_names[module_idx]
- expr = {
- 'module': module_name,
- 'output_type': _module_output_type[module_name],
- 'time_idx': t,
- 'batch_idx': batch_idx
- }
-
- input_num = _module_input_num[module_name]
- # Check if there are enough input in the stack
- if len(decoding_stack) < input_num:
- # Invalid expression. Not enough input.
- return self._invalid_expr(layout_tokens,
- 'not enough input for ' + module_name)
-
- # Get the input from stack
- for n_input in range(input_num - 1, -1, -1):
- stack_top = decoding_stack.pop()
- if stack_top['output_type'] != 'att':
- # Invalid expression. Input must be attention
- return self._invalid_expr(layout_tokens,
- 'input incompatible for ' + module_name)
- expr['input_%d' % n_input] = stack_top
-
- decoding_stack.append(expr)
-
- # After decoding the reverse polish expression, there should be exactly
- # one expression in the stack
- if len(decoding_stack) != 1:
- return self._invalid_expr(
- layout_tokens,
- 'final stack size not equal to 1 (%d remains)' % len(decoding_stack))
-
- result = decoding_stack[0]
- # The result type should be answer, not attention
- if result['output_type'] != 'ans':
- return self._invalid_expr(layout_tokens,
- 'result type must be ans, not att')
- return result
-
- def assemble(self, layout_tokens_batch):
- # layout_tokens_batch is a numpy array with shape [max_dec_len, batch_size],
- # containing module tokens and , in Reverse Polish Notation.
- _, batch_size = layout_tokens_batch.shape
- expr_list = [
- self._assemble_layout_tokens(layout_tokens_batch[:, batch_i], batch_i)
- for batch_i in range(batch_size)
- ]
- expr_validity = np.array(
- [expr['module'] != INVALID_EXPR for expr in expr_list], np.bool)
- return expr_list, expr_validity
diff --git a/research/qa_kg/model_n2nmn/model.py b/research/qa_kg/model_n2nmn/model.py
deleted file mode 100644
index 56896f438006ac28c82d8e92ded2d7bbf8cf3863..0000000000000000000000000000000000000000
--- a/research/qa_kg/model_n2nmn/model.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import numpy as np
-import tensorflow as tf
-import tensorflow_fold as td
-from model_n2nmn import netgen_att
-from model_n2nmn import assembler
-from model_n2nmn.modules import Modules
-
-
-class Model:
-
- def __init__(self,
- config,
- kb,
- text_seq_batch,
- seq_length_batch,
- num_vocab_txt,
- num_vocab_nmn,
- EOS_idx,
- num_choices,
- decoder_sampling,
- use_gt_layout=None,
- gt_layout_batch=None,
- scope='neural_module_network',
- reuse=None):
- with tf.variable_scope(scope, reuse=reuse):
- # Part 1: Seq2seq RNN to generate module layout tokens
-
- embedding_mat = tf.get_variable(
- 'embedding_mat', [num_vocab_txt, config.embed_dim_txt],
- initializer=tf.contrib.layers.xavier_initializer())
-
- with tf.variable_scope('layout_generation'):
- att_seq2seq = netgen_att.AttentionSeq2Seq(
- config, text_seq_batch, seq_length_batch, num_vocab_txt,
- num_vocab_nmn, EOS_idx, decoder_sampling, embedding_mat,
- use_gt_layout, gt_layout_batch)
- self.att_seq2seq = att_seq2seq
- predicted_tokens = att_seq2seq.predicted_tokens
- token_probs = att_seq2seq.token_probs
- word_vecs = att_seq2seq.word_vecs
- neg_entropy = att_seq2seq.neg_entropy
- self.atts = att_seq2seq.atts
-
- self.predicted_tokens = predicted_tokens
- self.token_probs = token_probs
- self.word_vecs = word_vecs
- self.neg_entropy = neg_entropy
-
- # log probability of each generated sequence
- self.log_seq_prob = tf.reduce_sum(tf.log(token_probs), axis=0)
-
- # Part 2: Neural Module Network
- with tf.variable_scope('layout_execution'):
- modules = Modules(config, kb, word_vecs, num_choices, embedding_mat)
- self.modules = modules
- # Recursion of modules
- att_shape = [len(kb)]
- # Forward declaration of module recursion
- att_expr_decl = td.ForwardDeclaration(td.PyObjectType(),
- td.TensorType(att_shape))
- # _key_find
- case_key_find = td.Record([('time_idx', td.Scalar(dtype='int32')),
- ('batch_idx', td.Scalar(dtype='int32'))])
- case_key_find = case_key_find >> td.ScopedLayer(
- modules.KeyFindModule, name_or_scope='KeyFindModule')
- # _key_filter
- case_key_filter = td.Record([('input_0', att_expr_decl()),
- ('time_idx', td.Scalar('int32')),
- ('batch_idx', td.Scalar('int32'))])
- case_key_filter = case_key_filter >> td.ScopedLayer(
- modules.KeyFilterModule, name_or_scope='KeyFilterModule')
- recursion_cases = td.OneOf(
- td.GetItem('module'),
- {'_key_find': case_key_find,
- '_key_filter': case_key_filter})
- att_expr_decl.resolve_to(recursion_cases)
- # _val_desc: output scores for choice (for valid expressions)
- predicted_scores = td.Record([('input_0', recursion_cases),
- ('time_idx', td.Scalar('int32')),
- ('batch_idx', td.Scalar('int32'))])
- predicted_scores = predicted_scores >> td.ScopedLayer(
- modules.ValDescribeModule, name_or_scope='ValDescribeModule')
-
- # For invalid expressions, define a dummy answer
- # so that all answers have the same form
- INVALID = assembler.INVALID_EXPR
- dummy_scores = td.Void() >> td.FromTensor(
- np.zeros(num_choices, np.float32))
- output_scores = td.OneOf(
- td.GetItem('module'),
- {'_val_desc': predicted_scores,
- INVALID: dummy_scores})
-
- # compile and get the output scores
- self.compiler = td.Compiler.create(output_scores)
- self.scores = self.compiler.output_tensors[0]
-
- # Regularization: Entropy + L2
- self.entropy_reg = tf.reduce_mean(neg_entropy)
- module_weights = [
- v for v in tf.trainable_variables()
- if (scope in v.op.name and v.op.name.endswith('weights'))
- ]
- self.l2_reg = tf.add_n([tf.nn.l2_loss(v) for v in module_weights])
diff --git a/research/qa_kg/model_n2nmn/modules.py b/research/qa_kg/model_n2nmn/modules.py
deleted file mode 100644
index 8c7a7370f81a5b81d08c87136688765dce556ada..0000000000000000000000000000000000000000
--- a/research/qa_kg/model_n2nmn/modules.py
+++ /dev/null
@@ -1,131 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import tensorflow as tf
-
-
-class Modules:
-
- def __init__(self, config, kb, word_vecs, num_choices, embedding_mat):
- self.config = config
-
- self.embedding_mat = embedding_mat
-
- # kb has shape [N_kb, 3]
- self.kb = kb
- self.embed_keys_e, self.embed_keys_r, self.embed_vals_e = self.embed_kb()
-
- # word_vecs has shape [T_decoder, N, D_txt]
- self.word_vecs = word_vecs
- self.num_choices = num_choices
-
- def embed_kb(self):
- keys_e, keys_r, vals_e = [], [], []
- for idx_sub, idx_rel, idx_obj in self.kb:
- keys_e.append(idx_sub)
- keys_r.append(idx_rel)
- vals_e.append(idx_obj)
- embed_keys_e = tf.nn.embedding_lookup(self.embedding_mat, keys_e)
- embed_keys_r = tf.nn.embedding_lookup(self.embedding_mat, keys_r)
- embed_vals_e = tf.nn.embedding_lookup(self.embedding_mat, vals_e)
- return embed_keys_e, embed_keys_r, embed_vals_e
-
- def _slice_word_vecs(self, time_idx, batch_idx):
- # this callable will be wrapped into a td.Function
- # In TF Fold, batch_idx and time_idx are both [N_batch, 1] tensors
- # time is highest dim in word_vecs
- joint_index = tf.stack([time_idx, batch_idx], axis=1)
- return tf.gather_nd(self.word_vecs, joint_index)
-
- # All the layers are wrapped with td.ScopedLayer
- def KeyFindModule(self,
- time_idx,
- batch_idx,
- scope='KeyFindModule',
- reuse=None):
- # In TF Fold, batch_idx and time_idx are both [N_batch, 1] tensors
- text_param = self._slice_word_vecs(time_idx, batch_idx)
-
- # Mapping: embed_keys_e x text_param -> att
- # Input:
- # embed_keys_e: [N_kb, D_txt]
- # text_param: [N, D_txt]
- # Output:
- # att: [N, N_kb]
- #
- # Implementation:
- # 1. Elementwise multiplication between embed_key_e and text_param
- # 2. L2-normalization
- with tf.variable_scope(scope, reuse=reuse):
- m = tf.matmul(text_param, self.embed_keys_e, transpose_b=True)
- att = tf.nn.l2_normalize(m, dim=1)
- return att
-
- def KeyFilterModule(self,
- input_0,
- time_idx,
- batch_idx,
- scope='KeyFilterModule',
- reuse=None):
- att_0 = input_0
- text_param = self._slice_word_vecs(time_idx, batch_idx)
-
- # Mapping: and(embed_keys_r x text_param, att) -> att
- # Input:
- # embed_keys_r: [N_kb, D_txt]
- # text_param: [N, D_txt]
- # att_0: [N, N_kb]
- # Output:
- # att: [N, N_kb]
- #
- # Implementation:
- # 1. Elementwise multiplication between embed_key_r and text_param
- # 2. L2-normalization
- # 3. Take the elementwise-min
- with tf.variable_scope(scope, reuse=reuse):
- m = tf.matmul(text_param, self.embed_keys_r, transpose_b=True)
- att_1 = tf.nn.l2_normalize(m, dim=1)
- att = tf.minimum(att_0, att_1)
- return att
-
- def ValDescribeModule(self,
- input_0,
- time_idx,
- batch_idx,
- scope='ValDescribeModule',
- reuse=None):
- att = input_0
-
- # Mapping: att -> answer probs
- # Input:
- # embed_vals_e: [N_kb, D_txt]
- # att: [N, N_kb]
- # embedding_mat: [self.num_choices, D_txt]
- # Output:
- # answer_scores: [N, self.num_choices]
- #
- # Implementation:
- # 1. Attention-weighted sum over values
- # 2. Compute cosine similarity scores between the weighted sum and
- # each candidate answer
- with tf.variable_scope(scope, reuse=reuse):
- # weighted_sum has shape [N, D_txt]
- weighted_sum = tf.matmul(att, self.embed_vals_e)
- # scores has shape [N, self.num_choices]
- scores = tf.matmul(
- weighted_sum,
- tf.nn.l2_normalize(self.embedding_mat, dim=1),
- transpose_b=True)
- return scores
diff --git a/research/qa_kg/model_n2nmn/netgen_att.py b/research/qa_kg/model_n2nmn/netgen_att.py
deleted file mode 100644
index df6509946a5457bb07f2dfdcfab44aaf67447d0f..0000000000000000000000000000000000000000
--- a/research/qa_kg/model_n2nmn/netgen_att.py
+++ /dev/null
@@ -1,295 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import tensorflow as tf
-from util.nn import fc_layer as fc
-
-
-def _get_lstm_cell(num_layers, lstm_dim):
- cell_list = [
- tf.contrib.rnn.BasicLSTMCell(lstm_dim, state_is_tuple=True)
- for _ in range(num_layers)
- ]
- cell = tf.contrib.rnn.MultiRNNCell(cell_list, state_is_tuple=True)
- return cell
-
-
-class AttentionSeq2Seq:
-
- def __init__(self,
- config,
- text_seq_batch,
- seq_length_batch,
- num_vocab_txt,
- num_vocab_nmn,
- EOS_token,
- decoder_sampling,
- embedding_mat,
- use_gt_layout=None,
- gt_layout_batch=None,
- scope='encoder_decoder',
- reuse=None):
- self.T_decoder = config.T_decoder
- self.encoder_num_vocab = num_vocab_txt
- self.encoder_embed_dim = config.embed_dim_txt
- self.decoder_num_vocab = num_vocab_nmn
- self.decoder_embed_dim = config.embed_dim_nmn
- self.lstm_dim = config.lstm_dim
- self.num_layers = config.num_layers
- self.EOS_token = EOS_token
- self.decoder_sampling = decoder_sampling
- self.embedding_mat = embedding_mat
-
- with tf.variable_scope(scope, reuse=reuse):
- self._build_encoder(text_seq_batch, seq_length_batch)
- self._build_decoder(use_gt_layout, gt_layout_batch)
-
- def _build_encoder(self,
- text_seq_batch,
- seq_length_batch,
- scope='encoder',
- reuse=None):
- lstm_dim = self.lstm_dim
- num_layers = self.num_layers
-
- with tf.variable_scope(scope, reuse=reuse):
- T = tf.shape(text_seq_batch)[0]
- N = tf.shape(text_seq_batch)[1]
- self.T_encoder = T
- self.N = N
-
- # text_seq has shape [T, N] and embedded_seq has shape [T, N, D]
- embedded_seq = tf.nn.embedding_lookup(self.embedding_mat, text_seq_batch)
- self.embedded_input_seq = embedded_seq
-
- # The RNN
- cell = _get_lstm_cell(num_layers, lstm_dim)
-
- # encoder_outputs has shape [T, N, lstm_dim]
- encoder_outputs, encoder_states = tf.nn.dynamic_rnn(
- cell,
- embedded_seq,
- seq_length_batch,
- dtype=tf.float32,
- time_major=True,
- scope='lstm')
- self.encoder_outputs = encoder_outputs
- self.encoder_states = encoder_states
-
- # transform the encoder outputs for further attention alignments
- # encoder_outputs_flat has shape [T, N, lstm_dim]
- encoder_h_transformed = fc(
- 'encoder_h_transform',
- tf.reshape(encoder_outputs, [-1, lstm_dim]),
- output_dim=lstm_dim)
- encoder_h_transformed = tf.reshape(encoder_h_transformed,
- [T, N, lstm_dim])
- self.encoder_h_transformed = encoder_h_transformed
-
- # seq_not_finished is a shape [T, N, 1] tensor,
- # where seq_not_finished[t, n]
- # is 1 iff sequence n is not finished at time t, and 0 otherwise
- seq_not_finished = tf.less(
- tf.range(T)[:, tf.newaxis, tf.newaxis],
- seq_length_batch[:, tf.newaxis])
- seq_not_finished = tf.cast(seq_not_finished, tf.float32)
- self.seq_not_finished = seq_not_finished
-
- def _build_decoder(self,
- use_gt_layout,
- gt_layout_batch,
- scope='decoder',
- reuse=None):
- # The main difference from before is that the decoders now takes another
- # input (the attention) when computing the next step
- # T_max is the maximum length of decoded sequence (including )
- #
- # This function is for decoding only. It performs greedy search or sampling.
- # the first input is (its embedding vector) and the subsequent inputs
- # are the outputs from previous time step
- # num_vocab does not include
- #
- # use_gt_layout is None or a bool tensor, and gt_layout_batch is a tensor
- # with shape [T_max, N].
- # If use_gt_layout is not None, then when use_gt_layout is true, predict
- # exactly the tokens in gt_layout_batch, regardless of actual probability.
- # Otherwise, if sampling is True, sample from the token probability
- # If sampling is False, do greedy decoding (beam size 1)
- N = self.N
- encoder_states = self.encoder_states
- T_max = self.T_decoder
- lstm_dim = self.lstm_dim
- num_layers = self.num_layers
- EOS_token = self.EOS_token
- sampling = self.decoder_sampling
-
- with tf.variable_scope(scope, reuse=reuse):
- embedding_mat = tf.get_variable(
- 'embedding_mat', [self.decoder_num_vocab, self.decoder_embed_dim])
- # we use a separate embedding for , as it is only used in the
- # beginning of the sequence
- go_embedding = tf.get_variable('go_embedding',
- [1, self.decoder_embed_dim])
-
- with tf.variable_scope('att_prediction'):
- v = tf.get_variable('v', [lstm_dim])
- W_a = tf.get_variable(
- 'weights', [lstm_dim, lstm_dim],
- initializer=tf.contrib.layers.xavier_initializer())
- b_a = tf.get_variable(
- 'biases', lstm_dim, initializer=tf.constant_initializer(0.))
-
- # The parameters to predict the next token
- with tf.variable_scope('token_prediction'):
- W_y = tf.get_variable(
- 'weights', [lstm_dim * 2, self.decoder_num_vocab],
- initializer=tf.contrib.layers.xavier_initializer())
- b_y = tf.get_variable(
- 'biases',
- self.decoder_num_vocab,
- initializer=tf.constant_initializer(0.))
-
- # Attentional decoding
- # Loop function is called at time t BEFORE the cell execution at time t,
- # and its next_input is used as the input at time t (not t+1)
- # c.f. https://www.tensorflow.org/api_docs/python/tf/nn/raw_rnn
- mask_range = tf.reshape(
- tf.range(self.decoder_num_vocab, dtype=tf.int32), [1, -1])
- all_eos_pred = EOS_token * tf.ones([N], tf.int32)
- all_one_prob = tf.ones([N], tf.float32)
- all_zero_entropy = tf.zeros([N], tf.float32)
- if use_gt_layout is not None:
- gt_layout_mult = tf.cast(use_gt_layout, tf.int32)
- pred_layout_mult = 1 - gt_layout_mult
-
- def loop_fn(time, cell_output, cell_state, loop_state):
- if cell_output is None: # time == 0
- next_cell_state = encoder_states
- next_input = tf.tile(go_embedding, [N, 1])
- else: # time > 0
- next_cell_state = cell_state
-
- # compute the attention map over the input sequence
- # a_raw has shape [T, N, 1]
- att_raw = tf.reduce_sum(
- tf.tanh(
- tf.nn.xw_plus_b(cell_output, W_a, b_a) +
- self.encoder_h_transformed) * v,
- axis=2,
- keep_dims=True)
- # softmax along the first dimension (T) over not finished examples
- # att has shape [T, N, 1]
- att = tf.nn.softmax(att_raw, dim=0) * self.seq_not_finished
- att = att / tf.reduce_sum(att, axis=0, keep_dims=True)
- # d has shape [N, lstm_dim]
- d2 = tf.reduce_sum(att * self.encoder_outputs, axis=0)
-
- # token_scores has shape [N, num_vocab]
- token_scores = tf.nn.xw_plus_b(
- tf.concat([cell_output, d2], axis=1), W_y, b_y)
- # predict the next token (behavior depending on parameters)
- if sampling:
- # predicted_token has shape [N]
- logits = token_scores
- predicted_token = tf.cast(
- tf.reshape(tf.multinomial(token_scores, 1), [-1]), tf.int32)
- else:
- # predicted_token has shape [N]
- predicted_token = tf.cast(tf.argmax(token_scores, 1), tf.int32)
- if use_gt_layout is not None:
- predicted_token = (gt_layout_batch[time - 1] * gt_layout_mult +
- predicted_token * pred_layout_mult)
-
- # token_prob has shape [N], the probability of the predicted token
- # although token_prob is not needed for predicting the next token
- # it is needed in output (for policy gradient training)
- # [N, num_vocab]
- # mask has shape [N, num_vocab]
- mask = tf.equal(mask_range, tf.reshape(predicted_token, [-1, 1]))
- all_token_probs = tf.nn.softmax(token_scores)
- token_prob = tf.reduce_sum(
- all_token_probs * tf.cast(mask, tf.float32), axis=1)
- neg_entropy = tf.reduce_sum(
- all_token_probs * tf.log(all_token_probs), axis=1)
-
- # is_eos_predicted is a [N] bool tensor, indicating whether
- # has already been predicted previously in each sequence
- is_eos_predicted = loop_state[2]
- predicted_token_old = predicted_token
- # if has already been predicted, now predict with
- # prob 1
- predicted_token = tf.where(is_eos_predicted, all_eos_pred,
- predicted_token)
- token_prob = tf.where(is_eos_predicted, all_one_prob, token_prob)
- neg_entropy = tf.where(is_eos_predicted, all_zero_entropy,
- neg_entropy)
- is_eos_predicted = tf.logical_or(is_eos_predicted,
- tf.equal(predicted_token_old,
- EOS_token))
-
- # the prediction is from the cell output of the last step
- # timestep (t-1), feed it as input into timestep t
- next_input = tf.nn.embedding_lookup(embedding_mat, predicted_token)
-
- elements_finished = tf.greater_equal(time, T_max)
-
- # loop_state is a 5-tuple, representing
- # 1) the predicted_tokens
- # 2) the prob of predicted_tokens
- # 3) whether has already been predicted
- # 4) the negative entropy of policy (accumulated across timesteps)
- # 5) the attention
- if loop_state is None: # time == 0
- # Write the predicted token into the output
- predicted_token_array = tf.TensorArray(
- dtype=tf.int32, size=T_max, infer_shape=False)
- token_prob_array = tf.TensorArray(
- dtype=tf.float32, size=T_max, infer_shape=False)
- att_array = tf.TensorArray(
- dtype=tf.float32, size=T_max, infer_shape=False)
- next_loop_state = (predicted_token_array, token_prob_array, tf.zeros(
- [N], dtype=tf.bool), tf.zeros([N], dtype=tf.float32), att_array)
- else: # time > 0
- t_write = time - 1
- next_loop_state = (
- loop_state[0].write(t_write, predicted_token),
- loop_state[1].write(t_write, token_prob),
- is_eos_predicted,
- loop_state[3] + neg_entropy,
- loop_state[4].write(t_write, att))
- return (elements_finished, next_input, next_cell_state, cell_output,
- next_loop_state)
-
- # The RNN
- cell = _get_lstm_cell(num_layers, lstm_dim)
- _, _, decodes_ta = tf.nn.raw_rnn(cell, loop_fn, scope='lstm')
- predicted_tokens = decodes_ta[0].stack()
- token_probs = decodes_ta[1].stack()
- neg_entropy = decodes_ta[3]
- # atts has shape [T_decoder, T_encoder, N, 1]
- atts = decodes_ta[4].stack()
- self.atts = atts
- # word_vec has shape [T_decoder, N, D]
- word_vecs = tf.reduce_sum(atts * self.embedded_input_seq, axis=1)
-
- predicted_tokens.set_shape([None, None])
- token_probs.set_shape([None, None])
- neg_entropy.set_shape([None])
- word_vecs.set_shape([None, None, self.encoder_embed_dim])
-
- self.predicted_tokens = predicted_tokens
- self.token_probs = token_probs
- self.neg_entropy = neg_entropy
- self.word_vecs = word_vecs
diff --git a/research/qa_kg/util/__init__.py b/research/qa_kg/util/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/qa_kg/util/data_reader.py b/research/qa_kg/util/data_reader.py
deleted file mode 100644
index 397390af6d95b350559fbd20cc55e85a12ce03c0..0000000000000000000000000000000000000000
--- a/research/qa_kg/util/data_reader.py
+++ /dev/null
@@ -1,231 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from collections import namedtuple
-try:
- from queue import Queue # Python 3
-except ImportError:
- from Queue import Queue # Python 2
-import re
-import threading
-import numpy as np
-import tensorflow as tf
-
-Data = namedtuple('Data', ['X', 'Y', 'MultiYs', 'qid'])
-
-
-class SampleBuilder:
-
- def __init__(self, config):
- self.config = config
-
- self.kb_raw = self.read_kb()
- self.data_raw = self.read_raw_data()
-
- # dictionary of entities, normal words, and relations
- self.dict_all = self.gen_dict()
- self.reverse_dict_all = dict(
- zip(self.dict_all.values(), self.dict_all.keys()))
-
- tf.logging.info('size of dict: %d' % len(self.dict_all))
-
- self.kb = self.build_kb()
- self.data_all = self.build_samples()
-
- def read_kb(self):
- kb_raw = []
- for line in open(self.config.KB_file):
- sub, rel, obj = line.strip().split('|')
- kb_raw.append((sub, rel, obj))
- tf.logging.info('# of KB records: %d' % len(kb_raw))
- return kb_raw
-
- def read_raw_data(self):
- data = dict()
- for name in self.config.data_files:
- raw = []
- tf.logging.info(
- 'Reading data file {}'.format(self.config.data_files[name]))
- for line in open(self.config.data_files[name]):
- question, answers = line.strip().split('\t')
- question = question.replace('],', ']') # ignore ',' in the template
- raw.append((question, answers))
- data[name] = raw
- return data
-
- def build_kb(self):
- tf.logging.info('Indexing KB...')
- kb = []
- for sub, rel, obj in self.kb_raw:
- kb.append([self.dict_all[sub], self.dict_all[rel], self.dict_all[obj]])
- return kb
-
- def gen_dict(self):
- s = set()
- for sub, rel, obj in self.kb_raw:
- s.add(sub)
- s.add(rel)
- s.add(obj)
- for name in self.data_raw:
- for question, answers in self.data_raw[name]:
- normal = re.split('\[[^\]]+\]', question)
- for phrase in normal:
- for word in phrase.split():
- s.add(word)
- s = list(s)
- d = {s[idx]: idx for idx in range(len(s))}
- return d
-
- def build_samples(self):
-
- def map_entity_idx(text):
- entities = re.findall('\[[^\]]+\]', text)
- for entity in entities:
- entity = entity[1:-1]
- index = self.dict_all[entity]
- text = text.replace('[%s]' % entity, '@%d' % index)
- return text
-
- data_all = dict()
-
- for name in self.data_raw:
- X, Y, MultiYs, qid = [], [], [], []
- for i, (question, answers) in enumerate(self.data_raw[name]):
- qdata, labels = [], []
- question = map_entity_idx(question)
- for word in question.split():
- if word[0] == '@':
- qdata.append(int(word[1:]))
- else:
- qdata.append(self.dict_all[word])
- for answer in answers.split('|'):
- labels.append(self.dict_all[answer])
- if len(qdata) > self.config.T_encoder:
- self.config.T_encoder = len(qdata)
- for label in labels:
- X.append(qdata)
- Y.append(label)
- MultiYs.append(set(labels))
- qid.append(i)
- data_all[name] = Data(X=X, Y=Y, MultiYs=MultiYs, qid=qid)
-
- return data_all
-
-
-def _run_prefetch(prefetch_queue, batch_loader, data, shuffle, one_pass,
- config):
- assert len(data.X) == len(data.Y) == len(data.MultiYs) == len(data.qid)
- num_samples = len(data.X)
- batch_size = config.batch_size
-
- n_sample = 0
- fetch_order = config.rng.permutation(num_samples)
- while True:
- sample_ids = fetch_order[n_sample:n_sample + batch_size]
- batch = batch_loader.load_one_batch(sample_ids)
- prefetch_queue.put(batch, block=True)
-
- n_sample += len(sample_ids)
- if n_sample >= num_samples:
- if one_pass:
- prefetch_queue.put(None, block=True)
- n_sample = 0
- if shuffle:
- fetch_order = config.rng.permutation(num_samples)
-
-
-class DataReader:
- def __init__(self,
- config,
- data,
- assembler,
- shuffle=True,
- one_pass=False,
- prefetch_num=10):
- self.config = config
-
- self.data = data
- self.assembler = assembler
- self.batch_loader = BatchLoader(self.config,
- self.data, self.assembler)
-
- self.shuffle = shuffle
- self.one_pass = one_pass
- self.prefetch_queue = Queue(maxsize=prefetch_num)
- self.prefetch_thread = threading.Thread(target=_run_prefetch,
- args=(self.prefetch_queue,
- self.batch_loader, self.data,
- self.shuffle, self.one_pass,
- self.config))
- self.prefetch_thread.daemon = True
- self.prefetch_thread.start()
-
- def batches(self):
- while True:
- if self.prefetch_queue.empty():
- tf.logging.warning('Waiting for data loading (IO is slow)...')
- batch = self.prefetch_queue.get(block=True)
- if batch is None:
- assert self.one_pass
- tf.logging.info('One pass finished!')
- raise StopIteration()
- yield batch
-
-
-class BatchLoader:
- def __init__(self, config,
- data, assembler):
- self.config = config
-
- self.data = data
- self.assembler = assembler
-
- self.T_encoder = config.T_encoder
- self.T_decoder = config.T_decoder
-
- tf.logging.info('T_encoder: %d' % self.T_encoder)
- tf.logging.info('T_decoder: %d' % self.T_decoder)
- tf.logging.info('batch size: %d' % self.config.batch_size)
-
- self.gt_layout_tokens = config.gt_layout_tokens
-
- def load_one_batch(self, sample_ids):
- actual_batch_size = len(sample_ids)
- input_seq_batch = np.zeros((self.T_encoder, actual_batch_size), np.int32)
- seq_len_batch = np.zeros(actual_batch_size, np.int32)
- ans_label_batch = np.zeros(actual_batch_size, np.int32)
- ans_set_labels_list = [None] * actual_batch_size
- question_id_list = [None] * actual_batch_size
- gt_layout_batch = np.zeros((self.T_decoder, actual_batch_size), np.int32)
-
- for batch_i in range(actual_batch_size):
- idx = sample_ids[batch_i]
- seq_len = len(self.data.X[idx])
- seq_len_batch[batch_i] = seq_len
- input_seq_batch[:seq_len, batch_i] = self.data.X[idx]
- ans_label_batch[batch_i] = self.data.Y[idx]
- ans_set_labels_list[batch_i] = self.data.MultiYs[idx]
- question_id_list[batch_i] = self.data.qid[idx]
-
- gt_layout_batch[:, batch_i] = self.assembler.module_list2tokens(
- self.gt_layout_tokens, self.T_decoder)
-
- batch = dict(input_seq_batch=input_seq_batch,
- seq_len_batch=seq_len_batch,
- ans_label_batch=ans_label_batch,
- gt_layout_batch=gt_layout_batch,
- ans_set_labels_list=ans_set_labels_list,
- question_id_list=question_id_list)
- return batch
diff --git a/research/qa_kg/util/misc.py b/research/qa_kg/util/misc.py
deleted file mode 100644
index 9a0199bb403709f3c04e58c17951459febdb40f4..0000000000000000000000000000000000000000
--- a/research/qa_kg/util/misc.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from datetime import datetime
-import json
-import logging
-import os
-import tensorflow as tf
-import tensorflow.contrib.slim as slim
-
-
-def prepare_dirs_and_logger(config):
- formatter = logging.Formatter('%(asctime)s:%(levelname)s::%(message)s')
- logger = logging.getLogger('tensorflow')
-
- for hdlr in logger.handlers:
- logger.removeHandler(hdlr)
-
- handler = logging.StreamHandler()
- handler.setFormatter(formatter)
-
- logger.addHandler(handler)
- logger.setLevel(tf.logging.INFO)
-
- config.log_dir = os.path.join(config.exp_dir, config.log_dir,
- config.train_tag)
- config.model_dir = os.path.join(config.exp_dir, config.model_dir,
- config.train_tag)
- config.output_dir = os.path.join(config.exp_dir, config.output_dir,
- config.train_tag)
-
- for path in [
- config.log_dir, config.model_dir, config.output_dir
- ]:
- if not os.path.exists(path):
- os.makedirs(path)
-
- config.data_files = {
- 'train': os.path.join(config.data_dir, config.train_data_file),
- 'dev': os.path.join(config.data_dir, config.dev_data_file),
- 'test': os.path.join(config.data_dir, config.test_data_file)
- }
-
- return config
-
-
-def get_time():
- return datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
-
-
-def show_all_variables():
- model_vars = tf.trainable_variables()
- slim.model_analyzer.analyze_vars(model_vars, print_info=True)
-
-
-def save_config(config):
- param_path = os.path.join(config.model_dir, 'params.json')
-
- tf.logging.info('log dir: %s' % config.log_dir)
- tf.logging.info('model dir: %s' % config.model_dir)
- tf.logging.info('param path: %s' % param_path)
- tf.logging.info('output dir: %s' % config.output_dir)
-
- with open(param_path, 'w') as f:
- f.write(json.dumps(config.__dict__, indent=4, sort_keys=True))
diff --git a/research/qa_kg/util/nn.py b/research/qa_kg/util/nn.py
deleted file mode 100644
index 38ba02b2ecac1cea51308287a48e3062ee51fa81..0000000000000000000000000000000000000000
--- a/research/qa_kg/util/nn.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Copyright 2017 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-import tensorflow as tf
-
-
-def fc_layer(name,
- bottom,
- output_dim,
- bias_term=True,
- weights_initializer=None,
- biases_initializer=None,
- reuse=None):
- # flatten bottom input
- shape = bottom.get_shape().as_list()
- input_dim = 1
- for d in shape[1:]:
- input_dim *= d
- flat_bottom = tf.reshape(bottom, [-1, input_dim])
-
- # weights and biases variables
- with tf.variable_scope(name, reuse=reuse):
- # initialize the variables
- if weights_initializer is None:
- weights_initializer = tf.contrib.layers.xavier_initializer()
- if bias_term and biases_initializer is None:
- biases_initializer = tf.constant_initializer(0.)
-
- # weights has shape [input_dim, output_dim]
- weights = tf.get_variable(
- 'weights', [input_dim, output_dim], initializer=weights_initializer)
- if bias_term:
- biases = tf.get_variable(
- 'biases', output_dim, initializer=biases_initializer)
- if not reuse:
- tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES,
- tf.nn.l2_loss(weights))
-
- if bias_term:
- fc = tf.nn.xw_plus_b(flat_bottom, weights, biases)
- else:
- fc = tf.matmul(flat_bottom, weights)
- return fc
diff --git a/research/real_nvp/README.md b/research/real_nvp/README.md
deleted file mode 100644
index c20ef111eb070be94bf6c11ea15ec1ce9d2ad686..0000000000000000000000000000000000000000
--- a/research/real_nvp/README.md
+++ /dev/null
@@ -1,282 +0,0 @@
-
-
-
-
-# Real NVP in TensorFlow
-
-*A Tensorflow implementation of the training procedure of*
-[*Density estimation using Real NVP*](https://arxiv.org/abs/1605.08803)*, by
-Laurent Dinh, Jascha Sohl-Dickstein and Samy Bengio, for Imagenet
-(32x32 and 64x64), CelebA and LSUN Including the scripts to
-put the datasets in `.tfrecords` format.*
-
-We are happy to open source the code for *Real NVP*, a novel approach to
-density estimation using deep neural networks that enables tractable density
-estimation and efficient one-pass inference and sampling. This model
-successfully decomposes images into hierarchical features ranging from
-high-level concepts to low-resolution details. Visualizations are available
-[here](http://goo.gl/yco14s).
-
-## Installation
-* python 2.7:
- * python 3 support is not available yet
-* pip (python package manager)
- * `apt-get install python-pip` on Ubuntu
- * `brew` installs pip along with python on OSX
-* Install the dependencies for [LSUN](https://github.com/fyu/lsun.git)
- * Install [OpenCV](http://opencv.org/)
- * `pip install numpy lmdb`
-* Install the python dependencies
- * `pip install scipy scikit-image Pillow`
-* Install the
-[latest Tensorflow Pip package](https://www.tensorflow.org/get_started/os_setup.html#using-pip)
-for Python 2.7
-
-## Getting Started
-Once you have successfully installed the dependencies, you can start by
-downloading the repository:
-```shell
-git clone --recursive https://github.com/tensorflow/models.git
-```
-Afterward, you can use the utilities in this folder prepare the datasets.
-
-## Preparing datasets
-### CelebA
-For [*CelebA*](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html), download
-`img_align_celeba.zip` from the Dropbox link on this
-[page](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) under the
-link *Align&Cropped Images* in the *Img* directory and `list_eval_partition.txt`
-under the link *Train/Val/Test Partitions* in the *Eval* directory. Then do:
-
-```shell
-mkdir celeba
-cd celeba
-unzip img_align_celeba.zip
-```
-
-We'll format the training subset:
-```shell
-python2.7 ../models/real_nvp/celeba_formatting.py \
- --partition_fn list_eval_partition.txt \
- --file_out celeba_train \
- --fn_root img_align_celeba \
- --set 0
-```
-
-Then the validation subset:
-```shell
-python2.7 ../models/real_nvp/celeba_formatting.py \
- --partition_fn list_eval_partition.txt \
- --file_out celeba_valid \
- --fn_root img_align_celeba \
- --set 1
-```
-
-And finally the test subset:
-```shell
-python2.7 ../models/real_nvp/celeba_formatting.py \
- --partition_fn list_eval_partition.txt \
- --file_out celeba_test \
- --fn_root img_align_celeba \
- --set 2
-```
-
-Afterward:
-```shell
-cd ..
-```
-
-### Small Imagenet
-Downloading the [*small Imagenet*](http://image-net.org/small/download.php)
-dataset is more straightforward and can be done
-entirely in Shell:
-```shell
-mkdir small_imnet
-cd small_imnet
-for FILENAME in train_32x32.tar valid_32x32.tar train_64x64.tar valid_64x64.tar
-do
- curl -O http://image-net.org/small/$FILENAME
- tar -xvf $FILENAME
-done
-```
-
-Then, you can format the datasets as follow:
-```shell
-for DIRNAME in train_32x32 valid_32x32 train_64x64 valid_64x64
-do
- python2.7 ../models/real_nvp/imnet_formatting.py \
- --file_out $DIRNAME \
- --fn_root $DIRNAME
-done
-cd ..
-```
-
-### LSUN
-To prepare the [*LSUN*](http://lsun.cs.princeton.edu/2016/) dataset, we will
-need to use the code associated:
-```shell
-git clone https://github.com/fyu/lsun.git
-cd lsun
-```
-Then we'll download the db files:
-```shell
-for CATEGORY in bedroom church_outdoor tower
-do
- python2.7 download.py -c $CATEGORY
- unzip "$CATEGORY"_train_lmdb.zip
- unzip "$CATEGORY"_val_lmdb.zip
- python2.7 data.py export "$CATEGORY"_train_lmdb \
- --out_dir "$CATEGORY"_train --flat
- python2.7 data.py export "$CATEGORY"_val_lmdb \
- --out_dir "$CATEGORY"_val --flat
-done
-```
-
-Finally, we then format the dataset into `.tfrecords`:
-```shell
-for CATEGORY in bedroom church_outdoor tower
-do
- python2.7 ../models/real_nvp/lsun_formatting.py \
- --file_out "$CATEGORY"_train \
- --fn_root "$CATEGORY"_train
- python2.7 ../models/real_nvp/lsun_formatting.py \
- --file_out "$CATEGORY"_val \
- --fn_root "$CATEGORY"_val
-done
-cd ..
-```
-
-
-## Training
-We'll give an example on how to train a model on the small Imagenet
-dataset (32x32):
-```shell
-cd models/real_nvp/
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 32 \
---hpconfig=n_scale=4,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset imnet \
---traindir /tmp/real_nvp_imnet32/train \
---logdir /tmp/real_nvp_imnet32/train \
---data_path ../../small_imnet/train_32x32_?????.tfrecords
-```
-In parallel, you can run the script to generate visualization from the model:
-```shell
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 32 \
---hpconfig=n_scale=4,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset imnet \
---traindir /tmp/real_nvp_imnet32/train \
---logdir /tmp/real_nvp_imnet32/sample \
---data_path ../../small_imnet/valid_32x32_?????.tfrecords \
---mode sample
-```
-Additionally, you can also run in the script to evaluate the model on the
-validation set:
-```shell
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 32 \
---hpconfig=n_scale=4,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset imnet \
---traindir /tmp/real_nvp_imnet32/train \
---logdir /tmp/real_nvp_imnet32/eval \
---data_path ../../small_imnet/valid_32x32_?????.tfrecords \
---eval_set_size 50000
---mode eval
-```
-The visualizations and validation set evaluation can be seen through
-[Tensorboard](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tensorboard/README.md).
-
-Another example would be how to run the model on LSUN (bedroom category):
-```shell
-# train the model
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 64 \
---hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset lsun \
---traindir /tmp/real_nvp_church_outdoor/train \
---logdir /tmp/real_nvp_church_outdoor/train \
---data_path ../../lsun/church_outdoor_train_?????.tfrecords
-```
-
-```shell
-# sample from the model
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 64 \
---hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset lsun \
---traindir /tmp/real_nvp_church_outdoor/train \
---logdir /tmp/real_nvp_church_outdoor/sample \
---data_path ../../lsun/church_outdoor_val_?????.tfrecords \
---mode sample
-```
-
-```shell
-# evaluate the model
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 64 \
---hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset lsun \
---traindir /tmp/real_nvp_church_outdoor/train \
---logdir /tmp/real_nvp_church_outdoor/eval \
---data_path ../../lsun/church_outdoor_val_?????.tfrecords \
---eval_set_size 300
---mode eval
-```
-
-Finally, we'll give the commands to run the model on the CelebA dataset:
-```shell
-# train the model
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 64 \
---hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset lsun \
---traindir /tmp/real_nvp_celeba/train \
---logdir /tmp/real_nvp_celeba/train \
---data_path ../../celeba/celeba_train.tfrecords
-```
-
-```shell
-# sample from the model
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 64 \
---hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset celeba \
---traindir /tmp/real_nvp_celeba/train \
---logdir /tmp/real_nvp_celeba/sample \
---data_path ../../celeba/celeba_valid.tfrecords \
---mode sample
-```
-
-```shell
-# evaluate the model on validation set
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 64 \
---hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset celeba \
---traindir /tmp/real_nvp_celeba/train \
---logdir /tmp/real_nvp_celeba/eval_valid \
---data_path ../../celeba/celeba_valid.tfrecords \
---eval_set_size 19867
---mode eval
-
-# evaluate the model on test set
-python2.7 real_nvp_multiscale_dataset.py \
---image_size 64 \
---hpconfig=n_scale=5,base_dim=32,clip_gradient=100,residual_blocks=4 \
---dataset celeba \
---traindir /tmp/real_nvp_celeba/train \
---logdir /tmp/real_nvp_celeba/eval_test \
---data_path ../../celeba/celeba_test.tfrecords \
---eval_set_size 19962
---mode eval
-```
-
-## Credits
-This code was written by Laurent Dinh
-([@laurent-dinh](https://github.com/laurent-dinh)) with
-the help of
-Jascha Sohl-Dickstein ([@Sohl-Dickstein](https://github.com/Sohl-Dickstein)
-and [jaschasd@google.com](mailto:jaschasd@google.com)),
-Samy Bengio, Jon Shlens, Sherry Moore and
-David Andersen.
diff --git a/research/real_nvp/__init__.py b/research/real_nvp/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/real_nvp/celeba_formatting.py b/research/real_nvp/celeba_formatting.py
deleted file mode 100644
index e03571086d88763264d7660aa5e9db5e9074dec5..0000000000000000000000000000000000000000
--- a/research/real_nvp/celeba_formatting.py
+++ /dev/null
@@ -1,96 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""CelebA dataset formating.
-
-Download img_align_celeba.zip from
-http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html under the
-link "Align&Cropped Images" in the "Img" directory and list_eval_partition.txt
-under the link "Train/Val/Test Partitions" in the "Eval" directory. Then do:
-unzip img_align_celeba.zip
-
-Use the script as follow:
-python celeba_formatting.py \
- --partition_fn [PARTITION_FILE_PATH] \
- --file_out [OUTPUT_FILE_PATH_PREFIX] \
- --fn_root [CELEBA_FOLDER] \
- --set [SUBSET_INDEX]
-
-"""
-
-from __future__ import print_function
-
-import os
-import os.path
-
-import scipy.io
-import scipy.io.wavfile
-import scipy.ndimage
-import tensorflow as tf
-
-
-tf.flags.DEFINE_string("file_out", "",
- "Filename of the output .tfrecords file.")
-tf.flags.DEFINE_string("fn_root", "", "Name of root file path.")
-tf.flags.DEFINE_string("partition_fn", "", "Partition file path.")
-tf.flags.DEFINE_string("set", "", "Name of subset.")
-
-FLAGS = tf.flags.FLAGS
-
-
-def _int64_feature(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-
-
-def _bytes_feature(value):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
-
-def main():
- """Main converter function."""
- # Celeb A
- with open(FLAGS.partition_fn, "r") as infile:
- img_fn_list = infile.readlines()
- img_fn_list = [elem.strip().split() for elem in img_fn_list]
- img_fn_list = [elem[0] for elem in img_fn_list if elem[1] == FLAGS.set]
- fn_root = FLAGS.fn_root
- num_examples = len(img_fn_list)
-
- file_out = "%s.tfrecords" % FLAGS.file_out
- writer = tf.python_io.TFRecordWriter(file_out)
- for example_idx, img_fn in enumerate(img_fn_list):
- if example_idx % 1000 == 0:
- print(example_idx, "/", num_examples)
- image_raw = scipy.ndimage.imread(os.path.join(fn_root, img_fn))
- rows = image_raw.shape[0]
- cols = image_raw.shape[1]
- depth = image_raw.shape[2]
- image_raw = image_raw.tostring()
- example = tf.train.Example(
- features=tf.train.Features(
- feature={
- "height": _int64_feature(rows),
- "width": _int64_feature(cols),
- "depth": _int64_feature(depth),
- "image_raw": _bytes_feature(image_raw)
- }
- )
- )
- writer.write(example.SerializeToString())
- writer.close()
-
-
-if __name__ == "__main__":
- main()
diff --git a/research/real_nvp/imnet_formatting.py b/research/real_nvp/imnet_formatting.py
deleted file mode 100644
index 1775dd54d368b62d047d1428d4bcf79ad4a68ae0..0000000000000000000000000000000000000000
--- a/research/real_nvp/imnet_formatting.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""LSUN dataset formatting.
-
-Download and format the Imagenet dataset as follow:
-mkdir [IMAGENET_PATH]
-cd [IMAGENET_PATH]
-for FILENAME in train_32x32.tar valid_32x32.tar train_64x64.tar valid_64x64.tar
-do
- curl -O http://image-net.org/small/$FILENAME
- tar -xvf $FILENAME
-done
-
-Then use the script as follow:
-for DIRNAME in train_32x32 valid_32x32 train_64x64 valid_64x64
-do
- python imnet_formatting.py \
- --file_out $DIRNAME \
- --fn_root $DIRNAME
-done
-
-"""
-
-from __future__ import print_function
-
-import os
-import os.path
-
-import scipy.io
-import scipy.io.wavfile
-import scipy.ndimage
-import tensorflow as tf
-
-
-tf.flags.DEFINE_string("file_out", "",
- "Filename of the output .tfrecords file.")
-tf.flags.DEFINE_string("fn_root", "", "Name of root file path.")
-
-FLAGS = tf.flags.FLAGS
-
-
-def _int64_feature(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-
-
-def _bytes_feature(value):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
-
-def main():
- """Main converter function."""
- # LSUN
- fn_root = FLAGS.fn_root
- img_fn_list = os.listdir(fn_root)
- img_fn_list = [img_fn for img_fn in img_fn_list
- if img_fn.endswith('.png')]
- num_examples = len(img_fn_list)
-
- n_examples_per_file = 10000
- for example_idx, img_fn in enumerate(img_fn_list):
- if example_idx % n_examples_per_file == 0:
- file_out = "%s_%05d.tfrecords"
- file_out = file_out % (FLAGS.file_out,
- example_idx // n_examples_per_file)
- print("Writing on:", file_out)
- writer = tf.python_io.TFRecordWriter(file_out)
- if example_idx % 1000 == 0:
- print(example_idx, "/", num_examples)
- image_raw = scipy.ndimage.imread(os.path.join(fn_root, img_fn))
- rows = image_raw.shape[0]
- cols = image_raw.shape[1]
- depth = image_raw.shape[2]
- image_raw = image_raw.astype("uint8")
- image_raw = image_raw.tostring()
- example = tf.train.Example(
- features=tf.train.Features(
- feature={
- "height": _int64_feature(rows),
- "width": _int64_feature(cols),
- "depth": _int64_feature(depth),
- "image_raw": _bytes_feature(image_raw)
- }
- )
- )
- writer.write(example.SerializeToString())
- if example_idx % n_examples_per_file == (n_examples_per_file - 1):
- writer.close()
- writer.close()
-
-
-if __name__ == "__main__":
- main()
diff --git a/research/real_nvp/lsun_formatting.py b/research/real_nvp/lsun_formatting.py
deleted file mode 100644
index 13a21c5e90f86dcdea777419d1df848fcea03d45..0000000000000000000000000000000000000000
--- a/research/real_nvp/lsun_formatting.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""LSUN dataset formatting.
-
-Download and format the LSUN dataset as follow:
-git clone https://github.com/fyu/lsun.git
-cd lsun
-python2.7 download.py -c [CATEGORY]
-
-Then unzip the downloaded .zip files before executing:
-python2.7 data.py export [IMAGE_DB_PATH] --out_dir [LSUN_FOLDER] --flat
-
-Then use the script as follow:
-python lsun_formatting.py \
- --file_out [OUTPUT_FILE_PATH_PREFIX] \
- --fn_root [LSUN_FOLDER]
-
-"""
-from __future__ import print_function
-
-import os
-import os.path
-
-import numpy
-import skimage.transform
-from PIL import Image
-import tensorflow as tf
-
-
-tf.flags.DEFINE_string("file_out", "",
- "Filename of the output .tfrecords file.")
-tf.flags.DEFINE_string("fn_root", "", "Name of root file path.")
-
-FLAGS = tf.flags.FLAGS
-
-
-def _int64_feature(value):
- return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
-
-
-def _bytes_feature(value):
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
-
-
-def main():
- """Main converter function."""
- fn_root = FLAGS.fn_root
- img_fn_list = os.listdir(fn_root)
- img_fn_list = [img_fn for img_fn in img_fn_list
- if img_fn.endswith('.webp')]
- num_examples = len(img_fn_list)
-
- n_examples_per_file = 10000
- for example_idx, img_fn in enumerate(img_fn_list):
- if example_idx % n_examples_per_file == 0:
- file_out = "%s_%05d.tfrecords"
- file_out = file_out % (FLAGS.file_out,
- example_idx // n_examples_per_file)
- print("Writing on:", file_out)
- writer = tf.python_io.TFRecordWriter(file_out)
- if example_idx % 1000 == 0:
- print(example_idx, "/", num_examples)
- image_raw = numpy.array(Image.open(os.path.join(fn_root, img_fn)))
- rows = image_raw.shape[0]
- cols = image_raw.shape[1]
- depth = image_raw.shape[2]
- downscale = min(rows / 96., cols / 96.)
- image_raw = skimage.transform.pyramid_reduce(image_raw, downscale)
- image_raw *= 255.
- image_raw = image_raw.astype("uint8")
- rows = image_raw.shape[0]
- cols = image_raw.shape[1]
- depth = image_raw.shape[2]
- image_raw = image_raw.tostring()
- example = tf.train.Example(
- features=tf.train.Features(
- feature={
- "height": _int64_feature(rows),
- "width": _int64_feature(cols),
- "depth": _int64_feature(depth),
- "image_raw": _bytes_feature(image_raw)
- }
- )
- )
- writer.write(example.SerializeToString())
- if example_idx % n_examples_per_file == (n_examples_per_file - 1):
- writer.close()
- writer.close()
-
-
-if __name__ == "__main__":
- main()
diff --git a/research/real_nvp/real_nvp_multiscale_dataset.py b/research/real_nvp/real_nvp_multiscale_dataset.py
deleted file mode 100644
index c0e1864f1988cd983cdba14ced2462dae1b67e29..0000000000000000000000000000000000000000
--- a/research/real_nvp/real_nvp_multiscale_dataset.py
+++ /dev/null
@@ -1,1639 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""Script for training, evaluation and sampling for Real NVP.
-
-$ python real_nvp_multiscale_dataset.py \
---alsologtostderr \
---image_size 64 \
---hpconfig=n_scale=5,base_dim=8 \
---dataset imnet \
---data_path [DATA_PATH]
-"""
-
-from __future__ import print_function
-
-import time
-from datetime import datetime
-import os
-
-import numpy
-from six.moves import xrange
-import tensorflow as tf
-
-from tensorflow import gfile
-
-from real_nvp_utils import (
- batch_norm, batch_norm_log_diff, conv_layer,
- squeeze_2x2, squeeze_2x2_ordered, standard_normal_ll,
- standard_normal_sample, unsqueeze_2x2, variable_on_cpu)
-
-
-tf.flags.DEFINE_string("master", "local",
- "BNS name of the TensorFlow master, or local.")
-
-tf.flags.DEFINE_string("logdir", "/tmp/real_nvp_multiscale",
- "Directory to which writes logs.")
-
-tf.flags.DEFINE_string("traindir", "/tmp/real_nvp_multiscale",
- "Directory to which writes logs.")
-
-tf.flags.DEFINE_integer("train_steps", 1000000000000000000,
- "Number of steps to train for.")
-
-tf.flags.DEFINE_string("data_path", "", "Path to the data.")
-
-tf.flags.DEFINE_string("mode", "train",
- "Mode of execution. Must be 'train', "
- "'sample' or 'eval'.")
-
-tf.flags.DEFINE_string("dataset", "imnet",
- "Dataset used. Must be 'imnet', "
- "'celeba' or 'lsun'.")
-
-tf.flags.DEFINE_integer("recursion_type", 2,
- "Type of the recursion.")
-
-tf.flags.DEFINE_integer("image_size", 64,
- "Size of the input image.")
-
-tf.flags.DEFINE_integer("eval_set_size", 0,
- "Size of evaluation dataset.")
-
-tf.flags.DEFINE_string(
- "hpconfig", "",
- "A comma separated list of hyperparameters for the model. Format is "
- "hp1=value1,hp2=value2,etc. If this FLAG is set, the model will be trained "
- "with the specified hyperparameters, filling in missing hyperparameters "
- "from the default_values in |hyper_params|.")
-
-FLAGS = tf.flags.FLAGS
-
-class HParams(object):
- """Dictionary of hyperparameters."""
- def __init__(self, **kwargs):
- self.dict_ = kwargs
- self.__dict__.update(self.dict_)
-
- def update_config(self, in_string):
- """Update the dictionary with a comma separated list."""
- pairs = in_string.split(",")
- pairs = [pair.split("=") for pair in pairs]
- for key, val in pairs:
- self.dict_[key] = type(self.dict_[key])(val)
- self.__dict__.update(self.dict_)
- return self
-
- def __getitem__(self, key):
- return self.dict_[key]
-
- def __setitem__(self, key, val):
- self.dict_[key] = val
- self.__dict__.update(self.dict_)
-
-
-def get_default_hparams():
- """Get the default hyperparameters."""
- return HParams(
- batch_size=64,
- residual_blocks=2,
- n_couplings=2,
- n_scale=4,
- learning_rate=0.001,
- momentum=1e-1,
- decay=1e-3,
- l2_coeff=0.00005,
- clip_gradient=100.,
- optimizer="adam",
- dropout_mask=0,
- base_dim=32,
- bottleneck=0,
- use_batch_norm=1,
- alternate=1,
- use_aff=1,
- skip=1,
- data_constraint=.9,
- n_opt=0)
-
-
-# RESNET UTILS
-def residual_block(input_, dim, name, use_batch_norm=True,
- train=True, weight_norm=True, bottleneck=False):
- """Residual convolutional block."""
- with tf.variable_scope(name):
- res = input_
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=dim, name="bn_in", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.nn.relu(res)
- if bottleneck:
- res = conv_layer(
- input_=res, filter_size=[1, 1], dim_in=dim, dim_out=dim,
- name="h_0", stddev=numpy.sqrt(2. / (dim)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=(not use_batch_norm),
- weight_norm=weight_norm, scale=False)
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=dim,
- name="bn_0", scale=False, train=train,
- epsilon=1e-4, axes=[0, 1, 2])
- res = tf.nn.relu(res)
- res = conv_layer(
- input_=res, filter_size=[3, 3], dim_in=dim,
- dim_out=dim, name="h_1", stddev=numpy.sqrt(2. / (1. * dim)),
- strides=[1, 1, 1, 1], padding="SAME", nonlinearity=None,
- bias=(not use_batch_norm),
- weight_norm=weight_norm, scale=False)
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=dim, name="bn_1", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.nn.relu(res)
- res = conv_layer(
- input_=res, filter_size=[1, 1], dim_in=dim, dim_out=dim,
- name="out", stddev=numpy.sqrt(2. / (1. * dim)),
- strides=[1, 1, 1, 1], padding="SAME", nonlinearity=None,
- bias=True, weight_norm=weight_norm, scale=True)
- else:
- res = conv_layer(
- input_=res, filter_size=[3, 3], dim_in=dim, dim_out=dim,
- name="h_0", stddev=numpy.sqrt(2. / (dim)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=(not use_batch_norm),
- weight_norm=weight_norm, scale=False)
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=dim, name="bn_0", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.nn.relu(res)
- res = conv_layer(
- input_=res, filter_size=[3, 3], dim_in=dim, dim_out=dim,
- name="out", stddev=numpy.sqrt(2. / (1. * dim)),
- strides=[1, 1, 1, 1], padding="SAME", nonlinearity=None,
- bias=True, weight_norm=weight_norm, scale=True)
- res += input_
-
- return res
-
-
-def resnet(input_, dim_in, dim, dim_out, name, use_batch_norm=True,
- train=True, weight_norm=True, residual_blocks=5,
- bottleneck=False, skip=True):
- """Residual convolutional network."""
- with tf.variable_scope(name):
- res = input_
- if residual_blocks != 0:
- res = conv_layer(
- input_=res, filter_size=[3, 3], dim_in=dim_in, dim_out=dim,
- name="h_in", stddev=numpy.sqrt(2. / (dim_in)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=True,
- weight_norm=weight_norm, scale=False)
- if skip:
- out = conv_layer(
- input_=res, filter_size=[1, 1], dim_in=dim, dim_out=dim,
- name="skip_in", stddev=numpy.sqrt(2. / (dim)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=True,
- weight_norm=weight_norm, scale=True)
-
- # residual blocks
- for idx_block in xrange(residual_blocks):
- res = residual_block(res, dim, "block_%d" % idx_block,
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- bottleneck=bottleneck)
- if skip:
- out += conv_layer(
- input_=res, filter_size=[1, 1], dim_in=dim, dim_out=dim,
- name="skip_%d" % idx_block, stddev=numpy.sqrt(2. / (dim)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=True,
- weight_norm=weight_norm, scale=True)
- # outputs
- if skip:
- res = out
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=dim, name="bn_pre_out", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.nn.relu(res)
- res = conv_layer(
- input_=res, filter_size=[1, 1], dim_in=dim,
- dim_out=dim_out,
- name="out", stddev=numpy.sqrt(2. / (1. * dim)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=True,
- weight_norm=weight_norm, scale=True)
- else:
- if bottleneck:
- res = conv_layer(
- input_=res, filter_size=[1, 1], dim_in=dim_in, dim_out=dim,
- name="h_0", stddev=numpy.sqrt(2. / (dim_in)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=(not use_batch_norm),
- weight_norm=weight_norm, scale=False)
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=dim, name="bn_0", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.nn.relu(res)
- res = conv_layer(
- input_=res, filter_size=[3, 3], dim_in=dim,
- dim_out=dim, name="h_1", stddev=numpy.sqrt(2. / (1. * dim)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None,
- bias=(not use_batch_norm),
- weight_norm=weight_norm, scale=False)
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=dim, name="bn_1", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.nn.relu(res)
- res = conv_layer(
- input_=res, filter_size=[1, 1], dim_in=dim, dim_out=dim_out,
- name="out", stddev=numpy.sqrt(2. / (1. * dim)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=True,
- weight_norm=weight_norm, scale=True)
- else:
- res = conv_layer(
- input_=res, filter_size=[3, 3], dim_in=dim_in, dim_out=dim,
- name="h_0", stddev=numpy.sqrt(2. / (dim_in)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=(not use_batch_norm),
- weight_norm=weight_norm, scale=False)
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=dim, name="bn_0", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.nn.relu(res)
- res = conv_layer(
- input_=res, filter_size=[3, 3], dim_in=dim, dim_out=dim_out,
- name="out", stddev=numpy.sqrt(2. / (1. * dim)),
- strides=[1, 1, 1, 1], padding="SAME",
- nonlinearity=None, bias=True,
- weight_norm=weight_norm, scale=True)
- return res
-
-
-# COUPLING LAYERS
-# masked convolution implementations
-def masked_conv_aff_coupling(input_, mask_in, dim, name,
- use_batch_norm=True, train=True, weight_norm=True,
- reverse=False, residual_blocks=5,
- bottleneck=False, use_width=1., use_height=1.,
- mask_channel=0., skip=True):
- """Affine coupling with masked convolution."""
- with tf.variable_scope(name) as scope:
- if reverse or (not train):
- scope.reuse_variables()
- shape = input_.get_shape().as_list()
- batch_size = shape[0]
- height = shape[1]
- width = shape[2]
- channels = shape[3]
-
- # build mask
- mask = use_width * numpy.arange(width)
- mask = use_height * numpy.arange(height).reshape((-1, 1)) + mask
- mask = mask.astype("float32")
- mask = tf.mod(mask_in + mask, 2)
- mask = tf.reshape(mask, [-1, height, width, 1])
- if mask.get_shape().as_list()[0] == 1:
- mask = tf.tile(mask, [batch_size, 1, 1, 1])
- res = input_ * tf.mod(mask_channel + mask, 2)
-
- # initial input
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=channels, name="bn_in", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res *= 2.
- res = tf.concat([res, -res], 3)
- res = tf.concat([res, mask], 3)
- dim_in = 2. * channels + 1
- res = tf.nn.relu(res)
- res = resnet(input_=res, dim_in=dim_in, dim=dim,
- dim_out=2 * channels,
- name="resnet", use_batch_norm=use_batch_norm,
- train=train, weight_norm=weight_norm,
- residual_blocks=residual_blocks,
- bottleneck=bottleneck, skip=skip)
- mask = tf.mod(mask_channel + mask, 2)
- res = tf.split(axis=3, num_or_size_splits=2, value=res)
- shift, log_rescaling = res[-2], res[-1]
- scale = variable_on_cpu(
- "rescaling_scale", [],
- tf.constant_initializer(0.))
- shift = tf.reshape(
- shift, [batch_size, height, width, channels])
- log_rescaling = tf.reshape(
- log_rescaling, [batch_size, height, width, channels])
- log_rescaling = scale * tf.tanh(log_rescaling)
- if not use_batch_norm:
- scale_shift = variable_on_cpu(
- "scale_shift", [],
- tf.constant_initializer(0.))
- log_rescaling += scale_shift
- shift *= (1. - mask)
- log_rescaling *= (1. - mask)
- if reverse:
- res = input_
- if use_batch_norm:
- mean, var = batch_norm_log_diff(
- input_=res * (1. - mask), dim=channels, name="bn_out",
- train=False, epsilon=1e-4, axes=[0, 1, 2])
- log_var = tf.log(var)
- res *= tf.exp(.5 * log_var * (1. - mask))
- res += mean * (1. - mask)
- res *= tf.exp(-log_rescaling)
- res -= shift
- log_diff = -log_rescaling
- if use_batch_norm:
- log_diff += .5 * log_var * (1. - mask)
- else:
- res = input_
- res += shift
- res *= tf.exp(log_rescaling)
- log_diff = log_rescaling
- if use_batch_norm:
- mean, var = batch_norm_log_diff(
- input_=res * (1. - mask), dim=channels, name="bn_out",
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- log_var = tf.log(var)
- res -= mean * (1. - mask)
- res *= tf.exp(-.5 * log_var * (1. - mask))
- log_diff -= .5 * log_var * (1. - mask)
-
- return res, log_diff
-
-
-def masked_conv_add_coupling(input_, mask_in, dim, name,
- use_batch_norm=True, train=True, weight_norm=True,
- reverse=False, residual_blocks=5,
- bottleneck=False, use_width=1., use_height=1.,
- mask_channel=0., skip=True):
- """Additive coupling with masked convolution."""
- with tf.variable_scope(name) as scope:
- if reverse or (not train):
- scope.reuse_variables()
- shape = input_.get_shape().as_list()
- batch_size = shape[0]
- height = shape[1]
- width = shape[2]
- channels = shape[3]
-
- # build mask
- mask = use_width * numpy.arange(width)
- mask = use_height * numpy.arange(height).reshape((-1, 1)) + mask
- mask = mask.astype("float32")
- mask = tf.mod(mask_in + mask, 2)
- mask = tf.reshape(mask, [-1, height, width, 1])
- if mask.get_shape().as_list()[0] == 1:
- mask = tf.tile(mask, [batch_size, 1, 1, 1])
- res = input_ * tf.mod(mask_channel + mask, 2)
-
- # initial input
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=channels, name="bn_in", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res *= 2.
- res = tf.concat([res, -res], 3)
- res = tf.concat([res, mask], 3)
- dim_in = 2. * channels + 1
- res = tf.nn.relu(res)
- shift = resnet(input_=res, dim_in=dim_in, dim=dim, dim_out=channels,
- name="resnet", use_batch_norm=use_batch_norm,
- train=train, weight_norm=weight_norm,
- residual_blocks=residual_blocks,
- bottleneck=bottleneck, skip=skip)
- mask = tf.mod(mask_channel + mask, 2)
- shift *= (1. - mask)
- # use_batch_norm = False
- if reverse:
- res = input_
- if use_batch_norm:
- mean, var = batch_norm_log_diff(
- input_=res * (1. - mask),
- dim=channels, name="bn_out", train=False, epsilon=1e-4)
- log_var = tf.log(var)
- res *= tf.exp(.5 * log_var * (1. - mask))
- res += mean * (1. - mask)
- res -= shift
- log_diff = tf.zeros_like(res)
- if use_batch_norm:
- log_diff += .5 * log_var * (1. - mask)
- else:
- res = input_
- res += shift
- log_diff = tf.zeros_like(res)
- if use_batch_norm:
- mean, var = batch_norm_log_diff(
- input_=res * (1. - mask), dim=channels,
- name="bn_out", train=train, epsilon=1e-4, axes=[0, 1, 2])
- log_var = tf.log(var)
- res -= mean * (1. - mask)
- res *= tf.exp(-.5 * log_var * (1. - mask))
- log_diff -= .5 * log_var * (1. - mask)
-
- return res, log_diff
-
-
-def masked_conv_coupling(input_, mask_in, dim, name,
- use_batch_norm=True, train=True, weight_norm=True,
- reverse=False, residual_blocks=5,
- bottleneck=False, use_aff=True,
- use_width=1., use_height=1.,
- mask_channel=0., skip=True):
- """Coupling with masked convolution."""
- if use_aff:
- return masked_conv_aff_coupling(
- input_=input_, mask_in=mask_in, dim=dim, name=name,
- use_batch_norm=use_batch_norm, train=train, weight_norm=weight_norm,
- reverse=reverse, residual_blocks=residual_blocks,
- bottleneck=bottleneck, use_width=use_width, use_height=use_height,
- mask_channel=mask_channel, skip=skip)
- else:
- return masked_conv_add_coupling(
- input_=input_, mask_in=mask_in, dim=dim, name=name,
- use_batch_norm=use_batch_norm, train=train, weight_norm=weight_norm,
- reverse=reverse, residual_blocks=residual_blocks,
- bottleneck=bottleneck, use_width=use_width, use_height=use_height,
- mask_channel=mask_channel, skip=skip)
-
-
-# channel-axis splitting implementations
-def conv_ch_aff_coupling(input_, dim, name,
- use_batch_norm=True, train=True, weight_norm=True,
- reverse=False, residual_blocks=5,
- bottleneck=False, change_bottom=True, skip=True):
- """Affine coupling with channel-wise splitting."""
- with tf.variable_scope(name) as scope:
- if reverse or (not train):
- scope.reuse_variables()
-
- if change_bottom:
- input_, canvas = tf.split(axis=3, num_or_size_splits=2, value=input_)
- else:
- canvas, input_ = tf.split(axis=3, num_or_size_splits=2, value=input_)
- shape = input_.get_shape().as_list()
- batch_size = shape[0]
- height = shape[1]
- width = shape[2]
- channels = shape[3]
- res = input_
-
- # initial input
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=channels, name="bn_in", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.concat([res, -res], 3)
- dim_in = 2. * channels
- res = tf.nn.relu(res)
- res = resnet(input_=res, dim_in=dim_in, dim=dim, dim_out=2 * channels,
- name="resnet", use_batch_norm=use_batch_norm,
- train=train, weight_norm=weight_norm,
- residual_blocks=residual_blocks,
- bottleneck=bottleneck, skip=skip)
- shift, log_rescaling = tf.split(axis=3, num_or_size_splits=2, value=res)
- scale = variable_on_cpu(
- "scale", [],
- tf.constant_initializer(1.))
- shift = tf.reshape(
- shift, [batch_size, height, width, channels])
- log_rescaling = tf.reshape(
- log_rescaling, [batch_size, height, width, channels])
- log_rescaling = scale * tf.tanh(log_rescaling)
- if not use_batch_norm:
- scale_shift = variable_on_cpu(
- "scale_shift", [],
- tf.constant_initializer(0.))
- log_rescaling += scale_shift
- if reverse:
- res = canvas
- if use_batch_norm:
- mean, var = batch_norm_log_diff(
- input_=res, dim=channels, name="bn_out", train=False,
- epsilon=1e-4, axes=[0, 1, 2])
- log_var = tf.log(var)
- res *= tf.exp(.5 * log_var)
- res += mean
- res *= tf.exp(-log_rescaling)
- res -= shift
- log_diff = -log_rescaling
- if use_batch_norm:
- log_diff += .5 * log_var
- else:
- res = canvas
- res += shift
- res *= tf.exp(log_rescaling)
- log_diff = log_rescaling
- if use_batch_norm:
- mean, var = batch_norm_log_diff(
- input_=res, dim=channels, name="bn_out", train=train,
- epsilon=1e-4, axes=[0, 1, 2])
- log_var = tf.log(var)
- res -= mean
- res *= tf.exp(-.5 * log_var)
- log_diff -= .5 * log_var
- if change_bottom:
- res = tf.concat([input_, res], 3)
- log_diff = tf.concat([tf.zeros_like(log_diff), log_diff], 3)
- else:
- res = tf.concat([res, input_], 3)
- log_diff = tf.concat([log_diff, tf.zeros_like(log_diff)], 3)
-
- return res, log_diff
-
-
-def conv_ch_add_coupling(input_, dim, name,
- use_batch_norm=True, train=True, weight_norm=True,
- reverse=False, residual_blocks=5,
- bottleneck=False, change_bottom=True, skip=True):
- """Additive coupling with channel-wise splitting."""
- with tf.variable_scope(name) as scope:
- if reverse or (not train):
- scope.reuse_variables()
-
- if change_bottom:
- input_, canvas = tf.split(axis=3, num_or_size_splits=2, value=input_)
- else:
- canvas, input_ = tf.split(axis=3, num_or_size_splits=2, value=input_)
- shape = input_.get_shape().as_list()
- channels = shape[3]
- res = input_
-
- # initial input
- if use_batch_norm:
- res = batch_norm(
- input_=res, dim=channels, name="bn_in", scale=False,
- train=train, epsilon=1e-4, axes=[0, 1, 2])
- res = tf.concat([res, -res], 3)
- dim_in = 2. * channels
- res = tf.nn.relu(res)
- shift = resnet(input_=res, dim_in=dim_in, dim=dim, dim_out=channels,
- name="resnet", use_batch_norm=use_batch_norm,
- train=train, weight_norm=weight_norm,
- residual_blocks=residual_blocks,
- bottleneck=bottleneck, skip=skip)
- if reverse:
- res = canvas
- if use_batch_norm:
- mean, var = batch_norm_log_diff(
- input_=res, dim=channels, name="bn_out", train=False,
- epsilon=1e-4, axes=[0, 1, 2])
- log_var = tf.log(var)
- res *= tf.exp(.5 * log_var)
- res += mean
- res -= shift
- log_diff = tf.zeros_like(res)
- if use_batch_norm:
- log_diff += .5 * log_var
- else:
- res = canvas
- res += shift
- log_diff = tf.zeros_like(res)
- if use_batch_norm:
- mean, var = batch_norm_log_diff(
- input_=res, dim=channels, name="bn_out", train=train,
- epsilon=1e-4, axes=[0, 1, 2])
- log_var = tf.log(var)
- res -= mean
- res *= tf.exp(-.5 * log_var)
- log_diff -= .5 * log_var
- if change_bottom:
- res = tf.concat([input_, res], 3)
- log_diff = tf.concat([tf.zeros_like(log_diff), log_diff], 3)
- else:
- res = tf.concat([res, input_], 3)
- log_diff = tf.concat([log_diff, tf.zeros_like(log_diff)], 3)
-
- return res, log_diff
-
-
-def conv_ch_coupling(input_, dim, name,
- use_batch_norm=True, train=True, weight_norm=True,
- reverse=False, residual_blocks=5,
- bottleneck=False, use_aff=True, change_bottom=True,
- skip=True):
- """Coupling with channel-wise splitting."""
- if use_aff:
- return conv_ch_aff_coupling(
- input_=input_, dim=dim, name=name,
- use_batch_norm=use_batch_norm, train=train, weight_norm=weight_norm,
- reverse=reverse, residual_blocks=residual_blocks,
- bottleneck=bottleneck, change_bottom=change_bottom, skip=skip)
- else:
- return conv_ch_add_coupling(
- input_=input_, dim=dim, name=name,
- use_batch_norm=use_batch_norm, train=train, weight_norm=weight_norm,
- reverse=reverse, residual_blocks=residual_blocks,
- bottleneck=bottleneck, change_bottom=change_bottom, skip=skip)
-
-
-# RECURSIVE USE OF COUPLING LAYERS
-def rec_masked_conv_coupling(input_, hps, scale_idx, n_scale,
- use_batch_norm=True, weight_norm=True,
- train=True):
- """Recursion on coupling layers."""
- shape = input_.get_shape().as_list()
- channels = shape[3]
- residual_blocks = hps.residual_blocks
- base_dim = hps.base_dim
- mask = 1.
- use_aff = hps.use_aff
- res = input_
- skip = hps.skip
- log_diff = tf.zeros_like(input_)
- dim = base_dim
- if FLAGS.recursion_type < 4:
- dim *= 2 ** scale_idx
- with tf.variable_scope("scale_%d" % scale_idx):
- # initial coupling layers
- res, inc_log_diff = masked_conv_coupling(
- input_=res,
- mask_in=mask, dim=dim,
- name="coupling_0",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=False, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=use_aff,
- use_width=1., use_height=1., skip=skip)
- log_diff += inc_log_diff
- res, inc_log_diff = masked_conv_coupling(
- input_=res,
- mask_in=1. - mask, dim=dim,
- name="coupling_1",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=False, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=use_aff,
- use_width=1., use_height=1., skip=skip)
- log_diff += inc_log_diff
- res, inc_log_diff = masked_conv_coupling(
- input_=res,
- mask_in=mask, dim=dim,
- name="coupling_2",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=False, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=True,
- use_width=1., use_height=1., skip=skip)
- log_diff += inc_log_diff
- if scale_idx < (n_scale - 1):
- with tf.variable_scope("scale_%d" % scale_idx):
- res = squeeze_2x2(res)
- log_diff = squeeze_2x2(log_diff)
- res, inc_log_diff = conv_ch_coupling(
- input_=res,
- change_bottom=True, dim=2 * dim,
- name="coupling_4",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=False, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=use_aff, skip=skip)
- log_diff += inc_log_diff
- res, inc_log_diff = conv_ch_coupling(
- input_=res,
- change_bottom=False, dim=2 * dim,
- name="coupling_5",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=False, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=use_aff, skip=skip)
- log_diff += inc_log_diff
- res, inc_log_diff = conv_ch_coupling(
- input_=res,
- change_bottom=True, dim=2 * dim,
- name="coupling_6",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=False, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=True, skip=skip)
- log_diff += inc_log_diff
- res = unsqueeze_2x2(res)
- log_diff = unsqueeze_2x2(log_diff)
- if FLAGS.recursion_type > 1:
- res = squeeze_2x2_ordered(res)
- log_diff = squeeze_2x2_ordered(log_diff)
- if FLAGS.recursion_type > 2:
- res_1 = res[:, :, :, :channels]
- res_2 = res[:, :, :, channels:]
- log_diff_1 = log_diff[:, :, :, :channels]
- log_diff_2 = log_diff[:, :, :, channels:]
- else:
- res_1, res_2 = tf.split(axis=3, num_or_size_splits=2, value=res)
- log_diff_1, log_diff_2 = tf.split(axis=3, num_or_size_splits=2, value=log_diff)
- res_1, inc_log_diff = rec_masked_conv_coupling(
- input_=res_1, hps=hps, scale_idx=scale_idx + 1, n_scale=n_scale,
- use_batch_norm=use_batch_norm, weight_norm=weight_norm,
- train=train)
- res = tf.concat([res_1, res_2], 3)
- log_diff_1 += inc_log_diff
- log_diff = tf.concat([log_diff_1, log_diff_2], 3)
- res = squeeze_2x2_ordered(res, reverse=True)
- log_diff = squeeze_2x2_ordered(log_diff, reverse=True)
- else:
- res = squeeze_2x2_ordered(res)
- log_diff = squeeze_2x2_ordered(log_diff)
- res, inc_log_diff = rec_masked_conv_coupling(
- input_=res, hps=hps, scale_idx=scale_idx + 1, n_scale=n_scale,
- use_batch_norm=use_batch_norm, weight_norm=weight_norm,
- train=train)
- log_diff += inc_log_diff
- res = squeeze_2x2_ordered(res, reverse=True)
- log_diff = squeeze_2x2_ordered(log_diff, reverse=True)
- else:
- with tf.variable_scope("scale_%d" % scale_idx):
- res, inc_log_diff = masked_conv_coupling(
- input_=res,
- mask_in=1. - mask, dim=dim,
- name="coupling_3",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=False, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=True,
- use_width=1., use_height=1., skip=skip)
- log_diff += inc_log_diff
- return res, log_diff
-
-
-def rec_masked_deconv_coupling(input_, hps, scale_idx, n_scale,
- use_batch_norm=True, weight_norm=True,
- train=True):
- """Recursion on inverting coupling layers."""
- shape = input_.get_shape().as_list()
- channels = shape[3]
- residual_blocks = hps.residual_blocks
- base_dim = hps.base_dim
- mask = 1.
- use_aff = hps.use_aff
- res = input_
- log_diff = tf.zeros_like(input_)
- skip = hps.skip
- dim = base_dim
- if FLAGS.recursion_type < 4:
- dim *= 2 ** scale_idx
- if scale_idx < (n_scale - 1):
- if FLAGS.recursion_type > 1:
- res = squeeze_2x2_ordered(res)
- log_diff = squeeze_2x2_ordered(log_diff)
- if FLAGS.recursion_type > 2:
- res_1 = res[:, :, :, :channels]
- res_2 = res[:, :, :, channels:]
- log_diff_1 = log_diff[:, :, :, :channels]
- log_diff_2 = log_diff[:, :, :, channels:]
- else:
- res_1, res_2 = tf.split(axis=3, num_or_size_splits=2, value=res)
- log_diff_1, log_diff_2 = tf.split(axis=3, num_or_size_splits=2, value=log_diff)
- res_1, log_diff_1 = rec_masked_deconv_coupling(
- input_=res_1, hps=hps,
- scale_idx=scale_idx + 1, n_scale=n_scale,
- use_batch_norm=use_batch_norm, weight_norm=weight_norm,
- train=train)
- res = tf.concat([res_1, res_2], 3)
- log_diff = tf.concat([log_diff_1, log_diff_2], 3)
- res = squeeze_2x2_ordered(res, reverse=True)
- log_diff = squeeze_2x2_ordered(log_diff, reverse=True)
- else:
- res = squeeze_2x2_ordered(res)
- log_diff = squeeze_2x2_ordered(log_diff)
- res, log_diff = rec_masked_deconv_coupling(
- input_=res, hps=hps,
- scale_idx=scale_idx + 1, n_scale=n_scale,
- use_batch_norm=use_batch_norm, weight_norm=weight_norm,
- train=train)
- res = squeeze_2x2_ordered(res, reverse=True)
- log_diff = squeeze_2x2_ordered(log_diff, reverse=True)
- with tf.variable_scope("scale_%d" % scale_idx):
- res = squeeze_2x2(res)
- log_diff = squeeze_2x2(log_diff)
- res, inc_log_diff = conv_ch_coupling(
- input_=res,
- change_bottom=True, dim=2 * dim,
- name="coupling_6",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=True, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=True, skip=skip)
- log_diff += inc_log_diff
- res, inc_log_diff = conv_ch_coupling(
- input_=res,
- change_bottom=False, dim=2 * dim,
- name="coupling_5",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=True, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=use_aff, skip=skip)
- log_diff += inc_log_diff
- res, inc_log_diff = conv_ch_coupling(
- input_=res,
- change_bottom=True, dim=2 * dim,
- name="coupling_4",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=True, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=use_aff, skip=skip)
- log_diff += inc_log_diff
- res = unsqueeze_2x2(res)
- log_diff = unsqueeze_2x2(log_diff)
- else:
- with tf.variable_scope("scale_%d" % scale_idx):
- res, inc_log_diff = masked_conv_coupling(
- input_=res,
- mask_in=1. - mask, dim=dim,
- name="coupling_3",
- use_batch_norm=use_batch_norm, train=train,
- weight_norm=weight_norm,
- reverse=True, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=True,
- use_width=1., use_height=1., skip=skip)
- log_diff += inc_log_diff
-
- with tf.variable_scope("scale_%d" % scale_idx):
- res, inc_log_diff = masked_conv_coupling(
- input_=res,
- mask_in=mask, dim=dim,
- name="coupling_2",
- use_batch_norm=use_batch_norm, train=train, weight_norm=weight_norm,
- reverse=True, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=True,
- use_width=1., use_height=1., skip=skip)
- log_diff += inc_log_diff
- res, inc_log_diff = masked_conv_coupling(
- input_=res,
- mask_in=1. - mask, dim=dim,
- name="coupling_1",
- use_batch_norm=use_batch_norm, train=train, weight_norm=weight_norm,
- reverse=True, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=use_aff,
- use_width=1., use_height=1., skip=skip)
- log_diff += inc_log_diff
- res, inc_log_diff = masked_conv_coupling(
- input_=res,
- mask_in=mask, dim=dim,
- name="coupling_0",
- use_batch_norm=use_batch_norm, train=train, weight_norm=weight_norm,
- reverse=True, residual_blocks=residual_blocks,
- bottleneck=hps.bottleneck, use_aff=use_aff,
- use_width=1., use_height=1., skip=skip)
- log_diff += inc_log_diff
-
- return res, log_diff
-
-
-# ENCODER AND DECODER IMPLEMENTATIONS
-# start the recursions
-def encoder(input_, hps, n_scale, use_batch_norm=True,
- weight_norm=True, train=True):
- """Encoding/gaussianization function."""
- res = input_
- log_diff = tf.zeros_like(input_)
- res, inc_log_diff = rec_masked_conv_coupling(
- input_=res, hps=hps, scale_idx=0, n_scale=n_scale,
- use_batch_norm=use_batch_norm, weight_norm=weight_norm,
- train=train)
- log_diff += inc_log_diff
-
- return res, log_diff
-
-
-def decoder(input_, hps, n_scale, use_batch_norm=True,
- weight_norm=True, train=True):
- """Decoding/generator function."""
- res, log_diff = rec_masked_deconv_coupling(
- input_=input_, hps=hps, scale_idx=0, n_scale=n_scale,
- use_batch_norm=use_batch_norm, weight_norm=weight_norm,
- train=train)
-
- return res, log_diff
-
-
-class RealNVP(object):
- """Real NVP model."""
-
- def __init__(self, hps, sampling=False):
- # DATA TENSOR INSTANTIATION
- device = "/cpu:0"
- if FLAGS.dataset == "imnet":
- with tf.device(
- tf.train.replica_device_setter(0, worker_device=device)):
- filename_queue = tf.train.string_input_producer(
- gfile.Glob(FLAGS.data_path), num_epochs=None)
- reader = tf.TFRecordReader()
- _, serialized_example = reader.read(filename_queue)
- features = tf.parse_single_example(
- serialized_example,
- features={
- "image_raw": tf.FixedLenFeature([], tf.string),
- })
- image = tf.decode_raw(features["image_raw"], tf.uint8)
- image.set_shape([FLAGS.image_size * FLAGS.image_size * 3])
- image = tf.cast(image, tf.float32)
- if FLAGS.mode == "train":
- images = tf.train.shuffle_batch(
- [image], batch_size=hps.batch_size, num_threads=1,
- capacity=1000 + 3 * hps.batch_size,
- # Ensures a minimum amount of shuffling of examples.
- min_after_dequeue=1000)
- else:
- images = tf.train.batch(
- [image], batch_size=hps.batch_size, num_threads=1,
- capacity=1000 + 3 * hps.batch_size)
- self.x_orig = x_orig = images
- image_size = FLAGS.image_size
- x_in = tf.reshape(
- x_orig,
- [hps.batch_size, FLAGS.image_size, FLAGS.image_size, 3])
- x_in = tf.clip_by_value(x_in, 0, 255)
- x_in = (tf.cast(x_in, tf.float32)
- + tf.random_uniform(tf.shape(x_in))) / 256.
- elif FLAGS.dataset == "celeba":
- with tf.device(
- tf.train.replica_device_setter(0, worker_device=device)):
- filename_queue = tf.train.string_input_producer(
- gfile.Glob(FLAGS.data_path), num_epochs=None)
- reader = tf.TFRecordReader()
- _, serialized_example = reader.read(filename_queue)
- features = tf.parse_single_example(
- serialized_example,
- features={
- "image_raw": tf.FixedLenFeature([], tf.string),
- })
- image = tf.decode_raw(features["image_raw"], tf.uint8)
- image.set_shape([218 * 178 * 3]) # 218, 178
- image = tf.cast(image, tf.float32)
- image = tf.reshape(image, [218, 178, 3])
- image = image[40:188, 15:163, :]
- if FLAGS.mode == "train":
- image = tf.image.random_flip_left_right(image)
- images = tf.train.shuffle_batch(
- [image], batch_size=hps.batch_size, num_threads=1,
- capacity=1000 + 3 * hps.batch_size,
- min_after_dequeue=1000)
- else:
- images = tf.train.batch(
- [image], batch_size=hps.batch_size, num_threads=1,
- capacity=1000 + 3 * hps.batch_size)
- self.x_orig = x_orig = images
- image_size = 64
- x_in = tf.reshape(x_orig, [hps.batch_size, 148, 148, 3])
- x_in = tf.image.resize_images(
- x_in, [64, 64], method=0, align_corners=False)
- x_in = (tf.cast(x_in, tf.float32)
- + tf.random_uniform(tf.shape(x_in))) / 256.
- elif FLAGS.dataset == "lsun":
- with tf.device(
- tf.train.replica_device_setter(0, worker_device=device)):
- filename_queue = tf.train.string_input_producer(
- gfile.Glob(FLAGS.data_path), num_epochs=None)
- reader = tf.TFRecordReader()
- _, serialized_example = reader.read(filename_queue)
- features = tf.parse_single_example(
- serialized_example,
- features={
- "image_raw": tf.FixedLenFeature([], tf.string),
- "height": tf.FixedLenFeature([], tf.int64),
- "width": tf.FixedLenFeature([], tf.int64),
- "depth": tf.FixedLenFeature([], tf.int64)
- })
- image = tf.decode_raw(features["image_raw"], tf.uint8)
- height = tf.reshape((features["height"], tf.int64)[0], [1])
- height = tf.cast(height, tf.int32)
- width = tf.reshape((features["width"], tf.int64)[0], [1])
- width = tf.cast(width, tf.int32)
- depth = tf.reshape((features["depth"], tf.int64)[0], [1])
- depth = tf.cast(depth, tf.int32)
- image = tf.reshape(image, tf.concat([height, width, depth], 0))
- image = tf.random_crop(image, [64, 64, 3])
- if FLAGS.mode == "train":
- image = tf.image.random_flip_left_right(image)
- images = tf.train.shuffle_batch(
- [image], batch_size=hps.batch_size, num_threads=1,
- capacity=1000 + 3 * hps.batch_size,
- # Ensures a minimum amount of shuffling of examples.
- min_after_dequeue=1000)
- else:
- images = tf.train.batch(
- [image], batch_size=hps.batch_size, num_threads=1,
- capacity=1000 + 3 * hps.batch_size)
- self.x_orig = x_orig = images
- image_size = 64
- x_in = tf.reshape(x_orig, [hps.batch_size, 64, 64, 3])
- x_in = (tf.cast(x_in, tf.float32)
- + tf.random_uniform(tf.shape(x_in))) / 256.
- else:
- raise ValueError("Unknown dataset.")
- x_in = tf.reshape(x_in, [hps.batch_size, image_size, image_size, 3])
- side_shown = int(numpy.sqrt(hps.batch_size))
- shown_x = tf.transpose(
- tf.reshape(
- x_in[:(side_shown * side_shown), :, :, :],
- [side_shown, image_size * side_shown, image_size, 3]),
- [0, 2, 1, 3])
- shown_x = tf.transpose(
- tf.reshape(
- shown_x,
- [1, image_size * side_shown, image_size * side_shown, 3]),
- [0, 2, 1, 3]) * 255.
- tf.summary.image(
- "inputs",
- tf.cast(shown_x, tf.uint8),
- max_outputs=1)
-
- # restrict the data
- FLAGS.image_size = image_size
- data_constraint = hps.data_constraint
- pre_logit_scale = numpy.log(data_constraint)
- pre_logit_scale -= numpy.log(1. - data_constraint)
- pre_logit_scale = tf.cast(pre_logit_scale, tf.float32)
- logit_x_in = 2. * x_in # [0, 2]
- logit_x_in -= 1. # [-1, 1]
- logit_x_in *= data_constraint # [-.9, .9]
- logit_x_in += 1. # [.1, 1.9]
- logit_x_in /= 2. # [.05, .95]
- # logit the data
- logit_x_in = tf.log(logit_x_in) - tf.log(1. - logit_x_in)
- transform_cost = tf.reduce_sum(
- tf.nn.softplus(logit_x_in) + tf.nn.softplus(-logit_x_in)
- - tf.nn.softplus(-pre_logit_scale),
- [1, 2, 3])
-
- # INFERENCE AND COSTS
- z_out, log_diff = encoder(
- input_=logit_x_in, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=True)
- if FLAGS.mode != "train":
- z_out, log_diff = encoder(
- input_=logit_x_in, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=False)
- final_shape = [image_size, image_size, 3]
- prior_ll = standard_normal_ll(z_out)
- prior_ll = tf.reduce_sum(prior_ll, [1, 2, 3])
- log_diff = tf.reduce_sum(log_diff, [1, 2, 3])
- log_diff += transform_cost
- cost = -(prior_ll + log_diff)
-
- self.x_in = x_in
- self.z_out = z_out
- self.cost = cost = tf.reduce_mean(cost)
-
- l2_reg = sum(
- [tf.reduce_sum(tf.square(v)) for v in tf.trainable_variables()
- if ("magnitude" in v.name) or ("rescaling_scale" in v.name)])
-
- bit_per_dim = ((cost + numpy.log(256.) * image_size * image_size * 3.)
- / (image_size * image_size * 3. * numpy.log(2.)))
- self.bit_per_dim = bit_per_dim
-
- # OPTIMIZATION
- momentum = 1. - hps.momentum
- decay = 1. - hps.decay
- if hps.optimizer == "adam":
- optimizer = tf.train.AdamOptimizer(
- learning_rate=hps.learning_rate,
- beta1=momentum, beta2=decay, epsilon=1e-08,
- use_locking=False, name="Adam")
- elif hps.optimizer == "rmsprop":
- optimizer = tf.train.RMSPropOptimizer(
- learning_rate=hps.learning_rate, decay=decay,
- momentum=momentum, epsilon=1e-04,
- use_locking=False, name="RMSProp")
- else:
- optimizer = tf.train.MomentumOptimizer(hps.learning_rate,
- momentum=momentum)
-
- step = tf.get_variable(
- "global_step", [], tf.int64,
- tf.zeros_initializer(),
- trainable=False)
- self.step = step
- grads_and_vars = optimizer.compute_gradients(
- cost + hps.l2_coeff * l2_reg,
- tf.trainable_variables())
- grads, vars_ = zip(*grads_and_vars)
- capped_grads, gradient_norm = tf.clip_by_global_norm(
- grads, clip_norm=hps.clip_gradient)
- gradient_norm = tf.check_numerics(gradient_norm,
- "Gradient norm is NaN or Inf.")
-
- l2_z = tf.reduce_sum(tf.square(z_out), [1, 2, 3])
- if not sampling:
- tf.summary.scalar("negative_log_likelihood", tf.reshape(cost, []))
- tf.summary.scalar("gradient_norm", tf.reshape(gradient_norm, []))
- tf.summary.scalar("bit_per_dim", tf.reshape(bit_per_dim, []))
- tf.summary.scalar("log_diff", tf.reshape(tf.reduce_mean(log_diff), []))
- tf.summary.scalar("prior_ll", tf.reshape(tf.reduce_mean(prior_ll), []))
- tf.summary.scalar(
- "log_diff_var",
- tf.reshape(tf.reduce_mean(tf.square(log_diff))
- - tf.square(tf.reduce_mean(log_diff)), []))
- tf.summary.scalar(
- "prior_ll_var",
- tf.reshape(tf.reduce_mean(tf.square(prior_ll))
- - tf.square(tf.reduce_mean(prior_ll)), []))
- tf.summary.scalar("l2_z_mean", tf.reshape(tf.reduce_mean(l2_z), []))
- tf.summary.scalar(
- "l2_z_var",
- tf.reshape(tf.reduce_mean(tf.square(l2_z))
- - tf.square(tf.reduce_mean(l2_z)), []))
-
-
- capped_grads_and_vars = zip(capped_grads, vars_)
- self.train_step = optimizer.apply_gradients(
- capped_grads_and_vars, global_step=step)
-
- # SAMPLING AND VISUALIZATION
- if sampling:
- # SAMPLES
- sample = standard_normal_sample([100] + final_shape)
- sample, _ = decoder(
- input_=sample, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=True)
- sample = tf.nn.sigmoid(sample)
-
- sample = tf.clip_by_value(sample, 0, 1) * 255.
- sample = tf.reshape(sample, [100, image_size, image_size, 3])
- sample = tf.transpose(
- tf.reshape(sample, [10, image_size * 10, image_size, 3]),
- [0, 2, 1, 3])
- sample = tf.transpose(
- tf.reshape(sample, [1, image_size * 10, image_size * 10, 3]),
- [0, 2, 1, 3])
- tf.summary.image(
- "samples",
- tf.cast(sample, tf.uint8),
- max_outputs=1)
-
- # CONCATENATION
- concatenation, _ = encoder(
- input_=logit_x_in, hps=hps,
- n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=False)
- concatenation = tf.reshape(
- concatenation,
- [(side_shown * side_shown), image_size, image_size, 3])
- concatenation = tf.transpose(
- tf.reshape(
- concatenation,
- [side_shown, image_size * side_shown, image_size, 3]),
- [0, 2, 1, 3])
- concatenation = tf.transpose(
- tf.reshape(
- concatenation,
- [1, image_size * side_shown, image_size * side_shown, 3]),
- [0, 2, 1, 3])
- concatenation, _ = decoder(
- input_=concatenation, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=False)
- concatenation = tf.nn.sigmoid(concatenation) * 255.
- tf.summary.image(
- "concatenation",
- tf.cast(concatenation, tf.uint8),
- max_outputs=1)
-
- # MANIFOLD
-
- # Data basis
- z_u, _ = encoder(
- input_=logit_x_in[:8, :, :, :], hps=hps,
- n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=False)
- u_1 = tf.reshape(z_u[0, :, :, :], [-1])
- u_2 = tf.reshape(z_u[1, :, :, :], [-1])
- u_3 = tf.reshape(z_u[2, :, :, :], [-1])
- u_4 = tf.reshape(z_u[3, :, :, :], [-1])
- u_5 = tf.reshape(z_u[4, :, :, :], [-1])
- u_6 = tf.reshape(z_u[5, :, :, :], [-1])
- u_7 = tf.reshape(z_u[6, :, :, :], [-1])
- u_8 = tf.reshape(z_u[7, :, :, :], [-1])
-
- # 3D dome
- manifold_side = 8
- angle_1 = numpy.arange(manifold_side) * 1. / manifold_side
- angle_2 = numpy.arange(manifold_side) * 1. / manifold_side
- angle_1 *= 2. * numpy.pi
- angle_2 *= 2. * numpy.pi
- angle_1 = angle_1.astype("float32")
- angle_2 = angle_2.astype("float32")
- angle_1 = tf.reshape(angle_1, [1, -1, 1])
- angle_1 += tf.zeros([manifold_side, manifold_side, 1])
- angle_2 = tf.reshape(angle_2, [-1, 1, 1])
- angle_2 += tf.zeros([manifold_side, manifold_side, 1])
- n_angle_3 = 40
- angle_3 = numpy.arange(n_angle_3) * 1. / n_angle_3
- angle_3 *= 2 * numpy.pi
- angle_3 = angle_3.astype("float32")
- angle_3 = tf.reshape(angle_3, [-1, 1, 1, 1])
- angle_3 += tf.zeros([n_angle_3, manifold_side, manifold_side, 1])
- manifold = tf.cos(angle_1) * (
- tf.cos(angle_2) * (
- tf.cos(angle_3) * u_1 + tf.sin(angle_3) * u_2)
- + tf.sin(angle_2) * (
- tf.cos(angle_3) * u_3 + tf.sin(angle_3) * u_4))
- manifold += tf.sin(angle_1) * (
- tf.cos(angle_2) * (
- tf.cos(angle_3) * u_5 + tf.sin(angle_3) * u_6)
- + tf.sin(angle_2) * (
- tf.cos(angle_3) * u_7 + tf.sin(angle_3) * u_8))
- manifold = tf.reshape(
- manifold,
- [n_angle_3 * manifold_side * manifold_side] + final_shape)
- manifold, _ = decoder(
- input_=manifold, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=False)
- manifold = tf.nn.sigmoid(manifold)
-
- manifold = tf.clip_by_value(manifold, 0, 1) * 255.
- manifold = tf.reshape(
- manifold,
- [n_angle_3,
- manifold_side * manifold_side,
- image_size,
- image_size,
- 3])
- manifold = tf.transpose(
- tf.reshape(
- manifold,
- [n_angle_3, manifold_side,
- image_size * manifold_side, image_size, 3]), [0, 1, 3, 2, 4])
- manifold = tf.transpose(
- tf.reshape(
- manifold,
- [n_angle_3, image_size * manifold_side,
- image_size * manifold_side, 3]),
- [0, 2, 1, 3])
- manifold = tf.transpose(manifold, [1, 2, 0, 3])
- manifold = tf.reshape(
- manifold,
- [1, image_size * manifold_side,
- image_size * manifold_side, 3 * n_angle_3])
- tf.summary.image(
- "manifold",
- tf.cast(manifold[:, :, :, :3], tf.uint8),
- max_outputs=1)
-
- # COMPRESSION
- z_complete, _ = encoder(
- input_=logit_x_in[:hps.n_scale, :, :, :], hps=hps,
- n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=False)
- z_compressed_list = [z_complete]
- z_noisy_list = [z_complete]
- z_lost = z_complete
- for scale_idx in xrange(hps.n_scale - 1):
- z_lost = squeeze_2x2_ordered(z_lost)
- z_lost, _ = tf.split(axis=3, num_or_size_splits=2, value=z_lost)
- z_compressed = z_lost
- z_noisy = z_lost
- for _ in xrange(scale_idx + 1):
- z_compressed = tf.concat(
- [z_compressed, tf.zeros_like(z_compressed)], 3)
- z_compressed = squeeze_2x2_ordered(
- z_compressed, reverse=True)
- z_noisy = tf.concat(
- [z_noisy, tf.random_normal(
- z_noisy.get_shape().as_list())], 3)
- z_noisy = squeeze_2x2_ordered(z_noisy, reverse=True)
- z_compressed_list.append(z_compressed)
- z_noisy_list.append(z_noisy)
- self.z_reduced = z_lost
- z_compressed = tf.concat(z_compressed_list, 0)
- z_noisy = tf.concat(z_noisy_list, 0)
- noisy_images, _ = decoder(
- input_=z_noisy, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=False)
- compressed_images, _ = decoder(
- input_=z_compressed, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=False)
- noisy_images = tf.nn.sigmoid(noisy_images)
- compressed_images = tf.nn.sigmoid(compressed_images)
-
- noisy_images = tf.clip_by_value(noisy_images, 0, 1) * 255.
- noisy_images = tf.reshape(
- noisy_images,
- [(hps.n_scale * hps.n_scale), image_size, image_size, 3])
- noisy_images = tf.transpose(
- tf.reshape(
- noisy_images,
- [hps.n_scale, image_size * hps.n_scale, image_size, 3]),
- [0, 2, 1, 3])
- noisy_images = tf.transpose(
- tf.reshape(
- noisy_images,
- [1, image_size * hps.n_scale, image_size * hps.n_scale, 3]),
- [0, 2, 1, 3])
- tf.summary.image(
- "noise",
- tf.cast(noisy_images, tf.uint8),
- max_outputs=1)
- compressed_images = tf.clip_by_value(compressed_images, 0, 1) * 255.
- compressed_images = tf.reshape(
- compressed_images,
- [(hps.n_scale * hps.n_scale), image_size, image_size, 3])
- compressed_images = tf.transpose(
- tf.reshape(
- compressed_images,
- [hps.n_scale, image_size * hps.n_scale, image_size, 3]),
- [0, 2, 1, 3])
- compressed_images = tf.transpose(
- tf.reshape(
- compressed_images,
- [1, image_size * hps.n_scale, image_size * hps.n_scale, 3]),
- [0, 2, 1, 3])
- tf.summary.image(
- "compression",
- tf.cast(compressed_images, tf.uint8),
- max_outputs=1)
-
- # SAMPLES x2
- final_shape[0] *= 2
- final_shape[1] *= 2
- big_sample = standard_normal_sample([25] + final_shape)
- big_sample, _ = decoder(
- input_=big_sample, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=True)
- big_sample = tf.nn.sigmoid(big_sample)
-
- big_sample = tf.clip_by_value(big_sample, 0, 1) * 255.
- big_sample = tf.reshape(
- big_sample,
- [25, image_size * 2, image_size * 2, 3])
- big_sample = tf.transpose(
- tf.reshape(
- big_sample,
- [5, image_size * 10, image_size * 2, 3]), [0, 2, 1, 3])
- big_sample = tf.transpose(
- tf.reshape(
- big_sample,
- [1, image_size * 10, image_size * 10, 3]),
- [0, 2, 1, 3])
- tf.summary.image(
- "big_sample",
- tf.cast(big_sample, tf.uint8),
- max_outputs=1)
-
- # SAMPLES x10
- final_shape[0] *= 5
- final_shape[1] *= 5
- extra_large = standard_normal_sample([1] + final_shape)
- extra_large, _ = decoder(
- input_=extra_large, hps=hps, n_scale=hps.n_scale,
- use_batch_norm=hps.use_batch_norm, weight_norm=True,
- train=True)
- extra_large = tf.nn.sigmoid(extra_large)
-
- extra_large = tf.clip_by_value(extra_large, 0, 1) * 255.
- tf.summary.image(
- "extra_large",
- tf.cast(extra_large, tf.uint8),
- max_outputs=1)
-
- def eval_epoch(self, hps):
- """Evaluate bits/dim."""
- n_eval_dict = {
- "imnet": 50000,
- "lsun": 300,
- "celeba": 19962,
- "svhn": 26032,
- }
- if FLAGS.eval_set_size == 0:
- num_examples_eval = n_eval_dict[FLAGS.dataset]
- else:
- num_examples_eval = FLAGS.eval_set_size
- n_epoch = num_examples_eval / hps.batch_size
- eval_costs = []
- bar_len = 70
- for epoch_idx in xrange(n_epoch):
- n_equal = epoch_idx * bar_len * 1. / n_epoch
- n_equal = numpy.ceil(n_equal)
- n_equal = int(n_equal)
- n_dash = bar_len - n_equal
- progress_bar = "[" + "=" * n_equal + "-" * n_dash + "]\r"
- print(progress_bar, end=' ')
- cost = self.bit_per_dim.eval()
- eval_costs.append(cost)
- print("")
- return float(numpy.mean(eval_costs))
-
-
-def train_model(hps, logdir):
- """Training."""
- with tf.Graph().as_default():
- with tf.device(tf.train.replica_device_setter(0)):
- with tf.variable_scope("model"):
- model = RealNVP(hps)
-
- saver = tf.train.Saver(tf.global_variables())
-
- # Build the summary operation from the last tower summaries.
- summary_op = tf.summary.merge_all()
-
- # Build an initialization operation to run below.
- init = tf.global_variables_initializer()
-
- # Start running operations on the Graph. allow_soft_placement must be set to
- # True to build towers on GPU, as some of the ops do not have GPU
- # implementations.
- sess = tf.Session(config=tf.ConfigProto(
- allow_soft_placement=True,
- log_device_placement=True))
- sess.run(init)
-
- ckpt_state = tf.train.get_checkpoint_state(logdir)
- if ckpt_state and ckpt_state.model_checkpoint_path:
- print("Loading file %s" % ckpt_state.model_checkpoint_path)
- saver.restore(sess, ckpt_state.model_checkpoint_path)
-
- # Start the queue runners.
- tf.train.start_queue_runners(sess=sess)
-
- summary_writer = tf.summary.FileWriter(
- logdir,
- graph=sess.graph)
-
- local_step = 0
- while True:
- fetches = [model.step, model.bit_per_dim, model.train_step]
- # The chief worker evaluates the summaries every 10 steps.
- should_eval_summaries = local_step % 100 == 0
- if should_eval_summaries:
- fetches += [summary_op]
-
-
- start_time = time.time()
- outputs = sess.run(fetches)
- global_step_val = outputs[0]
- loss = outputs[1]
- duration = time.time() - start_time
- assert not numpy.isnan(
- loss), 'Model diverged with loss = NaN'
-
- if local_step % 10 == 0:
- examples_per_sec = hps.batch_size / float(duration)
- format_str = ('%s: step %d, loss = %.2f '
- '(%.1f examples/sec; %.3f '
- 'sec/batch)')
- print(format_str % (datetime.now(), global_step_val, loss,
- examples_per_sec, duration))
-
- if should_eval_summaries:
- summary_str = outputs[-1]
- summary_writer.add_summary(summary_str, global_step_val)
-
- # Save the model checkpoint periodically.
- if local_step % 1000 == 0 or (local_step + 1) == FLAGS.train_steps:
- checkpoint_path = os.path.join(logdir, 'model.ckpt')
- saver.save(
- sess,
- checkpoint_path,
- global_step=global_step_val)
-
- if outputs[0] >= FLAGS.train_steps:
- break
-
- local_step += 1
-
-
-def evaluate(hps, logdir, traindir, subset="valid", return_val=False):
- """Evaluation."""
- hps.batch_size = 100
- with tf.Graph().as_default():
- with tf.device("/cpu:0"):
- with tf.variable_scope("model") as var_scope:
- eval_model = RealNVP(hps)
- summary_writer = tf.summary.FileWriter(logdir)
- var_scope.reuse_variables()
-
- saver = tf.train.Saver()
- sess = tf.Session(config=tf.ConfigProto(
- allow_soft_placement=True,
- log_device_placement=True))
- tf.train.start_queue_runners(sess)
-
- previous_global_step = 0 # don"t run eval for step = 0
-
- with sess.as_default():
- while True:
- ckpt_state = tf.train.get_checkpoint_state(traindir)
- if not (ckpt_state and ckpt_state.model_checkpoint_path):
- print("No model to eval yet at %s" % traindir)
- time.sleep(30)
- continue
- print("Loading file %s" % ckpt_state.model_checkpoint_path)
- saver.restore(sess, ckpt_state.model_checkpoint_path)
-
- current_step = tf.train.global_step(sess, eval_model.step)
- if current_step == previous_global_step:
- print("Waiting for the checkpoint to be updated.")
- time.sleep(30)
- continue
- previous_global_step = current_step
-
- print("Evaluating...")
- bit_per_dim = eval_model.eval_epoch(hps)
- print("Epoch: %d, %s -> %.3f bits/dim"
- % (current_step, subset, bit_per_dim))
- print("Writing summary...")
- summary = tf.Summary()
- summary.value.extend(
- [tf.Summary.Value(
- tag="bit_per_dim",
- simple_value=bit_per_dim)])
- summary_writer.add_summary(summary, current_step)
-
- if return_val:
- return current_step, bit_per_dim
-
-
-def sample_from_model(hps, logdir, traindir):
- """Sampling."""
- hps.batch_size = 100
- with tf.Graph().as_default():
- with tf.device("/cpu:0"):
- with tf.variable_scope("model") as var_scope:
- eval_model = RealNVP(hps, sampling=True)
- summary_writer = tf.summary.FileWriter(logdir)
- var_scope.reuse_variables()
-
- summary_op = tf.summary.merge_all()
- saver = tf.train.Saver()
- sess = tf.Session(config=tf.ConfigProto(
- allow_soft_placement=True,
- log_device_placement=True))
- coord = tf.train.Coordinator()
- threads = tf.train.start_queue_runners(sess=sess, coord=coord)
-
- previous_global_step = 0 # don"t run eval for step = 0
-
- initialized = False
- with sess.as_default():
- while True:
- ckpt_state = tf.train.get_checkpoint_state(traindir)
- if not (ckpt_state and ckpt_state.model_checkpoint_path):
- if not initialized:
- print("No model to eval yet at %s" % traindir)
- time.sleep(30)
- continue
- else:
- print ("Loading file %s"
- % ckpt_state.model_checkpoint_path)
- saver.restore(sess, ckpt_state.model_checkpoint_path)
-
- current_step = tf.train.global_step(sess, eval_model.step)
- if current_step == previous_global_step:
- print("Waiting for the checkpoint to be updated.")
- time.sleep(30)
- continue
- previous_global_step = current_step
-
- fetches = [summary_op]
-
- outputs = sess.run(fetches)
- summary_writer.add_summary(outputs[0], current_step)
- coord.request_stop()
- coord.join(threads)
-
-
-def main(unused_argv):
- hps = get_default_hparams().update_config(FLAGS.hpconfig)
- if FLAGS.mode == "train":
- train_model(hps=hps, logdir=FLAGS.logdir)
- elif FLAGS.mode == "sample":
- sample_from_model(hps=hps, logdir=FLAGS.logdir,
- traindir=FLAGS.traindir)
- else:
- hps.batch_size = 100
- evaluate(hps=hps, logdir=FLAGS.logdir,
- traindir=FLAGS.traindir, subset=FLAGS.mode)
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/research/real_nvp/real_nvp_utils.py b/research/real_nvp/real_nvp_utils.py
deleted file mode 100644
index d8240f0e98d5b0d91bab8a9027ac737eb425a999..0000000000000000000000000000000000000000
--- a/research/real_nvp/real_nvp_utils.py
+++ /dev/null
@@ -1,475 +0,0 @@
-# Copyright 2016 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-r"""Utility functions for Real NVP.
-"""
-
-# pylint: disable=dangerous-default-value
-
-import numpy
-from six.moves import xrange
-import tensorflow as tf
-from tensorflow.python.framework import ops
-
-DEFAULT_BN_LAG = .0
-
-
-def stable_var(input_, mean=None, axes=[0]):
- """Numerically more stable variance computation."""
- if mean is None:
- mean = tf.reduce_mean(input_, axes)
- res = tf.square(input_ - mean)
- max_sqr = tf.reduce_max(res, axes)
- res /= max_sqr
- res = tf.reduce_mean(res, axes)
- res *= max_sqr
-
- return res
-
-
-def variable_on_cpu(name, shape, initializer, trainable=True):
- """Helper to create a Variable stored on CPU memory.
-
- Args:
- name: name of the variable
- shape: list of ints
- initializer: initializer for Variable
- trainable: boolean defining if the variable is for training
- Returns:
- Variable Tensor
- """
- var = tf.get_variable(
- name, shape, initializer=initializer, trainable=trainable)
- return var
-
-
-# layers
-def conv_layer(input_,
- filter_size,
- dim_in,
- dim_out,
- name,
- stddev=1e-2,
- strides=[1, 1, 1, 1],
- padding="SAME",
- nonlinearity=None,
- bias=False,
- weight_norm=False,
- scale=False):
- """Convolutional layer."""
- with tf.variable_scope(name) as scope:
- weights = variable_on_cpu(
- "weights",
- filter_size + [dim_in, dim_out],
- tf.random_uniform_initializer(
- minval=-stddev, maxval=stddev))
- # weight normalization
- if weight_norm:
- weights /= tf.sqrt(tf.reduce_sum(tf.square(weights), [0, 1, 2]))
- if scale:
- magnitude = variable_on_cpu(
- "magnitude", [dim_out],
- tf.constant_initializer(
- stddev * numpy.sqrt(dim_in * numpy.prod(filter_size) / 12.)))
- weights *= magnitude
- res = input_
- # handling filter size bigger than image size
- if hasattr(input_, "shape"):
- if input_.get_shape().as_list()[1] < filter_size[0]:
- pad_1 = tf.zeros([
- input_.get_shape().as_list()[0],
- filter_size[0] - input_.get_shape().as_list()[1],
- input_.get_shape().as_list()[2],
- input_.get_shape().as_list()[3]
- ])
- pad_2 = tf.zeros([
- input_.get_shape().as_list[0],
- filter_size[0],
- filter_size[1] - input_.get_shape().as_list()[2],
- input_.get_shape().as_list()[3]
- ])
- res = tf.concat(axis=1, values=[pad_1, res])
- res = tf.concat(axis=2, values=[pad_2, res])
- res = tf.nn.conv2d(
- input=res,
- filter=weights,
- strides=strides,
- padding=padding,
- name=scope.name)
-
- if hasattr(input_, "shape"):
- if input_.get_shape().as_list()[1] < filter_size[0]:
- res = tf.slice(res, [
- 0, filter_size[0] - input_.get_shape().as_list()[1],
- filter_size[1] - input_.get_shape().as_list()[2], 0
- ], [-1, -1, -1, -1])
-
- if bias:
- biases = variable_on_cpu("biases", [dim_out], tf.constant_initializer(0.))
- res = tf.nn.bias_add(res, biases)
- if nonlinearity is not None:
- res = nonlinearity(res)
-
- return res
-
-
-def max_pool_2x2(input_):
- """Max pooling."""
- return tf.nn.max_pool(
- input_, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME")
-
-
-def depool_2x2(input_, stride=2):
- """Depooling."""
- shape = input_.get_shape().as_list()
- batch_size = shape[0]
- height = shape[1]
- width = shape[2]
- channels = shape[3]
- res = tf.reshape(input_, [batch_size, height, 1, width, 1, channels])
- res = tf.concat(
- axis=2, values=[res, tf.zeros([batch_size, height, stride - 1, width, 1, channels])])
- res = tf.concat(axis=4, values=[
- res, tf.zeros([batch_size, height, stride, width, stride - 1, channels])
- ])
- res = tf.reshape(res, [batch_size, stride * height, stride * width, channels])
-
- return res
-
-
-# random flip on a batch of images
-def batch_random_flip(input_):
- """Simultaneous horizontal random flip."""
- if isinstance(input_, (float, int)):
- return input_
- shape = input_.get_shape().as_list()
- batch_size = shape[0]
- height = shape[1]
- width = shape[2]
- channels = shape[3]
- res = tf.split(axis=0, num_or_size_splits=batch_size, value=input_)
- res = [elem[0, :, :, :] for elem in res]
- res = [tf.image.random_flip_left_right(elem) for elem in res]
- res = [tf.reshape(elem, [1, height, width, channels]) for elem in res]
- res = tf.concat(axis=0, values=res)
-
- return res
-
-
-# build a one hot representation corresponding to the integer tensor
-# the one-hot dimension is appended to the integer tensor shape
-def as_one_hot(input_, n_indices):
- """Convert indices to one-hot."""
- shape = input_.get_shape().as_list()
- n_elem = numpy.prod(shape)
- indices = tf.range(n_elem)
- indices = tf.cast(indices, tf.int64)
- indices_input = tf.concat(axis=0, values=[indices, tf.reshape(input_, [-1])])
- indices_input = tf.reshape(indices_input, [2, -1])
- indices_input = tf.transpose(indices_input)
- res = tf.sparse_to_dense(
- indices_input, [n_elem, n_indices], 1., 0., name="flat_one_hot")
- res = tf.reshape(res, [elem for elem in shape] + [n_indices])
-
- return res
-
-
-def squeeze_2x2(input_):
- """Squeezing operation: reshape to convert space to channels."""
- return squeeze_nxn(input_, n_factor=2)
-
-
-def squeeze_nxn(input_, n_factor=2):
- """Squeezing operation: reshape to convert space to channels."""
- if isinstance(input_, (float, int)):
- return input_
- shape = input_.get_shape().as_list()
- batch_size = shape[0]
- height = shape[1]
- width = shape[2]
- channels = shape[3]
- if height % n_factor != 0:
- raise ValueError("Height not divisible by %d." % n_factor)
- if width % n_factor != 0:
- raise ValueError("Width not divisible by %d." % n_factor)
- res = tf.reshape(
- input_,
- [batch_size,
- height // n_factor,
- n_factor, width // n_factor,
- n_factor, channels])
- res = tf.transpose(res, [0, 1, 3, 5, 2, 4])
- res = tf.reshape(
- res,
- [batch_size,
- height // n_factor,
- width // n_factor,
- channels * n_factor * n_factor])
-
- return res
-
-
-def unsqueeze_2x2(input_):
- """Unsqueezing operation: reshape to convert channels into space."""
- if isinstance(input_, (float, int)):
- return input_
- shape = input_.get_shape().as_list()
- batch_size = shape[0]
- height = shape[1]
- width = shape[2]
- channels = shape[3]
- if channels % 4 != 0:
- raise ValueError("Number of channels not divisible by 4.")
- res = tf.reshape(input_, [batch_size, height, width, channels // 4, 2, 2])
- res = tf.transpose(res, [0, 1, 4, 2, 5, 3])
- res = tf.reshape(res, [batch_size, 2 * height, 2 * width, channels // 4])
-
- return res
-
-
-# batch norm
-def batch_norm(input_,
- dim,
- name,
- scale=True,
- train=True,
- epsilon=1e-8,
- decay=.1,
- axes=[0],
- bn_lag=DEFAULT_BN_LAG):
- """Batch normalization."""
- # create variables
- with tf.variable_scope(name):
- var = variable_on_cpu(
- "var", [dim], tf.constant_initializer(1.), trainable=False)
- mean = variable_on_cpu(
- "mean", [dim], tf.constant_initializer(0.), trainable=False)
- step = variable_on_cpu("step", [], tf.constant_initializer(0.), trainable=False)
- if scale:
- gamma = variable_on_cpu("gamma", [dim], tf.constant_initializer(1.))
- beta = variable_on_cpu("beta", [dim], tf.constant_initializer(0.))
- # choose the appropriate moments
- if train:
- used_mean, used_var = tf.nn.moments(input_, axes, name="batch_norm")
- cur_mean, cur_var = used_mean, used_var
- if bn_lag > 0.:
- used_mean -= (1. - bn_lag) * (used_mean - tf.stop_gradient(mean))
- used_var -= (1 - bn_lag) * (used_var - tf.stop_gradient(var))
- used_mean /= (1. - bn_lag**(step + 1))
- used_var /= (1. - bn_lag**(step + 1))
- else:
- used_mean, used_var = mean, var
- cur_mean, cur_var = used_mean, used_var
-
- # normalize
- res = (input_ - used_mean) / tf.sqrt(used_var + epsilon)
- # de-normalize
- if scale:
- res *= gamma
- res += beta
-
- # update variables
- if train:
- with tf.name_scope(name, "AssignMovingAvg", [mean, cur_mean, decay]):
- with ops.colocate_with(mean):
- new_mean = tf.assign_sub(
- mean,
- tf.check_numerics(decay * (mean - cur_mean), "NaN in moving mean."))
- with tf.name_scope(name, "AssignMovingAvg", [var, cur_var, decay]):
- with ops.colocate_with(var):
- new_var = tf.assign_sub(
- var,
- tf.check_numerics(decay * (var - cur_var),
- "NaN in moving variance."))
- with tf.name_scope(name, "IncrementTime", [step]):
- with ops.colocate_with(step):
- new_step = tf.assign_add(step, 1.)
- res += 0. * new_mean * new_var * new_step
-
- return res
-
-
-# batch normalization taking into account the volume transformation
-def batch_norm_log_diff(input_,
- dim,
- name,
- train=True,
- epsilon=1e-8,
- decay=.1,
- axes=[0],
- reuse=None,
- bn_lag=DEFAULT_BN_LAG):
- """Batch normalization with corresponding log determinant Jacobian."""
- if reuse is None:
- reuse = not train
- # create variables
- with tf.variable_scope(name) as scope:
- if reuse:
- scope.reuse_variables()
- var = variable_on_cpu(
- "var", [dim], tf.constant_initializer(1.), trainable=False)
- mean = variable_on_cpu(
- "mean", [dim], tf.constant_initializer(0.), trainable=False)
- step = variable_on_cpu("step", [], tf.constant_initializer(0.), trainable=False)
- # choose the appropriate moments
- if train:
- used_mean, used_var = tf.nn.moments(input_, axes, name="batch_norm")
- cur_mean, cur_var = used_mean, used_var
- if bn_lag > 0.:
- used_var = stable_var(input_=input_, mean=used_mean, axes=axes)
- cur_var = used_var
- used_mean -= (1 - bn_lag) * (used_mean - tf.stop_gradient(mean))
- used_mean /= (1. - bn_lag**(step + 1))
- used_var -= (1 - bn_lag) * (used_var - tf.stop_gradient(var))
- used_var /= (1. - bn_lag**(step + 1))
- else:
- used_mean, used_var = mean, var
- cur_mean, cur_var = used_mean, used_var
-
- # update variables
- if train:
- with tf.name_scope(name, "AssignMovingAvg", [mean, cur_mean, decay]):
- with ops.colocate_with(mean):
- new_mean = tf.assign_sub(
- mean,
- tf.check_numerics(
- decay * (mean - cur_mean), "NaN in moving mean."))
- with tf.name_scope(name, "AssignMovingAvg", [var, cur_var, decay]):
- with ops.colocate_with(var):
- new_var = tf.assign_sub(
- var,
- tf.check_numerics(decay * (var - cur_var),
- "NaN in moving variance."))
- with tf.name_scope(name, "IncrementTime", [step]):
- with ops.colocate_with(step):
- new_step = tf.assign_add(step, 1.)
- used_var += 0. * new_mean * new_var * new_step
- used_var += epsilon
-
- return used_mean, used_var
-
-
-def convnet(input_,
- dim_in,
- dim_hid,
- filter_sizes,
- dim_out,
- name,
- use_batch_norm=True,
- train=True,
- nonlinearity=tf.nn.relu):
- """Chaining of convolutional layers."""
- dims_in = [dim_in] + dim_hid[:-1]
- dims_out = dim_hid
- res = input_
-
- bias = (not use_batch_norm)
- with tf.variable_scope(name):
- for layer_idx in xrange(len(dim_hid)):
- res = conv_layer(
- input_=res,
- filter_size=filter_sizes[layer_idx],
- dim_in=dims_in[layer_idx],
- dim_out=dims_out[layer_idx],
- name="h_%d" % layer_idx,
- stddev=1e-2,
- nonlinearity=None,
- bias=bias)
- if use_batch_norm:
- res = batch_norm(
- input_=res,
- dim=dims_out[layer_idx],
- name="bn_%d" % layer_idx,
- scale=(nonlinearity == tf.nn.relu),
- train=train,
- epsilon=1e-8,
- axes=[0, 1, 2])
- if nonlinearity is not None:
- res = nonlinearity(res)
-
- res = conv_layer(
- input_=res,
- filter_size=filter_sizes[-1],
- dim_in=dims_out[-1],
- dim_out=dim_out,
- name="out",
- stddev=1e-2,
- nonlinearity=None)
-
- return res
-
-
-# distributions
-# log-likelihood estimation
-def standard_normal_ll(input_):
- """Log-likelihood of standard Gaussian distribution."""
- res = -.5 * (tf.square(input_) + numpy.log(2. * numpy.pi))
-
- return res
-
-
-def standard_normal_sample(shape):
- """Samples from standard Gaussian distribution."""
- return tf.random_normal(shape)
-
-
-SQUEEZE_MATRIX = numpy.array([[[[1., 0., 0., 0.]], [[0., 0., 1., 0.]]],
- [[[0., 0., 0., 1.]], [[0., 1., 0., 0.]]]])
-
-
-def squeeze_2x2_ordered(input_, reverse=False):
- """Squeezing operation with a controlled ordering."""
- shape = input_.get_shape().as_list()
- batch_size = shape[0]
- height = shape[1]
- width = shape[2]
- channels = shape[3]
- if reverse:
- if channels % 4 != 0:
- raise ValueError("Number of channels not divisible by 4.")
- channels /= 4
- else:
- if height % 2 != 0:
- raise ValueError("Height not divisible by 2.")
- if width % 2 != 0:
- raise ValueError("Width not divisible by 2.")
- weights = numpy.zeros((2, 2, channels, 4 * channels))
- for idx_ch in xrange(channels):
- slice_2 = slice(idx_ch, (idx_ch + 1))
- slice_3 = slice((idx_ch * 4), ((idx_ch + 1) * 4))
- weights[:, :, slice_2, slice_3] = SQUEEZE_MATRIX
- shuffle_channels = [idx_ch * 4 for idx_ch in xrange(channels)]
- shuffle_channels += [idx_ch * 4 + 1 for idx_ch in xrange(channels)]
- shuffle_channels += [idx_ch * 4 + 2 for idx_ch in xrange(channels)]
- shuffle_channels += [idx_ch * 4 + 3 for idx_ch in xrange(channels)]
- shuffle_channels = numpy.array(shuffle_channels)
- weights = weights[:, :, :, shuffle_channels].astype("float32")
- if reverse:
- res = tf.nn.conv2d_transpose(
- value=input_,
- filter=weights,
- output_shape=[batch_size, height * 2, width * 2, channels],
- strides=[1, 2, 2, 1],
- padding="SAME",
- name="unsqueeze_2x2")
- else:
- res = tf.nn.conv2d(
- input=input_,
- filter=weights,
- strides=[1, 2, 2, 1],
- padding="SAME",
- name="squeeze_2x2")
-
- return res
diff --git a/research/sentiment_analysis/README.md b/research/sentiment_analysis/README.md
deleted file mode 100644
index f98c42751df2475b4dbdc1db0cc303b42d2b7b4c..0000000000000000000000000000000000000000
--- a/research/sentiment_analysis/README.md
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
-# Sentiment Analysis
-## Overview
-This is an implementation of the Sentiment Analysis model as described in the [this paper](https://arxiv.org/abs/1412.1058). The implementation is with the reference to [paddle version](https://github.com/mlperf/reference/tree/master/sentiment_analysis/paddle).
-
-The model makes use of concatenation of two CNN layers with different kernel sizes. Batch normalization and dropout layers are used to prevent over-fitting.
-
-## Dataset
-The [keras](https://keras.io)'s [IMDB Movie reviews sentiment classification](https://keras.io/datasets/#imdb-movie-reviews-sentiment-classification) dataset is used. The dataset file download is handled by keras module, and the downloaded files are stored at ``~/.keras/datasets` directory. The compressed file's filesize as of June 15 2018 is 17MB.
-
-## Running Code
-### Train and evaluate model
-To train and evaluate the model, issue the following command:
-```
-python sentiment_main.py
-```
-Arguments:
- * `--dataset`: The dataset name to be downloaded and preprocessed. By default, it is `imdb`.
-
-There are other arguments about models and training process. Use the `--help` or `-h` flag to get a full list of possible arguments with detailed descriptions.
-
-## Benchmarks
-The model was recorded to have the accuracy of 90.1% for the IMDB dataset.
diff --git a/research/sentiment_analysis/__init__.py b/research/sentiment_analysis/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/sentiment_analysis/data/__init__.py b/research/sentiment_analysis/data/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/sentiment_analysis/data/dataset.py b/research/sentiment_analysis/data/dataset.py
deleted file mode 100644
index 9ba4b9ac677296fbc7d1db549c5dd011117a16c4..0000000000000000000000000000000000000000
--- a/research/sentiment_analysis/data/dataset.py
+++ /dev/null
@@ -1,52 +0,0 @@
-"""Dataset module for sentiment analysis.
-
-Currently imdb dataset is available.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import data.imdb as imdb
-
-DATASET_IMDB = "imdb"
-
-
-def load(dataset, vocabulary_size, sentence_length):
- """Returns training and evaluation input.
-
- Args:
- dataset: Dataset to be trained and evaluated.
- Currently only imdb is supported.
- vocabulary_size: The number of the most frequent tokens
- to be used from the corpus.
- sentence_length: The number of words in each sentence.
- Longer sentences get cut, shorter ones padded.
- Raises:
- ValueError: if the dataset value is not valid.
- Returns:
- A tuple of length 4, for training sentences, labels,
- evaluation sentences, and evaluation labels,
- each being an numpy array.
- """
- if dataset == DATASET_IMDB:
- return imdb.load(vocabulary_size, sentence_length)
- else:
- raise ValueError("unsupported dataset: " + dataset)
-
-
-def get_num_class(dataset):
- """Returns an integer for the number of label classes.
-
- Args:
- dataset: Dataset to be trained and evaluated.
- Currently only imdb is supported.
- Raises:
- ValueError: if the dataset value is not valid.
- Returns:
- int: The number of label classes.
- """
- if dataset == DATASET_IMDB:
- return imdb.NUM_CLASS
- else:
- raise ValueError("unsupported dataset: " + dataset)
diff --git a/research/sentiment_analysis/data/imdb.py b/research/sentiment_analysis/data/imdb.py
deleted file mode 100644
index f8160ca2f71ac158a4ee42119a04e2dafec033ee..0000000000000000000000000000000000000000
--- a/research/sentiment_analysis/data/imdb.py
+++ /dev/null
@@ -1,54 +0,0 @@
-"""IMDB Dataset module for sentiment analysis."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-from data.util import OOV_CHAR
-from data.util import pad_sentence
-from data.util import START_CHAR
-
-NUM_CLASS = 2
-
-
-def load(vocabulary_size, sentence_length):
- """Returns training and evaluation input for imdb dataset.
-
- Args:
- vocabulary_size: The number of the most frequent tokens
- to be used from the corpus.
- sentence_length: The number of words in each sentence.
- Longer sentences get cut, shorter ones padded.
- Raises:
- ValueError: if the dataset value is not valid.
- Returns:
- A tuple of length 4, for training and evaluation data,
- each being an numpy array.
- """
- (x_train, y_train), (x_test, y_test) = tf.keras.datasets.imdb.load_data(
- path="imdb.npz",
- num_words=vocabulary_size,
- skip_top=0,
- maxlen=None,
- seed=113,
- start_char=START_CHAR,
- oov_char=OOV_CHAR,
- index_from=OOV_CHAR+1)
-
- x_train_processed = []
- for sen in x_train:
- sen = pad_sentence(sen, sentence_length)
- x_train_processed.append(np.array(sen))
- x_train_processed = np.array(x_train_processed)
-
- x_test_processed = []
- for sen in x_test:
- sen = pad_sentence(sen, sentence_length)
- x_test_processed.append(np.array(sen))
- x_test_processed = np.array(x_test_processed)
-
- return x_train_processed, np.eye(NUM_CLASS)[y_train], \
- x_test_processed, np.eye(NUM_CLASS)[y_test]
diff --git a/research/sentiment_analysis/data/util.py b/research/sentiment_analysis/data/util.py
deleted file mode 100644
index c8f8808f7e1d7f26876d052c108d1948cbd2fe9f..0000000000000000000000000000000000000000
--- a/research/sentiment_analysis/data/util.py
+++ /dev/null
@@ -1,32 +0,0 @@
-"""Utility module for sentiment analysis."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-START_CHAR = 1
-END_CHAR = 2
-OOV_CHAR = 3
-
-
-def pad_sentence(sentence, sentence_length):
- """Pad the given sentense at the end.
-
- If the input is longer than sentence_length,
- the remaining portion is dropped.
- END_CHAR is used for the padding.
-
- Args:
- sentence: A numpy array of integers.
- sentence_length: The length of the input after the padding.
- Returns:
- A numpy array of integers of the given length.
- """
- sentence = sentence[:sentence_length]
- if len(sentence) < sentence_length:
- sentence = np.pad(sentence, (0, sentence_length - len(sentence)),
- "constant", constant_values=(START_CHAR, END_CHAR))
-
- return sentence
diff --git a/research/sentiment_analysis/sentiment_main.py b/research/sentiment_analysis/sentiment_main.py
deleted file mode 100644
index 8b9ba5f921eef72377b480669bd81087fd1b160a..0000000000000000000000000000000000000000
--- a/research/sentiment_analysis/sentiment_main.py
+++ /dev/null
@@ -1,115 +0,0 @@
-"""Main function for the sentiment analysis model.
-
-The model makes use of concatenation of two CNN layers with
-different kernel sizes. See `sentiment_model.py`
-for more details about the models.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import argparse
-import os
-
-import tensorflow as tf
-
-from data import dataset
-import sentiment_model
-
-
-
-_DROPOUT_RATE = 0.95
-
-
-def run_model(dataset_name, emb_dim, voc_size, sen_len,
- hid_dim, batch_size, epochs, model_save_dir):
- """Run training loop and an evaluation at the end.
-
- Args:
- dataset_name: Dataset name to be trained and evaluated.
- emb_dim: The dimension of the Embedding layer.
- voc_size: The number of the most frequent tokens
- to be used from the corpus.
- sen_len: The number of words in each sentence.
- Longer sentences get cut, shorter ones padded.
- hid_dim: The dimension of the Embedding layer.
- batch_size: The size of each batch during training.
- epochs: The number of the iteration over the training set for training.
- """
-
- model = sentiment_model.CNN(emb_dim, voc_size, sen_len,
- hid_dim, dataset.get_num_class(dataset_name),
- _DROPOUT_RATE)
- model.summary()
-
- model.compile(loss="categorical_crossentropy",
- optimizer="rmsprop",
- metrics=["accuracy"])
-
- tf.logging.info("Loading the data")
- x_train, y_train, x_test, y_test = dataset.load(
- dataset_name, voc_size, sen_len)
-
- if not os.path.exists(model_save_dir):
- os.makedirs(model_save_dir)
-
- filepath=model_save_dir+"/model-{epoch:02d}.hdf5"
-
- checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_accuracy',
- verbose=1,save_best_only=True,
- save_weights_only=True,mode='auto')
-
-
- model.fit(x_train, y_train, batch_size=batch_size,
- validation_split=0.4, epochs=epochs, callbacks=[checkpoint_callback])
-
- score = model.evaluate(x_test, y_test, batch_size=batch_size)
-
- model.save(os.path.join(model_save_dir, "full-model.h5"))
-
- tf.logging.info("Score: {}".format(score))
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("-d", "--dataset", help="Dataset to be trained "
- "and evaluated.",
- type=str, choices=["imdb"], default="imdb")
-
- parser.add_argument("-e", "--embedding_dim",
- help="The dimension of the Embedding layer.",
- type=int, default=512)
-
- parser.add_argument("-v", "--vocabulary_size",
- help="The number of the words to be considered "
- "in the dataset corpus.",
- type=int, default=6000)
-
- parser.add_argument("-s", "--sentence_length",
- help="The number of words in a data point."
- "Entries of smaller length are padded.",
- type=int, default=600)
-
- parser.add_argument("-c", "--hidden_dim",
- help="The number of the CNN layer filters.",
- type=int, default=512)
-
- parser.add_argument("-b", "--batch_size",
- help="The size of each batch for training.",
- type=int, default=500)
-
- parser.add_argument("-p", "--epochs",
- help="The number of epochs for training.",
- type=int, default=55)
-
- parser.add_argument("-f", "--folder",
- help="folder/dir to save trained model",
- type=str, default=None)
- args = parser.parse_args()
-
- if args.folder is None:
- parser.error("-f argument folder/dir to save is None,provide path to save model.")
-
- run_model(args.dataset, args.embedding_dim, args.vocabulary_size,
- args.sentence_length, args.hidden_dim,
- args.batch_size, args.epochs, args.folder)
diff --git a/research/sentiment_analysis/sentiment_model.py b/research/sentiment_analysis/sentiment_model.py
deleted file mode 100644
index 586474992ab4521512bbbb5156be0b306f94c3bb..0000000000000000000000000000000000000000
--- a/research/sentiment_analysis/sentiment_model.py
+++ /dev/null
@@ -1,50 +0,0 @@
-"""Model for sentiment analysis.
-
-The model makes use of concatenation of two CNN layers with
-different kernel sizes.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-
-class CNN(tf.keras.models.Model):
- """CNN for sentimental analysis."""
-
- def __init__(self, emb_dim, num_words, sentence_length, hid_dim,
- class_dim, dropout_rate):
- """Initialize CNN model.
-
- Args:
- emb_dim: The dimension of the Embedding layer.
- num_words: The number of the most frequent tokens
- to be used from the corpus.
- sentence_length: The number of words in each sentence.
- Longer sentences get cut, shorter ones padded.
- hid_dim: The dimension of the Embedding layer.
- class_dim: The number of the CNN layer filters.
- dropout_rate: The portion of kept value in the Dropout layer.
- Returns:
- tf.keras.models.Model: A Keras model.
- """
-
- input_layer = tf.keras.layers.Input(shape=(sentence_length,), dtype=tf.int32)
-
- layer = tf.keras.layers.Embedding(num_words, output_dim=emb_dim)(input_layer)
-
- layer_conv3 = tf.keras.layers.Conv1D(hid_dim, 3, activation="relu")(layer)
- layer_conv3 = tf.keras.layers.GlobalMaxPooling1D()(layer_conv3)
-
- layer_conv4 = tf.keras.layers.Conv1D(hid_dim, 2, activation="relu")(layer)
- layer_conv4 = tf.keras.layers.GlobalMaxPooling1D()(layer_conv4)
-
- layer = tf.keras.layers.concatenate([layer_conv4, layer_conv3], axis=1)
- layer = tf.keras.layers.BatchNormalization()(layer)
- layer = tf.keras.layers.Dropout(dropout_rate)(layer)
-
- output = tf.keras.layers.Dense(class_dim, activation="softmax")(layer)
-
- super(CNN, self).__init__(inputs=[input_layer], outputs=output)
diff --git a/research/seq2species/README.md b/research/seq2species/README.md
deleted file mode 100644
index dbe473131d59fe990e1ff344ea69bb5bd05972e4..0000000000000000000000000000000000000000
--- a/research/seq2species/README.md
+++ /dev/null
@@ -1,187 +0,0 @@
-
-
-
-
-# Seq2Species: Neural Network Models for Species Classification
-
-*A deep learning solution for read-level taxonomic classification with 16s.*
-
-Recent improvements in sequencing technology have made possible large, public
-databases of biological sequencing data, bringing about new data richness for
-many important problems in bioinformatics. However, this growing availability of
-data creates a need for analysis methods capable of efficiently handling these
-large sequencing datasets. We on the [Genomics team in Google
-Brain](https://ai.google/research/teams/brain/healthcare-biosciences) are
-particularly interested in the class of problems which can be framed as
-assigning meaningful labels to short biological sequences, and are exploring the
-possiblity of creating a general deep learning solution for solving this class
-of sequence-labeling problems. We are excited to share our initial progress in
-this direction by releasing Seq2Species, an open-source neural network framework
-for [TensorFlow](https://www.tensorflow.org/) for predicting read-level
-taxonomic labels from genomic sequence. Our release includes all the code
-necessary to train new Seq2Species models.
-
-## About Seq2Species
-
-Briefly, Seq2Species provides a framework for training deep neural networks to
-predict database-derived labels directly from short reads of DNA. Thus far, our
-research has focused predominantly on demonstrating the value of this deep
-learning approach on the problem of determining the species of origin of
-next-generation sequencing reads from [16S ribosomal
-DNA](https://en.wikipedia.org/wiki/16S_ribosomal_RNA). We used this
-Seq2Species framework to train depthwise separable convolutional neural networks
-on short subsequences from the 16S genes of more than 13 thousand distinct
-species. The resulting classification model assign species-level probabilities
-to individual 16S reads.
-
-For more information about the use cases we have explored, or for technical
-details describing how Seq2Species work, please see our
-[preprint](https://www.biorxiv.org/content/early/2018/06/22/353474).
-
-## Installation
-
-Training Seq2Species models requires installing the following dependencies:
-
-* python 2.7
-
-* protocol buffers
-
-* numpy
-
-* absl
-
-### Dependencies
-
-Detailed instructions for installing TensorFlow are available on the [Installing
-TensorFlow](https://www.tensorflow.org/install/) website. Please follow the
-full instructions for installing TensorFlow with GPU support. For most
-users, the following command will suffice for continuing with CPU support only:
-```bash
-# For CPU
-pip install --upgrade tensorflow
-```
-
-The TensorFlow installation should also include installation of the numpy and
-absl libraries, which are two of TensorFlow's python dependencies. If
-necessary, instructions for standalone installation are available:
-
-* [numpy](https://scipy.org/install.html)
-
-* [absl](https://github.com/abseil/abseil-py)
-
-Information about protocol buffers, as well as download and installation
-intructions for the protocol buffer (protobuf) compiler, are available on the [Google
-Developers website](https://developers.google.com/protocol-buffers/). A typical
-Ubuntu user can install this library using `apt-get`:
-```bash
-sudo apt-get install protobuf-compiler
-```
-
-### Clone
-
-Now, clone `tensorflow/models` to start working with the code:
-```bash
-git clone https://github.com/tensorflow/models.git
-```
-
-### Protobuf Compilation
-
-Seq2Species uses protobufs to store and save dataset and model metadata. Before
-the framework can be used to build and train models, the protobuf libraries must
-be compiled. This can be accomplished using the following command:
-```bash
-# From tensorflow/models/research
-protoc seq2species/protos/seq2label.proto --python_out=.
-```
-
-### Testing the Installation
-
-One can test that Seq2Species has been installed correctly by running the
-following command:
-```bash
-python seq2species/run_training_test.py
-```
-
-## Usage Information
-
-Input data to Seq2Species models should be [tf.train.Example protocol messages](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto) stored in
-[TFRecord format](https://www.tensorflow.org/versions/r1.0/api_guides/python/python_io#tfrecords_format_details).
-Specifically, the input pipeline expects tf.train.Examples with a 'sequence' field
-containing a genomic sequence as an upper-case string, as one field for each
-target label (e.g. 'species'). There should also be an accompanying
-Seq2LabelDatasetInfo text protobuf containing metadata about the input, including
-the possible label values for each target.
-
-Below, we give an example command that could be used to launch training for 1000
-steps, assuming that appropriate data and metadata files are stored at
-`${TFRECORD}` and `${DATASET_INFO}`:
-```bash
-python seq2species/run_training.py --train_files ${TFRECORD}
---metadata_path ${DATASET_INFO} --hparams 'train_steps=1000'
---logdir $HOME/seq2species
-```
-This will output [TensorBoard
-summaries](https://www.tensorflow.org/guide/summaries_and_tensorboard), [TensorFlow
-checkpoints](https://www.tensorflow.org/guide/variables#checkpoint_files), Seq2LabelModelInfo and
-Seq2LabelExperimentMeasures metadata to the logdir `$HOME/seq2species`.
-
-### Preprocessed Seq2Species Data
-
-We have provided preprocessed data based on 16S reference sequences from the
-[NCBI RefSeq Targeted Loci
-Project](https://www.ncbi.nlm.nih.gov/refseq/targetedloci/) in a Seq2Species
-bucket on Google Cloud Storage. After installing the
-[Cloud SDK](https://cloud.google.com/sdk/install),
-one can download those data (roughly 25 GB) to a local directory `${DEST}` using
-the `gsutil` command:
-```bash
-BUCKET=gs://brain-genomics-public/research/seq2species
-mkdir -p ${DEST}
-gsutil -m cp ${BUCKET}/* ${DEST}
-```
-
-To check if the copy has completed successsfully, check the `${DEST}` directory:
-```bash
-ls -1 ${DEST}
-```
-which should produce:
-```bash
-ncbi_100bp_revcomp.dataset_info.pbtxt
-ncbi_100bp_revcomp.tfrecord
-```
-
-The following command can be used to train a copy of one of our best-perfoming
-deep neural network models for 100 base pair (bp) data. This command also
-illustrates how to set hyperparameter values explicitly from the commandline.
-The file `configuration.py` provides a full list of hyperparameters, their descriptions,
-and their default values. Additional flags are described at the top of
-`run_training.py`.
-```bash
-python seq2species/run_training.py \
---num_filters 3 \
---noise_rate 0.04 \
---train_files ${DEST}/ncbi_100bp_revcomp.tfrecord \
---metadata_path ${DEST}/ncbi_100bp_revcomp.dataset_info.pbtxt \
---logdir $HOME/seq2species \
---hparams 'filter_depths=[1,1,1],filter_widths=[5,9,13],grad_clip_norm=20.0,keep_prob=0.94017831318,
-lr_decay=0.0655052811,lr_init=0.000469689635793,lrelu_slope=0.0125376069918,min_read_length=100,num_fc_layers=2,num_fc_units=2828,optimizer=adam,optimizer_hp=0.885769367218,pointwise_depths=[84,58,180],pooling_type=avg,train_steps=3000000,use_depthwise_separable=true,weight_scale=1.18409526348'
-```
-
-### Visualization
-
-[TensorBoard](https://github.com/tensorflow/tensorboard) can be used to
-visualize training curves and other metrics stored in the summary files produced
-by `run_training.py`. Use the following command to launch a TensorBoard instance
-for the example model directory `$HOME/seq2species`:
-```bash
-tensorboard --logdir=$HOME/seq2species
-```
-
-## Contact
-
-Any issues with the Seq2Species framework should be filed with the
-[TensorFlow/models issue tracker](https://github.com/tensorflow/models/issues).
-Questions regarding Seq2Species capabilities can be directed to
-[seq2species-interest@google.com](mailto:seq2species-interest@google.com). This
-code is maintained by [@apbusia](https://github.com/apbusia) and
-[@depristo](https://github.com/depristo).
diff --git a/research/seq2species/build_model.py b/research/seq2species/build_model.py
deleted file mode 100644
index 9f4ae6b2eb2c1b4b4deeba99a0fbcf3be0ca644a..0000000000000000000000000000000000000000
--- a/research/seq2species/build_model.py
+++ /dev/null
@@ -1,506 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Defines convolutional model graph for Seq2Species.
-
-Builds TensorFlow computation graph for predicting the given taxonomic target
-labels from short reads of DNA using convolutional filters, followed by
-fully-connected layers and a softmax output layer.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import math
-
-import tensorflow as tf
-
-import input as seq2species_input
-import seq2label_utils
-
-
-class ConvolutionalNet(object):
- """Class to build and store the model's computational graph and operations.
-
- Attributes:
- read_length: int; the length in basepairs of the input reads of DNA.
- placeholders: dict; mapping from name to tf.Placeholder.
- global_step: tf.Variable tracking number of training iterations performed.
- train_op: operation to perform one training step by gradient descent.
- summary_op: operation to log model's performance metrics to TF event files.
- accuracy: tf.Variable giving the model's read-level accuracy for the
- current inputs.
- weighted_accuracy: tf.Variable giving the model's read-level weighted
- accuracy for the current inputs.
- loss: tf.Variable giving the model's current cross entropy loss.
- logits: tf.Variable containing the model's logits for the current inputs.
- predictions: tf.Variable containing the model's current predicted
- probability distributions for the current inputs.
- possible_labels: a dict of possible label values (list of strings), keyed by
- target name. Labels in the lists are the order used for integer encoding.
- use_tpu: whether model is to be run on TPU.
- """
-
- def __init__(self, hparams, dataset_info, targets, use_tpu=False):
- """Initializes the ConvolutionalNet according to provided hyperparameters.
-
- Does not build the graph---this is done by calling `build_graph` on the
- constructed object or using `model_fn`.
-
- Args:
- hparams: tf.contrib.training.Hparams object containing the model's
- hyperparamters; see configuration.py for hyperparameter definitions.
- dataset_info: a `Seq2LabelDatasetInfo` message reflecting the dataset
- metadata.
- targets: list of strings: the names of the prediction targets.
- use_tpu: whether we are running on TPU; if True, summaries will be
- disabled.
- """
- self._placeholders = {}
- self._targets = targets
- self._dataset_info = dataset_info
- self._hparams = hparams
- all_label_values = seq2label_utils.get_all_label_values(self.dataset_info)
- self._possible_labels = {
- target: all_label_values[target]
- for target in self.targets
- }
- self._use_tpu = use_tpu
-
- @property
- def hparams(self):
- return self._hparams
-
- @property
- def dataset_info(self):
- return self._dataset_info
-
- @property
- def possible_labels(self):
- return self._possible_labels
-
- @property
- def bases(self):
- return seq2species_input.DNA_BASES
-
- @property
- def n_bases(self):
- return seq2species_input.NUM_DNA_BASES
-
- @property
- def targets(self):
- return self._targets
-
- @property
- def read_length(self):
- return self.dataset_info.read_length
-
- @property
- def placeholders(self):
- return self._placeholders
-
- @property
- def global_step(self):
- return self._global_step
-
- @property
- def train_op(self):
- return self._train_op
-
- @property
- def summary_op(self):
- return self._summary_op
-
- @property
- def accuracy(self):
- return self._accuracy
-
- @property
- def weighted_accuracy(self):
- return self._weighted_accuracy
-
- @property
- def loss(self):
- return self._loss
-
- @property
- def total_loss(self):
- return self._total_loss
-
- @property
- def logits(self):
- return self._logits
-
- @property
- def predictions(self):
- return self._predictions
-
- @property
- def use_tpu(self):
- return self._use_tpu
-
- def _summary_scalar(self, name, scalar):
- """Adds a summary scalar, if the platform supports summaries."""
- if not self.use_tpu:
- return tf.summary.scalar(name, scalar)
- else:
- return None
-
- def _summary_histogram(self, name, values):
- """Adds a summary histogram, if the platform supports summaries."""
- if not self.use_tpu:
- return tf.summary.histogram(name, values)
- else:
- return None
-
- def _init_weights(self, shape, scale=1.0, name='weights'):
- """Randomly initializes a weight Tensor of the given shape.
-
- Args:
- shape: list; desired Tensor dimensions.
- scale: float; standard deviation scale with which to initialize weights.
- name: string name for the variable.
-
- Returns:
- TF Variable contining truncated random Normal initialized weights.
- """
- num_inputs = shape[0] if len(shape) < 3 else shape[0] * shape[1] * shape[2]
- stddev = scale / math.sqrt(num_inputs)
- return tf.get_variable(
- name,
- shape=shape,
- initializer=tf.truncated_normal_initializer(0., stddev))
-
- def _init_bias(self, size):
- """Initializes bias vector of given shape as zeros.
-
- Args:
- size: int; desired size of bias Tensor.
-
- Returns:
- TF Variable containing the initialized biases.
- """
- return tf.get_variable(
- name='b_{}'.format(size),
- shape=[size],
- initializer=tf.zeros_initializer())
-
- def _add_summaries(self, mode, gradient_norm, parameter_norm):
- """Defines TensorFlow operation for logging summaries to event files.
-
- Args:
- mode: the ModeKey string.
- gradient_norm: Tensor; norm of gradients produced during the current
- training operation.
- parameter_norm: Tensor; norm of the model parameters produced during the
- current training operation.
- """
- # Log summaries for TensorBoard.
- if mode == tf.estimator.ModeKeys.TRAIN:
- self._summary_scalar('norm_of_gradients', gradient_norm)
- self._summary_scalar('norm_of_parameters', parameter_norm)
- self._summary_scalar('total_loss', self.total_loss)
- self._summary_scalar('learning_rate', self._learn_rate)
- for target in self.targets:
- self._summary_scalar('per_read_weighted_accuracy/{}'.format(target),
- self.weighted_accuracy[target])
- self._summary_scalar('per_read_accuracy/{}'.format(target),
- self.accuracy[target])
- self._summary_histogram('prediction_frequency/{}'.format(target),
- self._predictions[target])
- self._summary_scalar('cross_entropy_loss/{}'.format(target),
- self._loss[target])
- self._summary_op = tf.summary.merge_all()
- else:
- # Log average performance metrics over many batches using placeholders.
- summaries = []
- for target in self.targets:
- accuracy_ph = tf.placeholder(tf.float32, shape=())
- weighted_accuracy_ph = tf.placeholder(tf.float32, shape=())
- cross_entropy_ph = tf.placeholder(tf.float32, shape=())
- self._placeholders.update({
- 'accuracy/{}'.format(target): accuracy_ph,
- 'weighted_accuracy/{}'.format(target): weighted_accuracy_ph,
- 'cross_entropy/{}'.format(target): cross_entropy_ph,
- })
- summaries += [
- self._summary_scalar('cross_entropy_loss/{}'.format(target),
- cross_entropy_ph),
- self._summary_scalar('per_read_accuracy/{}'.format(target),
- accuracy_ph),
- self._summary_scalar('per_read_weighted_accuracy/{}'.format(target),
- weighted_accuracy_ph)
- ]
-
- self._summary_op = tf.summary.merge(summaries)
-
- def _convolution(self,
- inputs,
- filter_dim,
- pointwise_dim=None,
- scale=1.0,
- padding='SAME'):
- """Applies convolutional filter of given dimensions to given input Tensor.
-
- If a pointwise dimension is specified, a depthwise separable convolution is
- performed.
-
- Args:
- inputs: 4D Tensor of shape (# reads, 1, # basepairs, # bases).
- filter_dim: integer tuple of the form (width, depth).
- pointwise_dim: int; output dimension for pointwise convolution.
- scale: float; standard deviation scale with which to initialize weights.
- padding: string; type of padding to use. One of "SAME" or "VALID".
-
- Returns:
- 4D Tensor result of applying the convolutional filter to the inputs.
- """
- in_channels = inputs.get_shape()[3].value
- filter_width, filter_depth = filter_dim
- filters = self._init_weights([1, filter_width, in_channels, filter_depth],
- scale)
- self._summary_histogram(filters.name.split(':')[0].split('/')[1], filters)
- if pointwise_dim is None:
- return tf.nn.conv2d(
- inputs,
- filters,
- strides=[1, 1, 1, 1],
- padding=padding,
- name='weights')
- pointwise_filters = self._init_weights(
- [1, 1, filter_depth * in_channels, pointwise_dim],
- scale,
- name='pointwise_weights')
- self._summary_histogram(
- pointwise_filters.name.split(':')[0].split('/')[1], pointwise_filters)
- return tf.nn.separable_conv2d(
- inputs,
- filters,
- pointwise_filters,
- strides=[1, 1, 1, 1],
- padding=padding)
-
- def _pool(self, inputs, pooling_type):
- """Performs pooling across width and height of the given inputs.
-
- Args:
- inputs: Tensor shaped (batch, height, width, channels) over which to pool.
- In our case, height is a unitary dimension and width can be thought of
- as the read dimension.
- pooling_type: string; one of "avg" or "max".
-
- Returns:
- Tensor result of performing pooling of the given pooling_type over the
- height and width dimensions of the given inputs.
- """
- if pooling_type == 'max':
- return tf.reduce_max(inputs, axis=[1, 2])
- if pooling_type == 'avg':
- return tf.reduce_sum(
- inputs, axis=[1, 2]) / tf.to_float(tf.shape(inputs)[2])
-
- def _leaky_relu(self, lrelu_slope, inputs):
- """Applies leaky ReLu activation to the given inputs with the given slope.
-
- Args:
- lrelu_slope: float; slope value for the activation function.
- A slope of 0.0 defines a standard ReLu activation, while a positive
- slope defines a leaky ReLu.
- inputs: Tensor upon which to apply the activation function.
-
- Returns:
- Tensor result of applying the activation function to the given inputs.
- """
- with tf.variable_scope('leaky_relu_activation'):
- return tf.maximum(lrelu_slope * inputs, inputs)
-
- def _dropout(self, inputs, keep_prob):
- """Applies dropout to the given inputs.
-
- Args:
- inputs: Tensor upon which to apply dropout.
- keep_prob: float; probability with which to randomly retain values in
- the given input.
-
- Returns:
- Tensor result of applying dropout to the given inputs.
- """
- with tf.variable_scope('dropout'):
- if keep_prob < 1.0:
- return tf.nn.dropout(inputs, keep_prob)
- return inputs
-
- def build_graph(self, features, labels, mode, batch_size):
- """Creates TensorFlow model graph.
-
- Args:
- features: a dict of input features Tensors.
- labels: a dict (by target name) of prediction labels.
- mode: the ModeKey string.
- batch_size: the integer batch size.
-
- Side Effect:
- Adds the following key Tensors and operations as class attributes:
- placeholders, global_step, train_op, summary_op, accuracy,
- weighted_accuracy, loss, logits, and predictions.
- """
- is_train = (mode == tf.estimator.ModeKeys.TRAIN)
- read = features['sequence']
-
- # Add a unitary dimension, so we can use conv2d.
- read = tf.expand_dims(read, 1)
- prev_out = read
-
- filters = zip(self.hparams.filter_widths, self.hparams.filter_depths)
- for i, f in enumerate(filters):
- with tf.variable_scope('convolution_' + str(i)):
- if self.hparams.use_depthwise_separable:
- p = self.hparams.pointwise_depths[i]
- else:
- p = None
- conv_out = self._convolution(
- prev_out, f, pointwise_dim=p, scale=self.hparams.weight_scale)
- conv_act_out = self._leaky_relu(self.hparams.lrelu_slope, conv_out)
- prev_out = (
- self._dropout(conv_act_out, self.hparams.keep_prob)
- if is_train else conv_act_out)
-
- for i in xrange(self.hparams.num_fc_layers):
- with tf.variable_scope('fully_connected_' + str(i)):
- # Create a convolutional layer which is equivalent to a fully-connected
- # layer when reads have length self.hparams.min_read_length.
- # The convolution will tile the layer appropriately for longer reads.
- biases = self._init_bias(self.hparams.num_fc_units)
- if i == 0:
- # Take entire min_read_length segment as input.
- # Output a single value per min_read_length_segment.
- filter_dimensions = (self.hparams.min_read_length,
- self.hparams.num_fc_units)
- else:
- # Take single output value of previous layer as input.
- filter_dimensions = (1, self.hparams.num_fc_units)
- fc_out = biases + self._convolution(
- prev_out,
- filter_dimensions,
- scale=self.hparams.weight_scale,
- padding='VALID')
- self._summary_histogram(biases.name.split(':')[0].split('/')[1], biases)
- fc_act_out = self._leaky_relu(self.hparams.lrelu_slope, fc_out)
- prev_out = (
- self._dropout(fc_act_out, self.hparams.keep_prob)
- if is_train else fc_act_out)
-
- # Pool to collapse tiling for reads longer than hparams.min_read_length.
- with tf.variable_scope('pool'):
- pool_out = self._pool(prev_out, self.hparams.pooling_type)
-
- with tf.variable_scope('output'):
- self._logits = {}
- self._predictions = {}
- self._weighted_accuracy = {}
- self._accuracy = {}
- self._loss = collections.OrderedDict()
-
- for target in self.targets:
- with tf.variable_scope(target):
- label = labels[target]
- possible_labels = self.possible_labels[target]
- weights = self._init_weights(
- [pool_out.get_shape()[1].value,
- len(possible_labels)],
- self.hparams.weight_scale,
- name='weights')
- biases = self._init_bias(len(possible_labels))
- self._summary_histogram(
- weights.name.split(':')[0].split('/')[1], weights)
- self._summary_histogram(
- biases.name.split(':')[0].split('/')[1], biases)
- logits = tf.matmul(pool_out, weights) + biases
- predictions = tf.nn.softmax(logits)
-
- gather_inds = tf.stack([tf.range(batch_size), label], axis=1)
- self._weighted_accuracy[target] = tf.reduce_mean(
- tf.gather_nd(predictions, gather_inds))
- argmax_prediction = tf.cast(tf.argmax(predictions, axis=1), tf.int32)
- self._accuracy[target] = tf.reduce_mean(
- tf.to_float(tf.equal(label, argmax_prediction)))
-
- losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
- labels=label, logits=logits)
- self._loss[target] = tf.reduce_mean(losses)
- self._logits[target] = logits
- self._predictions[target] = predictions
-
- # Compute total loss
- self._total_loss = tf.add_n(self._loss.values())
-
- # Define the optimizer.
-
- # tf.estimator framework builds the global_step for us, but if we aren't
- # using the framework we have to make it ourselves.
- self._global_step = tf.train.get_or_create_global_step()
- if self.hparams.lr_decay < 0:
- self._learn_rate = self.hparams.lr_init
- else:
- self._learn_rate = tf.train.exponential_decay(
- self.hparams.lr_init,
- self._global_step,
- int(self.hparams.train_steps),
- self.hparams.lr_decay,
- staircase=False)
- if self.hparams.optimizer == 'adam':
- opt = tf.train.AdamOptimizer(self._learn_rate, self.hparams.optimizer_hp)
- elif self.hparams.optimizer == 'momentum':
- opt = tf.train.MomentumOptimizer(self._learn_rate,
- self.hparams.optimizer_hp)
- if self.use_tpu:
- opt = tf.contrib.tpu.CrossShardOptimizer(opt)
-
- gradients, variables = zip(*opt.compute_gradients(self._total_loss))
- clipped_gradients, _ = tf.clip_by_global_norm(gradients,
- self.hparams.grad_clip_norm)
- with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
- self._train_op = opt.apply_gradients(
- zip(clipped_gradients, variables), global_step=self._global_step)
-
- if not self.use_tpu:
- grad_norm = tf.global_norm(gradients) if is_train else None
- param_norm = tf.global_norm(variables) if is_train else None
- self._add_summaries(mode, grad_norm, param_norm)
-
- def model_fn(self, features, labels, mode, params):
- """Function fulfilling the tf.estimator model_fn interface.
-
- Args:
- features: a dict containing the input features for prediction.
- labels: a dict from target name to Tensor-value prediction.
- mode: the ModeKey string.
- params: a dictionary of parameters for building the model; current params
- are params["batch_size"]: the integer batch size.
-
- Returns:
- A tf.estimator.EstimatorSpec object ready for use in training, inference.
- or evaluation.
- """
- self.build_graph(features, labels, mode, params['batch_size'])
-
- return tf.estimator.EstimatorSpec(
- mode,
- predictions=self.predictions,
- loss=self.total_loss,
- train_op=self.train_op,
- eval_metric_ops={})
diff --git a/research/seq2species/configuration.py b/research/seq2species/configuration.py
deleted file mode 100644
index a4dd626e27246a79292a80d165b199c42a154df5..0000000000000000000000000000000000000000
--- a/research/seq2species/configuration.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Defines hyperparameter configuration for ConvolutionalNet models.
-
-Specifically, provides methods for defining and initializing TensorFlow
-hyperparameters objects for a convolutional model as defined in:
-seq2species.build_model
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-
-
-def parse_hparams(hparam_values='', num_filters=1):
- """Initializes TensorFlow hyperparameters object with default values.
-
- In addition, default hyperparameter values are overwritten with the specified
- ones, where necessary.
-
- Args:
- hparam_values: comma-separated string of name=value pairs for setting
- particular hyperparameters.
- num_filters: int; number of filters in the model.
- Must be fixed outside of hyperparameter/study object as Vizier does not
- support having inter-hyperparameter dependencies.
-
- Returns:
- tf.contrib.training.Hparams object containing the model's hyperparameters.
- """
- hparams = tf.contrib.training.HParams()
-
- # Specify model architecture option.
- hparams.add_hparam('use_depthwise_separable', True)
-
- # Specify number of model parameters.
- hparams.add_hparam('filter_widths', [3] * num_filters)
- hparams.add_hparam('filter_depths', [1] * num_filters)
- hparams.add_hparam('pointwise_depths', [64] * num_filters)
- hparams.add_hparam('num_fc_layers', 2)
- hparams.add_hparam('num_fc_units', 455)
- hparams.add_hparam('min_read_length', 100)
- hparams.add_hparam('pooling_type', 'avg')
-
- # Specify activation options.
- hparams.add_hparam('lrelu_slope', 0.0) # Negative slope for leaky relu.
-
- # Specify training options.
- hparams.add_hparam('keep_prob', 1.0)
- hparams.add_hparam('weight_scale', 1.0)
- hparams.add_hparam('grad_clip_norm', 20.0)
- hparams.add_hparam('lr_init', 0.001)
- hparams.add_hparam('lr_decay', 0.1)
- hparams.add_hparam('optimizer', 'adam')
- # optimizer_hp is decay rate for 1st moment estimates for ADAM, and
- # momentum for SGD.
- hparams.add_hparam('optimizer_hp', 0.9)
- hparams.add_hparam('train_steps', 400000)
-
- # Overwrite defaults with specified values.
- hparams.parse(hparam_values)
- return hparams
diff --git a/research/seq2species/input.py b/research/seq2species/input.py
deleted file mode 100644
index f1636c87501175d7a6940693e00ae5713a1780ff..0000000000000000000000000000000000000000
--- a/research/seq2species/input.py
+++ /dev/null
@@ -1,325 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Input pipe for feeding examples to a Seq2Label model graph."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-from google.protobuf import text_format
-
-from protos import seq2label_pb2
-import seq2label_utils
-
-DNA_BASES = tuple('ACGT')
-NUM_DNA_BASES = len(DNA_BASES)
-# Possible FASTA characters/IUPAC ambiguity codes.
-# See https://en.wikipedia.org/wiki/Nucleic_acid_notation.
-AMBIGUITY_CODES = {
- 'K': 'GT',
- 'M': 'AC',
- 'R': 'AG',
- 'Y': 'CT',
- 'S': 'CG',
- 'W': 'AT',
- 'B': 'CGT',
- 'V': 'ACG',
- 'H': 'ACT',
- 'D': 'AGT',
- 'X': 'ACGT',
- 'N': 'ACGT'
-}
-
-
-def load_dataset_info(dataset_info_path):
- """Load a `Seq2LabelDatasetInfo` from a serialized text proto file."""
- dataset_info = seq2label_pb2.Seq2LabelDatasetInfo()
- with tf.gfile.Open(dataset_info_path, 'r') as f:
- text_format.Parse(f.read(), dataset_info)
- return dataset_info
-
-
-class _InputEncoding(object):
- """A helper class providing the graph operations needed to encode input.
-
- Instantiation of an _InputEncoding will write on the default TF graph, so it
- should only be instantiated inside the `input_fn`.
-
- Attributes:
- mode: `tf.estimator.ModeKeys`; the execution mode {TRAIN, EVAL, INFER}.
- targets: list of strings; the names of the labels of interest (e.g.
- "species").
- dna_bases: a tuple of the recognized DNA alphabet.
- n_bases: the size of the DNA alphabet.
- all_characters: list of recognized alphabet, including ambiguity codes.
- label_values: a tuple of strings, the possible label values of the
- prediction target.
- n_labels: the size of label_values
- fixed_read_length: an integer value of the statically-known read length, or
- None if the read length is to be determined dynamically.
- """
-
- def __init__(self,
- dataset_info,
- mode,
- targets,
- noise_rate=0.0,
- fixed_read_length=None):
- self.mode = mode
- self.targets = targets
- self.dna_bases = DNA_BASES
- self.n_bases = NUM_DNA_BASES
- self.all_characters = list(DNA_BASES) + sorted(AMBIGUITY_CODES.keys())
- self.character_encodings = np.concatenate(
- [[self._character_to_base_distribution(char)]
- for char in self.all_characters],
- axis=0)
- all_legal_label_values = seq2label_utils.get_all_label_values(dataset_info)
- # TF lookup tables.
- self.characters_table = tf.contrib.lookup.index_table_from_tensor(
- mapping=self.all_characters)
- self.label_tables = {
- target: tf.contrib.lookup.index_table_from_tensor(
- all_legal_label_values[target])
- for target in targets
- }
- self.fixed_read_length = fixed_read_length
- self.noise_rate = noise_rate
-
- def _character_to_base_distribution(self, char):
- """Maps the given character to a probability distribution over DNA bases.
-
- Args:
- char: character to be encoded as a probability distribution over bases.
-
- Returns:
- Array of size (self.n_bases,) representing the identity of the given
- character as a distribution over the possible DNA bases, self.dna_bases.
-
- Raises:
- ValueError: if the given character is not contained in the recognized
- alphabet, self.all_characters.
- """
- if char not in self.all_characters:
- raise ValueError(
- 'Base distribution requested for unrecognized character %s.' % char)
- possible_bases = AMBIGUITY_CODES[char] if char in AMBIGUITY_CODES else char
- base_indices = [self.dna_bases.index(base) for base in possible_bases]
- probability_weight = 1.0 / len(possible_bases)
- distribution = np.zeros((self.n_bases))
- distribution[base_indices] = probability_weight
- return distribution
-
- def encode_read(self, string_seq):
- """Converts the input read sequence to one-hot encoding.
-
- Args:
- string_seq: tf.String; input read sequence.
-
- Returns:
- Input read sequence as a one-hot encoded Tensor, with depth and ordering
- of one-hot encoding determined by the given bases. Ambiguous characters
- such as "N" and "S" are encoded as a probability distribution over the
- possible bases they represent.
- """
- with tf.variable_scope('encode_read'):
- read = tf.string_split([string_seq], delimiter='').values
- read = self.characters_table.lookup(read)
- read = tf.cast(tf.gather(self.character_encodings, read), tf.float32)
- if self.fixed_read_length:
- read = tf.reshape(read, (self.fixed_read_length, self.n_bases))
- return read
-
- def encode_label(self, target, string_label):
- """Converts the label value to an integer encoding.
-
- Args:
- target: str; the target name.
- string_label: tf.String; value of the label for the current input read.
-
- Returns:
- Given label value as an index into the possible_target_values.
- """
- with tf.variable_scope('encode_label/{}'.format(target)):
- return tf.cast(self.label_tables[target].lookup(string_label), tf.int32)
-
- def _empty_label(self):
- return tf.constant((), dtype=tf.int32, shape=())
-
- def parse_single_tfexample(self, serialized_example):
- """Parses a tf.train.Example proto to a one-hot encoded read, label pair.
-
- Injects noise into the incoming tf.train.Example's read sequence
- when noise_rate is non-zero.
-
- Args:
- serialized_example: string; the serialized tf.train.Example proto
- containing the read sequence and label value of interest as
- tf.FixedLenFeatures.
-
- Returns:
- Tuple (features, labels) of dicts for the input features and prediction
- targets.
- """
- with tf.variable_scope('parse_single_tfexample'):
- features_spec = {'sequence': tf.FixedLenFeature([], tf.string)}
- for target in self.targets:
- features_spec[target] = tf.FixedLenFeature([], tf.string)
- features = tf.parse_single_example(
- serialized_example, features=features_spec)
- if self.noise_rate > 0.0:
- read_sequence = tf.py_func(seq2label_utils.add_read_noise,
- [features['sequence'], self.noise_rate],
- (tf.string))
- else:
- read_sequence = features['sequence']
- read_sequence = self.encode_read(read_sequence)
- read_features = {'sequence': read_sequence}
- if self.mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL):
- label = {
- target: self.encode_label(target, features[target])
- for target in self.targets
- }
- else:
- label = {target: self._empty_label() for target in self.targets}
- return read_features, label
-
-
-class InputDataset(object):
- """A class providing access to input data for the Seq2Label model.
-
- Attributes:
- mode: `tf.estimator.ModeKeys`; the execution mode {TRAIN, EVAL, INFER}.
- targets: list of strings; the names of the labels of interest (e.g.
- "species").
- dataset_info: a `Seq2LabelDatasetInfo` message reflecting the dataset
- metadata.
- initializer: the TF initializer op for the underlying iterator, which
- will rewind the iterator.
- is_train: Boolean indicating whether or not the execution mode is TRAIN.
- """
-
- def __init__(self,
- mode,
- targets,
- dataset_info,
- train_epochs=None,
- noise_rate=0.0,
- random_seed=None,
- input_tfrecord_files=None,
- fixed_read_length=None,
- ensure_constant_batch_size=False,
- num_parallel_calls=32):
- """Constructor for InputDataset.
-
- Args:
- mode: `tf.estimator.ModeKeys`; the execution mode {TRAIN, EVAL, INFER}.
- targets: list of strings; the names of the labels of interest (e.g.
- "species").
- dataset_info: a `Seq2LabelDatasetInfo` message reflecting the dataset
- metadata.
- train_epochs: the number of training epochs to perform, if mode==TRAIN.
- noise_rate: float [0.0, 1.0] specifying rate at which to inject
- base-flipping noise into the read sequences.
- random_seed: seed to be used for shuffling, if mode==TRAIN.
- input_tfrecord_files: a list of filenames for TFRecords of TF examples.
- fixed_read_length: an integer value of the statically-known read length,
- or None if the read length is to be determined dynamically. The read
- length must be known statically for TPU execution.
- ensure_constant_batch_size: ensure a constant batch size at the expense of
- discarding the last "short" batch. This also gives us a statically
- constant batch size, which is essential for e.g. the TPU platform.
- num_parallel_calls: the number of dataset elements to process in parallel.
- If None, elements will be processed sequentially.
- """
- self.input_tfrecord_files = input_tfrecord_files
- self.mode = mode
- self.targets = targets
- self.dataset_info = dataset_info
- self._train_epochs = train_epochs
- self._noise_rate = noise_rate
- self._random_seed = random_seed
- if random_seed is not None:
- np.random.seed(random_seed)
- self._fixed_read_length = fixed_read_length
- self._ensure_constant_batch_size = ensure_constant_batch_size
- self._num_parallel_calls = num_parallel_calls
-
- @staticmethod
- def from_tfrecord_files(input_tfrecord_files, *args, **kwargs):
- return InputDataset(
- *args, input_tfrecord_files=input_tfrecord_files, **kwargs)
-
- @property
- def is_train(self):
- return self.mode == tf.estimator.ModeKeys.TRAIN
-
- def input_fn(self, params):
- """Supplies input for the model.
-
- This function supplies input to our model as a function of the mode.
-
- Args:
- params: a dictionary, containing:
- - params['batch_size']: the integer batch size.
-
- Returns:
- A tuple of two values as follows:
- 1) the *features* dict, containing a tensor value for keys as follows:
- - "sequence" - the encoded read input sequence.
- 2) the *labels* dict. containing a key for `target`, whose value is:
- - a string Tensor value (in TRAIN/EVAL mode), or
- - a blank Tensor (PREDICT mode).
- """
- randomize_input = self.is_train
- batch_size = params['batch_size']
-
- encoding = _InputEncoding(
- self.dataset_info,
- self.mode,
- self.targets,
- noise_rate=self._noise_rate,
- fixed_read_length=self._fixed_read_length)
-
- dataset = tf.data.TFRecordDataset(self.input_tfrecord_files)
- dataset = dataset.map(
- encoding.parse_single_tfexample,
- num_parallel_calls=self._num_parallel_calls)
-
- dataset = dataset.repeat(self._train_epochs if self.is_train else 1)
- if randomize_input:
- dataset = dataset.shuffle(
- buffer_size=max(1000, batch_size), seed=self._random_seed)
-
- if self._ensure_constant_batch_size:
- # Only take batches of *exactly* size batch_size; then we get a
- # statically knowable batch shape.
- dataset = dataset.batch(batch_size, drop_remainder=True)
- else:
- dataset = dataset.batch(batch_size)
-
- # Prefetch to allow infeed to be in parallel with model computations.
- dataset = dataset.prefetch(2)
-
- # Use initializable iterator to support table lookups.
- iterator = dataset.make_initializable_iterator()
- self.initializer = iterator.initializer
- tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer)
-
- features, labels = iterator.get_next()
- return (features, labels)
diff --git a/research/seq2species/protos/BUILD b/research/seq2species/protos/BUILD
deleted file mode 100644
index 5628d4c41a79fc3fb6fb61fc9089f1524c49f175..0000000000000000000000000000000000000000
--- a/research/seq2species/protos/BUILD
+++ /dev/null
@@ -1,16 +0,0 @@
-# Protos for Tensorflow Seq2Species API.
-
-package(
- default_visibility = ["//visibility:public"],
-)
-
-py_proto_library(
- name = "seq2label_py_pb2",
- api_version = 2,
- deps = [":seq2label_proto"],
-)
-
-proto_library(
- name = "seq2label_proto",
- srcs = ["seq2label.proto"],
-)
diff --git a/research/seq2species/protos/__init__.py b/research/seq2species/protos/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/research/seq2species/protos/seq2label.proto b/research/seq2species/protos/seq2label.proto
deleted file mode 100644
index 531c4ad75e06db119c547cde34323c08f772e7fa..0000000000000000000000000000000000000000
--- a/research/seq2species/protos/seq2label.proto
+++ /dev/null
@@ -1,49 +0,0 @@
-syntax = "proto2";
-
-package seq2species.protos;
-
-// Summarizes metadata information for a dataset that can be used for running
-// training or inference.
-message Seq2LabelDatasetInfo {
- // Summarizes all possible values for a given label in the dataset.
- message LabelInfo {
- optional string name = 1;
- repeated string values = 2;
- // Per-value weights used to normalize the classes in a dataset.
- repeated float weights = 3;
- }
- repeated LabelInfo labels = 3;
- // Length (in basepairs) of the reads in the dataset.
- optional int32 read_length = 4;
- // Stride (in number of basepairs) in the moving window.
- optional int32 read_stride = 7;
- // Total number of examples in the dataset.
- optional int64 num_examples = 5;
- // Full path to the dataset.
- optional string dataset_path = 6;
-}
-
-// Summarizes metadata information about a model trained on a Seq2Label dataset.
-message Seq2LabelModelInfo {
- optional string hparams_string = 1;
- optional string model_type = 2;
- repeated string targets = 3;
- optional int32 num_filters = 4;
- optional int32 batch_size = 5;
- optional string metadata_path = 6;
- optional float training_noise_rate = 7;
-}
-
-// Summarizes resulting measures of modelling experiments.
-message Seq2LabelExperimentMeasures {
- optional string checkpoint_path = 1;
- optional int64 steps = 2;
- optional float wall_time = 3;
- optional bool experiment_infeasible = 4;
-
- message Measure {
- optional string name = 1;
- optional float value = 2;
- }
- repeated Measure measures = 5;
-}
diff --git a/research/seq2species/run_training.py b/research/seq2species/run_training.py
deleted file mode 100644
index f03bb09ecf8b1b65f9899f20cb91e3f0363a2f18..0000000000000000000000000000000000000000
--- a/research/seq2species/run_training.py
+++ /dev/null
@@ -1,293 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Defines training scheme for neural networks for Seq2Species prediction.
-
-Defines and runs the loop for training a (optionally) depthwise separable
-convolutional model for predicting taxonomic labels from short reads of DNA.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import time
-
-from absl import flags
-import numpy as np
-import tensorflow as tf
-from google.protobuf import text_format
-
-import build_model
-import configuration
-import input as seq2species_input
-from protos import seq2label_pb2
-import seq2label_utils
-
-# Define non-tunable parameters.
-flags.DEFINE_integer('num_filters', 1, 'Number of filters for conv model')
-flags.DEFINE_string('hparams', '',
- 'Comma-separated list of name=value hyperparameter '
- "pairs ('hp1=value1,hp2=value2'). Unspecified "
- 'hyperparameters will be filled with defaults.')
-flags.DEFINE_integer('batch_size', 512, 'Size of batches during training.')
-flags.DEFINE_integer('min_train_steps', 1000,
- 'Minimum number of training steps to run.')
-flags.DEFINE_float('max_task_loss', 10.0,
- "Terminate trial if task loss doesn't fall below this "
- 'within --min_train_steps.')
-flags.DEFINE_integer('n_print_progress_every', 1000,
- 'Print training progress every '
- '--n_print_progress_every global steps.')
-flags.DEFINE_list('targets', ['species'],
- 'Names of taxonomic ranks to use as training targets.')
-flags.DEFINE_float(
- 'noise_rate', 0.0, 'Rate [0.0, 1.0] at which to inject '
- 'base-flipping noise into input read sequences.')
-
-# Define paths to logs and data.
-flags.DEFINE_list(
- 'train_files', [], 'Full paths to the TFRecords containing the '
- 'training examples.')
-flags.DEFINE_string(
- 'metadata_path', '', 'Full path of the text proto containing configuration '
- 'information about the set of training examples.')
-flags.DEFINE_string('logdir', '/tmp/seq2species',
- 'Directory to which to write logs.')
-
-# Define supervisor/checkpointing options.
-flags.DEFINE_integer('task', 0, 'Task ID of the replica running the training.')
-flags.DEFINE_string('master', '', 'Name of the TF master to use.')
-flags.DEFINE_integer(
- 'save_model_secs', 900, 'Rate at which to save model parameters. '
- 'Set to 0 to disable checkpointing.')
-flags.DEFINE_integer('recovery_wait_secs', 30,
- 'Wait to recover model from checkpoint '
- 'before timing out.')
-flags.DEFINE_integer('save_summaries_secs', 900,
- 'Rate at which to save Tensorboard summaries.')
-flags.DEFINE_integer('ps_tasks', 0,
- 'Number of tasks in the ps job; 0 if no ps is used.')
-
-FLAGS = flags.FLAGS
-RANDOM_SEED = 42
-
-
-def wait_until(time_sec):
- """Stalls execution until a given time.
-
- Args:
- time_sec: time, in seconds, until which to loop idly.
- """
- while time.time() < time_sec:
- pass
-
-
-def update_measures(measures, new_measures, loss_val, max_loss=None):
- """Updates tracking of experimental measures and infeasibilty.
-
- Args:
- measures: dict; mapping from measure name to measure value.
- new_measures: dict; mapping from measure name to new measure values.
- loss_val: float; value of loss metric by which to determine fesibility.
- max_loss: float; maximum value at which to consider the loss feasible.
-
- Side Effects:
- Updates the given mapping of measures and values based on the current
- experimental metrics stored in new_measures, and determines current
- feasibility of the experiment based on the provided loss value.
- """
- max_loss = max_loss if max_loss else np.finfo('f').max
- measures['is_infeasible'] = (
- loss_val >= max_loss or not np.isfinite(loss_val))
- measures.update(new_measures)
-
-
-def run_training(model, hparams, training_dataset, logdir, batch_size):
- """Trains the given model on random mini-batches of reads.
-
- Args:
- model: ConvolutionalNet instance containing the model graph and operations.
- hparams: tf.contrib.training.Hparams object containing the model's
- hyperparamters; see configuration.py for hyperparameter definitions.
- training_dataset: an `InputDataset` that can feed labelled examples.
- logdir: string; full path of directory to which to save checkpoints.
- batch_size: integer batch size.
-
- Yields:
- Tuple comprising a dictionary of experimental measures and the save path
- for train checkpoints and summaries.
- """
- input_params = dict(batch_size=batch_size)
- features, labels = training_dataset.input_fn(input_params)
- model.build_graph(features, labels, tf.estimator.ModeKeys.TRAIN, batch_size)
-
- is_chief = FLAGS.task == 0
- scaffold = tf.train.Scaffold(
- saver=tf.train.Saver(
- tf.global_variables(),
- max_to_keep=5,
- keep_checkpoint_every_n_hours=1.0),
- init_op=tf.global_variables_initializer(),
- summary_op=model.summary_op)
- with tf.train.MonitoredTrainingSession(
- master=FLAGS.master,
- checkpoint_dir=logdir,
- is_chief=is_chief,
- scaffold=scaffold,
- save_summaries_secs=FLAGS.save_summaries_secs,
- save_checkpoint_secs=FLAGS.save_model_secs,
- max_wait_secs=FLAGS.recovery_wait_secs) as sess:
- global_step = sess.run(model.global_step)
- print('Initialized model at global step ', global_step)
- init_time = time.time()
- measures = {'is_infeasible': False}
-
- if is_chief:
- model_info = seq2label_utils.construct_seq2label_model_info(
- hparams, 'conv', FLAGS.targets, FLAGS.metadata_path, FLAGS.batch_size,
- FLAGS.num_filters, FLAGS.noise_rate)
- write_message(model_info, os.path.join(logdir, 'model_info.pbtxt'))
-
- ops = [
- model.accuracy, model.weighted_accuracy, model.total_loss,
- model.global_step, model.train_op
- ]
-
- while not sess.should_stop() and global_step < hparams.train_steps:
- accuracy, weighted_accuracy, loss, global_step, _ = sess.run(ops)
-
- def gather_measures():
- """Updates the measures dictionary from this batch."""
- new_measures = {'train_loss': loss, 'global_step': global_step}
- for target in FLAGS.targets:
- new_measures.update({
- ('train_accuracy/%s' % target): accuracy[target],
- ('train_weighted_accuracy/%s' % target): weighted_accuracy[target]
- })
- update_measures(
- measures, new_measures, loss, max_loss=FLAGS.max_task_loss)
-
- # Periodically track measures according to current mini-batch performance.
-
- # Log a message.
- if global_step % FLAGS.n_print_progress_every == 0:
- log_message = ('\tstep: %d (%d sec), loss: %f' %
- (global_step, time.time() - init_time, loss))
- for target in FLAGS.targets:
- log_message += (', accuracy/%s: %f ' % (target, accuracy[target]))
- log_message += (', weighted_accuracy/%s: %f ' %
- (target, weighted_accuracy[target]))
- print(log_message)
-
- # Gather new measures and update the measures dictionary.
- gather_measures()
- yield measures, scaffold.saver.last_checkpoints[-1]
-
- # Check for additional stopping criteria.
- if not np.isfinite(loss) or (loss >= FLAGS.max_task_loss and
- global_step > FLAGS.min_train_steps):
- break
-
- # Always yield once at the end.
- gather_measures()
- yield measures, scaffold.saver.last_checkpoints[-1]
-
-
-def write_message(message, filename):
- """Writes contents of the given message to the given filename as a text proto.
-
- Args:
- message: the proto message to save.
- filename: full path of file to which to save the text proto.
-
- Side Effects:
- Outputs a text proto file to the given filename.
- """
- message_string = text_format.MessageToString(message)
- with tf.gfile.GFile(filename, 'w') as f:
- f.write(message_string)
-
-
-def write_measures(measures, checkpoint_file, init_time):
- """Writes performance measures to file.
-
- Args:
- measures: dict; mapping from measure name to measure value.
- checkpoint_file: string; full save path for checkpoints and summaries.
- init_time: int; start time for work on the current experiment.
-
- Side Effects:
- Writes given dictionary of performance measures for the current experiment
- to a 'measures.pbtxt' file in the checkpoint directory.
- """
- # Save experiment measures.
- print('global_step: ', measures['global_step'])
- experiment_measures = seq2label_pb2.Seq2LabelExperimentMeasures(
- checkpoint_path=checkpoint_file,
- steps=measures['global_step'],
- experiment_infeasible=measures['is_infeasible'],
- wall_time=time.time() - init_time) # Inaccurate for restarts.
- for name, value in measures.iteritems():
- if name not in ['is_infeasible', 'global_step']:
- experiment_measures.measures.add(name=name, value=value)
- measures_file = os.path.join(
- os.path.dirname(checkpoint_file), 'measures.pbtxt')
- write_message(experiment_measures, measures_file)
- print('Wrote ', measures_file,
- ' containing the following experiment measures:\n', experiment_measures)
-
-
-def main(unused_argv):
- dataset_info = seq2species_input.load_dataset_info(FLAGS.metadata_path)
-
- init_time = time.time()
-
- # Determine model hyperparameters.
- hparams = configuration.parse_hparams(FLAGS.hparams, FLAGS.num_filters)
- print('Current Hyperparameters:')
- for hp_name, hp_val in hparams.values().items():
- print('\t', hp_name, ': ', hp_val)
-
- # Initialize the model graph.
- print('Constructing TensorFlow Graph.')
- tf.reset_default_graph()
-
- input_dataset = seq2species_input.InputDataset.from_tfrecord_files(
- FLAGS.train_files,
- 'train',
- FLAGS.targets,
- dataset_info,
- noise_rate=FLAGS.noise_rate,
- random_seed=RANDOM_SEED)
-
- with tf.device(tf.train.replica_device_setter(FLAGS.ps_tasks)):
- model = build_model.ConvolutionalNet(
- hparams, dataset_info, targets=FLAGS.targets)
-
- # Run the experiment.
- measures, checkpoint_file = None, None
- print('Starting model training.')
- for cur_measures, cur_file in run_training(
- model, hparams, input_dataset, FLAGS.logdir, batch_size=FLAGS.batch_size):
- measures, checkpoint_file = cur_measures, cur_file
-
- # Save experiment results.
- write_measures(measures, checkpoint_file, init_time)
-
-
-if __name__ == '__main__':
- tf.app.run(main)
diff --git a/research/seq2species/run_training_test.py b/research/seq2species/run_training_test.py
deleted file mode 100644
index 754d2e0174e730f57309a52ab2b80d1e84e7ab15..0000000000000000000000000000000000000000
--- a/research/seq2species/run_training_test.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Tests for run_training."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import time
-
-from absl import flags
-from absl.testing import absltest
-from absl.testing import flagsaver
-from absl.testing import parameterized
-import numpy as np
-import tensorflow as tf
-from google.protobuf import text_format
-
-import run_training
-from protos import seq2label_pb2
-import test_utils
-
-FLAGS = flags.FLAGS
-
-
-class RunTrainingTest(parameterized.TestCase):
-
- @parameterized.parameters(2, 4, 7)
- def test_wait_until(self, wait_sec):
- end_time = time.time() + wait_sec
- run_training.wait_until(end_time)
- self.assertEqual(round(time.time() - end_time), 0)
-
- @parameterized.parameters(
- ({}, {'a': 0.7, 'b': 12.3}, 12.3, None,
- {'a': 0.7, 'b': 12.3, 'is_infeasible': False}),
- ({'a': 0.42}, {'b': 24.5}, 24.5, 32.0,
- {'a': 0.42, 'b': 24.5, 'is_infeasible': False}),
- ({'a': 0.503}, {'a': 0.82, 'b': 7.2}, 7.2, 0.1,
- {'a': 0.82, 'b': 7.2, 'is_infeasible': True}),
- ({}, {'a': 0.7, 'b': 12.3}, float('Inf'), None,
- {'a': 0.7, 'b': 12.3, 'is_infeasible': True})
- )
- def test_update_measures(self, measures, new_measures, loss, max_loss,
- expected):
- run_training.update_measures(measures, new_measures, loss, max_loss)
- self.assertEqual(measures, expected)
-
- def test_write_measures(self):
- init_time = time.time()
- measures = {
- 'global_step': 311448,
- 'train_loss': np.float32(18.36),
- 'train_weighted_accuracy': np.float32(0.3295),
- 'train_accuracy': 0.8243,
- 'is_infeasible': False
- }
- tmp_path = os.path.join(FLAGS.test_tmpdir, 'measures.pbtxt')
- run_training.write_measures(measures, tmp_path, init_time)
- experiment_measures = seq2label_pb2.Seq2LabelExperimentMeasures()
- with tf.gfile.Open(tmp_path) as f:
- text_format.Parse(f.read(), experiment_measures)
- self.assertEqual(experiment_measures.checkpoint_path, tmp_path)
- self.assertFalse(experiment_measures.experiment_infeasible)
- self.assertEqual(experiment_measures.steps, measures['global_step'])
- self.assertGreater(experiment_measures.wall_time, 0)
- self.assertEqual(len(experiment_measures.measures), 3)
- for measure in experiment_measures.measures:
- self.assertAlmostEqual(measure.value, measures[measure.name])
-
- @parameterized.parameters((test_utils.TEST_TARGETS[:1],),
- (test_utils.TEST_TARGETS,))
- def test_run_training(self, targets):
- """Tests whether the training loop can be run successfully.
-
- Generates test input files and runs the main driving code.
-
- Args:
- targets: the targets to train on.
- """
- # Create test input and metadata files.
- num_examples, read_len = 20, 5
- train_file = test_utils.create_tmp_train_file(num_examples, read_len)
- metadata_path = test_utils.create_tmp_metadata(num_examples, read_len)
-
- # Check that the training loop runs as expected.
- logdir = os.path.join(FLAGS.test_tmpdir, 'train:{}'.format(len(targets)))
- with flagsaver.flagsaver(
- train_files=train_file,
- metadata_path=metadata_path,
- targets=targets,
- logdir=logdir,
- hparams='train_steps=10,min_read_length=5',
- batch_size=10):
- run_training.main(FLAGS)
- # Check training loop ran by confirming existence of a checkpoint file.
- self.assertIsNotNone(tf.train.latest_checkpoint(FLAGS.logdir))
- # Check training loop ran by confiming existence of a measures file.
- self.assertTrue(
- os.path.exists(os.path.join(FLAGS.logdir, 'measures.pbtxt')))
-
-
-if __name__ == '__main__':
- absltest.main()
diff --git a/research/seq2species/seq2label_utils.py b/research/seq2species/seq2label_utils.py
deleted file mode 100644
index b975b7f17d0dfd4798ed798509e7b3d3447cfd6b..0000000000000000000000000000000000000000
--- a/research/seq2species/seq2label_utils.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utilities for working with Seq2Label datasets and models.
-
-This library provides utilities for parsing and generating Seq2Label protos.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from protos import seq2label_pb2
-
-
-def get_all_label_values(dataset_info):
- """Retrieves possible values for modeled labels from a `Seq2LabelDatasetInfo`.
-
- Args:
- dataset_info: a `Seq2LabelDatasetInfo` message.
-
- Returns:
- A dictionary mapping each label name to a tuple of its permissible values.
- """
- return {
- label_info.name: tuple(label_info.values)
- for label_info in dataset_info.labels
- }
-
-
-def construct_seq2label_model_info(hparams, model_type, targets, metadata_path,
- batch_size, num_filters,
- training_noise_rate):
- """Constructs a Seq2LabelModelInfo proto with the given properties.
-
- Args:
- hparams: initialized tf.contrib.training.Hparams object.
- model_type: string; descriptive tag indicating type of model, ie. "conv".
- targets: list of names of the targets the model is trained to predict.
- metadata_path: string; full path to Seq2LabelDatasetInfo text proto used
- to initialize the model.
- batch_size: int; number of reads per mini-batch.
- num_filters: int; number of filters for convolutional model.
- training_noise_rate: float; rate [0.0, 1.0] of base-flipping noise injected
- into input read sequenced at training time.
-
- Returns:
- The Seq2LabelModelInfo proto with the hparams, model_type, targets,
- num_filters, batch_size, metadata_path, and training_noise_rate fields
- set to the given values.
- """
- return seq2label_pb2.Seq2LabelModelInfo(
- hparams_string=hparams.to_json(),
- model_type=model_type,
- targets=sorted(targets),
- num_filters=num_filters,
- batch_size=batch_size,
- metadata_path=metadata_path,
- training_noise_rate=training_noise_rate)
-
-
-def add_read_noise(read, base_flip_probability=0.01):
- """Adds base-flipping noise to the given read sequence.
-
- Args:
- read: string; the read sequence to which to add noise.
- base_flip_probability: float; probability of a base flip at each position.
-
- Returns:
- The given read with base-flipping noise added at the provided
- base_flip_probability rate.
- """
- base_flips = np.random.binomial(1, base_flip_probability, len(read))
- if sum(base_flips) == 0:
- return read
-
- read = np.array(list(read))
- possible_mutations = np.char.replace(['ACTG'] * sum(base_flips),
- read[base_flips == 1], '')
- mutations = map(np.random.choice, map(list, possible_mutations))
- read[base_flips == 1] = mutations
- return ''.join(read)
diff --git a/research/seq2species/test_utils.py b/research/seq2species/test_utils.py
deleted file mode 100644
index f02798fb533c99c8545e3a51f56de31b629ca2f4..0000000000000000000000000000000000000000
--- a/research/seq2species/test_utils.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# Copyright 2018 The TensorFlow Authors All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Utility methods for accessing and operating on test data."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-from absl import flags
-import tensorflow as tf
-from google.protobuf import text_format
-
-import input as seq2species_input
-from protos import seq2label_pb2
-
-FLAGS = flags.FLAGS
-
-# Target names included in the example inputs.
-TEST_TARGETS = ['test_target_1', 'test_target_2']
-
-
-def _as_bytes_feature(in_string):
- """Converts the given string to a tf.train.BytesList feature.
-
- Args:
- in_string: string to be converted to BytesList Feature.
-
- Returns:
- The TF BytesList Feature representing the given string.
- """
- return tf.train.Feature(bytes_list=tf.train.BytesList(value=[in_string]))
-
-
-def create_tmp_train_file(num_examples,
- read_len,
- characters=seq2species_input.DNA_BASES,
- name='test.tfrecord'):
- """Write a test TFRecord of input examples to temporary test directory.
-
- The generated input examples are test tf.train.Example protos, each comprised
- of a toy sequence of length read_len and non-meaningful labels for targets in
- TEST_TARGETS.
-
- Args:
- num_examples: int; number of examples to write to test input file.
- read_len: int; length of test read sequences.
- characters: string; set of characters from which to construct test reads.
- Defaults to canonical DNA bases.
- name: string; filename for the test input file.
-
- Returns:
- Full path to the generated temporary test input file.
- """
- tmp_path = os.path.join(FLAGS.test_tmpdir, name)
- with tf.python_io.TFRecordWriter(tmp_path) as writer:
- for i in xrange(num_examples):
- char = characters[i % len(characters)]
- features_dict = {'sequence': _as_bytes_feature(char * read_len)}
- for target_name in TEST_TARGETS:
- nonsense_label = _as_bytes_feature(str(i))
- features_dict[target_name] = nonsense_label
- tf_features = tf.train.Features(feature=features_dict)
- example = tf.train.Example(features=tf_features)
- writer.write(example.SerializeToString())
- return tmp_path
-
-
-def create_tmp_metadata(num_examples, read_len):
- """Write a test Seq2LabelDatasetInfo test proto to temporary test directory.
-
- Args:
- num_examples: int; number of example labels to write into test metadata.
- read_len: int; length of test read sequences.
-
- Returns:
- Full path to the generated temporary test file containing the
- Seq2LabelDatasetInfo text proto.
- """
- dataset_info = seq2label_pb2.Seq2LabelDatasetInfo(
- read_length=read_len,
- num_examples=num_examples,
- read_stride=1,
- dataset_path='test.tfrecord')
-
- for target in TEST_TARGETS:
- dataset_info.labels.add(
- name=target, values=[str(i) for i in xrange(num_examples)])
-
- tmp_path = os.path.join(FLAGS.test_tmpdir, 'test.pbtxt')
- with tf.gfile.GFile(tmp_path, 'w') as f:
- f.write(text_format.MessageToString(dataset_info))
- return tmp_path
diff --git a/research/seq_flow_lite/.bazelrc b/research/seq_flow_lite/.bazelrc
new file mode 100644
index 0000000000000000000000000000000000000000..73cf8a5a7a9ddcdee386e5ece97dcf4c004ecd60
--- /dev/null
+++ b/research/seq_flow_lite/.bazelrc
@@ -0,0 +1,82 @@
+# gRPC using libcares in opensource has some issues.
+build --define=grpc_no_ares=true
+
+# Suppress all warning messages.
+build:short_logs --output_filter=DONT_MATCH_ANYTHING
+
+# Force python3
+build --action_env=PYTHON_BIN_PATH=/usr/bin/python3
+build --repo_env=PYTHON_BIN_PATH=/usr/bin/python3
+build --python_path=/usr/bin/python3
+
+# Enable using platform specific build settings
+build --enable_platform_specific_config
+
+# Flag to enable remote config. Required starting from TF 2.2.
+common --experimental_repo_remote_exec
+
+build:manylinux2010 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010:toolchain
+
+build -c opt
+build --cxxopt="-std=c++14"
+build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"
+build --auto_output_filter=subpackages
+build --copt="-Wall" --copt="-Wno-sign-compare"
+build --linkopt="-lrt -lm"
+
+# TF isn't built in dbg mode, so our dbg builds will segfault due to inconsistency
+# of defines when using tf's headers. In particular in refcount.h.
+build --cxxopt="-DNDEBUG"
+
+
+build --define=use_fast_cpp_protos=true
+build --define=allow_oversize_protos=true
+
+build --spawn_strategy=standalone
+build -c opt
+
+# Adding "--cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0" creates parity with TF
+# compilation options. It also addresses memory use due to
+# copy-on-write semantics of std::strings of the older ABI.
+build --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0
+
+# Make Bazel print out all options from rc files.
+build --announce_rc
+
+# Other build flags.
+build --define=grpc_no_ares=true
+
+# See https://github.com/bazelbuild/bazel/issues/7362 for information on what
+# --incompatible_remove_legacy_whole_archive flag does.
+# This flag is set to true in Bazel 1.0 and newer versions. We tried to migrate
+# Tensorflow to the default, however test coverage wasn't enough to catch the
+# errors.
+# There is ongoing work on Bazel team's side to provide support for transitive
+# shared libraries. As part of migrating to transitive shared libraries, we
+# hope to provide a better mechanism for control over symbol exporting, and
+# then tackle this issue again.
+#
+# TODO: Remove this line once TF doesn't depend on Bazel wrapping all library
+# archives in -whole_archive -no_whole_archive.
+build --noincompatible_remove_legacy_whole_archive
+
+# These are bazel 2.0's incompatible flags. Tensorflow needs to use bazel 2.0.0
+# to use cc_shared_library, as part of the Tensorflow Build Improvements RFC:
+# https://github.com/tensorflow/community/pull/179
+build --noincompatible_prohibit_aapt1
+
+# Build TF with C++ 17 features.
+build:c++17 --cxxopt=-std=c++1z
+build:c++17 --cxxopt=-stdlib=libc++
+build:c++1z --config=c++17
+
+# Enable using platform specific build settings, except when cross-compiling for
+# mobile platforms.
+build --enable_platform_specific_config
+
+
+# Options from ./configure
+try-import %workspace%/.tf_configure.bazelrc
+
+# Put user-specific options in .bazelrc.user
+try-import %workspace%/.bazelrc.user
diff --git a/research/seq_flow_lite/BUILD b/research/seq_flow_lite/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..e3d4564aa86e0e577a726dbcf90087752c334999
--- /dev/null
+++ b/research/seq_flow_lite/BUILD
@@ -0,0 +1,54 @@
+licenses(["notice"])
+
+package(
+ default_visibility = [":friends"],
+)
+
+package_group(
+ name = "friends",
+ packages = [
+ "//...",
+ ],
+)
+
+py_library(
+ name = "metric_functions",
+ srcs = ["metric_functions.py"],
+ srcs_version = "PY3",
+)
+
+py_library(
+ name = "input_fn_reader",
+ srcs = ["input_fn_reader.py"],
+ srcs_version = "PY3",
+ deps = [
+ "//layers:projection_layers",
+ ],
+)
+
+py_binary(
+ name = "trainer",
+ srcs = ["trainer.py"],
+ python_version = "PY3",
+ srcs_version = "PY3",
+ deps = [
+ ":input_fn_reader",
+ ":metric_functions",
+ "//models:prado",
+ ],
+)
+
+py_binary(
+ name = "export_to_tflite",
+ srcs = ["export_to_tflite.py"],
+ python_version = "PY3",
+ srcs_version = "PY3",
+ deps = [
+ ":input_fn_reader",
+ ":metric_functions",
+ "//layers:base_layers",
+ "//layers:projection_layers",
+ "//models:prado",
+ "//utils:tflite_utils",
+ ],
+)
diff --git a/research/seq_flow_lite/CONTRIBUTING.md b/research/seq_flow_lite/CONTRIBUTING.md
new file mode 100644
index 0000000000000000000000000000000000000000..9a86ba025d83dfc39db62d84ee7cac0a9da1ac08
--- /dev/null
+++ b/research/seq_flow_lite/CONTRIBUTING.md
@@ -0,0 +1,28 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution;
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+## Community Guidelines
+
+This project follows
+[Google's Open Source Community Guidelines](https://opensource.google/conduct/).
diff --git a/research/seq_flow_lite/README.md b/research/seq_flow_lite/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..93ca02cccb00f83100d4d61b9b2b5ae5c9f63310
--- /dev/null
+++ b/research/seq_flow_lite/README.md
@@ -0,0 +1,87 @@
+# Sequence Projection Models
+
+This repository contains implementation of the following papers.
+
+* [*PRADO: Projection Attention Networks for Document Classification On-Device*](https://www.aclweb.org/anthology/D19-1506/)
+* [*Self-Governing Neural Networks for On-Device Short Text Classification*](https://www.aclweb.org/anthology/D18-1105/)
+
+## Description
+
+We provide a family of models that projects sequence to fixed sized features.
+The idea behind is to build embedding-free models that minimize the model size.
+Instead of using embedding table to lookup embeddings, sequence projection
+models computes them on the fly.
+
+
+## History
+
+### August 24, 2020
+* Add PRADO and SGNN implementation.
+
+## Authors or Maintainers
+
+* Prabhu Kaliamoorthi
+* Yicheng Fan ([@thunderfyc](https://github.com/thunderfyc))
+
+
+## Requirements
+
+[](https://github.com/tensorflow/tensorflow/releases/tag/v2.3.0)
+[](https://www.python.org/downloads/release/python-360/)
+
+
+## Training
+
+Train a PRADO model on civil comments dataset
+
+```shell
+bazel run -c opt :trainer -- \
+--config_path=$(pwd)/configs/civil_comments_prado.txt \
+--runner_mode=train --logtostderr --output_dir=/tmp/prado
+```
+
+Train a SGNN model to detect languages:
+
+```shell
+bazel run -c opt sgnn:train -- --logtostderr --output_dir=/tmp/sgnn
+```
+
+## Evaluation
+
+Evaluate PRADO model:
+
+```shell
+bazel run -c opt :trainer -- \
+--config_path=$(pwd)/configs/civil_comments_prado.txt \
+--runner_mode=eval --logtostderr --output_dir=/tmp/prado
+```
+
+Evaluate SGNN model:
+```shell
+bazel run -c opt sgnn:run_tflite -- --model=/tmp/sgnn/model.tflite "Hello world"
+```
+
+
+## References
+
+1. **Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift**
+ Sergey Ioffe, Christian Szegedy
+ [[link]](https://arxiv.org/abs/1502.03167). In ICML, 2015.
+
+2. **Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference**
+ Benoit Jacob, Skirmantas Kligys, Bo Chen, Menglong Zhu, Matthew Tang, Andrew Howard, Hartwig Adam, Dmitry Kalenichenko
+ [[link]](https://arxiv.org/abs/1712.05877). In CVPR, 2018.
+
+3. **PRADO: Projection Attention Networks for Document Classification On-Device**
+ Prabhu Kaliamoorthi, Sujith Ravi, Zornitsa Kozareva
+ [[link]](https://www.aclweb.org/anthology/D19-1506/). In EMNLP-IJCNLP, 2019
+
+4. **Self-Governing Neural Networks for On-Device Short Text Classification**
+ Sujith Ravi, Zornitsa Kozareva
+ [[link]](https://www.aclweb.org/anthology/D18-1105). In EMNLP, 2018
+
+## License
+
+[](https://opensource.org/licenses/Apache-2.0)
+
+This project is licensed under the terms of the **Apache License 2.0**.
diff --git a/research/seq_flow_lite/WORKSPACE b/research/seq_flow_lite/WORKSPACE
new file mode 100644
index 0000000000000000000000000000000000000000..d29ee07c65b810015f3f54c9aff18c8f3b302db6
--- /dev/null
+++ b/research/seq_flow_lite/WORKSPACE
@@ -0,0 +1,185 @@
+workspace(name = "tensorflow_models_seq_flow_lite")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@//third_party/py:python_configure.bzl", "python_configure")
+
+
+http_archive(
+ name = "io_bazel_rules_closure",
+ sha256 = "5b00383d08dd71f28503736db0500b6fb4dda47489ff5fc6bed42557c07c6ba9",
+ strip_prefix = "rules_closure-308b05b2419edb5c8ee0471b67a40403df940149",
+ urls = [
+ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz",
+ "https://github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz", # 2019-06-13
+ ],
+)
+
+http_archive(
+ name = "org_tensorflow",
+ sha256 = "fc6d7c57cd9427e695a38ad00fb6ecc3f623bac792dd44ad73a3f85b338b68be",
+ strip_prefix = "tensorflow-8a4ffe2e1ae722cff5306778df0cfca8b7f503fe",
+ urls = [
+ "https://github.com/tensorflow/tensorflow/archive/8a4ffe2e1ae722cff5306778df0cfca8b7f503fe.tar.gz",
+ ],
+)
+
+
+http_archive(
+ name = "org_tflite_support",
+ strip_prefix = "tflite-support-0861599711ef31de58f62ed3ff6bbcc1e4817ef6",
+ sha256 = "ef5e33d00930f3b0bad843d550476049faa3c77ca598dbb94acf81d01ba8badd",
+ urls = ["https://github.com/tensorflow/tflite-support/archive/0861599711ef31de58f62ed3ff6bbcc1e4817ef6.zip"],
+)
+
+http_archive(
+ name = "org_tensorflow_text",
+ sha256 = "f64647276f7288d1b1fe4c89581d51404d0ce4ae97f2bcc4c19bd667549adca8",
+ strip_prefix = "text-2.2.0",
+ urls = [
+ "https://github.com/tensorflow/text/archive/v2.2.0.zip",
+ ],
+ patches = ["@//third_party:tensorflow_text_fix_local_config_tf.patch"],
+ patch_args = ["-p1"],
+ repo_mapping = {"@com_google_re2": "@com_googlesource_code_re2"},
+)
+
+load("//tf_ops:repo.bzl", "cc_tf_configure", "reverb_protoc_deps")
+cc_tf_configure()
+PROTOC_VERSION = "3.9.0"
+PROTOC_SHA256 = "15e395b648a1a6dda8fd66868824a396e9d3e89bc2c8648e3b9ab9801bea5d55"
+reverb_protoc_deps(version = PROTOC_VERSION, sha256 = PROTOC_SHA256)
+
+# ABSL cpp library.
+http_archive(
+ name = "com_google_absl",
+ sha256 = "f368a8476f4e2e0eccf8a7318b98dafbe30b2600f4e3cf52636e5eb145aba06a", # SHARED_ABSL_SHA
+ strip_prefix = "abseil-cpp-df3ea785d8c30a9503321a3d35ee7d35808f190d",
+ urls = [
+ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-cpp/archive/df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz",
+ "https://github.com/abseil/abseil-cpp/archive/df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz",
+ ],
+)
+
+http_archive(
+ name = "rules_cc",
+ strip_prefix = "rules_cc-master",
+ urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
+)
+
+# GoogleTest/GoogleMock framework. Used by most unit-tests.
+http_archive(
+ name = "com_google_googletest",
+ urls = ["https://github.com/google/googletest/archive/master.zip"],
+ strip_prefix = "googletest-master",
+)
+
+# gflags needed by glog
+http_archive(
+ name = "com_github_gflags_gflags",
+ sha256 = "6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe",
+ strip_prefix = "gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a",
+ urls = [
+ "https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
+ "https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
+ ],
+)
+
+# glog
+http_archive(
+ name = "com_google_glog",
+ sha256 = "f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c",
+ strip_prefix = "glog-0.4.0",
+ urls = ["https://github.com/google/glog/archive/v0.4.0.tar.gz"],
+)
+
+http_archive(
+ name = "absl_py",
+ sha256 = "603febc9b95a8f2979a7bdb77d2f5e4d9b30d4e0d59579f88eba67d4e4cc5462",
+ strip_prefix = "abseil-py-pypi-v0.9.0",
+ urls = [
+ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-py/archive/pypi-v0.9.0.tar.gz",
+ "https://github.com/abseil/abseil-py/archive/pypi-v0.9.0.tar.gz",
+ ],
+)
+
+http_archive(
+ name = "utf_archive",
+ build_file = "@//third_party:utf.BUILD",
+ sha256 = "262a902f622dcd28e05b8a4be10da0aa3899050d0be8f4a71780eed6b2ea65ca",
+ urls = [
+ "https://mirror.bazel.build/9fans.github.io/plan9port/unix/libutf.tgz",
+ "https://9fans.github.io/plan9port/unix/libutf.tgz",
+ ],
+)
+
+
+#-----------------------------------------------------------------------------
+# proto
+#-----------------------------------------------------------------------------
+# proto_library, cc_proto_library and java_proto_library rules implicitly depend
+# on @com_google_protobuf//:proto, @com_google_protobuf//:cc_toolchain and
+# @com_google_protobuf//:java_toolchain, respectively.
+# This statement defines the @com_google_protobuf repo.
+http_archive(
+ name = "com_google_protobuf",
+ strip_prefix = "protobuf-3.8.0",
+ urls = ["https://github.com/google/protobuf/archive/v3.8.0.zip"],
+ sha256 = "1e622ce4b84b88b6d2cdf1db38d1a634fe2392d74f0b7b74ff98f3a51838ee53",
+)
+
+load("//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo")
+flatbuffers()
+
+load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
+tf_workspace(tf_repo_name = "org_tensorflow")
+
+
+# TF submodule compilation doesn't take care of grpc deps. Do it manually here.
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
+grpc_deps()
+
+load(
+ "@build_bazel_rules_apple//apple:repositories.bzl",
+ "apple_rules_dependencies",
+)
+apple_rules_dependencies()
+
+load(
+ "@build_bazel_apple_support//lib:repositories.bzl",
+ "apple_support_dependencies",
+)
+apple_support_dependencies()
+
+load("@upb//bazel:repository_defs.bzl", "bazel_version_repository")
+bazel_version_repository(name = "bazel_version")
+
+
+# Set up Android.
+load("//third_party/android:android_configure.bzl", "android_configure")
+android_configure(name="local_config_android")
+load("@local_config_android//:android.bzl", "android_workspace")
+android_workspace()
+
+python_configure(name = "local_config_python")
+
+new_git_repository(
+ name = "icu4c",
+ tag = "release-66-1",
+ remote = "https://github.com/unicode-org/icu",
+ build_file = "@//third_party:icu.BUILD",
+ patch_cmds = [
+ "find . -type f -exec sed -i 's/#\s*include \"unicode/#include \"icu4c\/source\/common\/unicode/g' {} \;",
+ ],
+)
+
+
+http_archive(
+ name = "farmhash_archive",
+ build_file = "//third_party:farmhash.BUILD",
+ sha256 = "6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0", # SHARED_FARMHASH_SHA
+ strip_prefix = "farmhash-816a4ae622e964763ca0862d9dbd19324a1eaf45",
+ urls = [
+ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz",
+ "https://github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz",
+ ],
+)
diff --git a/research/seq_flow_lite/colab/BUILD b/research/seq_flow_lite/colab/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..b234bf3ca67bafd5b11573dabef22ad5967fbea2
--- /dev/null
+++ b/research/seq_flow_lite/colab/BUILD
@@ -0,0 +1,7 @@
+sh_binary(
+ name = "move_ops",
+ srcs = ["move_ops.sh"],
+ data = [
+ "//tf_ops:sequence_string_projection_op_py",
+ ],
+)
diff --git a/research/seq_flow_lite/colab/move_ops.sh b/research/seq_flow_lite/colab/move_ops.sh
new file mode 100755
index 0000000000000000000000000000000000000000..a1300bf62137ae12780dcb1a9f941ff946873aba
--- /dev/null
+++ b/research/seq_flow_lite/colab/move_ops.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+RUNFILES_DIR=$(pwd)
+cp -f "${RUNFILES_DIR}/tf_ops/libsequence_string_projection_op_py_gen_op.so" \
+ "${BUILD_WORKSPACE_DIRECTORY}/tf_ops"
+cp -f "${RUNFILES_DIR}/tf_ops/sequence_string_projection_op.py" \
+ "${BUILD_WORKSPACE_DIRECTORY}/tf_ops"
+
diff --git a/research/seq_flow_lite/colab/setup.py b/research/seq_flow_lite/colab/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..f4604e3d0618f9e458ea47601f82b8dcc7ccb304
--- /dev/null
+++ b/research/seq_flow_lite/colab/setup.py
@@ -0,0 +1,53 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import os
+import subprocess
+from setuptools import find_packages
+from setuptools import setup
+from distutils import spawn
+from distutils.command import build
+
+
+class _BuildCommand(build.build):
+ sub_commands = [
+ ('bazel_build', lambda self: True),
+ ] + build.build.sub_commands
+
+
+class _BazelBuildCommand(setuptools.Command):
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ self._bazel_cmd = spawn.find_executable('bazel')
+
+ def run(self):
+ subprocess.check_call(
+ [self._bazel_cmd, 'run', '-c', 'opt', '//colab:move_ops'],
+ cwd=os.path.dirname(os.path.realpath(__file__)))
+
+
+setup(
+ name='seq_flow_lite',
+ version='0.1',
+ packages=['tf_ops'],
+ package_data={'': ['*.so']},
+ cmdclass={
+ 'build': _BuildCommand,
+ 'bazel_build': _BazelBuildCommand,
+ },
+ description='Test')
diff --git a/research/seq_flow_lite/colab/setup_workspace.sh b/research/seq_flow_lite/colab/setup_workspace.sh
new file mode 100755
index 0000000000000000000000000000000000000000..de76d02a55296c03087f6c6099e0cc65dfa6e652
--- /dev/null
+++ b/research/seq_flow_lite/colab/setup_workspace.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+cd "$(dirname "$0")"
+mv setup.py ..
+touch ../tf_ops/__init__.py
diff --git a/research/seq_flow_lite/configs/civil_comments_prado.txt b/research/seq_flow_lite/configs/civil_comments_prado.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b15f5b91741f2557abed89c48592921dc2c4ef9a
--- /dev/null
+++ b/research/seq_flow_lite/configs/civil_comments_prado.txt
@@ -0,0 +1,27 @@
+{
+ "model_config" : {
+ "labels": ["identity_attack", "insult", "obscene", "severe_toxicity", "sexual_explicit", "threat", "toxicity"],
+ "multilabel": true,
+ "quantize": true,
+ "max_seq_len": 128,
+ "max_seq_len_inference": 128,
+ "split_on_space": true,
+ "embedding_regularizer_scale": 35e-3,
+ "embedding_size": 64,
+ "bigram_channels": 64,
+ "trigram_channels": 64,
+ "feature_size": 512,
+ "network_regularizer_scale": 1e-4,
+ "keep_prob": 0.5,
+ "distortion_probability": 0.25
+ },
+ "name": "models.prado",
+ "batch_size": 1024,
+ "save_checkpoints_steps": 100,
+ "train_steps": 100000,
+ "learning_rate": 1e-3,
+ "learning_rate_decay_steps": 42000,
+ "learning_rate_decay_rate": 0.7,
+ "iterations_per_loop": 100,
+ "dataset": "civil_comments"
+}
diff --git a/research/seq_flow_lite/configs/go_emotion_prado.txt b/research/seq_flow_lite/configs/go_emotion_prado.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c159c50ee9a0a3c1d6e0834cab51a08ba1e60055
--- /dev/null
+++ b/research/seq_flow_lite/configs/go_emotion_prado.txt
@@ -0,0 +1,28 @@
+{
+ "model_config" : {
+ "labels": ["admiration", "amusement", "anger", "annoyance", "approval", "caring", "confusion", "curiosity", "desire", "disappointment", "disapproval", "disgust", "embarrassment", "excitement", "fear", "gratitude", "grief", "joy", "love", "nervousness", "optimism", "pride", "realization", "relief", "remorse", "sadness", "surprise", "neutral"],
+ "multilabel": true,
+ "quantize": true,
+ "max_seq_len": 128,
+ "max_seq_len_inference": 128,
+ "split_on_space": true,
+ "embedding_regularizer_scale": 35e-3,
+ "embedding_size": 64,
+ "bigram_channels": 64,
+ "trigram_channels": 64,
+ "feature_size": 512,
+ "network_regularizer_scale": 1e-4,
+ "keep_prob": 0.5,
+ "distortion_probability": 0.0
+ },
+ "name": "models.prado",
+ "batch_size": 1024,
+ "save_checkpoints_steps": 100,
+ "train_steps": 100000,
+ "learning_rate": 0.0006,
+ "learning_rate_decay_steps": 340,
+ "learning_rate_decay_rate": 0.7,
+ "iterations_per_loop": 100,
+ "dataset": "goemotions"
+}
+
diff --git a/research/seq_flow_lite/demo/colab/BUILD b/research/seq_flow_lite/demo/colab/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..b23440bda97c9f2ba34d558503196bdff1c938ce
--- /dev/null
+++ b/research/seq_flow_lite/demo/colab/BUILD
@@ -0,0 +1,9 @@
+sh_binary(
+ name = "move_ops",
+ srcs = ["move_ops.sh"],
+ data = [
+ "//tf_ops:sequence_string_projection_op_py",
+ "//tf_ops:sequence_string_projection_op_v2_py",
+ "//tf_ops:tf_custom_ops_py",
+ ],
+)
diff --git a/research/seq_flow_lite/demo/colab/move_ops.sh b/research/seq_flow_lite/demo/colab/move_ops.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5018797cf198295ff6b86ef6bdd2832a4fe966f8
--- /dev/null
+++ b/research/seq_flow_lite/demo/colab/move_ops.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+RUNFILES_DIR=$(pwd)
+cp -f "${RUNFILES_DIR}/tf_ops/libsequence_string_projection_op_py_gen_op.so" \
+ "${BUILD_WORKSPACE_DIRECTORY}/tf_ops"
+cp -f "${RUNFILES_DIR}/tf_ops/sequence_string_projection_op.py" \
+ "${BUILD_WORKSPACE_DIRECTORY}/tf_ops"
+
+cp -f "${RUNFILES_DIR}/tf_ops/libsequence_string_projection_op_v2_py_gen_op.so" \
+ "${BUILD_WORKSPACE_DIRECTORY}/tf_ops"
+cp -f "${RUNFILES_DIR}/tf_ops/sequence_string_projection_op_v2.py" \
+ "${BUILD_WORKSPACE_DIRECTORY}/tf_ops"
+
+cp -f "${RUNFILES_DIR}/tf_ops/libtf_custom_ops_py_gen_op.so" \
+ "${BUILD_WORKSPACE_DIRECTORY}/tf_ops"
+cp -f "${RUNFILES_DIR}/tf_ops/tf_custom_ops_py.py" \
+ "${BUILD_WORKSPACE_DIRECTORY}/tf_ops"
+
diff --git a/research/seq_flow_lite/demo/colab/setup.py b/research/seq_flow_lite/demo/colab/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..691e4c78c5c2805eb1f588dae9c57d47080313f2
--- /dev/null
+++ b/research/seq_flow_lite/demo/colab/setup.py
@@ -0,0 +1,53 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from distutils import spawn
+from distutils.command import build
+import os
+import subprocess
+
+import setuptools
+
+
+class _BuildCommand(build.build):
+ sub_commands = [
+ ('bazel_build', lambda self: True),
+ ] + build.build.sub_commands
+
+
+class _BazelBuildCommand(setuptools.Command):
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ self._bazel_cmd = spawn.find_executable('bazel')
+
+ def run(self):
+ subprocess.check_call(
+ [self._bazel_cmd, 'run', '-c', 'opt', '//demo/colab:move_ops'],
+ cwd=os.path.dirname(os.path.realpath(__file__)))
+
+
+setuptools.setup(
+ name='seq_flow_lite',
+ version='0.1',
+ packages=['tf_ops'],
+ package_data={'': ['*.so']},
+ cmdclass={
+ 'build': _BuildCommand,
+ 'bazel_build': _BazelBuildCommand,
+ },
+ description='Test')
diff --git a/research/seq_flow_lite/demo/colab/setup_workspace.sh b/research/seq_flow_lite/demo/colab/setup_workspace.sh
new file mode 100755
index 0000000000000000000000000000000000000000..ec71325c57f5e90750b2d378016ff7a8327c31ce
--- /dev/null
+++ b/research/seq_flow_lite/demo/colab/setup_workspace.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+cd "$(dirname "$0")"
+mv setup.py ../..
+touch ../../tf_ops/__init__.py
diff --git a/research/seq_flow_lite/demo/prado/BUILD b/research/seq_flow_lite/demo/prado/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..26f2e718151ca97861b578cfa7dd96fd2cb65a76
--- /dev/null
+++ b/research/seq_flow_lite/demo/prado/BUILD
@@ -0,0 +1,22 @@
+# A demo app for invoking a PRADO TFLite model.
+
+licenses(["notice"])
+
+package(
+ default_visibility = ["//:friends"], # sequence projection
+)
+
+cc_binary(
+ name = "prado_tflite_example",
+ srcs = ["prado_tflite_example.cc"],
+ data = [
+ "data/tflite.fb",
+ ],
+ deps = [
+ "@org_tensorflow//tensorflow/lite:framework",
+ "@org_tensorflow//tensorflow/lite:string_util",
+ "//tflite_ops:expected_value", # sequence projection
+ "//tflite_ops:quantization_util", # sequence projection
+ "//tflite_ops:sequence_string_projection", # sequence projection
+ ],
+)
diff --git a/research/seq_flow_lite/demo/prado/data/tflite.fb b/research/seq_flow_lite/demo/prado/data/tflite.fb
new file mode 100644
index 0000000000000000000000000000000000000000..70fb9c2bb28b5cf5fae5a39c95e576145f7b19ad
Binary files /dev/null and b/research/seq_flow_lite/demo/prado/data/tflite.fb differ
diff --git a/research/seq_flow_lite/demo/prado/prado_tflite_example.cc b/research/seq_flow_lite/demo/prado/prado_tflite_example.cc
new file mode 100644
index 0000000000000000000000000000000000000000..1fa523a892bcf77ad32c3283b98baae61e985e6e
--- /dev/null
+++ b/research/seq_flow_lite/demo/prado/prado_tflite_example.cc
@@ -0,0 +1,150 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/string_util.h"
+#include "tflite_ops/expected_value.h" // seq_flow_lite
+#include "tflite_ops/quantization_util.h" // seq_flow_lite
+#include "tflite_ops/sequence_string_projection.h" // seq_flow_lite
+
+namespace {
+const int kTextInput = 0;
+const int kClassOutput = 0;
+const int kNumberOfInputs = 1;
+const int kNumberOfOutputs = 1;
+const int kClassOutputRank = 2;
+const int kClassOutputBatchSizeIndex = 0;
+const int kBatchSize = 1;
+const int kClassOutputClassIndex = 1;
+constexpr char kTfliteDemoFile[] =
+ "demo/prado/data/tflite.fb";
+
+std::unique_ptr CreateInterpreter(
+ const std::string& tflite_flat_buffer) {
+ // This pointer points to a memory location contained in tflite_flat_buffer,
+ // hence it need not be deleted.
+ const tflite::Model* model = tflite::GetModel(tflite_flat_buffer.data());
+ std::unique_ptr interpreter;
+ tflite::ops::builtin::BuiltinOpResolver resolver;
+ resolver.AddCustom(
+ "SEQUENCE_STRING_PROJECTION",
+ tflite::ops::custom::Register_SEQUENCE_STRING_PROJECTION());
+ resolver.AddCustom("ExpectedValueOp",
+ tflite::ops::custom::Register_EXPECTED_VALUE());
+ tflite::InterpreterBuilder(model, resolver,
+ /*error_reporter=*/nullptr)(&interpreter);
+ if (!interpreter) {
+ std::cout << "Unable to create tflite interpreter\n";
+ }
+ return interpreter;
+}
+
+std::vector InvokeModel(
+ const std::string& text,
+ std::unique_ptr& interpreter) {
+ std::vector classes;
+ auto inputs = interpreter->inputs();
+ if (inputs.size() != kNumberOfInputs) {
+ std::cerr << "Model does not accept the right number of inputs.";
+ return classes;
+ }
+ // Set input to the model.
+ TfLiteTensor* input = interpreter->tensor(inputs[kTextInput]);
+ tflite::DynamicBuffer buf;
+ buf.AddString(text.data(), text.length());
+ buf.WriteToTensorAsVector(input);
+
+ // Allocate buffers.
+ interpreter->AllocateTensors();
+
+ // Invoke inference on the model.
+ interpreter->Invoke();
+
+ // Extract outputs and perform sanity checks on them.
+ auto outputs = interpreter->outputs();
+ if (outputs.size() != kNumberOfOutputs) {
+ std::cerr << "Model does not produce right number of outputs.";
+ return classes;
+ }
+ TfLiteTensor* class_output = interpreter->tensor(outputs[kClassOutput]);
+ if (class_output->type != kTfLiteUInt8) {
+ std::cerr << "Tensor output types are not as expected.";
+ return classes;
+ }
+ if (class_output->dims->size != kClassOutputRank) {
+ std::cerr << "Tensor output should be rank " << kClassOutputRank;
+ return classes;
+ }
+ const auto output_dims = class_output->dims->data;
+ if (output_dims[kClassOutputBatchSizeIndex] != kBatchSize) {
+ std::cerr << "Batch size is expected to be " << kBatchSize;
+ return classes;
+ }
+
+ // Extract output from the output tensor and populate results.
+ const size_t num_classes = output_dims[kClassOutputClassIndex];
+ for (int i = 0; i < num_classes; ++i) {
+ // Find class probability or log probability for the class index
+ classes.push_back(tflite::PodDequantize(*class_output, i));
+ }
+ return classes;
+}
+
+std::string GetTfliteDemoFile() {
+ std::string tflite_flat_buffer;
+ std::ifstream file(kTfliteDemoFile,
+ std::ios::in | std::ios::binary | std::ios::ate);
+ if (!file.is_open()) {
+ std::cerr << "Unable to open demo tflite file.\n";
+ return tflite_flat_buffer;
+ }
+ size_t size = file.tellg();
+ file.seekg(0, file.beg);
+ tflite_flat_buffer.resize(size);
+ file.read(const_cast(tflite_flat_buffer.data()), size);
+ file.close();
+ return tflite_flat_buffer;
+}
+} // namespace
+
+int main(int argc, char** argv) {
+ // The flatbuffer must remain valid until the interpreter is destroyed.
+ std::string tflite_flat_buffer = GetTfliteDemoFile();
+ if (tflite_flat_buffer.empty()) {
+ return EXIT_FAILURE;
+ }
+ auto interpreter = CreateInterpreter(tflite_flat_buffer);
+ if (!interpreter) {
+ return EXIT_FAILURE;
+ }
+ while (true) {
+ std::string sentence;
+ std::cout << "Enter input: ";
+ std::getline(std::cin, sentence);
+ std::vector classes = InvokeModel(sentence, interpreter);
+ for (float class_value : classes) {
+ std::cout << class_value << std::endl;
+ }
+ }
+ return EXIT_SUCCESS;
+}
diff --git a/research/seq_flow_lite/export_to_tflite.py b/research/seq_flow_lite/export_to_tflite.py
new file mode 100644
index 0000000000000000000000000000000000000000..41bebb87877502a35171f8b22bd75380bbe676e0
--- /dev/null
+++ b/research/seq_flow_lite/export_to_tflite.py
@@ -0,0 +1,66 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""A tool to export TFLite model."""
+
+import importlib
+import json
+import os
+
+from absl import app
+from absl import flags
+import tensorflow.compat.v1 as tf
+
+from layers import base_layers # import seq_flow_lite module
+from layers import projection_layers # import seq_flow_lite module
+from utils import tflite_utils # import seq_flow_lite module
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("output_dir", None, "The output or model directory.")
+
+
+def load_runner_config():
+ config = os.path.join(FLAGS.output_dir, "runner_config.txt")
+ with tf.gfile.Open(config, "r") as f:
+ return json.loads(f.read())
+
+
+def main(_):
+ runner_config = load_runner_config()
+ model_config = runner_config["model_config"]
+ rel_module_path = "" # empty base dir
+ model = importlib.import_module(rel_module_path + runner_config["name"])
+ with tf.Graph().as_default() as graph:
+ with tf.Session(graph=graph) as session:
+ text = tf.placeholder(tf.string, shape=[1], name="Input")
+ prxlayer = projection_layers.ProjectionLayer(model_config,
+ base_layers.TFLITE)
+ encoder = model.Encoder(model_config, base_layers.TFLITE)
+ projection, seq_lengh = prxlayer(text)
+ logits = encoder(projection, seq_lengh)
+
+ session.run(tf.global_variables_initializer())
+ session.run(tf.local_variables_initializer())
+ saver = tf.train.Saver()
+ saver.restore(session, tf.train.latest_checkpoint(FLAGS.output_dir))
+ tflite_fb = tflite_utils.generate_tflite(session, graph, [text], [logits])
+ output_file_name = os.path.join(FLAGS.output_dir, "tflite.fb")
+ with tf.gfile.Open(output_file_name, "wb") as f:
+ f.write(tflite_fb)
+
+
+if __name__ == "__main__":
+ app.run(main)
diff --git a/research/seq_flow_lite/input_fn_reader.py b/research/seq_flow_lite/input_fn_reader.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e17ae5331af122371c5fb386d64a1040f93465f
--- /dev/null
+++ b/research/seq_flow_lite/input_fn_reader.py
@@ -0,0 +1,81 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""Methods related to input datasets and readers."""
+
+import functools
+import sys
+
+from absl import logging
+
+import tensorflow as tf
+import tensorflow_datasets as tfds
+
+from layers import projection_layers # import seq_flow_lite module
+from utils import misc_utils # import seq_flow_lite module
+
+
+def imdb_reviews(features, _):
+ return features["text"], features["label"]
+
+
+def civil_comments(features, runner_config):
+ labels = runner_config["model_config"]["labels"]
+ label_tensor = tf.stack([features[label] for label in labels], axis=1)
+ label_tensor = tf.floor(label_tensor + 0.5)
+ return features["text"], label_tensor
+
+
+def goemotions(features, runner_config):
+ labels = runner_config["model_config"]["labels"]
+ label_tensor = tf.stack([features[label] for label in labels], axis=1)
+ return features["comment_text"], tf.cast(label_tensor, tf.float32)
+
+
+def create_input_fn(runner_config, mode, drop_remainder):
+ """Returns an input function to use in the instantiation of tf.estimator.*."""
+
+ def _post_processor(features, batch_size):
+ """Post process the data to a form expected by model_fn."""
+ data_processor = getattr(sys.modules[__name__], runner_config["dataset"])
+ text, label = data_processor(features, runner_config)
+ model_config = runner_config["model_config"]
+ if "max_seq_len" in model_config:
+ max_seq_len = model_config["max_seq_len"]
+ logging.info("Truncating text to have at most %d tokens", max_seq_len)
+ text = misc_utils.random_substr(text, max_seq_len)
+ text = tf.reshape(text, [batch_size])
+ num_classes = len(model_config["labels"])
+ label = tf.reshape(label, [batch_size, num_classes])
+ prxlayer = projection_layers.ProjectionLayer(model_config, mode)
+ projection, seq_length = prxlayer(text)
+ return {"projection": projection, "seq_length": seq_length, "label": label}
+
+ def _input_fn(params):
+ """Method to be used for reading the data."""
+ assert mode != tf.estimator.ModeKeys.PREDICT
+ split = "train" if mode == tf.estimator.ModeKeys.TRAIN else "test"
+ ds = tfds.load(runner_config["dataset"], split=split)
+ ds = ds.batch(params["batch_size"], drop_remainder=drop_remainder)
+ ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
+ ds = ds.shuffle(buffer_size=100)
+ ds = ds.repeat(count=1 if mode == tf.estimator.ModeKeys.EVAL else None)
+ ds = ds.map(
+ functools.partial(_post_processor, batch_size=params["batch_size"]),
+ num_parallel_calls=tf.data.experimental.AUTOTUNE,
+ deterministic=False)
+ return ds
+
+ return _input_fn
diff --git a/research/seq_flow_lite/layers/BUILD b/research/seq_flow_lite/layers/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..48e70a22038ae90b4d5173053b5983ef75773baa
--- /dev/null
+++ b/research/seq_flow_lite/layers/BUILD
@@ -0,0 +1,78 @@
+py_strict_library = py_library
+
+licenses(["notice"])
+
+package(
+ default_visibility = ["//:friends"], # sequence projection
+)
+
+py_strict_library(
+ name = "base_layers",
+ srcs = ["base_layers.py"],
+ srcs_version = "PY3",
+ deps = [
+ # package tensorflow
+ ],
+)
+
+py_strict_library(
+ name = "quantization_layers",
+ srcs = ["quantization_layers.py"],
+ srcs_version = "PY3",
+ deps = [
+ ":base_layers",
+ # package tensorflow
+ ],
+)
+
+py_strict_library(
+ name = "normalization_layers",
+ srcs = ["normalization_layers.py"],
+ srcs_version = "PY3",
+ deps = [
+ ":base_layers",
+ ":quantization_layers",
+ # package tensorflow
+ # "//tf_ops:tf_custom_ops" # sequence projection
+ "//tf_ops:tf_custom_ops_py", # sequence projection
+ ],
+)
+
+py_strict_library(
+ name = "dense_layers",
+ srcs = ["dense_layers.py"],
+ srcs_version = "PY3",
+ deps = [
+ ":base_layers",
+ ":normalization_layers",
+ ":quantization_layers",
+ # package tensorflow
+ ],
+)
+
+py_strict_library(
+ name = "conv_layers",
+ srcs = ["conv_layers.py"],
+ srcs_version = "PY3",
+ deps = [
+ ":base_layers",
+ ":normalization_layers",
+ ":quantization_layers",
+ # package tensorflow
+ ],
+)
+
+py_strict_library(
+ name = "projection_layers",
+ srcs = ["projection_layers.py"],
+ srcs_version = "PY3",
+ deps = [
+ ":base_layers",
+ # package absl/logging
+ # package tensorflow
+ # "//tf_ops:sequence_string_projection_op" # sequence projection
+ "//tf_ops:sequence_string_projection_op_py", # sequence projection
+ # "//tf_ops:sequence_string_projection_op_v2" # sequence projection
+ "//tf_ops:sequence_string_projection_op_v2_py", # sequence projection
+ ],
+)
diff --git a/research/seq_flow_lite/layers/base_layers.py b/research/seq_flow_lite/layers/base_layers.py
new file mode 100644
index 0000000000000000000000000000000000000000..40e339986a6d7f7c8ce691d46136aa6e18cafb25
--- /dev/null
+++ b/research/seq_flow_lite/layers/base_layers.py
@@ -0,0 +1,118 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""Base layer for building models trained with quantization."""
+
+import tensorflow as tf
+
+TRAIN = "train"
+EVAL = "eval"
+PREDICT = "infer"
+TFLITE = "tflite"
+_MODE = [TRAIN, EVAL, PREDICT, TFLITE]
+
+
+class Parameters:
+ """A class that encapsulates parameters."""
+
+ def __init__(self,
+ mode,
+ quantize=True,
+ regularizer_scale=0.0,
+ invalid_logit=-1e6,
+ initializer=None):
+ assert isinstance(quantize, bool)
+ self.quantize = quantize
+ assert mode in _MODE
+ self.mode = mode
+ self.regularizer_scale = regularizer_scale
+ self.invalid_logit = invalid_logit
+ self.initializer = initializer
+
+
+class BaseLayer(tf.keras.layers.Layer):
+ """Base class for encoders."""
+
+ def __init__(self, parameters, **kwargs):
+ assert isinstance(parameters, Parameters)
+ self.parameters = parameters
+ super(BaseLayer, self).__init__(**kwargs)
+
+ def _assert_rank_and_type(self, tensor, rank, dtype=tf.float32):
+ assert len(tensor.get_shape().as_list()) == rank
+ assert tensor.dtype == dtype
+
+ def add_qweight(self, shape, num_bits=8):
+ """Return a quantized weight variable for the given shape."""
+ if self.parameters.initializer is not None:
+ initializer = self.parameters.initializer
+ else:
+ initializer = tf.keras.initializers.GlorotUniform()
+ weight = self.add_weight(
+ "weight", shape, initializer=initializer, trainable=True)
+ self.add_reg_loss(weight)
+ return self._weight_quantization(weight, num_bits=num_bits)
+
+ def _weight_quantization(self, tensor, num_bits=8):
+ """Quantize weights when enabled."""
+ # For infer mode, toco computes the min/max from the weights offline to
+ # quantize it. During train/eval this is computed from the current value
+ # in the session by the graph itself.
+ if self.parameters.quantize and self.parameters.mode in [TRAIN, EVAL]:
+ # Toco expects 0.0 to be part of the quantization range.
+ batch_min = tf.minimum(tf.reduce_min(tensor), 0.0)
+ batch_max = tf.maximum(tf.reduce_max(tensor), 0.0)
+
+ return tf.quantization.fake_quant_with_min_max_vars(
+ tensor, batch_min, batch_max, num_bits=num_bits)
+ else:
+ return tensor
+
+ def add_bias(self, shape):
+ weight = self.add_weight(
+ "bias",
+ shape,
+ initializer=tf.keras.initializers.Zeros(),
+ trainable=True)
+ self.add_reg_loss(weight)
+ return weight
+
+ def add_reg_loss(self, weight):
+ if self.parameters.regularizer_scale > 0.0:
+ reg_scale = tf.convert_to_tensor(self.parameters.regularizer_scale)
+ reg_loss = tf.nn.l2_loss(weight) * reg_scale
+ self.add_loss(reg_loss)
+
+ def assign_moving_average(self, var, update, ema_decay):
+ return var.assign(var.read_value() * (1 - ema_decay) + (ema_decay) * update)
+
+ def qrange_sigmoid(self, tensor):
+ if self.parameters.quantize:
+ return tf.quantization.fake_quant_with_min_max_args(tensor, 0.0, 1.0)
+ return tensor
+
+ def qrange_tanh(self, tensor):
+ if self.parameters.quantize:
+ return tf.quantization.fake_quant_with_min_max_args(tensor, -1.0, 1.0)
+ return tensor
+
+ def quantized_tanh(self, tensor):
+ return self.qrange_tanh(tf.tanh(tensor))
+
+ def quantized_sigmoid(self, tensor):
+ return self.qrange_sigmoid(tf.sigmoid(tensor))
+
+ def get_batch_dimension(self, tensor):
+ return tensor.get_shape().as_list()[0] or tf.shape(tensor)[0]
diff --git a/research/seq_flow_lite/layers/conv_layers.py b/research/seq_flow_lite/layers/conv_layers.py
new file mode 100644
index 0000000000000000000000000000000000000000..19c2adbcb8051307119527f7eb4ce72175f33003
--- /dev/null
+++ b/research/seq_flow_lite/layers/conv_layers.py
@@ -0,0 +1,118 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""Base layer for convolution."""
+import tensorflow as tf
+
+from layers import base_layers # import seq_flow_lite module
+from layers import normalization_layers # import seq_flow_lite module
+from layers import quantization_layers # import seq_flow_lite module
+
+
+class EncoderQConvolution(base_layers.BaseLayer):
+ """Quantized encoder convolution layers."""
+
+ def __init__(self,
+ filters,
+ ksize,
+ stride=1,
+ padding="SAME",
+ dilations=None,
+ activation=tf.keras.layers.ReLU(),
+ bias=True,
+ rank=4,
+ **kwargs):
+ self.out_filters = filters
+ assert rank >= 3 and rank <= 4
+ self.rank = rank
+ self.ksize = self._unpack(ksize)
+ self.strides = self._unpack(stride)
+ self.dilations = [1] + self._unpack(dilations) + [1] if dilations else None
+ self.activation = activation
+ self.bias = bias
+ self.padding = padding
+ self.qoutput = quantization_layers.ActivationQuantization(**kwargs)
+ self._create_normalizer(**kwargs)
+ super(EncoderQConvolution, self).__init__(**kwargs)
+
+ def _unpack(self, value):
+ if not isinstance(value, list):
+ assert isinstance(value, int)
+ return [1 if self.rank == 3 else value, value]
+ else:
+ assert len(value) == 2 and self.rank == 4
+ assert isinstance(value[0], int) and isinstance(value[1], int)
+ return value
+
+ def build(self, input_shapes):
+ assert len(input_shapes) == self.rank
+ self.in_filters = input_shapes[-1]
+ shape = self.ksize + [self.in_filters, self.out_filters]
+ self.filters = self.add_qweight(shape=shape)
+ if self.bias:
+ self.b = self.add_bias(shape=[self.out_filters])
+
+ def _create_normalizer(self, **kwargs):
+ self.normalization = normalization_layers.BatchNormalization(**kwargs)
+
+ def _conv_r4(self, inputs, normalize_method):
+ outputs = tf.nn.conv2d(
+ inputs,
+ self.filters,
+ strides=self.strides,
+ padding=self.padding,
+ dilations=self.dilations)
+ if self.bias:
+ outputs = tf.nn.bias_add(outputs, self.b)
+ outputs = normalize_method(outputs)
+ if self.activation:
+ outputs = self.activation(outputs)
+ return self.qoutput(outputs)
+
+ def _conv_r3(self, inputs, normalize_method):
+ bsz = self.get_batch_dimension(inputs)
+ inputs_r4 = tf.reshape(inputs, [bsz, 1, -1, self.in_filters])
+ outputs = self._conv_r4(inputs_r4, normalize_method)
+ return tf.reshape(outputs, [bsz, -1, self.out_filters])
+
+ def call(self, inputs):
+
+ def normalize_method(tensor):
+ return self.normalization(tensor)
+
+ return self._do_call(inputs, normalize_method)
+
+ def _do_call(self, inputs, normalize_method):
+ if self.rank == 3:
+ return self._conv_r3(inputs, normalize_method)
+ return self._conv_r4(inputs, normalize_method)
+
+ def quantize_using_output_range(self, tensor):
+ return self.qoutput.quantize_using_range(tensor)
+
+
+class EncoderQConvolutionVarLen(EncoderQConvolution):
+ """Convolution on variable length sequence."""
+
+ def _create_normalizer(self, **kwargs):
+ self.normalization = normalization_layers.VarLenBatchNormalization(
+ rank=4, **kwargs)
+
+ def call(self, inputs, mask, inverse_normalizer):
+
+ def normalize_method(tensor):
+ return self.normalization(tensor, mask, inverse_normalizer)
+
+ return self._do_call(inputs, normalize_method)
diff --git a/research/seq_flow_lite/layers/dense_layers.py b/research/seq_flow_lite/layers/dense_layers.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c55d215fdbbdc939060bc5887d8426ea926f6d3
--- /dev/null
+++ b/research/seq_flow_lite/layers/dense_layers.py
@@ -0,0 +1,107 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""Basic dense layers."""
+import tensorflow as tf
+
+from layers import base_layers # import seq_flow_lite module
+from layers import normalization_layers # import seq_flow_lite module
+from layers import quantization_layers # import seq_flow_lite module
+
+
+class BaseQDense(base_layers.BaseLayer):
+ """Quantized encoder dense layers."""
+
+ def __init__(self,
+ units,
+ activation=tf.keras.layers.ReLU(),
+ bias=True,
+ rank=2,
+ normalize=True,
+ **kwargs):
+ self.units = units
+ self.rank = rank
+ assert rank >= 2 and rank <= 4
+ self.activation = activation
+ self.bias = bias
+ self.normalize = normalize
+ self.qoutput = quantization_layers.ActivationQuantization(**kwargs)
+ self._create_normalizer(**kwargs)
+ super(BaseQDense, self).__init__(**kwargs)
+
+ def build(self, input_shapes):
+ assert len(input_shapes) == self.rank
+ if self.rank == 4:
+ assert input_shapes[1] == 1 or input_shapes[2] == 1
+ self.in_units = input_shapes[-1]
+ shape = [self.in_units, self.units]
+ self.w = self.add_qweight(shape=shape)
+ if self.bias:
+ self.b = self.add_bias(shape=[self.units])
+
+ def _create_normalizer(self, **kwargs):
+ self.normalization = normalization_layers.BatchNormalization(**kwargs)
+
+ def _dense_r2(self, inputs, normalize_method):
+ outputs = tf.matmul(inputs, self.w)
+ if self.bias:
+ outputs = tf.nn.bias_add(outputs, self.b)
+ if self.normalize:
+ outputs = normalize_method(outputs)
+ if self.activation:
+ outputs = self.activation(outputs)
+ return self.qoutput(outputs)
+
+ def _dense_r34(self, inputs, normalize_method):
+ bsz = self.get_batch_dimension(inputs)
+ outputs = tf.reshape(inputs, [-1, self.in_units])
+ outputs = self._dense_r2(outputs, normalize_method)
+ if self.rank == 3:
+ return tf.reshape(outputs, [bsz, -1, self.units])
+ elif inputs.get_shape().as_list()[1] == 1:
+ return tf.reshape(outputs, [bsz, 1, -1, self.units])
+ else:
+ return tf.reshape(outputs, [bsz, -1, 1, self.units])
+
+ def call(self, inputs):
+
+ def normalize_method(tensor):
+ return self.normalization(tensor)
+
+ return self._do_call(inputs, normalize_method)
+
+ def _do_call(self, inputs, normalize_method):
+ if self.rank == 2:
+ return self._dense_r2(inputs, normalize_method)
+ return self._dense_r34(inputs, normalize_method)
+
+ def quantize_using_output_range(self, tensor):
+ return self.qoutput.quantize_using_range(tensor)
+
+
+class BaseQDenseVarLen(BaseQDense):
+ """Dense on variable length sequence."""
+
+ def _create_normalizer(self, **kwargs):
+ self.normalization = normalization_layers.VarLenBatchNormalization(
+ rank=2, **kwargs)
+
+ def call(self, inputs, mask, inverse_normalizer):
+
+ def normalize_method(tensor):
+ maskr2 = tf.reshape(mask, [-1, 1])
+ return self.normalization(tensor, maskr2, inverse_normalizer)
+
+ return self._do_call(inputs, normalize_method)
diff --git a/research/seq_flow_lite/layers/normalization_layers.py b/research/seq_flow_lite/layers/normalization_layers.py
new file mode 100644
index 0000000000000000000000000000000000000000..248143a096f536d2eb539bfa9a8d39fab15a547d
--- /dev/null
+++ b/research/seq_flow_lite/layers/normalization_layers.py
@@ -0,0 +1,140 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""Layers for normalization."""
+import tensorflow as tf
+
+from layers import base_layers # import seq_flow_lite module
+from layers import quantization_layers # import seq_flow_lite module
+from tf_ops import tf_custom_ops_py # import seq_flow_lite module
+
+
+class BatchNormalization(base_layers.BaseLayer):
+ """A class that applies batch normalization to the input tensor."""
+
+ def __init__(self, ema_decay=0.999, **kwargs):
+ self.ema_decay = ema_decay
+ super(BatchNormalization, self).__init__(**kwargs)
+
+ def build(self, input_shapes):
+ self.reduce_dims = list(range(len(input_shapes) - 1))
+ shape = [input_shapes[-1]]
+ self.offset = self.add_weight(
+ "offset",
+ shape=shape,
+ initializer=tf.keras.initializers.Zeros(),
+ trainable=True)
+ self.scale = self.add_weight(
+ "scale",
+ shape=shape,
+ initializer=tf.keras.initializers.Ones(),
+ trainable=True)
+ self.mva_mean = self.add_weight(
+ "mva_mean",
+ shape=shape,
+ initializer=tf.keras.initializers.Zeros(),
+ trainable=False)
+ self.mva_var = self.add_weight(
+ "mva_variance",
+ shape=shape,
+ initializer=tf.keras.initializers.Ones(),
+ trainable=False)
+
+ def call(self, inputs):
+ mean_mom, var_mom = None, None
+ if self.parameters.mode == base_layers.TRAIN:
+ mean_mom, var_mom = tf.nn.moments(inputs, self.reduce_dims)
+ return self._batch_norm(inputs, mean_mom, var_mom)
+
+ def _batch_norm(self, inputs, mean_mom, var_mom):
+ if self.parameters.mode == base_layers.TRAIN:
+ # During training compute summay stats, update them to moving average
+ # variables and use the summary stas for batch normalization.
+ with tf.control_dependencies([
+ self.assign_moving_average(self.mva_mean, mean_mom, self.ema_decay),
+ self.assign_moving_average(self.mva_var, var_mom, self.ema_decay)
+ ]):
+ tensor = tf.nn.batch_normalization(inputs, mean_mom, var_mom,
+ self.offset, self.scale, 1e-9)
+ else:
+ # During eval/inference use the moving average variable for batch
+ # normalization. The variables would be frozen to constants before
+ # saving graph.
+ tensor = tf.nn.batch_normalization(inputs, self.mva_mean, self.mva_var,
+ self.offset, self.scale, 1e-9)
+ return tensor
+
+
+class VarLenBatchNormalization(BatchNormalization):
+ """A class that applies batch normalization to the input tensor."""
+
+ def __init__(self, rank=2, **kwargs):
+ self.rank = rank
+ assert rank == 2 or rank == 4
+ super(VarLenBatchNormalization, self).__init__(**kwargs)
+
+ def _reduce(self, tensor, multiplier):
+ return tf.reduce_sum(tensor, axis=self.reduce_dims) * multiplier
+
+ def call(self, inputs, mask, inverse_normalizer):
+ if self.parameters.mode == base_layers.TRAIN:
+ self._assert_rank_and_type(inputs, self.rank)
+ self._assert_rank_and_type(mask, self.rank)
+ inputs = mask * inputs
+ mean_mom = self._reduce(inputs, inverse_normalizer)
+ var_mom = self._reduce(inputs * inputs, inverse_normalizer)
+ return mask * self._batch_norm(inputs, mean_mom, var_mom)
+ elif self.parameters.mode == base_layers.EVAL:
+ return mask * self._batch_norm(inputs, None, None)
+ return self._batch_norm(inputs, None, None)
+
+
+class LayerNormalization(base_layers.BaseLayer):
+ """A class that applies layer normalization to the input tensor."""
+
+ def __init__(self, axes=None, **kwargs):
+ self.axes = axes or [-1]
+ self.qactivation = quantization_layers.ActivationQuantization(**kwargs)
+ super(LayerNormalization, self).__init__(**kwargs)
+
+ def build(self, input_shape):
+ self.rank = len(input_shape)
+ for i, axis in enumerate(self.axes):
+ if axis < 0:
+ self.axes[i] += self.rank
+ assert (self.axes[i] > 0 and self.axes[i] < self.rank)
+ self.offset = self.add_weight(
+ "offset",
+ shape=[1],
+ initializer=tf.keras.initializers.Zeros(),
+ trainable=True)
+ self.scale = self.add_weight(
+ "scale",
+ shape=[1],
+ initializer=tf.keras.initializers.Ones(),
+ trainable=True)
+
+ def call(self, tensor):
+ tensor = self.qactivation(tensor)
+ if self.parameters.mode != base_layers.TFLITE:
+ mean, variance = tf.nn.moments(tensor, self.axes, keepdims=True)
+ # If all the values in the tensor are same, variance will be 0. Adding a
+ # small epsilon to variance ensures that we get 0 as the normalized result
+ # instead of NaN in the resulting tensor.
+ tensor = (tensor - mean) / tf.sqrt(variance + 1e-6)
+ return tensor * self.scale + self.offset
+ else:
+ return tf_custom_ops_py.layer_norm(
+ tensor, self.scale, self.offset, axes=self.axes)
diff --git a/research/seq_flow_lite/layers/projection_layers.py b/research/seq_flow_lite/layers/projection_layers.py
new file mode 100644
index 0000000000000000000000000000000000000000..9dff8adc85274fb5c4656065c406229ebf7e8145
--- /dev/null
+++ b/research/seq_flow_lite/layers/projection_layers.py
@@ -0,0 +1,119 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tensorflow projection creator for PRADO model."""
+
+from absl import logging
+import tensorflow as tf
+
+from layers import base_layers # import seq_flow_lite module
+from tf_ops import sequence_string_projection_op as ssp # import seq_flow_lite module
+from tf_ops import sequence_string_projection_op_v2 as sspv2 # import seq_flow_lite module
+
+
+class ProjectionLayer(base_layers.BaseLayer):
+ """Base class for encoders."""
+
+ def __init__(self, model_config, mode):
+ """Create projection."""
+
+ def _get_params(varname, default_value=None):
+ value = model_config[varname] if varname in model_config else default_value
+ default = "" if varname in model_config else " (default)"
+ logging.info("%s = %s%s", varname, value, default)
+ setattr(self, varname, value)
+
+ self.mode = mode
+ _get_params("feature_size")
+ _get_params("max_seq_len", 0)
+ _get_params("add_eos_tag", False)
+ _get_params("add_bos_tag", False)
+ _get_params("hashtype", "murmur")
+ _get_params("split_on_space", True)
+ _get_params("token_separators", "")
+ _get_params("vocabulary", "")
+ _get_params("quantize")
+ _get_params("word_novelty_bits", 0)
+ _get_params("doc_size_levels", 0)
+ self.distortion_probability = 0.0
+ if mode == base_layers.TRAIN:
+ _get_params("distortion_probability", 0.0)
+ parameters = base_layers.Parameters(mode, self.quantize)
+ super(ProjectionLayer, self).__init__(parameters=parameters)
+
+ def call(self, inputs):
+ projection, _, seq_length = ssp.sequence_string_projection(
+ input=inputs,
+ feature_size=self.feature_size,
+ max_splits=self.max_seq_len - 1,
+ hashtype=self.hashtype,
+ distortion_probability=self.distortion_probability,
+ split_on_space=self.split_on_space,
+ token_separators=self.token_separators,
+ word_novelty_bits=self.word_novelty_bits,
+ doc_size_levels=self.doc_size_levels,
+ add_eos_tag=self.add_eos_tag,
+ add_bos_tag=self.add_bos_tag,
+ vocabulary=self.vocabulary)
+
+ modes = [base_layers.PREDICT, base_layers.TFLITE]
+ if self.mode not in modes and self.max_seq_len > 0:
+ short_by = self.max_seq_len - tf.shape(projection)[1]
+ projection = tf.pad(projection, [[0, 0], [0, short_by], [0, 0]])
+ batch_size = self.get_batch_dimension(inputs)
+ projection = tf.reshape(projection,
+ [batch_size, self.max_seq_len, self.feature_size])
+ if self.mode in modes:
+ projection = self.qrange_tanh(projection)
+ return projection, seq_length
+
+
+class ProjectionLayerPreSegmented(base_layers.BaseLayer):
+ """Base class for encoders."""
+
+ def __init__(self, model_config, mode):
+ """Create projection."""
+
+ def _get_params(varname, default_value=None):
+ value = model_config[varname] if varname in model_config else default_value
+ default = "" if varname in model_config else " (default)"
+ logging.info("%s = %s%s", varname, value, default)
+ setattr(self, varname, value)
+
+ self.mode = mode
+ _get_params("feature_size")
+ _get_params("add_eos_tag", False)
+ _get_params("add_bos_tag", False)
+ _get_params("vocabulary", "")
+ _get_params("quantize")
+ self.distortion_probability = 0.0
+ if mode == base_layers.TRAIN:
+ _get_params("distortion_probability", 0.0)
+ parameters = base_layers.Parameters(mode, self.quantize)
+ super(ProjectionLayerPreSegmented, self).__init__(parameters=parameters)
+
+ def call(self, inputs, sequence_length):
+ projection = sspv2.sequence_string_projection_v2(
+ input=inputs,
+ sequence_length=sequence_length,
+ feature_size=self.feature_size,
+ distortion_probability=self.distortion_probability,
+ add_eos_tag=self.add_eos_tag,
+ add_bos_tag=self.add_bos_tag,
+ vocabulary=self.vocabulary)
+
+ modes = [base_layers.PREDICT, base_layers.TFLITE]
+ if self.mode in modes:
+ projection = self.qrange_tanh(projection)
+ return projection
diff --git a/research/seq_flow_lite/layers/quantization_layers.py b/research/seq_flow_lite/layers/quantization_layers.py
new file mode 100644
index 0000000000000000000000000000000000000000..9990eb5b74b616b4d9be3aeb8387ce0ff63996ad
--- /dev/null
+++ b/research/seq_flow_lite/layers/quantization_layers.py
@@ -0,0 +1,97 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""Layers for quantization."""
+
+import tensorflow as tf
+
+from layers import base_layers # import seq_flow_lite module
+
+
+class ActivationQuantization(base_layers.BaseLayer):
+ """A class that applies quantization to a activation tensor."""
+
+ def __init__(self, ema_decay=0.99, num_bits=8, **kwargs):
+ self.ema_decay = ema_decay
+ self.num_bits = num_bits
+ super(ActivationQuantization, self).__init__(**kwargs)
+ if self.parameters.quantize:
+ self.min_var = self.add_weight(
+ "min", initializer=tf.keras.initializers.Zeros(), trainable=False)
+ self.max_var = self.add_weight(
+ "max", initializer=tf.keras.initializers.Ones(), trainable=False)
+
+ def call(self, inputs):
+ if self.parameters.quantize:
+ if self.parameters.mode == base_layers.TRAIN:
+ # Toco expects 0.0 to be part of the quantization range.
+ batch_min = tf.minimum(tf.reduce_min(inputs), 0.0)
+ min_var = self.assign_moving_average(self.min_var, batch_min,
+ self.ema_decay)
+
+ batch_max = tf.maximum(tf.reduce_max(inputs), 0.0)
+ max_var = self.assign_moving_average(self.max_var, batch_max,
+ self.ema_decay)
+ with tf.control_dependencies([min_var, max_var]):
+ return tf.quantization.fake_quant_with_min_max_vars(
+ inputs, batch_min, batch_max, num_bits=self.num_bits)
+ else:
+ return tf.quantization.fake_quant_with_min_max_vars(
+ inputs, self.min_var, self.max_var, num_bits=self.num_bits)
+ return inputs
+
+ def quantize_using_range(self, inputs):
+ if self.parameters.quantize:
+ return tf.quantization.fake_quant_with_min_max_vars(
+ inputs, self.min_var, self.max_var, num_bits=self.num_bits)
+ return inputs
+
+
+class ConcatQuantization(ActivationQuantization):
+ """A class that applies quantization to a activation tensor."""
+
+ def __init__(self, axis=2, **kwargs):
+ self.axis = axis
+ super(ConcatQuantization, self).__init__(**kwargs)
+
+ def reduce_list(self, tensor_list, functor):
+ reduce_result = [functor(tensor) for tensor in tensor_list]
+ # Toco expects 0.0 to be part of the quantization range.
+ reduce_result.append(tf.constant(0.0))
+ return functor(tf.stack(reduce_result))
+
+ def call(self, tensors):
+ if self.parameters.quantize:
+ if self.parameters.mode == base_layers.TRAIN:
+ # Toco expects 0.0 to be part of the quantization range.
+ batch_min = self.reduce_list(tensors, tf.reduce_min)
+ min_var = self.assign_moving_average(self.min_var, batch_min,
+ self.ema_decay)
+
+ batch_max = self.reduce_list(tensors, tf.reduce_max)
+ max_var = self.assign_moving_average(self.max_var, batch_max,
+ self.ema_decay)
+ else:
+ min_var, max_var = self.min_var, self.max_var
+
+ tensors = [
+ tf.quantization.fake_quant_with_min_max_vars(
+ tensor, min_var, max_var, num_bits=self.num_bits)
+ for tensor in tensors
+ ]
+ tensor = tf.concat(tensors, axis=self.axis)
+ return tf.quantization.fake_quant_with_min_max_vars(
+ tensor, min_var, max_var, num_bits=self.num_bits)
+ return tf.concat(tensors, axis=self.axis)
diff --git a/research/seq_flow_lite/metric_functions.py b/research/seq_flow_lite/metric_functions.py
new file mode 100644
index 0000000000000000000000000000000000000000..819fd69516fbc8d9691d82b73dc390278f18fba8
--- /dev/null
+++ b/research/seq_flow_lite/metric_functions.py
@@ -0,0 +1,47 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""Metric functions."""
+import tensorflow.compat.v1 as tf
+
+
+def classification_metric(per_example_loss, label_ids, logits):
+ """Compute eval metrics."""
+ return {
+ "accuracy":
+ tf.metrics.accuracy(label_ids, tf.math.argmax(logits, axis=-1)),
+ "eval_loss":
+ tf.metrics.mean(per_example_loss)
+ }
+
+
+THRESHOLDS = [0.5]
+
+
+def labeling_metric(per_example_loss, label_ids, logits):
+ """Compute eval metrics."""
+ scores = tf.math.sigmoid(logits)
+ num_classes = label_ids.get_shape().as_list()[-1]
+ return_dict = {"eval_loss": tf.metrics.mean(per_example_loss)}
+ for idx in range(num_classes):
+ return_dict["auc/" + str(idx)] = tf.metrics.auc(label_ids[:, idx],
+ scores[:, idx])
+ return_dict["precision@" + str(THRESHOLDS) + "/" +
+ str(idx)] = tf.metrics.precision_at_thresholds(
+ label_ids[:, idx], scores[:, idx], thresholds=THRESHOLDS)
+ return_dict["recall@" + str(THRESHOLDS) + "/" +
+ str(idx)] = tf.metrics.recall_at_thresholds(
+ label_ids[:, idx], scores[:, idx], thresholds=THRESHOLDS)
+ return return_dict
diff --git a/research/seq_flow_lite/models/BUILD b/research/seq_flow_lite/models/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..cb8c75fe8d2d7e518648e5012c935890844f2bab
--- /dev/null
+++ b/research/seq_flow_lite/models/BUILD
@@ -0,0 +1,22 @@
+licenses(["notice"])
+
+package(
+ default_visibility = ["//:friends"], # sequence projection
+)
+
+py_library(
+ name = "prado",
+ srcs = ["prado.py"],
+ srcs_version = "PY3",
+ deps = [
+ # package absl/logging
+ # package tensorflow
+ "//layers:base_layers", # sequence projection
+ "//layers:conv_layers", # sequence projection
+ "//layers:dense_layers", # sequence projection
+ "//layers:projection_layers", # sequence projection
+ "//layers:quantization_layers", # sequence projection
+ # "//tf_ops:tf_custom_ops" # sequence projection
+ "//tf_ops:tf_custom_ops_py", # sequence projection
+ ],
+)
diff --git a/research/seq_flow_lite/models/prado.py b/research/seq_flow_lite/models/prado.py
new file mode 100644
index 0000000000000000000000000000000000000000..366b774a1f55ee6bd429b8654e6e5f3a2acf0446
--- /dev/null
+++ b/research/seq_flow_lite/models/prado.py
@@ -0,0 +1,193 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Lint as: python3
+"""Implementation of PRADO model."""
+
+import copy
+from absl import logging
+import numpy as np
+import tensorflow as tf
+
+from layers import base_layers # import seq_flow_lite module
+from layers import conv_layers # import seq_flow_lite module
+from layers import dense_layers # import seq_flow_lite module
+from layers import projection_layers # import seq_flow_lite module
+from layers import quantization_layers # import seq_flow_lite module
+from tf_ops import tf_custom_ops_py # import seq_flow_lite module
+
+
+class PaddedMaskedVarLenConv(conv_layers.EncoderQConvolutionVarLen):
+ """A layer that performs padded masked convolution."""
+
+ def __init__(self, invalid_value, ngram=2, skip_bigram=None, **kwargs):
+ self.invalid_value = invalid_value
+ assert ngram is None or (ngram >= 1 and ngram <= 5)
+ assert skip_bigram is None or skip_bigram == 1 or skip_bigram == 2
+ assert bool(ngram is None) != bool(skip_bigram is None)
+ self.kwidth = ngram if ngram is not None else (skip_bigram + 2)
+ mask = [1] * self.kwidth
+ if skip_bigram is not None:
+ mask[1], mask[skip_bigram] = 0, 0
+ self.mask = np.array(mask, dtype="float32").reshape((1, self.kwidth, 1, 1))
+ self.zero_pad = tf.keras.layers.ZeroPadding1D(padding=[0, self.kwidth - 1])
+ super(PaddedMaskedVarLenConv, self).__init__(
+ ksize=self.kwidth, rank=3, padding="VALID", activation=None, **kwargs)
+
+ def call(self, inputs, mask, inverse_normalizer):
+ self._assert_rank_and_type(inputs, 3)
+ self._assert_rank_and_type(mask, 3)
+ maskr4 = tf.expand_dims(mask, axis=1)
+ inputs_padded = self.zero_pad(inputs)
+ result = super(PaddedMaskedVarLenConv, self).call(inputs_padded, maskr4,
+ inverse_normalizer)
+ if self.parameters.mode not in [base_layers.PREDICT, base_layers.TFLITE]:
+ return result * mask + (1 - mask) * self.invalid_value
+ return result
+
+ def add_qweight(self, shape, num_bits=8):
+ weight = super(PaddedMaskedVarLenConv, self).add_qweight(
+ shape=shape, num_bits=num_bits)
+ return weight * tf.convert_to_tensor(self.mask)
+
+
+class AttentionPoolReduce(base_layers.BaseLayer):
+ """Attention pooling and reduce."""
+
+ def __init__(self, filters, ngram=2, skip_bigram=None, **kwargs):
+ super(AttentionPoolReduce, self).__init__(**kwargs)
+ self.filters = filters
+ self.value = PaddedMaskedVarLenConv(
+ 0, filters=filters, ngram=ngram, skip_bigram=skip_bigram, **kwargs)
+ self.attention_logits = PaddedMaskedVarLenConv(
+ self.parameters.invalid_logit,
+ filters=filters,
+ ngram=ngram,
+ skip_bigram=skip_bigram,
+ **kwargs)
+
+ def call(self, values_in, attention_in, mask, inverse_normalizer):
+ self._assert_rank_and_type(values_in, 3)
+ self._assert_rank_and_type(attention_in, 3)
+ self._assert_rank_and_type(mask, 3)
+ values = self.value(values_in, mask, inverse_normalizer)
+ attention_logits = self.attention_logits(attention_in, mask,
+ inverse_normalizer)
+
+ if self.parameters.mode == base_layers.TFLITE:
+ return tf_custom_ops_py.expected_value_op(attention_logits, values)
+ else:
+ attention_logits = tf.transpose(attention_logits, [0, 2, 1])
+ values = tf.transpose(values, [0, 2, 1])
+ attention = tf.nn.softmax(attention_logits)
+ return tf.reduce_sum(attention * values, axis=2)
+
+
+class Encoder(tf.keras.layers.Layer):
+ """A PRADO keras model."""
+
+ def __init__(self, config, mode):
+ super(Encoder, self).__init__()
+
+ def _get_params(varname, default_value=None):
+ value = config[varname] if varname in config else default_value
+ default = "" if varname in config else " (default)"
+ logging.info("%s = %s%s", varname, value, default)
+ setattr(self, varname, value)
+
+ _get_params("labels")
+ _get_params("quantize", True)
+ _get_params("embedding_regularizer_scale", 35e-3)
+ _get_params("embedding_size", 64)
+ _get_params("unigram_channels", 0)
+ _get_params("bigram_channels", 0)
+ _get_params("trigram_channels", 0)
+ _get_params("fourgram_channels", 0)
+ _get_params("fivegram_channels", 0)
+ _get_params("skip1bigram_channels", 0)
+ _get_params("skip2bigram_channels", 0)
+ _get_params("network_regularizer_scale", 1e-4)
+ _get_params("keep_prob", 0.5)
+ self.num_classes = len(self.labels)
+
+ self.parameters = base_layers.Parameters(
+ mode,
+ quantize=self.quantize,
+ regularizer_scale=self.embedding_regularizer_scale)
+ self.values_fc = dense_layers.BaseQDenseVarLen(
+ units=self.embedding_size, rank=3, parameters=self.parameters)
+ self.attention_fc = dense_layers.BaseQDenseVarLen(
+ units=self.embedding_size, rank=3, parameters=self.parameters)
+ self.dropout = tf.keras.layers.Dropout(rate=(1 - self.keep_prob))
+
+ self.parameters = copy.copy(self.parameters)
+ self.parameters.regularizer_scale = self.network_regularizer_scale
+ self.attention_pool_layers = []
+ self._add_attention_pool_layer(self.unigram_channels, 1)
+ self._add_attention_pool_layer(self.bigram_channels, 2)
+ self._add_attention_pool_layer(self.trigram_channels, 3)
+ self._add_attention_pool_layer(self.fourgram_channels, 4)
+ self._add_attention_pool_layer(self.fivegram_channels, 5)
+ self._add_attention_pool_layer(self.skip1bigram_channels, None, 1)
+ self._add_attention_pool_layer(self.skip2bigram_channels, None, 2)
+
+ self.concat_quantizer = quantization_layers.ConcatQuantization(
+ axis=1, parameters=self.parameters)
+ self.final_fc = dense_layers.BaseQDense(
+ units=self.num_classes,
+ rank=2,
+ parameters=self.parameters,
+ activation=None)
+
+ def _add_attention_pool_layer(self, channels, ngram, skip_bigram=None):
+ if channels > 0:
+ self.attention_pool_layers.append(
+ AttentionPoolReduce(
+ filters=channels,
+ skip_bigram=skip_bigram,
+ ngram=ngram,
+ parameters=self.parameters))
+
+ def _apply_fc_dropout(self, layer, inputs, mask, inverse_normalizer):
+ outputs = layer(inputs, mask, inverse_normalizer)
+ if self.parameters.mode == base_layers.TRAIN:
+ return self.dropout(outputs)
+ return outputs
+
+ def call(self, projection, seq_length):
+ mask = tf.sequence_mask(
+ seq_length, tf.shape(projection)[1], dtype=tf.float32)
+ inverse_normalizer = tf.math.reciprocal(tf.reduce_sum(mask))
+ maskr3 = tf.expand_dims(mask, axis=2)
+ values_in = self._apply_fc_dropout(self.values_fc, projection, mask,
+ inverse_normalizer)
+ attention_in = self._apply_fc_dropout(self.attention_fc, projection, mask,
+ inverse_normalizer)
+ tensors = [
+ layer(values_in, attention_in, maskr3, inverse_normalizer)
+ for layer in self.attention_pool_layers
+ ]
+ pre_logits = self.concat_quantizer(tensors)
+ return self.final_fc(pre_logits)
+
+
+class Model(Encoder):
+
+ def __init__(self, config, mode):
+ super(Model, self).__init__(config, mode)
+ self.projection = projection_layers.ProjectionLayer(config, mode)
+
+ def call(self, inputs):
+ projection, seq_length = self.projection(inputs)
+ return super(Model, self).call(projection, seq_length)
diff --git a/research/seq_flow_lite/models/sgnn/BUILD b/research/seq_flow_lite/models/sgnn/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..8c63240438ab8bfac11da5947124630e55e818e7
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/BUILD
@@ -0,0 +1,110 @@
+licenses(["notice"])
+
+package(
+ default_visibility = [
+ "//visibility:public",
+ ],
+)
+
+cc_library(
+ name = "sgnn_projection",
+ srcs = ["sgnn_projection.cc"],
+ hdrs = ["sgnn_projection.h"],
+ deps = [
+ "@org_tensorflow//tensorflow/lite:context",
+ "@org_tensorflow//tensorflow/lite:string_util",
+ "@org_tensorflow//tensorflow/lite/kernels:kernel_util",
+ "@org_tensorflow//tensorflow/lite/kernels/internal:tensor",
+ "@farmhash_archive//:farmhash",
+ "@flatbuffers",
+ ],
+)
+
+cc_library(
+ name = "sgnn_projection_op_resolver",
+ srcs = ["sgnn_projection_op_resolver.cc"],
+ hdrs = ["sgnn_projection_op_resolver.h"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":sgnn_projection",
+ "@org_tensorflow//tensorflow/lite:framework",
+ ],
+ alwayslink = 1,
+)
+
+cc_test(
+ name = "sgnn_projection_test",
+ srcs = ["sgnn_projection_test.cc"],
+ deps = [
+ ":sgnn_projection",
+ "@org_tensorflow//tensorflow/lite:string_util",
+ "@org_tensorflow//tensorflow/lite/kernels:test_util",
+ "@org_tensorflow//tensorflow/lite/schema:schema_fbs",
+ "@com_google_googletest//:gtest_main",
+ "@flatbuffers",
+ ],
+)
+
+py_library(
+ name = "sgnn",
+ srcs = [
+ "sgnn.py",
+ ],
+ srcs_version = "PY3",
+ deps = [
+ # package tensorflow
+ "@org_tflite_support//tensorflow_lite_support/custom_ops/python:tflite_text_api",
+ # Expect tensorflow text installed
+ ],
+)
+
+py_test(
+ name = "sgnn_test",
+ srcs = [
+ "sgnn_test.py",
+ ],
+ deps = [
+ ":sgnn",
+ # package tensorflow
+ # Expect tensorflow text installed
+ ],
+)
+
+py_binary(
+ name = "train",
+ srcs = [
+ "train.py",
+ ],
+ main = "train.py",
+ python_version = "PY3",
+ deps = [
+ ":sgnn",
+ # package tensorflow
+ # package tensorflow_datasets
+ ],
+)
+
+py_binary(
+ name = "run_tflite",
+ srcs = ["run_tflite.py"],
+ main = "run_tflite.py",
+ python_version = "PY3",
+ deps = [
+ # Expect numpy installed
+ # package TFLite flex delegate
+ # package TFLite interpreter
+ "@org_tflite_support//tensorflow_lite_support/custom_ops/kernel:ngrams_op_resolver",
+ "@org_tflite_support//tensorflow_lite_support/custom_ops/kernel:whitespace_tokenizer_op_resolver",
+ # Expect tensorflow text installed
+ ],
+)
+
+# pip install numpy
+py_library(
+ name = "expect_numpy_installed",
+)
+
+# pip install tensroflow_text
+py_library(
+ name = "expect_tensorflow_text_installed",
+)
diff --git a/research/seq_flow_lite/models/sgnn/run_tflite.py b/research/seq_flow_lite/models/sgnn/run_tflite.py
new file mode 100644
index 0000000000000000000000000000000000000000..d7b74448e5bb91a09efe2e5821157a152cf9f055
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/run_tflite.py
@@ -0,0 +1,54 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Script to run a langid TFLite model."""
+
+from absl import app
+from absl import flags
+import numpy as np
+from tensorflow.lite.python import interpreter as interpreter_wrapper # pylint: disable=g-direct-tensorflow-import
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string('model', '/tmp/langid/model.tflite',
+ 'Path to LangID TFLite model.')
+
+LANGIDS = ['ar', 'en', 'es', 'fr', 'ru', 'zh', 'unk']
+
+
+def main(argv):
+ with open(FLAGS.model, 'rb') as file:
+ model = file.read()
+ interpreter = interpreter_wrapper.InterpreterWithCustomOps(
+ model_content=model,
+ custom_op_registerers=[
+ 'AddWhitespaceTokenizerCustomOp', 'AddNgramsCustomOp',
+ 'AddSgnnProjectionCustomOp',
+ ])
+ interpreter.resize_tensor_input(0, [1, 1])
+ interpreter.allocate_tensors()
+ input_string = ' '.join(argv[1:])
+ print('Input: "{}"'.format(input_string))
+ input_array = np.array([[input_string]], dtype=np.str)
+ interpreter.set_tensor(interpreter.get_input_details()[0]['index'],
+ input_array)
+ interpreter.invoke()
+ output = interpreter.get_tensor(interpreter.get_output_details()[0]['index'])
+ for x in range(output.shape[0]):
+ for y in range(output.shape[1]):
+ print('{:>3s}: {:.4f}'.format(LANGIDS[y], output[x][y]))
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/research/seq_flow_lite/models/sgnn/sgnn.py b/research/seq_flow_lite/models/sgnn/sgnn.py
new file mode 100644
index 0000000000000000000000000000000000000000..240558cbdc0d923bfd91ce272d35f92bb2ea4413
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/sgnn.py
@@ -0,0 +1,228 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Builds SGNN model.
+
+[1] Sujith Ravi and Zornitsa Kozareva. 2018. "Self-governing neural networks for
+on-device short text
+classification." In Proceedings of the 2018 Conference on Empirical Methods in
+Natural Language
+Processing, pages 887-893. Association for Computational Linguistics
+
+The model will be constructed in this way:
+* Projects text to float features, the size is defined by projection_size
+* Fully connected layer predicts the class of predictions.
+"""
+
+import collections
+import tensorflow.compat.v2 as tf
+import tensorflow_text as tf_text
+
+from tensorflow_lite_support.custom_ops.python import tflite_text_api
+
+# Hparam collections that will be used to tune the model.
+Hparams = collections.namedtuple(
+ 'Hparams',
+ [
+ # Learning rate for the optimizer.
+ 'learning_rate'
+ ])
+
+
+def preprocess(text):
+ """Normalize the text, and return tokens."""
+ assert len(text.get_shape().as_list()) == 2
+ assert text.get_shape().as_list()[-1] == 1
+ text = tf.reshape(text, [-1])
+ text = tf_text.case_fold_utf8(text)
+ tokenizer = tflite_text_api.WhitespaceTokenizer()
+ return tokenizer.tokenize(text)
+
+
+def get_ngrams(tokens, n):
+ """Generates character ngrams from tokens.
+
+ Args:
+ tokens: A string ragged tensor for tokens, in shape of [batch_size,
+ num_token].
+ n: ngram size for char ngrams.
+
+ Returns:
+ A string ragged tensor for ngrams, in shape of [batch_size, num_token,
+ ngrams].
+ """
+ chars_split = tf.strings.unicode_split('^' + tokens + '$', 'UTF-8')
+ chars_joined = tflite_text_api.ngrams(
+ chars_split,
+ width=n,
+ axis=-1,
+ reduction_type=tf_text.Reduction.STRING_JOIN,
+ string_separator='')
+ flat_row_splits = tf.nn.embedding_lookup(chars_joined.values.row_splits,
+ chars_joined.row_splits)
+ return tf.RaggedTensor.from_row_splits(chars_joined.values.values,
+ flat_row_splits)
+
+
+def project(ngrams, hash_seed, buckets):
+ """Projects a ngram RaggedTensor to float tensor.
+
+ Args:
+ ngrams: A string ragged tensor, in shape of [batch_size, num_token, ngrams].
+ hash_seed: A python int list, in shape of [num_hash].
+ buckets: An int for the max value of projected integers.
+
+ Returns:
+ A float tensor that projects ngrams to the space represented by hash_seed,
+ in shape of [batch_size, num_hash].
+ """
+ num_hash = len(hash_seed)
+ # Hash ngrams string tensor to hash signatures.
+ signatures = tf.ragged.map_flat_values(tf.strings.to_hash_bucket_fast, ngrams,
+ buckets)
+
+ # Each ngram signature will be multiplied by a different hash seed,
+ # mod by hash buckets, and linear mapping.
+ # value = abs(signature * seed % bucket)
+ # if value > bucket / 2: value -= buckets
+ hash_tensor = tf.constant(hash_seed, dtype=tf.int64)
+ value = tf.math.floormod(
+ tf.abs(signatures.values * tf.reshape(hash_tensor, [-1, 1])), buckets)
+ value = value - tf.cast(tf.greater(value, buckets >> 1), tf.int64) * buckets
+
+ # Wrap values to ragged tensor, and calculates
+ # output_i,j = mean(value_i,j,k) for k-th ngram in i-th text
+ # computed with j-th hash seed
+ row_lengths = tf.repeat(
+ tf.reshape(signatures.row_lengths(), [1, -1]), num_hash, axis=0)
+ row_lengths = tf.cast(tf.reshape(row_lengths, [-1]), tf.int32)
+ result = tf.RaggedTensor.from_row_lengths(
+ tf.RaggedTensor.from_row_lengths(tf.reshape(value, [-1]), row_lengths),
+ tf.repeat(tf.shape(signatures.row_lengths()), num_hash))
+ result = tf.reduce_mean(result, 2) / (buckets >> 1)
+ return tf.transpose(tf.reshape(result.values, [num_hash, -1]))
+
+
+def fused_project(ngrams, hash_seed, buckets):
+ """A wrapper to fuse project method when converting to TFLite model.
+
+ Args:
+ ngrams: A string ragged tensor, in shape of [batch_size, num_token, ngrams].
+ hash_seed: A python int list, in shape of [num_hash].
+ buckets: An int for the max value of projected integers.
+
+ Returns:
+ A float tensor that projects ngrams to the space represented by hash_seed,
+ in shape of [batch_size, num_hash].
+ """
+ hash_seed_attr = ' '.join(['i: %d' % seed for seed in hash_seed])
+ experimental_implements = [
+ 'name: "tftext:custom:SgnnProjection"',
+ 'attr { key: "hash_seed" value { list {%s} } }' % hash_seed_attr,
+ 'attr { key: "buckets" value { i: %d } }' % buckets,
+ ]
+ experimental_implements = ' '.join(experimental_implements)
+
+ @tf.function(experimental_implements=experimental_implements)
+ def func(ngrams_values, *ngrams_row_splits):
+ ngrams = tf.RaggedTensor.from_nested_row_splits(
+ flat_values=ngrams_values, nested_row_splits=ngrams_row_splits)
+ return project(ngrams, hash_seed, buckets)
+ return func(ngrams.flat_values, *ngrams.nested_row_splits)
+
+
+def sgnn(texts, hash_seed, ngram_size):
+ """Projects the string text to float features.
+
+ It first generasts N ngrams of the tokens from given text,
+ then projects each ngram tensor with a partion of the seeds.
+
+ Args:
+ texts: a string tensor, in shape of [batch_size].
+ hash_seed: a list of integers, in shape of [projection_size].
+ ngram_size: max size of ngram to generate features.
+
+ Returns:
+ A float tensor that projects ngrams to the space represented by hash_seed,
+ in shape of [batch_size, projection_size].
+ """
+ projection_size = len(hash_seed)
+ partition_size = int(projection_size / ((ngram_size + 1) * ngram_size / 2))
+ if partition_size == 0:
+ raise ValueError(
+ 'projection size %d is not enough for %d ngram partitions' %
+ (projection_size, ngram_size))
+ indices = [int(i * (i + 1) / 2) * partition_size for i in range(ngram_size)]
+ indices.append(projection_size)
+ projection_layer = []
+ tokens = preprocess(texts)
+
+ for i in range(ngram_size):
+ ngram = get_ngrams(tokens, i + 1)
+ projection = fused_project(ngram, hash_seed[indices[i]:indices[i + 1]],
+ 0x7FFFFFFF)
+ projection_layer.append(projection)
+
+ return tf.cast(tf.concat(projection_layer, -1), tf.float32)
+
+
+class ProjectLayer(tf.keras.layers.Layer):
+ """Projects the texts to a fixed sized features."""
+
+ def __init__(self, seed, ngram_size, **kwargs):
+ self.seed = seed
+ self.ngram_size = ngram_size
+ super(ProjectLayer, self).__init__(**kwargs)
+
+ def get_config(self):
+ return {
+ 'seed': self.seed,
+ 'ngram_size': self.ngram_size,
+ }
+
+ def call(self, x):
+ return sgnn(x, self.seed, self.ngram_size)
+
+ def compute_output_shape(self, input_shape):
+ return (input_shape[0], len(self.seed))
+
+
+def keras_model(hash_seed, ngram_size, fc_size_list, hparams):
+ """Compiles a keras model from projected features to labels.
+
+ Args:
+ hash_seed: a list of int used to project the feature.
+ ngram_size: maximum size of ngram to generate features from texts.
+ fc_size_list: a list of int, sizes of each fully connected layer.
+ hparams: hyper parameters for the model.
+
+ Returns:
+ A keras model that predicts the language id.
+
+ """
+ if not fc_size_list:
+ raise ValueError(
+ 'Must specify one or more fully connected layers via fc_size_list')
+ model = tf.keras.Sequential()
+ model.add(ProjectLayer(hash_seed, ngram_size))
+ for size in fc_size_list[:-1]:
+ model.add(tf.keras.layers.Dense(size))
+ model.add(tf.keras.layers.Dense(fc_size_list[-1], activation='softmax'))
+
+ model.compile(
+ optimizer=tf.keras.optimizers.Adam(lr=hparams.learning_rate),
+ loss=tf.keras.losses.SparseCategoricalCrossentropy(),
+ metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
+ return model
diff --git a/research/seq_flow_lite/models/sgnn/sgnn_projection.cc b/research/seq_flow_lite/models/sgnn/sgnn_projection.cc
new file mode 100644
index 0000000000000000000000000000000000000000..a805bd82c3a9a40370f82990979a15f7f22a6028
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/sgnn_projection.cc
@@ -0,0 +1,139 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "models/sgnn/sgnn_projection.h" // seq_flow_lite
+
+#include
+#include
+
+#include "flatbuffers/flexbuffers.h" // flatbuffer
+#include "farmhash.h"
+#include "tensorflow/lite/context.h"
+#include "tensorflow/lite/kernels/internal/tensor_ctypes.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+#include "tensorflow/lite/string_util.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace sgnn {
+
+// This TFLite op implements the SGNN Projection
+//
+// Input:
+// * data: A ragged string tensor of rank 2 (a 1D string value tensor and
+// a 1D int64 row_split tensor).
+//
+// Attributes:
+// * hash_seed: list of integers
+// Hash seeds to project features
+// * buckets: scalar integer
+// Bucketize computed hash signatures.
+//
+// Output:
+// * output: A 2D float tensor, 1st dimension is the batch of `data`,
+// 2nd dimension is the size of `hash_seed`.
+
+constexpr int kValues = 0;
+constexpr int kRowSplits = 1;
+
+struct SgnnProjectionAttributes {
+ int buckets;
+ std::vector hash_seed;
+
+ explicit SgnnProjectionAttributes(const flexbuffers::Map& m)
+ : buckets(m["buckets"].AsInt32()) {
+ buckets = m["buckets"].AsInt32();
+ auto hash_seed_attr = m["hash_seed"].AsTypedVector();
+ hash_seed = std::vector(hash_seed_attr.size());
+ for (int i = 0; i < hash_seed_attr.size(); ++i) {
+ hash_seed[i] = hash_seed_attr[i].AsInt32();
+ }
+ }
+};
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+ const uint8_t* buffer_t = reinterpret_cast(buffer);
+ return new SgnnProjectionAttributes(
+ flexbuffers::GetRoot(buffer_t, length).AsMap());
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+ delete reinterpret_cast(buffer);
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+ const auto& attributes =
+ *reinterpret_cast(node->user_data);
+ const TfLiteTensor* input_row_splits = GetInput(context, node, kRowSplits);
+ TfLiteTensor* output = GetOutput(context, node, 0);
+ TfLiteIntArray* output_shape = TfLiteIntArrayCreate(2);
+ output_shape->data[0] = SizeOfDimension(input_row_splits, 0) - 1;
+ output_shape->data[1] = attributes.hash_seed.size();
+ TF_LITE_ENSURE_OK(context,
+ context->ResizeTensor(context, output, output_shape));
+ return kTfLiteOk;
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+ const auto& attributes =
+ *reinterpret_cast(node->user_data);
+ const TfLiteTensor* ngrams = GetInput(context, node, kValues);
+ const TfLiteTensor* row_splits = GetInput(context, node, kRowSplits);
+
+ auto row_splits_values = GetTensorData(row_splits);
+ auto output_values = GetTensorData(GetOutput(context, node, 0));
+ int output_idx = 0;
+ for (int i = 1; i < SizeOfDimension(row_splits, 0); ++i) {
+ int len = row_splits_values[i] - row_splits_values[i - 1];
+ std::vector hash_signature(len);
+
+ // Follow the implementation from
+ // tensorflow/core/kernels/string_to_hash_bucket_op.h
+ for (int j = 0; j < len; ++j) {
+ int index = row_splits->data.i64[i - 1] + j;
+ StringRef str = GetString(ngrams, index);
+ hash_signature[j] =
+ util::Fingerprint64(str.str, str.len) % attributes.buckets;
+ }
+ for (int k = 0; k < attributes.hash_seed.size(); ++k) {
+ double result = 0;
+ for (int j = 0; j < len; ++j) {
+ int64_t tmp = hash_signature[j] * attributes.hash_seed[k];
+ int64_t value = abs(tmp) % attributes.buckets;
+ if (value > attributes.buckets / 2) {
+ value -= attributes.buckets;
+ }
+ result += value;
+ }
+ output_values[output_idx] =
+ static_cast(result) / (attributes.buckets / 2) / len;
+ output_idx++;
+ }
+ }
+ return kTfLiteOk;
+}
+
+} // namespace sgnn
+
+TfLiteRegistration* Register_tftext_SGNN_PROJECTION() {
+ static TfLiteRegistration r = {sgnn::Init, sgnn::Free, sgnn::Prepare,
+ sgnn::Eval};
+ return &r;
+}
+
+} // namespace custom
+} // namespace ops
+} // namespace tflite
diff --git a/research/seq_flow_lite/models/sgnn/sgnn_projection.h b/research/seq_flow_lite/models/sgnn/sgnn_projection.h
new file mode 100644
index 0000000000000000000000000000000000000000..af4a25c1f883e3c9f60ece6cc81e8fc2ce5f4eb6
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/sgnn_projection.h
@@ -0,0 +1,31 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_MODELS_SEQUENCE_PROJECTION_SGNN_SGNN_PROJECTION_H_
+#define TENSORFLOW_MODELS_SEQUENCE_PROJECTION_SGNN_SGNN_PROJECTION_H_
+
+#include "tensorflow/lite/context.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+
+TfLiteRegistration* Register_tftext_SGNN_PROJECTION();
+
+} // namespace custom
+} // namespace ops
+} // namespace tflite
+
+#endif // TENSORFLOW_MODELS_SEQUENCE_PROJECTION_SGNN_SGNN_PROJECTION_H_
diff --git a/research/seq_flow_lite/models/sgnn/sgnn_projection_op_resolver.cc b/research/seq_flow_lite/models/sgnn/sgnn_projection_op_resolver.cc
new file mode 100644
index 0000000000000000000000000000000000000000..040e1d1f4bc727eea72e89985949ddedc4a3572e
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/sgnn_projection_op_resolver.cc
@@ -0,0 +1,32 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "models/sgnn/sgnn_projection_op_resolver.h" // seq_flow_lite
+
+#include "tensorflow/lite/mutable_op_resolver.h"
+#include "models/sgnn/sgnn_projection.h" // seq_flow_lite
+
+namespace tflite {
+namespace ops {
+namespace custom {
+
+void AddSgnnProjectionCustomOp(MutableOpResolver* resolver) {
+ resolver->AddCustom("tftext:custom:SgnnProjection",
+ Register_tftext_SGNN_PROJECTION());
+}
+
+} // namespace custom
+} // namespace ops
+} // namespace tflite
diff --git a/research/seq_flow_lite/models/sgnn/sgnn_projection_op_resolver.h b/research/seq_flow_lite/models/sgnn/sgnn_projection_op_resolver.h
new file mode 100644
index 0000000000000000000000000000000000000000..0d5a7f15da4ecaa0dab20ba0ffca59943aa0c6ce
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/sgnn_projection_op_resolver.h
@@ -0,0 +1,34 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_MODELS_SEQUENCE_PROJECTION_SGNN_SGNN_PROJECTION_OP_RESOLVER_H_
+#define TENSORFLOW_MODELS_SEQUENCE_PROJECTION_SGNN_SGNN_PROJECTION_OP_RESOLVER_H_
+
+#include "tensorflow/lite/mutable_op_resolver.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+
+// Adds the SgnnProjection custom op to an op resolver.
+// This function can be loaded using dlopen. Since C++ function names get
+// mangled, declare this function as extern C, so its name is unchanged.
+extern "C" void AddSgnnProjectionCustomOp(MutableOpResolver* resolver);
+
+} // namespace custom
+} // namespace ops
+} // namespace tflite
+
+#endif // TENSORFLOW_MODELS_SEQUENCE_PROJECTION_SGNN_SGNN_PROJECTION_OP_RESOLVER_H_
diff --git a/research/seq_flow_lite/models/sgnn/sgnn_projection_test.cc b/research/seq_flow_lite/models/sgnn/sgnn_projection_test.cc
new file mode 100644
index 0000000000000000000000000000000000000000..2fb25eb1160adf873d65c50b89881afc8d685a8f
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/sgnn_projection_test.cc
@@ -0,0 +1,101 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "models/sgnn/sgnn_projection.h" // seq_flow_lite
+
+#include
+#include
+
+#include
+#include
+#include "flatbuffers/flexbuffers.h" // flatbuffer
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/string_util.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace sgnn_projection {
+namespace test {
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
+
+} // namespace
+
+class SgnnProjectionModel : public SingleOpModel {
+ public:
+ // Constructor for testing the op with a tf.Tensor
+ SgnnProjectionModel(const std::vector& input_values,
+ const std::vector& input_row_splits,
+ const std::vector& hash_seed, int64_t buckets) {
+ input_values_index_ = AddInput(TensorType_STRING);
+ input_row_splits_index_ = AddInput(TensorType_INT64);
+ output_values_index_ = AddOutput(TensorType_FLOAT32);
+ BuildCustomOp(hash_seed, buckets);
+ BuildInterpreter({{static_cast(input_values.size())},
+ {static_cast(input_row_splits.size())}});
+ PopulateStringTensor(input_values_index_, input_values);
+ PopulateTensor(input_row_splits_index_, input_row_splits);
+ Invoke();
+ }
+
+ std::vector GetOutputShape() {
+ return GetTensorShape(output_values_index_);
+ }
+
+ std::vector ExtractOutputValue() {
+ return ExtractVector(output_values_index_);
+ }
+
+ private:
+ void BuildCustomOp(const std::vector& hash_seed, int64_t buckets) {
+ flexbuffers::Builder fbb;
+ size_t start_map = fbb.StartMap();
+ auto vector_start = fbb.StartVector("hash_seed");
+ for (int i = 0; i < hash_seed.size(); i++) {
+ fbb.Add(hash_seed[i]);
+ }
+ fbb.EndVector(vector_start, /*typed=*/true, /*fixed=*/false);
+ fbb.Int("buckets", buckets);
+ fbb.EndMap(start_map);
+ fbb.Finish();
+ SetCustomOp("tftext:custom:SgnnProjection", fbb.GetBuffer(),
+ Register_tftext_SGNN_PROJECTION);
+ }
+
+ int input_values_index_;
+ int input_row_splits_index_;
+ int output_values_index_;
+};
+
+// Keep same result of test_projection in sgnn_test.py
+TEST(SgnnProjectionTest, TensorSgnnProjection) {
+ SgnnProjectionModel m({"^h", "he", "el", "ll", "lo", "o$", "^h", "hi", "i$"},
+ /*input_row_splits=*/{0, 6, 9}, /*hash_seed=*/{5, 7},
+ /*buckets=*/0x7FFFFFFF);
+ EXPECT_THAT(m.GetOutputShape(), ElementsAre(2, 2));
+ EXPECT_THAT(m.ExtractOutputValue(),
+ ElementsAreArray(ArrayFloatNear(
+ { 0.448691, -0.238499, -0.037561, 0.080748})));
+}
+
+} // namespace test
+} // namespace sgnn_projection
+} // namespace custom
+} // namespace ops
+} // namespace tflite
diff --git a/research/seq_flow_lite/models/sgnn/sgnn_test.py b/research/seq_flow_lite/models/sgnn/sgnn_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e2db82e82cfeaeb8b6eeba3c93b95100cfca8bc
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/sgnn_test.py
@@ -0,0 +1,73 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# Lint as: python3
+"""Tests for seq_flow_lite.sgnn."""
+
+import tensorflow as tf
+from tensorflow.python.framework import test_util # pylint: disable=g-direct-tensorflow-import
+from models import sgnn # import seq_flow_lite module
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SgnnTest(tf.test.TestCase):
+
+ def test_preprocess(self):
+ self.assertAllEqual(
+ sgnn.preprocess(
+ tf.constant([['Hello World!'], [u'你好'],
+ [u'مرحبا بالعالم']])),
+ [['hello'.encode(), 'world!'.encode()], [u'你好'.encode()],
+ [u'مرحبا'.encode(), u'بالعالم'.encode()]])
+
+ def test_get_ngram(self):
+ tokens = tf.ragged.constant([['hello', 'world'], [u'你好'],
+ [u'مرحبا', u'بالعالم']])
+ self.assertAllEqual(
+ sgnn.get_ngrams(tokens, 3),
+ [[
+ b'^he', b'hel', b'ell', b'llo', b'lo$', b'^wo', b'wor', b'orl',
+ b'rld', b'ld$'
+ ], [u'^你好'.encode(), u'你好$'.encode()],
+ [
+ u'^مر'.encode(), u'مرح'.encode(), u'رحب'.encode(),
+ u'حبا'.encode(), u'با$'.encode(), u'^با'.encode(),
+ u'بال'.encode(), u'الع'.encode(), u'لعا'.encode(),
+ u'عال'.encode(), u'الم'.encode(), u'لم$'.encode()
+ ]])
+
+ def test_project(self):
+ ngrams = tf.ragged.constant([[b'^h', b'he', b'el', b'll', b'lo', b'o$'],
+ [b'^h', b'hi', b'i$']])
+ self.assertAllClose(
+ sgnn.fused_project(ngrams, [5, 7], 0x7FFFFFFF),
+ [[0.448691, -0.238499], [-0.037561, 0.080748]])
+ self.assertAllClose(
+ sgnn.fused_project(ngrams, [5, 7], 0x7FFFFFFF),
+ sgnn.project(ngrams, [5, 7], 0x7FFFFFFF))
+
+ def test_sgnn(self):
+ self.assertAllClose(
+ sgnn.sgnn(tf.constant([['hello'], ['hi']]), [3, 5, 7], 2),
+ [[0.268503, 0.448691, -0.238499], [0.093143, -0.037561, 0.080748]])
+
+ def test_keras_model(self):
+ hparams = sgnn.Hparams(learning_rate=2e-4)
+ model = sgnn.keras_model([1, 2, 3, 4], 2, [100, 50], hparams)
+ self.assertIsNotNone(model)
+
+
+if __name__ == '__main__':
+ tf.test.main()
diff --git a/research/seq_flow_lite/models/sgnn/train.py b/research/seq_flow_lite/models/sgnn/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..d6fd9f6913fdbbae8e089d1e4d260c75b638736a
--- /dev/null
+++ b/research/seq_flow_lite/models/sgnn/train.py
@@ -0,0 +1,121 @@
+# Copyright 2020 The TensorFlow Authors All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Script to train langid model.
+
+The script builds language detection from wikipedia dataset,
+builds SGNN model to train an on-device model to
+predict the language of the given text.
+"""
+
+import os
+from absl import app
+from absl import flags
+import numpy as np
+import tensorflow.compat.v2 as tf
+import tensorflow_datasets as tfds
+
+from models import sgnn # import seq_flow_lite module
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string('output_dir', '/tmp/langid',
+ 'Path for the output directory.')
+
+flags.DEFINE_integer('projection_size', 600, 'Size of projection layer.')
+flags.DEFINE_integer('ngram_size', 3, 'Max size of ngram to project features.')
+flags.DEFINE_string('fc_layer', '256,128',
+ 'Size of fully connected layer, separated by comma.')
+
+flags.DEFINE_integer('batch_size', 160, 'Batch size for training.')
+flags.DEFINE_integer('epochs', 10, 'Num of epochs for training.')
+flags.DEFINE_float('learning_rate', 2e-4, 'learning rate for optimizer.')
+
+LANGIDS = ['ar', 'en', 'es', 'fr', 'ru', 'zh']
+
+
+def dataset_fn(batch_size, is_training, split, try_gcs, max_input_len):
+ """Creates dataset to train and evaluate.
+
+ Args:
+ batch_size: Batch size for training or evaluation.
+ is_training: True if the dataset is for training.
+ split: Split of dataset, follow the pattern defined in
+ https://www.tensorflow.org/datasets/splits
+ try_gcs: True if loading the data from gcs.
+ max_input_len: Max length of input string.
+
+ Returns:
+ Dataset object.
+ """
+
+ def _get_text(item):
+ return tf.strings.substr(item['text'], 0, max_input_len)
+
+ all_data = []
+ for idx, langid in enumerate(LANGIDS):
+ dataset = tfds.load(
+ 'wikipedia/20190301.%s' % langid, try_gcs=try_gcs, split=split)
+
+ map_fn = lambda item: (_get_text(item), idx) # pylint: disable=cell-var-from-loop
+ dataset = dataset.map(map_fn)
+ all_data.append(dataset)
+
+ datasets = tf.data.experimental.sample_from_datasets(
+ all_data, [1. / len(all_data)] * len(LANGIDS))
+ repeat_count = None if is_training else 1
+ return datasets.cache().shuffle(100000).batch(batch_size).repeat(repeat_count)
+
+
+def save_and_convert(model, output_dir):
+ """Save keras model and convert to tflite."""
+ saved_model_path = os.path.join(output_dir, 'saved_model')
+ tf.saved_model.save(model, saved_model_path)
+ converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_path)
+ converter.allow_custom_ops = True
+ converter.target_spec.supported_ops = [
+ tf.lite.OpsSet.TFLITE_BUILTINS, tf.lite.OpsSet.SELECT_TF_OPS
+ ]
+ data = converter.convert()
+ with open(os.path.join(output_dir, 'model.tflite'), 'wb') as f:
+ f.write(data)
+
+
+def train_and_evaluate():
+ """Train and evaluate the model."""
+ hash_seed = np.random.uniform(-1, 1, FLAGS.projection_size) * 0x7FFFFFFF
+ fc_layer = [int(fc) for fc in FLAGS.fc_layer.split(',')]
+ fc_layer.append(len(LANGIDS) + 1)
+ hparams = sgnn.Hparams(learning_rate=FLAGS.learning_rate)
+
+ model = sgnn.keras_model(hash_seed, FLAGS.ngram_size, fc_layer, hparams)
+ model.fit(
+ dataset_fn(FLAGS.batch_size, True, 'train[:10%]', True, 100),
+ epochs=FLAGS.epochs,
+ steps_per_epoch=1000,
+ validation_steps=100,
+ validation_data=dataset_fn(FLAGS.batch_size, False, 'train[10:11%]', True,
+ 100),
+ )
+ save_and_convert(model, FLAGS.output_dir)
+
+
+def main(_):
+ if not os.path.exists(FLAGS.output_dir):
+ os.mkdir(FLAGS.output_dir)
+
+ train_and_evaluate()
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/research/seq_flow_lite/tf_ops/BUILD b/research/seq_flow_lite/tf_ops/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..f6254d33d47791bc039ee4798e8a8fc3d6348367
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/BUILD
@@ -0,0 +1,147 @@
+# Tensorflow ops for sequence string projection.
+
+load("//tf_ops:build_def.bzl", "gen_op_wrapper_py")
+
+licenses(["notice"])
+
+package(
+ default_visibility = [
+ "//:__subpackages__",
+ ],
+)
+
+py_library(
+ name = "text_projection",
+ srcs = ["text_projection.py"],
+ srcs_version = "PY3",
+ deps = [
+ ":sequence_string_projection_op_py",
+ ],
+)
+
+cc_library(
+ name = "sequence_string_projection_op",
+ srcs = [
+ "sequence_string_projection.cc",
+ ],
+ deps = [
+ ":projection_normalizer_util",
+ ":projection_tokenizer_util",
+ ":projection_util",
+ ":text_distorter",
+ "@com_google_absl//absl/container:flat_hash_map",
+ "@com_google_absl//absl/random",
+ "@tensorflow_includes//:includes",
+ "@tensorflow_solib//:framework_lib",
+ ],
+ alwayslink = 1,
+)
+
+cc_library(
+ name = "projection_util",
+ srcs = ["projection_util.cc"],
+ hdrs = ["projection_util.h"],
+ deps = [
+ "@utf_archive//:utf",
+ ],
+)
+
+cc_library(
+ name = "projection_tokenizer_util",
+ srcs = ["projection_tokenizer_util.cc"],
+ hdrs = ["projection_tokenizer_util.h"],
+ deps = [
+ ":projection_util",
+ "@utf_archive//:utf",
+ ],
+)
+
+cc_library(
+ name = "projection_normalizer_util",
+ srcs = ["projection_normalizer_util.cc"],
+ hdrs = ["projection_normalizer_util.h"],
+ deps = [
+ ":projection_util",
+ "@utf_archive//:utf",
+ ],
+)
+
+cc_library(
+ name = "text_distorter",
+ srcs = ["text_distorter.cc"],
+ hdrs = ["text_distorter.h"],
+ deps = [
+ "@com_google_absl//absl/strings",
+ "@icu4c",
+ "@tensorflow_includes//:includes",
+ "@tensorflow_solib//:framework_lib",
+ "@utf_archive//:utf",
+ ],
+)
+
+cc_test(
+ name = "sequence_string_projection_test",
+ size = "small",
+ srcs = ["sequence_string_projection_test.cc"],
+ deps = [
+ ":sequence_string_projection_op",
+ "@tensorflow_includes//:includes",
+ "@tensorflow_solib//:framework_lib",
+ ],
+)
+
+cc_library(
+ name = "sequence_string_projection_op_v2",
+ srcs = [
+ "sequence_string_projection_op_v2.cc",
+ ],
+ deps = [
+ ":projection_normalizer_util",
+ ":projection_util",
+ ":text_distorter",
+ "@tensorflow_includes//:includes",
+ "@tensorflow_solib//:framework_lib",
+ "@com_google_absl//absl/container:flat_hash_map",
+ "@com_google_absl//absl/random",
+ ],
+ alwayslink = 1,
+)
+
+cc_test(
+ name = "sequence_string_projection_op_v2_test",
+ size = "small",
+ srcs = ["sequence_string_projection_op_v2_test.cc"],
+ deps = [
+ ":sequence_string_projection_op_v2",
+ "@tensorflow_includes//:includes",
+ "@tensorflow_solib//:framework_lib",
+ ],
+)
+
+gen_op_wrapper_py(
+ name = "sequence_string_projection_op_v2_py",
+ out = "sequence_string_projection_op_v2.py",
+ kernel_lib = ":sequence_string_projection_op_v2",
+)
+
+gen_op_wrapper_py(
+ name = "sequence_string_projection_op_py",
+ out = "sequence_string_projection_op.py",
+ kernel_lib = ":sequence_string_projection_op",
+)
+
+cc_library(
+ name = "tf_custom_ops",
+ srcs = ["tf_custom_ops.cc"],
+ deps = [
+ "@tensorflow_includes//:includes",
+ "@tensorflow_solib//:framework_lib",
+ ],
+ alwayslink = 1,
+)
+
+gen_op_wrapper_py(
+ name = "tf_custom_ops_py",
+ out = "tf_custom_ops_py.py",
+ kernel_lib = ":tf_custom_ops",
+)
diff --git a/research/seq_flow_lite/tf_ops/build_def.bzl b/research/seq_flow_lite/tf_ops/build_def.bzl
new file mode 100644
index 0000000000000000000000000000000000000000..cb945f330664f63fe960ffcd1048172897c17bf5
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/build_def.bzl
@@ -0,0 +1,86 @@
+def tf_deps():
+ return [
+ "@tensorflow_includes//:includes",
+ "@tensorflow_solib//:framework_lib",
+ ]
+
+def tf_copts():
+ return ["-Wno-sign-compare"]
+
+def _make_search_paths(prefix, levels_to_root):
+ return ",".join(
+ [
+ "-rpath,%s/%s" % (prefix, "/".join([".."] * search_level))
+ for search_level in range(levels_to_root + 1)
+ ],
+ )
+
+def _rpath_linkopts(name):
+ # Search parent directories up to the TensorFlow root directory for shared
+ # object dependencies, even if this op shared object is deeply nested
+ # (e.g. tensorflow/contrib/package:python/ops/_op_lib.so). tensorflow/ is then
+ # the root and tensorflow/libtensorflow_framework.so should exist when
+ # deployed. Other shared object dependencies (e.g. shared between contrib/
+ # ops) are picked up as long as they are in either the same or a parent
+ # directory in the tensorflow/ tree.
+ levels_to_root = native.package_name().count("/") + name.count("/")
+ return ["-Wl,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),)]
+
+def gen_op_wrapper_py(name, out, kernel_lib, linkopts = [], **kwargs):
+ """Generates the py_library `name` with a data dep on the ops in kernel_lib.
+
+ The resulting py_library creates file `$out`, and has a dependency on a
+ symbolic library called lib{$name}_gen_op.so, which contains the kernels
+ and ops and can be loaded via `tf.load_op_library`.
+
+ Args:
+ name: The name of the py_library.
+ out: The name of the python file. Use "gen_{name}_ops.py".
+ kernel_lib: A cc_kernel_library target to generate for.
+ **kwargs: Any args to the `cc_binary` and `py_library` internal rules.
+ """
+ if not out.endswith(".py"):
+ fail("Argument out must end with '.py', but saw: {}".format(out))
+
+ module_name = "lib{}_gen_op".format(name)
+ version_script_file = "%s-version-script.lds" % module_name
+ native.genrule(
+ name = module_name + "_version_script",
+ outs = [version_script_file],
+ cmd = "echo '{global:\n *tensorflow*;\n *deepmind*;\n local: *;};' >$@",
+ output_licenses = ["unencumbered"],
+ visibility = ["//visibility:private"],
+ )
+ native.cc_binary(
+ name = "{}.so".format(module_name),
+ deps = [kernel_lib] + tf_deps() + [version_script_file],
+ copts = tf_copts() + [
+ "-fno-strict-aliasing", # allow a wider range of code [aliasing] to compile.
+ "-fvisibility=hidden", # avoid symbol clashes between DSOs.
+ ],
+ linkshared = 1,
+ linkopts = linkopts + _rpath_linkopts(module_name) + [
+ "-Wl,--version-script",
+ "$(location %s)" % version_script_file,
+ ],
+ **kwargs
+ )
+ native.genrule(
+ name = "{}_genrule".format(out),
+ outs = [out],
+ cmd = """
+ echo 'import tensorflow as tf
+_reverb_gen_op = tf.load_op_library(
+ tf.compat.v1.resource_loader.get_path_to_datafile(
+ "lib{}_gen_op.so"))
+_locals = locals()
+for k in dir(_reverb_gen_op):
+ _locals[k] = getattr(_reverb_gen_op, k)
+del _locals' > $@""".format(name),
+ )
+ native.py_library(
+ name = name,
+ srcs = [out],
+ data = [":lib{}_gen_op.so".format(name)],
+ **kwargs
+ )
diff --git a/research/seq_flow_lite/tf_ops/projection_normalizer_util.cc b/research/seq_flow_lite/tf_ops/projection_normalizer_util.cc
new file mode 100644
index 0000000000000000000000000000000000000000..3303ade4523b0072255e35e5e09517ca341f20c8
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/projection_normalizer_util.cc
@@ -0,0 +1,158 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tf_ops/projection_normalizer_util.h" // seq_flow_lite
+
+#include
+#include
+#include
+#include
+#include
+
+#include "tf_ops/projection_util.h" // seq_flow_lite
+
+// Returns true if the given text contains a number.
+bool IsDigit(const std::string& text) {
+ Rune rune;
+ for (size_t i = 0; i < text.length();) {
+ const int bytes_read = chartorune(&rune, const_cast(text.data()));
+ if (rune == Runeerror || bytes_read == 0) break;
+ if (rune >= static_cast('0') && rune <= static_cast('9')) {
+ return true;
+ }
+ i += bytes_read;
+ }
+ return false;
+}
+
+// Gets the string containing |num_chars| characters from |start| position.
+std::string GetCharToken(const std::vector& char_tokens,
+ size_t start, size_t num_chars) {
+ std::string char_token = "";
+ if (start + num_chars <= char_tokens.size()) {
+ for (size_t i = 0; i < num_chars; ++i) {
+ char_token.append(char_tokens[start + i]);
+ }
+ }
+ return char_token;
+}
+
+// Counts how many times |pattern| appeared from |start| position.
+int GetNumPattern(const std::vector& char_tokens, size_t start,
+ size_t num_chars, const std::string& pattern) {
+ int count = 0;
+ for (size_t i = start; i < char_tokens.size(); i += num_chars) {
+ std::string cur_pattern = GetCharToken(char_tokens, i, num_chars);
+ if (pattern == cur_pattern) {
+ ++count;
+ } else {
+ break;
+ }
+ }
+ return count;
+}
+
+std::string ContractToken(const char* input_ptr, size_t len, size_t num_chars) {
+ // This function contracts patterns whose length is |num_chars| and appeared
+ // more than twice. So if the input is shorter than 3 * |num_chars|, do not
+ // apply any contraction.
+ if (len < 3 * num_chars) {
+ return input_ptr;
+ }
+ std::vector char_tokens = SplitByChar(input_ptr, len, len);
+
+ std::string token;
+ token.reserve(len);
+ for (size_t i = 0; i < char_tokens.size();) {
+ std::string cur_pattern = GetCharToken(char_tokens, i, num_chars);
+
+ // Count how many times this pattern appeared.
+ int num_cur_patterns = 0;
+ if (cur_pattern.find(" ") == std::string::npos && !IsDigit(cur_pattern)) {
+ num_cur_patterns =
+ GetNumPattern(char_tokens, i + num_chars, num_chars, cur_pattern);
+ }
+
+ if (num_cur_patterns >= 2) {
+ // If this pattern is repeated, store it only twice.
+ token.append(cur_pattern);
+ token.append(cur_pattern);
+ i += (num_cur_patterns + 1) * num_chars;
+ } else {
+ token.append(char_tokens[i]);
+ ++i;
+ }
+ }
+
+ return token;
+}
+
+void ProjectionNormalizer::InitializeSeparators(const std::string& separators) {
+ for (size_t i = 0; i < separators.length(); ++i) {
+ if (separators[i] != ' ') {
+ separators_.insert(separators[i]);
+ }
+ }
+}
+
+std::string ProjectionNormalizer::NormalizeInternal(const char* input_ptr,
+ size_t len) {
+ std::string normalized;
+ normalized.reserve(len * 2);
+ for (size_t i = 0; i < len; ++i) {
+ char c = input_ptr[i];
+ bool matched_separator = separators_.find(c) != separators_.end();
+ if (matched_separator) {
+ if (i > 0 && input_ptr[i - 1] != ' ' && normalized.back() != ' ') {
+ normalized.append(" ");
+ }
+ }
+ normalized.append(1, c);
+ if (matched_separator) {
+ if (i + 1 < len && input_ptr[i + 1] != ' ' && c != '\'') {
+ normalized.append(" ");
+ }
+ }
+ }
+ return normalized;
+}
+
+std::string ProjectionNormalizer::Normalize(const std::string& input,
+ size_t max_input) {
+ return Normalize(input.c_str(), input.size(), max_input);
+}
+
+std::string ProjectionNormalizer::Normalize(const char* input_ptr, size_t len,
+ size_t max_input) {
+ std::string normalized(input_ptr, std::min(len, max_input));
+
+ if (normalize_repetition_) {
+ // Remove repeated 1 char (e.g. soooo => soo)
+ normalized = ContractToken(normalized.data(), normalized.length(), 1);
+
+ // Remove repeated 2 chars from the beginning (e.g. hahaha =>
+ // haha, xhahaha => xhaha, xyhahaha => xyhaha).
+ normalized = ContractToken(normalized.data(), normalized.length(), 2);
+
+ // Remove repeated 3 chars from the beginning
+ // (e.g. wowwowwow => wowwow, abcdbcdbcd => abcdbcd).
+ normalized = ContractToken(normalized.data(), normalized.length(), 3);
+ }
+
+ if (!separators_.empty()) {
+ // Add space around separators_.
+ normalized = NormalizeInternal(normalized.data(), normalized.length());
+ }
+ return normalized;
+}
diff --git a/research/seq_flow_lite/tf_ops/projection_normalizer_util.h b/research/seq_flow_lite/tf_ops/projection_normalizer_util.h
new file mode 100644
index 0000000000000000000000000000000000000000..4a7bed706c5c933748e5bed2b3e98b53a70bdf6f
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/projection_normalizer_util.h
@@ -0,0 +1,54 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_NORMALIZER_UTIL_H_
+#define TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_NORMALIZER_UTIL_H_
+
+#include
+#include
+#include
+
+#include "libutf/utf.h"
+
+// Normalizes the input with the given |separators| by adding a space before and
+// after each separator. When |normalize_repetition| is true, it removes the
+// repeated characters (except numbers) which consecutively appeared more than
+// twice in a word.
+// Examples: arwwwww -> arww, good!!!!! -> good!!, hahaha => haha.
+class ProjectionNormalizer {
+ public:
+ explicit ProjectionNormalizer(const std::string& separators,
+ bool normalize_repetition = false) {
+ InitializeSeparators(separators);
+ normalize_repetition_ = normalize_repetition;
+ }
+
+ // Normalizes the repeated characters (except numbers) which consecutively
+ // appeared more than twice in a word.
+ std::string Normalize(const std::string& input, size_t max_input = 300);
+ std::string Normalize(const char* input_ptr, size_t len,
+ size_t max_input = 300);
+
+ private:
+ // Parses and extracts supported separators.
+ void InitializeSeparators(const std::string& separators);
+
+ // Removes repeated chars.
+ std::string NormalizeInternal(const char* input_ptr, size_t len);
+
+ std::unordered_set separators_;
+ bool normalize_repetition_;
+};
+
+#endif // TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_NORMALIZER_UTIL_H_
diff --git a/research/seq_flow_lite/tf_ops/projection_tokenizer_util.cc b/research/seq_flow_lite/tf_ops/projection_tokenizer_util.cc
new file mode 100644
index 0000000000000000000000000000000000000000..25885630a60de41108dd88d3d31141c957df9e0e
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/projection_tokenizer_util.cc
@@ -0,0 +1,101 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tf_ops/projection_tokenizer_util.h" // seq_flow_lite
+
+#include
+#include
+#include
+#include
+#include
+
+#include "tf_ops/projection_util.h" // seq_flow_lite
+
+
+namespace {
+constexpr char kApostrophe = '\'';
+constexpr char kSpace = ' ';
+constexpr char kComma = ',';
+constexpr char kDot = '.';
+constexpr size_t kInvalid = -1;
+} // namespace
+
+// Returns true if the input |c| is ascii number.
+bool is_numeric(char c) { return c >= '0' && c <= '9'; }
+
+// Returns true if we want to prepend the separator to the next token.
+bool prepend_separator(char separator) { return separator == kApostrophe; }
+
+void ProjectionTokenizer::InitializeSeparators(const std::string& separators) {
+ for (size_t i = 0; i < separators.length(); ++i) {
+ separators_.insert(separators[i]);
+ }
+}
+
+size_t ProjectionTokenizer::FindNextSeparator(const char* input_ptr,
+ size_t from,
+ size_t length) const {
+ auto index = from;
+ while (index < length) {
+ char c = input_ptr[index];
+ // Do not break a number (e.g. "10,000", "0.23").
+ if (c == kComma || c == kDot) {
+ if (index + 1 < length && is_numeric(input_ptr[index + 1])) {
+ c = input_ptr[++index];
+ }
+ }
+ if (separators_.find(c) != separators_.end()) {
+ break;
+ }
+ ++index;
+ }
+ return index == length ? kInvalid : index;
+}
+
+std::vector ProjectionTokenizer::Tokenize(
+ const char* input_ptr, size_t len, size_t max_input,
+ size_t max_tokens) const {
+ // If separators_ is not given, tokenize the input with a space.
+ if (separators_.empty()) {
+ return SplitBySpace(input_ptr, len, max_input, max_tokens);
+ }
+
+ std::vector tokens;
+ size_t last_index =
+ max_input == kEntireString ? len : (len < max_input ? len : max_input);
+ size_t start = 0;
+ // Skip leading spaces.
+ while (start < last_index && input_ptr[start] == kSpace) {
+ start++;
+ }
+ auto end = FindNextSeparator(input_ptr, start, last_index);
+
+ while (end != kInvalid &&
+ (max_tokens == kAllTokens || tokens.size() < max_tokens - 1)) {
+ auto length = end - start;
+ if (length > 0) tokens.emplace_back(input_ptr + start, length);
+
+ // Add the separator (except space and apostrophe) as a token
+ char separator = input_ptr[end];
+ if (separator != kSpace && separator != kApostrophe) {
+ tokens.emplace_back(input_ptr + end, 1);
+ }
+
+ start = end + (prepend_separator(separator) ? 0 : 1);
+ end = FindNextSeparator(input_ptr, end + 1, last_index);
+ }
+ auto length = end == kInvalid ? (last_index - start) : (end - start);
+ if (length > 0) tokens.emplace_back(input_ptr + start, length);
+ return tokens;
+}
diff --git a/research/seq_flow_lite/tf_ops/projection_tokenizer_util.h b/research/seq_flow_lite/tf_ops/projection_tokenizer_util.h
new file mode 100644
index 0000000000000000000000000000000000000000..ca6ac553198b5bce9b5fce2d783e58da321519dd
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/projection_tokenizer_util.h
@@ -0,0 +1,58 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_TOKENIZER_UTIL_H_
+#define TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_TOKENIZER_UTIL_H_
+
+#include
+#include
+#include
+
+#include "libutf/utf.h"
+
+// Tokenizes the input with the given separators. To properly tokenize a text
+// containing contractions in English (e.g. I'm), it combines the apostrophe
+// with the token coming after it. For example, the text "I'm happy" is
+// tokenized into three tokens: "I", "'m", "happy". When |separators| is not
+// given, use the space to tokenize the input.
+// Note) This tokenization supports only English.
+class ProjectionTokenizer {
+ public:
+ explicit ProjectionTokenizer(const std::string& separators) {
+ InitializeSeparators(separators);
+ }
+
+ // Tokenizes the input by separators_. Limit to max_tokens, when it is not -1.
+ std::vector Tokenize(const std::string& input, size_t max_input,
+ size_t max_tokens) const {
+ return Tokenize(input.c_str(), input.size(), max_input, max_tokens);
+ }
+
+ std::vector Tokenize(const char* input_ptr, size_t len,
+ size_t max_input, size_t max_tokens) const;
+
+ private:
+ // Parses and extracts supported separators.
+ void InitializeSeparators(const std::string& separators);
+
+ // Starting from input_ptr[from], search for the next occurrence of
+ // separators_. Don't search beyond input_ptr[length](non-inclusive). Return
+ // -1 if not found.
+ size_t FindNextSeparator(const char* input_ptr, size_t from,
+ size_t length) const;
+
+ std::unordered_set separators_;
+};
+
+#endif // TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_TOKENIZER_UTIL_H_
diff --git a/research/seq_flow_lite/tf_ops/projection_util.cc b/research/seq_flow_lite/tf_ops/projection_util.cc
new file mode 100644
index 0000000000000000000000000000000000000000..0434af090449f74cbd0c741feba7bf73344761f7
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/projection_util.cc
@@ -0,0 +1,412 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tf_ops/projection_util.h" // seq_flow_lite
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace {
+constexpr int kInvalid = -1;
+constexpr char kSpace = ' ';
+} // namespace
+
+class MurmurHash : public HashEngine {
+ public:
+ void GetHashCodes(const std::string& word, std::vector* hash_codes,
+ int feature_size) override {
+ uint64_t hash_low = 0;
+ uint64_t hash_high = 0;
+ for (int i = 0; i < feature_size; i += 64) {
+ if (i == 0) {
+ auto hash = MurmurHash128(word.c_str(), word.size());
+ hash_low = hash.first;
+ hash_high = hash.second;
+ } else {
+ GetMoreBits(hash_low, hash_high, &hash_low, &hash_high);
+ }
+ hash_codes->push_back(hash_low);
+ hash_codes->push_back(hash_high);
+ }
+ }
+
+ private:
+ static constexpr uint64_t kMul = 0xc6a4a7935bd1e995ULL;
+ static constexpr uint64_t kMul2 = 0x9e3779b97f4a7835ULL;
+ inline uint64_t ShiftMix(uint64_t val) { return val ^ (val >> 47); }
+ inline uint64_t MurmurStep(uint64_t hash, uint64_t data) {
+ hash ^= ShiftMix(data * kMul) * kMul;
+ hash *= kMul;
+ return hash;
+ }
+ inline uint64_t Load64VariableLength(const void* p, int len) {
+ assert(len >= 1 && len <= 8);
+ const char* buf = static_cast(p);
+ uint64_t val = 0;
+ --len;
+ do {
+ val = (val << 8) | buf[len];
+ // (--len >= 0) is about 10 % faster than (len--) in some benchmarks.
+ } while (--len >= 0);
+ // No ToHost64(...) needed. The bytes are accessed in little-endian manner
+ // on every architecture.
+ return val;
+ }
+ void GetMoreBits(uint64_t hash, uint64_t hash2, uint64_t* rlow,
+ uint64_t* rhigh) {
+ hash = ShiftMix(hash) * kMul;
+ hash2 ^= hash;
+ *rhigh = ShiftMix(hash);
+ *rlow = ShiftMix(hash2 * kMul2) * kMul2;
+ }
+ std::pair MurmurHash128(const char* buf,
+ const size_t len) {
+ // Initialize the hashing value.
+ uint64_t hash = len * kMul;
+ // hash2 will be xored by hash during the hash computation iterations.
+ // In the end we use an alternative mixture multiplier for mixing
+ // the bits in hash2.
+ uint64_t hash2 = 0;
+ // Let's remove the bytes not divisible by the sizeof(uint64_t).
+ // This allows the inner loop to process the data as 64 bit integers.
+ const size_t len_aligned = len & ~0x7;
+ const char* end = buf + len_aligned;
+
+ for (const char* p = buf; p != end; p += 8) {
+ // Manually unrolling this loop 2x did not help on Intel Core 2.
+ hash = MurmurStep(hash, Load64VariableLength(p, 8));
+ hash2 ^= hash;
+ }
+ if ((len & 0x7) != 0) {
+ const uint64_t data = Load64VariableLength(end, len & 0x7);
+ hash ^= data;
+ hash *= kMul;
+ hash2 ^= hash;
+ }
+ hash = ShiftMix(hash) * kMul;
+ hash2 ^= hash;
+ hash = ShiftMix(hash);
+
+ // mul2 is a prime just above golden ratio. mul2 is used to ensure that the
+ // impact of the last few bytes is different to the upper and lower 64 bits.
+ hash2 = ShiftMix(hash2 * kMul2) * kMul2;
+
+ return std::make_pair(hash, hash2);
+ }
+};
+
+class XFixHash : public HashEngine {
+ public:
+ explicit XFixHash(int bits_per_char)
+ : bits_per_char_(bits_per_char), bit_mask_((1ULL << bits_per_char) - 1) {}
+
+ void GetHashCodes(const std::string& word, std::vector* hash_codes,
+ int feature_size) override {
+ auto token_ptr = reinterpret_cast(word.c_str());
+ size_t token_size = word.size();
+ int token_idx = 0;
+ uint64_t hash_low = token_size * kMul;
+ uint64_t hash_high = token_size * kMul2;
+ uint64_t frhash = kMul;
+ uint64_t brhash = kMul2;
+ for (int i = 0; i < feature_size; i += 64) {
+ for (int j = i ? 0 : bits_per_char_; j < 64;
+ j += bits_per_char_, token_idx = (token_idx + 1) % token_size) {
+ frhash = ((frhash << 8) | token_ptr[token_idx]) * kMul;
+ brhash =
+ ((brhash << 8) | token_ptr[token_size - 1 - token_idx]) * kMul2;
+ hash_low = (hash_low << bits_per_char_) | (frhash & bit_mask_);
+ hash_high = (hash_high << bits_per_char_) | (brhash & bit_mask_);
+ }
+ hash_codes->push_back(hash_low);
+ hash_codes->push_back(hash_high);
+ }
+ }
+
+ private:
+ const uint64_t kMul = 0xc6a4a7935bd1e995ULL;
+ const uint64_t kMul2 = 0x9e3779b97f4a7835ULL;
+ const int bits_per_char_;
+ const uint64_t bit_mask_;
+};
+
+class UnicodeHash : public HashEngine {
+ public:
+ // bits_per_unicode should be a divisor of 64.
+ explicit UnicodeHash(int bits_per_unicode)
+ : bits_per_unicode_(bits_per_unicode),
+ bit_mask_(((1ULL << bits_per_unicode) - 1) << (64 - bits_per_unicode)) {
+ }
+
+ void GetHashCodes(const std::string& word, std::vector* hash_codes,
+ int feature_size) override {
+ auto word_ptr = word.c_str();
+ int utflength = utflen(const_cast(word_ptr));
+ // Both `feature_size` and `bits_per_unicode` are bit lengths.
+ const int max_usable_runes = feature_size * 2 / bits_per_unicode_;
+ if (max_usable_runes < utflength) {
+ const int unicode_skip = (utflength - max_usable_runes) / 2;
+ for (int i = 0; i < unicode_skip; ++i) {
+ Rune rune;
+ word_ptr += chartorune(&rune, const_cast(word_ptr));
+ }
+ utflength = max_usable_runes;
+ }
+
+ std::vector unicode_hashes;
+ unicode_hashes.reserve(utflength);
+ for (int i = 0; i < utflength; ++i) {
+ Rune rune;
+ word_ptr += chartorune(&rune, const_cast(word_ptr));
+ unicode_hashes.push_back((rune * kMul) & bit_mask_);
+ }
+
+ uint64_t hash = 0;
+ int k = 0;
+ for (int i = 0; i < feature_size * 2; i += 64) {
+ for (int j = 0; j < 64; j += bits_per_unicode_) {
+ if (k < unicode_hashes.size()) {
+ hash = (hash >> bits_per_unicode_) | unicode_hashes[k++];
+ } else {
+ hash = hash >> bits_per_unicode_;
+ }
+ }
+ hash_codes->push_back(hash);
+ }
+ }
+
+ private:
+ const uint64_t kMul = 0xc6a4a7935bd1e995ULL;
+ const int bits_per_unicode_;
+ const uint64_t bit_mask_;
+};
+
+bool Hasher::SupportedHashType(const std::string& hash_type) {
+ std::unordered_set supported({kMurmurHash, kUnicodeHash8,
+ kUnicodeHash16, kXfixHash8,
+ kXfixHash16, kXfixHash32});
+ return supported.find(hash_type) != supported.end();
+}
+
+Hasher* Hasher::CreateHasher(int feature_size, const std::string& hash_type) {
+ if (SupportedHashType(hash_type)) {
+ if (hash_type == kMurmurHash) {
+ return new Hasher(feature_size, new MurmurHash());
+ } else if (hash_type == kUnicodeHash8) {
+ return new Hasher(feature_size, new UnicodeHash(8));
+ } else if (hash_type == kUnicodeHash16) {
+ return new Hasher(feature_size, new UnicodeHash(16));
+ } else if (hash_type == kXfixHash8) {
+ return new Hasher(feature_size, new XFixHash(8));
+ } else if (hash_type == kXfixHash16) {
+ return new Hasher(feature_size, new XFixHash(16));
+ } else {
+ return new Hasher(feature_size, new XFixHash(32));
+ }
+ }
+ return nullptr;
+}
+
+Hasher::Hasher(int feature_size, HashEngine* hash_engine)
+ : feature_size_(feature_size), hash_engine_(hash_engine) {
+ hash_engine_->GetHashCodes(empty_string_, &null_hash_codes_, feature_size_);
+}
+
+std::string ProjectionUnicodeHandler::LowerCaseUTF8WithSupportedUnicodes(
+ const std::pair& source, bool* first_cap,
+ bool* all_caps) const {
+ // Ideally the size of target should be less than or equal to source. But
+ // when we do to_lower the number of bytes needed to encode a unicode
+ // character could increase. To account for this 4 times the source length
+ // is allocated for target.
+ const char* csource = source.first;
+ int len = source.second;
+ auto target = std::unique_ptr(new char[len * 4]);
+ auto target_ptr = target.get();
+ int i = 0;
+ bool first_char = true;
+ bool first_cap_value = false;
+ bool all_caps_value = false;
+ while (i < len) {
+ Rune rune;
+ const int bytes_read = chartorune(&rune, const_cast(csource + i));
+ if (bytes_read == 0 || bytes_read > len - i) {
+ break;
+ }
+ i += bytes_read;
+ if (rune != Runeerror) {
+ Rune lower = tolowerrune(rune);
+ // Skip processing the unicode if exclude_nonalphaspace_unicodes_ is
+ // true and the unicode is not alpha and not space.
+ const Rune kSpaceRune = ' ';
+ if (exclude_nonalphaspace_unicodes_ && !isalpharune(lower) &&
+ lower != kSpaceRune) {
+ continue;
+ }
+ if (IsUnrestrictedVocabulary() || IsValidUnicode(lower)) {
+ const int bytes_written = runetochar(target_ptr, &lower);
+ target_ptr += bytes_written;
+
+ const bool lower_case = (lower == rune);
+ if (first_char) {
+ first_cap_value = !lower_case;
+ all_caps_value = !lower_case;
+ } else {
+ first_cap_value &= lower_case;
+ all_caps_value &= !lower_case;
+ }
+ first_char = false;
+ }
+ }
+ }
+ if (first_cap) {
+ *first_cap = first_cap_value;
+ }
+ if (all_caps) {
+ *all_caps = all_caps_value;
+ }
+ return std::string(target.get(), target_ptr);
+}
+
+void ProjectionUnicodeHandler::InitializeVocabulary(
+ const std::string& vocabulary) {
+ for (size_t i = 0, index = 0; i < vocabulary.length();) {
+ Rune rune;
+ const int bytes_read =
+ chartorune(&rune, const_cast(vocabulary.c_str() + i));
+ if (!bytes_read || bytes_read > (vocabulary.length() - i)) {
+ break;
+ }
+ i += bytes_read;
+ // Include novel lower case unicode segments as part of valid chars.
+ if (rune == Runeerror) {
+ std::clog << "Invalid rune in vocabulary.";
+ } else if (IsValidUnicode(rune)) {
+ std::clog << "Duplicate rune " << rune << " found in vocabulary.";
+ } else if (rune != tolowerrune(rune)) {
+ std::clog << "Upper case rune " << rune << " found in vocabulary.";
+ } else {
+ valid_chars_[rune] = index++;
+ }
+ }
+}
+
+// Starting from input_ptr[from], search for the next occurrence of ' ',
+// Don't search beyond input_ptr[length](non-inclusive), return -1 if not
+// found.
+inline size_t FindNextSpace(const char* input_ptr, size_t from, size_t length) {
+ size_t space_index;
+ for (space_index = from; space_index < length; space_index++) {
+ if (input_ptr[space_index] == kSpace) {
+ break;
+ }
+ }
+ return space_index == length ? kInvalid : space_index;
+}
+
+template
+void SplitBySpaceInternal(std::vector* tokens, const char* input_ptr,
+ size_t len, size_t max_input, size_t max_tokens) {
+ size_t last_index =
+ max_input == kEntireString ? len : (len < max_input ? len : max_input);
+ size_t start = 0;
+ // skip leading spaces
+ while (start < last_index && input_ptr[start] == kSpace) {
+ start++;
+ }
+ auto end = FindNextSpace(input_ptr, start, last_index);
+ while (end != kInvalid &&
+ (max_tokens == kAllTokens || tokens->size() < max_tokens - 1)) {
+ auto length = end - start;
+ if (length > 0) {
+ tokens->emplace_back(input_ptr + start, length);
+ }
+
+ start = end + 1;
+ end = FindNextSpace(input_ptr, start, last_index);
+ }
+ auto length = end == kInvalid ? (last_index - start) : (end - start);
+ if (length > 0) {
+ tokens->emplace_back(input_ptr + start, length);
+ }
+}
+
+std::vector> SplitBySpaceAsPairs(
+ const char* input_ptr, size_t len, size_t max_tokens) {
+ std::vector> tokens;
+ SplitBySpaceInternal(&tokens, input_ptr, len, kEntireString, max_tokens);
+ return tokens;
+}
+
+std::vector SplitBySpace(const char* input_ptr, size_t len,
+ size_t max_input, size_t max_tokens) {
+ std::vector tokens;
+ SplitBySpaceInternal(&tokens, input_ptr, len, max_input, max_tokens);
+ return tokens;
+}
+
+template
+void SplitByCharInternal(std::vector* tokens, const char* input_ptr,
+ size_t len, size_t max_tokens) {
+ Rune rune;
+ for (size_t i = 0; i < len;) {
+ auto bytes_read = chartorune(&rune, const_cast(input_ptr + i));
+ if (bytes_read == 0 || bytes_read > (len - i)) break;
+ tokens->emplace_back(input_ptr + i, bytes_read);
+ if (max_tokens != kInvalid && tokens->size() == max_tokens) {
+ break;
+ }
+ i += bytes_read;
+ }
+}
+
+std::vector> SplitByCharAsPairs(
+ const char* input_ptr, size_t len, size_t max_tokens) {
+ std::vector> tokens;
+ SplitByCharInternal(&tokens, input_ptr, len, max_tokens);
+ return tokens;
+}
+
+std::vector SplitByChar(const char* input_ptr, size_t len,
+ size_t max_tokens) {
+ std::vector tokens;
+ SplitByCharInternal(&tokens, input_ptr, len, max_tokens);
+ return tokens;
+}
+
+std::string JoinPairsBySpace(
+ std::vector> words) {
+ std::stringstream ss;
+ bool first = true;
+ for (auto& str_pair : words) {
+ if (first) {
+ ss << std::string(str_pair.first, str_pair.second);
+ first = false;
+ } else {
+ ss << kSpace << std::string(str_pair.first, str_pair.second);
+ }
+ }
+ return ss.str();
+}
+
+std::vector> ProjectionUnicodeHandler::Tokenize(
+ const char* str, size_t len, bool by_space, int max_tokens) {
+ return by_space ? SplitBySpaceAsPairs(str, len, max_tokens)
+ : SplitByCharAsPairs(str, len, max_tokens);
+}
diff --git a/research/seq_flow_lite/tf_ops/projection_util.h b/research/seq_flow_lite/tf_ops/projection_util.h
new file mode 100644
index 0000000000000000000000000000000000000000..5cfa0f2de14a140729b455e7dea4781742e3a8a4
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/projection_util.h
@@ -0,0 +1,146 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_UTIL_H_
+#define TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_UTIL_H_
+#include
+#include
+#include
+#include
+
+#include "libutf/utf.h"
+
+constexpr int kFirstCapOffset = 3;
+constexpr int kAllCapsOffset = 4;
+constexpr int kWordNoveltyOffset = 1;
+constexpr int kDocSizeOffset = 2;
+
+const char kMurmurHash[] = "murmur";
+const char kXfixHash8[] = "xfixhash8";
+const char kXfixHash16[] = "xfixhash16";
+const char kXfixHash32[] = "xfixhash32";
+const char kUnicodeHash8[] = "unicodehash8";
+const char kUnicodeHash16[] = "unicodehash16";
+
+class HashEngine {
+ public:
+ virtual void GetHashCodes(const std::string& word,
+ std::vector* hash_codes,
+ int feature_size) = 0;
+ virtual ~HashEngine() {}
+};
+
+// A hashing wrapper class that can hash a string and generate a hash code with
+// requested number of features (two bit values). Some of the implementations
+// are copied from murmurhash.
+class Hasher {
+ public:
+ static Hasher* CreateHasher(int feature_size,
+ const std::string& hash_type = kMurmurHash);
+ static bool SupportedHashType(const std::string& hash_type);
+ bool GetHashCodes(const std::string& word,
+ std::vector* hash_codes) {
+ if (!hash_engine_) return false;
+ if (word.empty()) {
+ *hash_codes = null_hash_codes_;
+ } else {
+ hash_codes->clear();
+ hash_engine_->GetHashCodes(word, hash_codes, feature_size_);
+ }
+ return true;
+ }
+
+ private:
+ explicit Hasher(int feature_size, HashEngine* hash_engine);
+ const std::string empty_string_ = "";
+ const int feature_size_;
+ std::unique_ptr hash_engine_;
+ std::vector null_hash_codes_;
+};
+
+// Unicode processor for tensorflow and tflite string projection ops.
+class ProjectionUnicodeHandler {
+ public:
+ // Takes an utf8 string which lists the unicodes that are supported and are
+ // part of the vocabulary of this instance. When the utf8 string is empty,
+ // all unicode segments are supported by this instance. The boolean
+ // flag exclude_nonalphaspace_unicodes is used to indicate if nonalpha and
+ // space unicode segments from the input should be stripped out.
+ // Another way to analyse the filtering logic is as below.
+ // Vocabulary acts as a allowlist when provided and all unicode set when
+ // empty. The flag exclude_nonalphaspace_unicodes when true acts as a
+ // allowlist on all alpha characters and space. It includes the entire unicode
+ // set when false. Valid unicode segments are the intersection of these 2
+ // sets.
+ explicit ProjectionUnicodeHandler(const std::string& vocabulary,
+ bool exclude_nonalphaspace_unicodes = false)
+ : exclude_nonalphaspace_unicodes_(exclude_nonalphaspace_unicodes) {
+ InitializeVocabulary(vocabulary);
+ }
+
+ // Performs language independent lower case and returns a string with
+ // supported unicode segments.
+ std::string LowerCaseUTF8WithSupportedUnicodes(
+ const std::pair& source, bool* first_cap = nullptr,
+ bool* all_caps = nullptr) const;
+
+ // Returns a boolean flag indicating if the unicode segment is part of the
+ // vocabulary.
+ bool IsValidUnicode(Rune rune) const {
+ return valid_chars_.find(rune) != valid_chars_.end();
+ }
+
+ // Returns an index in [0, |vocabulary|), if the unicode is part of the
+ // vocabulary and -1 if it's not.
+ int UnicodeIndex(Rune rune) const {
+ return IsValidUnicode(rune) ? valid_chars_.at(rune) : -1;
+ }
+
+ // Returns |vocabulary|.
+ size_t NumberOfValidUnicodes() const { return valid_chars_.size(); }
+
+ // Returns true if the vocabulary is empty which means all unicode segments
+ // are supported.
+ bool IsUnrestrictedVocabulary() const { return valid_chars_.empty(); }
+
+ // Tokenizes input by space or unicode point segmentation. Limit to
+ // max_tokens, when it is not -1.
+ static std::vector> Tokenize(
+ const std::string& input, bool by_space, int max_tokens) {
+ return Tokenize(input.c_str(), input.size(), by_space, max_tokens);
+ }
+ static std::vector> Tokenize(const char* str,
+ size_t len,
+ bool by_space,
+ int max_tokens);
+
+ private:
+ // Parses and extracts supported unicode segments from a utf8 string.
+ void InitializeVocabulary(const std::string& vocabulary);
+ std::unordered_map valid_chars_;
+ bool exclude_nonalphaspace_unicodes_;
+};
+
+static constexpr size_t kEntireString = SIZE_MAX;
+static constexpr size_t kAllTokens = SIZE_MAX;
+
+std::vector SplitBySpace(const char* input_ptr, size_t len,
+ size_t max_input, size_t max_tokens);
+
+std::vector SplitByChar(const char* input_ptr, size_t len,
+ size_t max_tokens);
+
+std::string JoinPairsBySpace(std::vector> words);
+
+#endif // TENSORFLOW_MODELS_SEQUENCE_PROJECTION_TF_OPS_PROJECTION_UTIL_H_
diff --git a/research/seq_flow_lite/tf_ops/repo.bzl b/research/seq_flow_lite/tf_ops/repo.bzl
new file mode 100644
index 0000000000000000000000000000000000000000..862f3fa5114ec7ab8d291c528c4e5ad839cc7ed8
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/repo.bzl
@@ -0,0 +1,335 @@
+"""Reverb custom external dependencies."""
+
+# Sanitize a dependency so that it works correctly from code that includes
+# reverb as a submodule.
+def clean_dep(dep):
+ return str(Label(dep))
+
+def get_python_path(ctx):
+ path = ctx.os.environ.get("PYTHON_BIN_PATH")
+ if not path:
+ fail(
+ "Could not get environment variable PYTHON_BIN_PATH. " +
+ "Check your .bazelrc file.",
+ )
+ return path
+
+def _find_tf_include_path(repo_ctx):
+ exec_result = repo_ctx.execute(
+ [
+ get_python_path(repo_ctx),
+ "-c",
+ "import tensorflow as tf; import sys; " +
+ "sys.stdout.write(tf.sysconfig.get_include())",
+ ],
+ quiet = True,
+ )
+ if exec_result.return_code != 0:
+ fail("Could not locate tensorflow installation path:\n{}"
+ .format(exec_result.stderr))
+ return exec_result.stdout.splitlines()[-1]
+
+def _find_tf_lib_path(repo_ctx):
+ exec_result = repo_ctx.execute(
+ [
+ get_python_path(repo_ctx),
+ "-c",
+ "import tensorflow as tf; import sys; " +
+ "sys.stdout.write(tf.sysconfig.get_lib())",
+ ],
+ quiet = True,
+ )
+ if exec_result.return_code != 0:
+ fail("Could not locate tensorflow installation path:\n{}"
+ .format(exec_result.stderr))
+ return exec_result.stdout.splitlines()[-1]
+
+def _find_numpy_include_path(repo_ctx):
+ exec_result = repo_ctx.execute(
+ [
+ get_python_path(repo_ctx),
+ "-c",
+ "import numpy; import sys; " +
+ "sys.stdout.write(numpy.get_include())",
+ ],
+ quiet = True,
+ )
+ if exec_result.return_code != 0:
+ fail("Could not locate numpy includes path:\n{}"
+ .format(exec_result.stderr))
+ return exec_result.stdout.splitlines()[-1]
+
+def _find_python_include_path(repo_ctx):
+ exec_result = repo_ctx.execute(
+ [
+ get_python_path(repo_ctx),
+ "-c",
+ "from distutils import sysconfig; import sys; " +
+ "sys.stdout.write(sysconfig.get_python_inc())",
+ ],
+ quiet = True,
+ )
+ if exec_result.return_code != 0:
+ fail("Could not locate python includes path:\n{}"
+ .format(exec_result.stderr))
+ return exec_result.stdout.splitlines()[-1]
+
+def _find_python_solib_path(repo_ctx):
+ exec_result = repo_ctx.execute(
+ [
+ get_python_path(repo_ctx),
+ "-c",
+ "import sys; vi = sys.version_info; " +
+ "sys.stdout.write('python{}.{}'.format(vi.major, vi.minor))",
+ ],
+ )
+ if exec_result.return_code != 0:
+ fail("Could not locate python shared library path:\n{}"
+ .format(exec_result.stderr))
+ version = exec_result.stdout.splitlines()[-1]
+ basename = "lib{}.so".format(version)
+ exec_result = repo_ctx.execute(
+ ["{}-config".format(version), "--configdir"],
+ quiet = True,
+ )
+ if exec_result.return_code != 0:
+ fail("Could not locate python shared library path:\n{}"
+ .format(exec_result.stderr))
+ solib_dir = exec_result.stdout.splitlines()[-1]
+ full_path = repo_ctx.path("{}/{}".format(solib_dir, basename))
+ if not full_path.exists:
+ fail("Unable to find python shared library file:\n{}/{}"
+ .format(solib_dir, basename))
+ return struct(dir = solib_dir, basename = basename)
+
+def _eigen_archive_repo_impl(repo_ctx):
+ tf_include_path = _find_tf_include_path(repo_ctx)
+ repo_ctx.symlink(tf_include_path, "tf_includes")
+ repo_ctx.file(
+ "BUILD",
+ content = """
+cc_library(
+ name = "includes",
+ hdrs = glob(["tf_includes/Eigen/**/*.h",
+ "tf_includes/Eigen/**",
+ "tf_includes/unsupported/Eigen/**/*.h",
+ "tf_includes/unsupported/Eigen/**"]),
+ # https://groups.google.com/forum/#!topic/bazel-discuss/HyyuuqTxKok
+ includes = ["tf_includes"],
+ visibility = ["//visibility:public"],
+)
+""",
+ executable = False,
+ )
+
+def _nsync_includes_repo_impl(repo_ctx):
+ tf_include_path = _find_tf_include_path(repo_ctx)
+ repo_ctx.symlink(tf_include_path + "/external", "nsync_includes")
+ repo_ctx.file(
+ "BUILD",
+ content = """
+cc_library(
+ name = "includes",
+ hdrs = glob(["nsync_includes/nsync/public/*.h"]),
+ includes = ["nsync_includes"],
+ visibility = ["//visibility:public"],
+)
+""",
+ executable = False,
+ )
+
+def _zlib_includes_repo_impl(repo_ctx):
+ tf_include_path = _find_tf_include_path(repo_ctx)
+ repo_ctx.symlink(
+ tf_include_path + "/external/zlib",
+ "zlib",
+ )
+ repo_ctx.file(
+ "BUILD",
+ content = """
+cc_library(
+ name = "includes",
+ hdrs = glob(["zlib/**/*.h"]),
+ includes = ["zlib"],
+ visibility = ["//visibility:public"],
+)
+""",
+ executable = False,
+ )
+
+def _snappy_includes_repo_impl(repo_ctx):
+ tf_include_path = _find_tf_include_path(repo_ctx)
+ repo_ctx.symlink(
+ tf_include_path + "/external/snappy",
+ "snappy",
+ )
+ repo_ctx.file(
+ "BUILD",
+ content = """
+cc_library(
+ name = "includes",
+ hdrs = glob(["snappy/*.h"]),
+ includes = ["snappy"],
+ visibility = ["//visibility:public"],
+)
+""",
+ executable = False,
+ )
+
+def _protobuf_includes_repo_impl(repo_ctx):
+ tf_include_path = _find_tf_include_path(repo_ctx)
+ repo_ctx.symlink(tf_include_path, "tf_includes")
+ repo_ctx.symlink(Label("//third_party:protobuf.BUILD"), "BUILD")
+
+def _tensorflow_includes_repo_impl(repo_ctx):
+ tf_include_path = _find_tf_include_path(repo_ctx)
+ repo_ctx.symlink(tf_include_path, "tensorflow_includes")
+ repo_ctx.file(
+ "BUILD",
+ content = """
+cc_library(
+ name = "includes",
+ hdrs = glob(
+ [
+ "tensorflow_includes/**/*.h",
+ "tensorflow_includes/third_party/eigen3/**",
+ ],
+ exclude = ["tensorflow_includes/absl/**/*.h"],
+ ),
+ includes = ["tensorflow_includes"],
+ deps = [
+ "@eigen_archive//:eigen",
+ "@protobuf_archive//:includes",
+ "@zlib_includes//:includes",
+ "@snappy_includes//:includes",
+ ],
+ visibility = ["//visibility:public"],
+)
+filegroup(
+ name = "protos",
+ srcs = glob(["tensorflow_includes/**/*.proto"]),
+ visibility = ["//visibility:public"],
+)
+""",
+ executable = False,
+ )
+
+def _tensorflow_solib_repo_impl(repo_ctx):
+ tf_lib_path = _find_tf_lib_path(repo_ctx)
+ repo_ctx.symlink(tf_lib_path, "tensorflow_solib")
+ repo_ctx.file(
+ "BUILD",
+ content = """
+cc_library(
+ name = "framework_lib",
+ srcs = ["tensorflow_solib/libtensorflow_framework.so.2"],
+ deps = ["@python_includes", "@python_includes//:numpy_includes"],
+ visibility = ["//visibility:public"],
+)
+""",
+ )
+
+def _python_includes_repo_impl(repo_ctx):
+ python_include_path = _find_python_include_path(repo_ctx)
+ python_solib = _find_python_solib_path(repo_ctx)
+ repo_ctx.symlink(python_include_path, "python_includes")
+ numpy_include_path = _find_numpy_include_path(repo_ctx)
+ repo_ctx.symlink(numpy_include_path, "numpy_includes")
+ repo_ctx.symlink(
+ "{}/{}".format(python_solib.dir, python_solib.basename),
+ python_solib.basename,
+ )
+
+ # Note, "@python_includes" is a misnomer since we include the
+ # libpythonX.Y.so in the srcs, so we can get access to python's various
+ # symbols at link time.
+ repo_ctx.file(
+ "BUILD",
+ content = """
+cc_library(
+ name = "python_includes",
+ hdrs = glob(["python_includes/**/*.h"]),
+ srcs = ["{}"],
+ includes = ["python_includes"],
+ visibility = ["//visibility:public"],
+)
+cc_library(
+ name = "numpy_includes",
+ hdrs = glob(["numpy_includes/**/*.h"]),
+ includes = ["numpy_includes"],
+ visibility = ["//visibility:public"],
+)
+""".format(python_solib.basename),
+ executable = False,
+ )
+
+def cc_tf_configure():
+ """Autoconf pre-installed tensorflow repo."""
+ make_nsync_repo = repository_rule(
+ implementation = _nsync_includes_repo_impl,
+ )
+ make_nsync_repo(name = "nsync_includes")
+ make_zlib_repo = repository_rule(
+ implementation = _zlib_includes_repo_impl,
+ )
+ make_zlib_repo(name = "zlib_includes")
+ make_snappy_repo = repository_rule(
+ implementation = _snappy_includes_repo_impl,
+ )
+ make_snappy_repo(name = "snappy_includes")
+ make_protobuf_repo = repository_rule(
+ implementation = _protobuf_includes_repo_impl,
+ )
+ make_protobuf_repo(name = "protobuf_archive")
+ make_tfinc_repo = repository_rule(
+ implementation = _tensorflow_includes_repo_impl,
+ )
+ make_tfinc_repo(name = "tensorflow_includes")
+ make_tflib_repo = repository_rule(
+ implementation = _tensorflow_solib_repo_impl,
+ )
+ make_tflib_repo(name = "tensorflow_solib")
+ make_python_inc_repo = repository_rule(
+ implementation = _python_includes_repo_impl,
+ )
+ make_python_inc_repo(name = "python_includes")
+
+def _reverb_protoc_archive(ctx):
+ version = ctx.attr.version
+ sha256 = ctx.attr.sha256
+
+ override_version = ctx.os.environ.get("REVERB_PROTOC_VERSION")
+ if override_version:
+ sha256 = ""
+ version = override_version
+
+ urls = [
+ "https://github.com/protocolbuffers/protobuf/releases/download/v%s/protoc-%s-linux-x86_64.zip" % (version, version),
+ ]
+ ctx.download_and_extract(
+ url = urls,
+ sha256 = sha256,
+ )
+
+ ctx.file(
+ "BUILD",
+ content = """
+filegroup(
+ name = "protoc_bin",
+ srcs = ["bin/protoc"],
+ visibility = ["//visibility:public"],
+)
+""",
+ executable = False,
+ )
+
+reverb_protoc_archive = repository_rule(
+ implementation = _reverb_protoc_archive,
+ attrs = {
+ "version": attr.string(mandatory = True),
+ "sha256": attr.string(mandatory = True),
+ },
+)
+
+def reverb_protoc_deps(version, sha256):
+ reverb_protoc_archive(name = "protobuf_protoc", version = version, sha256 = sha256)
diff --git a/research/seq_flow_lite/tf_ops/sequence_string_projection.cc b/research/seq_flow_lite/tf_ops/sequence_string_projection.cc
new file mode 100644
index 0000000000000000000000000000000000000000..dd99bbe6af2af9976b59d77c7442ad86ad8bd086
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/sequence_string_projection.cc
@@ -0,0 +1,362 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tf_ops/projection_normalizer_util.h" // seq_flow_lite
+#include "tf_ops/projection_tokenizer_util.h" // seq_flow_lite
+#include "tf_ops/projection_util.h" // seq_flow_lite
+#include "tf_ops/text_distorter.h" // seq_flow_lite
+#include "absl/container/flat_hash_map.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+
+using ::tensorflow::int32;
+using ::tensorflow::int64;
+using ::tensorflow::OpKernel;
+using ::tensorflow::OpKernelConstruction;
+using ::tensorflow::OpKernelContext;
+using ::tensorflow::Tensor;
+using ::tensorflow::TensorShape;
+using ::tensorflow::TensorShapeUtils;
+using ::tensorflow::uint64;
+using ::tensorflow::errors::InvalidArgument;
+
+using tensorflow::shape_inference::DimensionHandle;
+using tensorflow::shape_inference::InferenceContext;
+
+constexpr char kBeginTokenTSP[] = "";
+constexpr char kEndTokenTSP[] = "";
+
+float* AllocateTensor(OpKernelContext* ctx, const std::string& tensor_name,
+ const TensorShape& tensor_shape) {
+ Tensor* tensor = nullptr;
+ auto status = ctx->allocate_output(tensor_name, tensor_shape, &tensor);
+ if (!TF_PREDICT_TRUE(status.ok())) {
+ ctx->CtxFailureWithWarning(__FILE__, __LINE__, status);
+ return nullptr;
+ }
+ return &tensor->flat()(0);
+}
+
+class SequenceStringProjectionOp : public OpKernel {
+ public:
+ explicit SequenceStringProjectionOp(OpKernelConstruction* context)
+ : OpKernel(context) {
+ OP_REQUIRES_OK(context, context->GetAttr("feature_size", &feature_size_));
+ std::string hashtype;
+ OP_REQUIRES_OK(context, context->GetAttr("hashtype", &hashtype));
+ hasher_ =
+ absl::WrapUnique(Hasher::CreateHasher(feature_size_, hashtype));
+ CHECK(hasher_);
+ float distortion_probability = 0.0;
+ OP_REQUIRES_OK(context, context->GetAttr("distortion_probability",
+ &distortion_probability));
+ text_distorter_ = absl::make_unique(distortion_probability);
+ OP_REQUIRES_OK(context,
+ context->GetAttr("split_on_space", &split_on_space_));
+ OP_REQUIRES_OK(context, context->GetAttr("max_splits", &max_splits_));
+ OP_REQUIRES_OK(context, context->GetAttr("vocabulary", &vocabulary_));
+ bool add_bos_tag;
+ OP_REQUIRES_OK(context, context->GetAttr("add_bos_tag", &add_bos_tag));
+ bos_tag_ = add_bos_tag ? 1 : 0;
+ bool add_eos_tag;
+ OP_REQUIRES_OK(context, context->GetAttr("add_eos_tag", &add_eos_tag));
+ eos_tag_ = add_eos_tag ? 1 : 0;
+ // When word_novelty_bits is set to a positive integer, the last feature
+ // generated by the op captures the token frequency.
+ OP_REQUIRES_OK(context,
+ context->GetAttr("word_novelty_bits", &word_novelty_bits_));
+ CHECK_GE(word_novelty_bits_, 0);
+ CHECK_LE(word_novelty_bits_, 7);
+ if (word_novelty_bits_ != 0) {
+ CHECK_GE(feature_size_, 1);
+ }
+ // When doc_size_levels is set to a positive integer, the second to last
+ // feature generated by the op is derived from the log of the document
+ // size.
+ OP_REQUIRES_OK(context,
+ context->GetAttr("doc_size_levels", &doc_size_levels_));
+ CHECK_GE(doc_size_levels_, 0);
+ CHECK_LE(doc_size_levels_, 16);
+ if (doc_size_levels_ != 0) {
+ CHECK_GE(feature_size_, 2);
+ }
+ word_novelty_offset_ = 1.0f / (1 << word_novelty_bits_);
+ bool exclude_nonalphaspace_unicodes;
+ OP_REQUIRES_OK(context, context->GetAttr("exclude_nonalphaspace_unicodes",
+ &exclude_nonalphaspace_unicodes));
+ if (!vocabulary_.empty()) {
+ CHECK(!exclude_nonalphaspace_unicodes);
+ }
+ unicode_handler_ = absl::make_unique(
+ vocabulary_, exclude_nonalphaspace_unicodes);
+ vocabulary_size_ = unicode_handler_->NumberOfValidUnicodes();
+
+ bool normalize_repetition;
+ OP_REQUIRES_OK(context, context->GetAttr("normalize_repetition",
+ &normalize_repetition));
+ std::string separators;
+ OP_REQUIRES_OK(context, context->GetAttr("token_separators", &separators));
+ if (!separators.empty() || normalize_repetition) {
+ projection_normalizer_ = absl::make_unique(
+ separators, normalize_repetition);
+ }
+
+ OP_REQUIRES_OK(context, context->GetAttr("add_first_cap_feature",
+ &add_first_cap_feature_));
+ CHECK_GE(add_first_cap_feature_, 0.0);
+ CHECK_LE(add_first_cap_feature_, 1.0);
+ if (add_first_cap_feature_ > 0.0) {
+ CHECK_GE(feature_size_, 3);
+ }
+
+ OP_REQUIRES_OK(context, context->GetAttr("add_all_caps_feature",
+ &add_all_caps_feature_));
+ CHECK_GE(add_all_caps_feature_, 0.0);
+ CHECK_LE(add_all_caps_feature_, 1.0);
+ if (add_all_caps_feature_ > 0.0) {
+ CHECK_GE(feature_size_, 4);
+ }
+ }
+
+ void Compute(OpKernelContext* ctx) override {
+ const Tensor* input_tensor;
+ OP_REQUIRES_OK(ctx, ctx->input("input", &input_tensor));
+ OP_REQUIRES(ctx, TensorShapeUtils::IsVector(input_tensor->shape()),
+ InvalidArgument("input must be a vector, got shape: ",
+ input_tensor->shape().DebugString()));
+
+ auto input_vec = input_tensor->vec<::tensorflow::tstring>();
+ const int64 batch_size = input_vec.dimension(0);
+ std::vector>> words_batches;
+ int64 max_seq_len = 0;
+ words_batches.reserve(batch_size);
+ std::vector normalized_input_vec(batch_size);
+ for (int64 i = 0; i < batch_size; ++i) {
+ std::vector> words;
+ if (projection_normalizer_ == nullptr) {
+ words =
+ unicode_handler_->Tokenize(input_vec(i).data(), input_vec(i).size(),
+ split_on_space_, max_splits_);
+ } else {
+ normalized_input_vec[i] = projection_normalizer_->Normalize(
+ input_vec(i).data(), input_vec(i).size(), SIZE_MAX);
+ words = unicode_handler_->Tokenize(normalized_input_vec[i],
+ split_on_space_, max_splits_);
+ }
+ const int64 seq_len =
+ static_cast(bos_tag_ + words.size() + eos_tag_);
+ CHECK_GT(seq_len, 0);
+ max_seq_len = std::max(max_seq_len, seq_len);
+ words_batches.emplace_back(std::move(words));
+ }
+
+ auto projection =
+ AllocateTensor(ctx, "projection",
+ TensorShape({batch_size, max_seq_len, feature_size_}));
+ AllocateTensor(ctx, "dummy_output", TensorShape({1}));
+ auto sequence_length =
+ AllocateTensor(ctx, "sequence_length", TensorShape({batch_size}));
+ if (!projection || !sequence_length) {
+ LOG(ERROR) << "Unable to create buffer!";
+ return;
+ }
+
+ const float mapping_table[4] = {0, 1, -1, 0};
+ const int increment = 32;
+ std::vector hash_codes;
+ absl::flat_hash_map word_counter;
+ for (int64 i = 0; i < batch_size; ++i) {
+ word_counter.clear();
+ const int64 num_tokens = words_batches[i].size();
+ sequence_length[i] = bos_tag_ + num_tokens + eos_tag_;
+ int64 offset0 = i * max_seq_len * feature_size_;
+ // Calculate doc_size_feature in [0, infinity)
+ float doc_size_feature =
+ (doc_size_levels_ != 0)
+ ? std::log2(static_cast(num_tokens)) / doc_size_levels_
+ : 0.0f;
+ // Rescale doc_size_feature to [-1, 1].
+ doc_size_feature = std::min(doc_size_feature, 1.0f) * 2.0f - 1.0f;
+ for (int64 j = -bos_tag_; j < num_tokens + eos_tag_; ++j) {
+ std::string word;
+ bool first_cap = false;
+ bool all_caps = false;
+ if (j < 0) {
+ // Use a special tag for begin of sentence.
+ word = kBeginTokenTSP;
+ } else if (j < num_tokens) {
+ auto uword = icu::UnicodeString::fromUTF8(
+ unicode_handler_->LowerCaseUTF8WithSupportedUnicodes(
+ words_batches[i][j], &first_cap, &all_caps));
+ word = text_distorter_->DistortText(&uword);
+ } else {
+ // Use a special tag for end of sentence.
+ CHECK_EQ(eos_tag_, 1);
+ word = kEndTokenTSP;
+ }
+ hasher_->GetHashCodes(word, &hash_codes);
+ for (int hindex = 0, k = 0; hindex < hash_codes.size(); hindex++) {
+ auto hash = hash_codes[hindex];
+ for (int kmax = std::min(k + increment, feature_size_); k < kmax;) {
+ projection[offset0 + k++] = mapping_table[hash & 0x3];
+ hash >>= 2;
+ }
+ }
+ if (word_novelty_bits_ != 0 && !hash_codes.empty()) {
+ const auto word_hash = hash_codes[0];
+ projection[offset0 + feature_size_ - kWordNoveltyOffset] =
+ std::min((word_counter[word_hash]++ * word_novelty_offset_),
+ 1.0f) *
+ 2.0f -
+ 1.0f;
+ }
+ if (doc_size_levels_ != 0) {
+ projection[offset0 + feature_size_ - kDocSizeOffset] =
+ doc_size_feature;
+ }
+ if (add_first_cap_feature_ > 0.0f) {
+ if (text_distorter_->BernouilleSample(add_first_cap_feature_)) {
+ projection[offset0 + feature_size_ - kFirstCapOffset] =
+ first_cap ? 1.0 : -1.0;
+ } else {
+ projection[offset0 + feature_size_ - kFirstCapOffset] = 0.0;
+ }
+ }
+ if (add_all_caps_feature_ > 0.0f) {
+ if (text_distorter_->BernouilleSample(add_all_caps_feature_)) {
+ projection[offset0 + feature_size_ - kAllCapsOffset] =
+ all_caps ? 1.0 : -1.0;
+ } else {
+ projection[offset0 + feature_size_ - kAllCapsOffset] = 0.0;
+ }
+ }
+ offset0 += feature_size_;
+ }
+ const int pending = (max_seq_len - (bos_tag_ + num_tokens + eos_tag_));
+ memset(projection + offset0, 0, pending * feature_size_ * sizeof(float));
+ }
+ }
+
+ private:
+ int32 feature_size_;
+ std::unique_ptr hasher_;
+ std::unique_ptr text_distorter_;
+ std::unique_ptr unicode_handler_;
+ std::unique_ptr projection_normalizer_;
+ std::string vocabulary_;
+ int vocabulary_size_;
+ int32 max_splits_;
+ bool split_on_space_;
+ int eos_tag_;
+ int bos_tag_;
+ int word_novelty_bits_;
+ int doc_size_levels_;
+ float word_novelty_offset_;
+ float add_first_cap_feature_;
+ float add_all_caps_feature_;
+};
+
+REGISTER_KERNEL_BUILDER(
+ Name("SequenceStringProjection").Device(::tensorflow::DEVICE_CPU),
+ SequenceStringProjectionOp);
+
+REGISTER_OP("SequenceStringProjection")
+ .Input("input: string")
+ .Output("projection: float32")
+ .Output("dummy_output: float32")
+ .Output("sequence_length: float32")
+ .Attr("feature_size: int")
+ .Attr("distortion_probability: float = 0.0")
+ .Attr("vocabulary: string = ''")
+ .Attr("hashtype: string = 'murmur'")
+ .Attr("max_splits: int = -1")
+ .Attr("exclude_nonalphaspace_unicodes: bool = False")
+ .Attr("add_bos_tag: bool = False")
+ .Attr("add_eos_tag: bool = True")
+ .Attr("add_first_cap_feature: float = 0.0")
+ .Attr("add_all_caps_feature: float = 0.0")
+ .Attr("word_novelty_bits: int = 0")
+ .Attr("doc_size_levels: int = 0")
+ .Attr("split_on_space: bool = True")
+ .Attr("token_separators: string = ''")
+ .Attr("normalize_repetition: bool = false")
+ .SetShapeFn([](InferenceContext* c) {
+ DimensionHandle size;
+
+ int32 feature_size;
+ TF_RETURN_IF_ERROR(c->GetAttr("feature_size", &feature_size));
+ const int kMaxFeatureSize = 4096;
+ CHECK_GE(feature_size, 0);
+ CHECK_LE(feature_size, kMaxFeatureSize);
+ auto batch_size = c->Dim(c->input(0), 0);
+ c->set_output(0, c->MakeShape({batch_size, InferenceContext::kUnknownDim,
+ feature_size}));
+ c->set_output(1, c->MakeShape({1}));
+ c->set_output(2, c->MakeShape({batch_size}));
+ return tensorflow::Status::OK();
+ })
+ .Doc(R"doc(
+This op referred to as Ternary Sequence String Projection op (TSP), tokenizes
+input text either on space or unicode boundary. Fingerprint for each token is
+computed using murmur hash and bit features are extracted from the fingerprint
+that maps every 2 bits to the ternary output {-1, 0, 1}. This effectively turns
+a batch of text input into a ternary rank 3 tensor (in float format) of shape
+[batch size, max token length, requested number of features].
+
+Input(s):
+- input: A string tensor with batch size number of elements.
+
+Attribute(s):
+- feature_size: Length of the ternary vector generated for each token.
+- distortion_probability: When non zero distort the input text with this
+ probability. Helps as a regularization method when training data set is
+ small.
+- vocabulary: When not empty provides a list of unique unicode characters that
+ will be allowed in the input text before fingerprinting. Another way to
+ say it is that the vocabulary is an optional character allowlist for the
+ input text. It helps normalize the text.
+- hashtype: Hashing method to use for projection.
+- max_splits: Maximum number of tokens that are allowed. It helps restrict the
+ max token length of the projection output. When the value is -1 the op
+ does not restrict the number of tokens in the output.
+- exclude_nonalphaspace_unicodes: When true excludes all unicodes that are
+ not alphabets or space character. This is multilingual. Though the effect
+ of this flag can be achieved using vocabulary, the vocabulary will have to
+ be very large for multilingual input.
+- add_bos_tag: When true inserts a begin of sentence tag.
+- add_eos_tag: When true inserts a end of sentence tag.
+- word_novelty_bits: When true adds a special feature to the ternary output
+ that captures the frequency of occurrence of a particular token. This is an
+ experimental feature.
+- doc_size_levels: When true adds a special feature to the ternary projection
+ output the document size in log scale. This is an experimental feature.
+- split_on_space: When true tokenization is done on space segmentation.
+ Otherwise tokenization is done by segmenting on unicode boundary.
+- add_first_cap_feature: Specifies the probability with which a feature to the
+ resulting projection tensor that helps discriminate if the input token is
+ Camel case will be added.
+- add_all_caps_feature: Specifies the probability with which a feature to the
+ resulting projection tensor that helps discriminate if the input token is
+ ALLCAPS will be added.
+
+Output(s):
+- projection: Floating point tensor with ternary values of shape
+ [batch size, max token length, requested number of features].
+- dummy_output: Ignore this output, will be eliminated in a subsequent version.
+- sequence_length: Batch size length vector containing the number of tokens for
+ each input text entry.
+)doc");
diff --git a/research/seq_flow_lite/tf_ops/sequence_string_projection_op_v2.cc b/research/seq_flow_lite/tf_ops/sequence_string_projection_op_v2.cc
new file mode 100644
index 0000000000000000000000000000000000000000..ac3ec3f4f0ed4f205ae45c92eda8b699180714b8
--- /dev/null
+++ b/research/seq_flow_lite/tf_ops/sequence_string_projection_op_v2.cc
@@ -0,0 +1,233 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tf_ops/projection_normalizer_util.h" // seq_flow_lite
+#include "tf_ops/projection_util.h" // seq_flow_lite
+#include "tf_ops/text_distorter.h" // seq_flow_lite
+
+using ::tensorflow::int32;
+using ::tensorflow::int64;
+using ::tensorflow::OpKernel;
+using ::tensorflow::OpKernelConstruction;
+using ::tensorflow::OpKernelContext;
+using ::tensorflow::Tensor;
+using ::tensorflow::TensorShapeUtils;
+using ::tensorflow::uint64;
+using ::tensorflow::errors::InvalidArgument;
+
+using ::tensorflow::shape_inference::DimensionHandle;
+using ::tensorflow::shape_inference::InferenceContext;
+using ::tensorflow::shape_inference::ShapeHandle;
+
+constexpr char kBeginTokenTSP[] = "";
+constexpr char kEndTokenTSP[] = "";
+constexpr float kMappingTable[4] = {0, 1, -1, 0};
+constexpr int kIncrement = 32;
+
+template
+class SequenceStringProjectionOpV2 : public OpKernel {
+ public:
+ explicit SequenceStringProjectionOpV2(OpKernelConstruction* context)
+ : OpKernel(context) {
+ OP_REQUIRES_OK(context, context->GetAttr("feature_size", &feature_size_));
+ std::string hashtype;
+ OP_REQUIRES_OK(context, context->GetAttr("hashtype", &hashtype));
+ hasher_ =
+ absl::WrapUnique(Hasher::CreateHasher(feature_size_, hashtype));
+
+ float distortion_probability = 0.0;
+ OP_REQUIRES_OK(context, context->GetAttr("distortion_probability",
+ &distortion_probability));
+ text_distorter_ = absl::make_unique(distortion_probability);
+
+ OP_REQUIRES_OK(context, context->GetAttr("vocabulary", &vocabulary_));
+ unicode_handler_ = absl::make_unique(vocabulary_);
+
+ bool add_bos_tag;
+ OP_REQUIRES_OK(context, context->GetAttr("add_bos_tag", &add_bos_tag));
+ bos_tag_ = add_bos_tag ? 1 : 0;
+
+ bool add_eos_tag;
+ OP_REQUIRES_OK(context, context->GetAttr("add_eos_tag", &add_eos_tag));
+ eos_tag_ = add_eos_tag ? 1 : 0;
+
+ bool normalize_repetition;
+ OP_REQUIRES_OK(context, context->GetAttr("normalize_repetition",
+ &normalize_repetition));
+ if (normalize_repetition) {
+ projection_normalizer_ = absl::make_unique
![](\"https://www.tensorflow.org/images/tf_logo_32px.png\"){kind=logo}
View on TensorFlow.org\n",
+ " ![](\"https://www.tensorflow.org/images/colab_logo_32px.png\"){kind=logo}
Run in Google Colab\n",
+ " ![](\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\")
View source on GitHub\n",
+ " ![](\"https://www.tensorflow.org/images/download_logo_32px.png\"){kind=logo}
Download notebook\n",
+ " 
+
-
-
+